{"_id": {"$oid": "6a26c67c3572c06bc1455df6"}, "test_id": "T0958", "benchmark": "business_letters", "date": "2026-04-21", "tags": {"document-type": ["letter"], "writing": ["typed", "handwritten"], "century": [20], "language": ["de"], "layout": ["prose"], "entry-type": ["person", "location"], "task": ["information-extraction"]}, "contributors": ["eric_decker", "maximilian_hindermann", "lea_kasper", "anthea_alberto", "pema_frick", "jose_luis_losada_palenzuela", "sorin_marti", "elena_spadini"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3.5-plus-02-15", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge and an expert in the field of 20th century Swiss history", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "            IDENTITY and PURPOSE:\n\n            You are presented with a series of images constituting a historical letter. Your task is to extract the\n            values of the following keys from the letter and return them in a JSON file where the values\n            corresponding to each key should be stored as a list, even if there is only a single value for a\n            key:\n\n            - letter_title: Title of the letter.\n            - sender_persons: Name(s) of the person(s) who wrote the letter.\n            - send_date: The exact or approximate date the letter was written.\n            - receiver_persons: Name(s) of the person(s) who received the letter.\n\n            Take a deep breath and think step by step about how to best accomplish this goal. Map out all the claims\n            and implications on a virtual whiteboard in your mind. Do not use OCR. Use the ISO format YYYY-MM-DD for\n            dates. If a piece of information is not included in the letter, set the value for the corresponding key\n            to \"null\". Do not return anything except the JSON file.\n\n            EXAMPLE:\n\n            {\n                \"letter_title\": [\"Petition for Environmental Protection\"],\n                \"send_date\": [\"1993-03-12\"],\n                \"sender_persons\": [\"Lisa Simpson\"],\n                \"receiver_persons\": [\"Mayor Joe Quimby\", \"Seymour Skinner\"],\n            }\n\n            OUTPUT:", "results": {"text": "{\"metadata\":{\"send_date\":[\"1928-04-02\"],\"letter_title\":[\"Betrifft Schiffahrt Basel-Rheinfelden\"],\"sender_persons\":[\"Basler Rheinschiffahrt-Aktiengesellschaft\"],\"receiver_persons\":[\"Basler Personenschiffahrtsgesellschaft\",\"Kur- und Verkehrsverein Rheinfelden\"]}}", "model": "qwen/qwen3.5-plus-20260216", "provider": "openrouter", "finish_reason": "stop", "usage": {"input_tokens": 10697, "output_tokens": 599, "total_tokens": 11296, "input_cost_usd": 0.00278122, "output_cost_usd": 0.00093444, "estimated_cost_usd": 0.00371566}, "duration": 15.597443580627441, "timestamp": "2026-04-21T13:44:00.622864", "parsed": {"metadata": {"send_date": ["1928-04-02"], "letter_title": ["Betrifft Schiffahrt Basel-Rheinfelden"], "sender_persons": ["Basler Rheinschiffahrt-Aktiengesellschaft"], "receiver_persons": ["Basler Personenschiffahrtsgesellschaft", "Kur- und Verkehrsverein Rheinfelden"], "document_number": "letter22"}}, "conversation_id": "7554c728-23e5-4ee3-906b-b31a3133f113", "score": {"send_date_tp": 1, "send_date_fp": 0, "send_date_fn": 0, "sender_persons_tp": 0, "sender_persons_fp": 1, "sender_persons_fn": 2, "receiver_persons_tp": 0, "receiver_persons_fp": 2, "receiver_persons_fn": 0}, "raw_response": "{\"id\":\"gen-1776771825-4Zd85gaCmvTavyTJMxgc\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"metadata\\\": {\\n    \\\"send_date\\\": [\\\"1928-04-02\\\"],\\n    \\\"letter_title\\\": [\\\"Betrifft Schiffahrt Basel-Rheinfelden\\\"],\\n    \\\"sender_persons\\\": [\\\"Basler Rheinschiffahrt-Aktiengesellschaft\\\"],\\n    \\\"receiver_persons\\\": [\\\"Basler Personenschiffahrtsgesellschaft\\\", \\\"Kur- und Verkehrsverein Rheinfelden\\\"]\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"metadata\":{\"send_date\":[\"1928-04-02\"],\"letter_title\":[\"Betrifft Schiffahrt Basel-Rheinfelden\"],\"sender_persons\":[\"Basler Rheinschiffahrt-Aktiengesellschaft\"],\"receiver_persons\":[\"Basler Personenschiffahrtsgesellschaft\",\"Kur- und Verkehrsverein Rheinfelden\"]}},\"reasoning\":\"\uff0c\u6211\u73b0\u5728\u9700\u8981\u5904\u7406\u7528\u6237\u63d0\u4f9b\u7684\u8fd9\u4e2a\u5386\u53f2\u4fe1\u4ef6\uff0c\u5e76\u63d0\u53d6\u51fa\u6307\u5b9a\u7684\u4fe1\u606f\u3002\u9996\u5148\uff0c\u7528\u6237\u8981\u6c42\u4ece\u56db\u5f20\u56fe\u7247\u4e2d\u63d0\u53d6\u4fe1\u4ef6\u7684\u6807\u9898\u3001\u53d1\u9001\u65e5\u671f\u3001\u53d1\u4ef6\u4eba\u548c\u6536\u4ef6\u4eba\uff0c\u5e76\u4ee5JSON\u683c\u5f0f\u8fd4\u56de\u3002\u6211\u9700\u8981\u4ed4\u7ec6\u5206\u6790\u6bcf\u4e00\u9875\u7684\u5185\u5bb9\uff0c\u786e\u4fdd\u4e0d\u9057\u6f0f\u4efb\u4f55\u5173\u952e\u4fe1\u606f\u3002\\n\\n\u9996\u5148\u770b\u7b2c\u4e00\u5f20\u56fe\u7247\u3002\u4fe1\u5934\u663e\u793a\u53d1\u4ef6\u4eba\u662fBasler Rheinschiffahrt-Aktiengesellschaft\uff0c\u5730\u5740\u5728Basel\u3002\u65e5\u671f\u662f1928\u5e744\u67082\u65e5\uff0c\u5199\u5728\u53f3\u4e0a\u89d2\u3002\u6536\u4ef6\u4eba\u662fBasler Personenschiffahrtsgesellschaft\uff0c\u540c\u6837\u5728Basel\u3002\u8fd9\u91cc\u53ef\u80fd\u9700\u8981\u6ce8\u610f\uff0c\u867d\u7136\u53d1\u4ef6\u4eba\u548c\u6536\u4ef6\u4eba\u90fd\u662f\u516c\u53f8\uff0c\u4f46\u7528\u6237\u8981\u6c42\u7684\u662f\u4eba\u540d\uff0c\u6240\u4ee5\u53ef\u80fd\u9700\u8981\u786e\u8ba4\u662f\u5426\u6709\u5177\u4f53\u7684\u4eba\u540d\u3002\u4e0d\u8fc7\u4fe1\u4ef6\u672b\u5c3e\u7684\u7b7e\u540d\u90e8\u5206\u6709\u624b\u5199\u7b7e\u540d\uff0c\u4f46\u53ef\u80fd\u65e0\u6cd5\u8fa8\u8ba4\u5177\u4f53\u540d\u5b57\uff0c\u6240\u4ee5\u53d1\u4ef6\u4eba\u53ef\u80fd\u53ea\u80fd\u586b\u516c\u53f8\u540d\u79f0\u3002\\n\\n\u7b2c\u4e8c\u5f20\u56fe\u7247\u7684\u6807\u9898\u662f\u201cBetrifft Schiffahrt Basel-Rheinfelden\u201d\uff0c\u8fd9\u5e94\u8be5\u662f\u4fe1\u4ef6\u7684\u6807\u9898\u3002\u6536\u4ef6\u4eba\u90e8\u5206\u63d0\u5230Kur- und Verkehrsverein, Rheinfelden\uff0c\u6240\u4ee5\u6536\u4ef6\u4eba\u5e94\u8be5\u662f\u8fd9\u4e2a\u534f\u4f1a\u3002\u53d1\u4ef6\u4eba\u4ecd\u7136\u662fBasler Rheinschiffahrt-Aktiengesellschaft\u3002\\n\\n\u7b2c\u4e09\u548c\u7b2c\u56db\u5f20\u56fe\u7247\u7ee7\u7eed\u8ba8\u8bba\u540c\u4e00\u4e3b\u9898\uff0c\u65e5\u671f\u90fd\u662f1928\u5e744\u67082\u65e5\uff0c\u786e\u8ba4\u53d1\u9001\u65e5\u671f\u6b63\u786e\u3002\u6536\u4ef6\u4eba\u4ecd\u7136\u662fKur- und Verkehrsverein, Rheinfelden\uff0c\u800c\u53d1\u4ef6\u4eba\u6ca1\u6709\u53d8\u5316\u3002\\n\\n\u73b0\u5728\u9700\u8981\u6574\u7406\u8fd9\u4e9b\u4fe1\u606f\u3002\u6807\u9898\u5e94\u8be5\u662f\u201cBetrifft Schiffahrt Basel-Rheinfelden\u201d\uff0c\u53d1\u9001\u65e5\u671f\u662f1928-04-02\u3002\u53d1\u4ef6\u4eba\u662fBasler Rheinschiffahrt-Aktiengesellschaft\uff0c\u4f46\u7528\u6237\u8981\u6c42\u7684\u662f\u4eba\u540d\uff0c\u8fd9\u91cc\u53ef\u80fd\u9700\u8981\u68c0\u67e5\u662f\u5426\u6709\u7b7e\u540d\u7684\u4eba\u540d\u3002\u7b2c\u4e00\u9875\u672b\u5c3e\u6709\u4e24\u4e2a\u7b7e\u540d\uff0c\u4f46\u53ef\u80fd\u65e0\u6cd5\u8bc6\u522b\u5177\u4f53\u540d\u5b57\uff0c\u6240\u4ee5\u53ef\u80fd\u53ea\u80fd\u586b\u516c\u53f8\u540d\u79f0\u3002\u6536\u4ef6\u4eba\u662fBasler Personenschiffahrtsgesellschaft\u548cKur- und Verkehrsverein, Rheinfelden\uff0c\u4f46\u6839\u636e\u7b2c\u4e8c\u9875\u7684\u6536\u4ef6\u4eba\u90e8\u5206\uff0c\u4e3b\u8981\u6536\u4ef6\u4eba\u662fKur- und Verkehrsverein\uff0c\u800cBasler Personenschiffahrtsgesellschaft\u53ef\u80fd\u662f\u6284\u9001\u5bf9\u8c61\u3002\u9700\u8981\u786e\u8ba4\u7528\u6237\u662f\u5426\u9700\u8981\u6240\u6709\u63d0\u5230\u7684\u6536\u4ef6\u4eba\uff0c\u8fd8\u662f\u4e3b\u8981\u6536\u4ef6\u4eba\u3002\u6839\u636e\u793a\u4f8b\uff0c\u6536\u4ef6\u4eba\u53ef\u80fd\u6709\u591a\u4e2a\uff0c\u6240\u4ee5\u5e94\u8be5\u90fd\u5305\u62ec\u8fdb\u53bb\u3002\\n\\n\u6700\u540e\uff0c\u786e\u4fdd\u6240\u6709\u4fe1\u606f\u90fd\u6309\u5217\u8868\u683c\u5f0f\uff0c\u5373\u4f7f\u53ea\u6709\u4e00\u4e2a\u5143\u7d20\u3002\u65e5\u671f\u7528ISO\u683c\u5f0f\uff0c\u6ca1\u6709\u7684\u4fe1\u606f\u586bnull\u3002\u68c0\u67e5\u662f\u5426\u6709\u9057\u6f0f\uff0c\u6bd4\u5982\u6807\u9898\u662f\u5426\u6b63\u786e\uff0c\u65e5\u671f\u662f\u5426\u4e00\u81f4\uff0c\u53d1\u4ef6\u4eba\u548c\u6536\u4ef6\u4eba\u662f\u5426\u51c6\u786e\u3002\u786e\u8ba4\u65e0\u8bef\u540e\u751f\u6210JSON\u3002\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"\uff0c\u6211\u73b0\u5728\u9700\u8981\u5904\u7406\u7528\u6237\u63d0\u4f9b\u7684\u8fd9\u4e2a\u5386\u53f2\u4fe1\u4ef6\uff0c\u5e76\u63d0\u53d6\u51fa\u6307\u5b9a\u7684\u4fe1\u606f\u3002\u9996\u5148\uff0c\u7528\u6237\u8981\u6c42\u4ece\u56db\u5f20\u56fe\u7247\u4e2d\u63d0\u53d6\u4fe1\u4ef6\u7684\u6807\u9898\u3001\u53d1\u9001\u65e5\u671f\u3001\u53d1\u4ef6\u4eba\u548c\u6536\u4ef6\u4eba\uff0c\u5e76\u4ee5JSON\u683c\u5f0f\u8fd4\u56de\u3002\u6211\u9700\u8981\u4ed4\u7ec6\u5206\u6790\u6bcf\u4e00\u9875\u7684\u5185\u5bb9\uff0c\u786e\u4fdd\u4e0d\u9057\u6f0f\u4efb\u4f55\u5173\u952e\u4fe1\u606f\u3002\\n\\n\u9996\u5148\u770b\u7b2c\u4e00\u5f20\u56fe\u7247\u3002\u4fe1\u5934\u663e\u793a\u53d1\u4ef6\u4eba\u662fBasler Rheinschiffahrt-Aktiengesellschaft\uff0c\u5730\u5740\u5728Basel\u3002\u65e5\u671f\u662f1928\u5e744\u67082\u65e5\uff0c\u5199\u5728\u53f3\u4e0a\u89d2\u3002\u6536\u4ef6\u4eba\u662fBasler Personenschiffahrtsgesellschaft\uff0c\u540c\u6837\u5728Basel\u3002\u8fd9\u91cc\u53ef\u80fd\u9700\u8981\u6ce8\u610f\uff0c\u867d\u7136\u53d1\u4ef6\u4eba\u548c\u6536\u4ef6\u4eba\u90fd\u662f\u516c\u53f8\uff0c\u4f46\u7528\u6237\u8981\u6c42\u7684\u662f\u4eba\u540d\uff0c\u6240\u4ee5\u53ef\u80fd\u9700\u8981\u786e\u8ba4\u662f\u5426\u6709\u5177\u4f53\u7684\u4eba\u540d\u3002\u4e0d\u8fc7\u4fe1\u4ef6\u672b\u5c3e\u7684\u7b7e\u540d\u90e8\u5206\u6709\u624b\u5199\u7b7e\u540d\uff0c\u4f46\u53ef\u80fd\u65e0\u6cd5\u8fa8\u8ba4\u5177\u4f53\u540d\u5b57\uff0c\u6240\u4ee5\u53d1\u4ef6\u4eba\u53ef\u80fd\u53ea\u80fd\u586b\u516c\u53f8\u540d\u79f0\u3002\\n\\n\u7b2c\u4e8c\u5f20\u56fe\u7247\u7684\u6807\u9898\u662f\u201cBetrifft Schiffahrt Basel-Rheinfelden\u201d\uff0c\u8fd9\u5e94\u8be5\u662f\u4fe1\u4ef6\u7684\u6807\u9898\u3002\u6536\u4ef6\u4eba\u90e8\u5206\u63d0\u5230Kur- und Verkehrsverein, Rheinfelden\uff0c\u6240\u4ee5\u6536\u4ef6\u4eba\u5e94\u8be5\u662f\u8fd9\u4e2a\u534f\u4f1a\u3002\u53d1\u4ef6\u4eba\u4ecd\u7136\u662fBasler Rheinschiffahrt-Aktiengesellschaft\u3002\\n\\n\u7b2c\u4e09\u548c\u7b2c\u56db\u5f20\u56fe\u7247\u7ee7\u7eed\u8ba8\u8bba\u540c\u4e00\u4e3b\u9898\uff0c\u65e5\u671f\u90fd\u662f1928\u5e744\u67082\u65e5\uff0c\u786e\u8ba4\u53d1\u9001\u65e5\u671f\u6b63\u786e\u3002\u6536\u4ef6\u4eba\u4ecd\u7136\u662fKur- und Verkehrsverein, Rheinfelden\uff0c\u800c\u53d1\u4ef6\u4eba\u6ca1\u6709\u53d8\u5316\u3002\\n\\n\u73b0\u5728\u9700\u8981\u6574\u7406\u8fd9\u4e9b\u4fe1\u606f\u3002\u6807\u9898\u5e94\u8be5\u662f\u201cBetrifft Schiffahrt Basel-Rheinfelden\u201d\uff0c\u53d1\u9001\u65e5\u671f\u662f1928-04-02\u3002\u53d1\u4ef6\u4eba\u662fBasler Rheinschiffahrt-Aktiengesellschaft\uff0c\u4f46\u7528\u6237\u8981\u6c42\u7684\u662f\u4eba\u540d\uff0c\u8fd9\u91cc\u53ef\u80fd\u9700\u8981\u68c0\u67e5\u662f\u5426\u6709\u7b7e\u540d\u7684\u4eba\u540d\u3002\u7b2c\u4e00\u9875\u672b\u5c3e\u6709\u4e24\u4e2a\u7b7e\u540d\uff0c\u4f46\u53ef\u80fd\u65e0\u6cd5\u8bc6\u522b\u5177\u4f53\u540d\u5b57\uff0c\u6240\u4ee5\u53ef\u80fd\u53ea\u80fd\u586b\u516c\u53f8\u540d\u79f0\u3002\u6536\u4ef6\u4eba\u662fBasler Personenschiffahrtsgesellschaft\u548cKur- und Verkehrsverein, Rheinfelden\uff0c\u4f46\u6839\u636e\u7b2c\u4e8c\u9875\u7684\u6536\u4ef6\u4eba\u90e8\u5206\uff0c\u4e3b\u8981\u6536\u4ef6\u4eba\u662fKur- und Verkehrsverein\uff0c\u800cBasler Personenschiffahrtsgesellschaft\u53ef\u80fd\u662f\u6284\u9001\u5bf9\u8c61\u3002\u9700\u8981\u786e\u8ba4\u7528\u6237\u662f\u5426\u9700\u8981\u6240\u6709\u63d0\u5230\u7684\u6536\u4ef6\u4eba\uff0c\u8fd8\u662f\u4e3b\u8981\u6536\u4ef6\u4eba\u3002\u6839\u636e\u793a\u4f8b\uff0c\u6536\u4ef6\u4eba\u53ef\u80fd\u6709\u591a\u4e2a\uff0c\u6240\u4ee5\u5e94\u8be5\u90fd\u5305\u62ec\u8fdb\u53bb\u3002\\n\\n\u6700\u540e\uff0c\u786e\u4fdd\u6240\u6709\u4fe1\u606f\u90fd\u6309\u5217\u8868\u683c\u5f0f\uff0c\u5373\u4f7f\u53ea\u6709\u4e00\u4e2a\u5143\u7d20\u3002\u65e5\u671f\u7528ISO\u683c\u5f0f\uff0c\u6ca1\u6709\u7684\u4fe1\u606f\u586bnull\u3002\u68c0\u67e5\u662f\u5426\u6709\u9057\u6f0f\uff0c\u6bd4\u5982\u6807\u9898\u662f\u5426\u6b63\u786e\uff0c\u65e5\u671f\u662f\u5426\u4e00\u81f4\uff0c\u53d1\u4ef6\u4eba\u548c\u6536\u4ef6\u4eba\u662f\u5426\u51c6\u786e\u3002\u786e\u8ba4\u65e0\u8bef\u540e\u751f\u6210JSON\u3002\",\"format\":\"unknown\",\"index\":0}]},\"native_finish_reason\":\"stop\"}],\"created\":1776771825,\"model\":\"qwen/qwen3.5-plus-20260216\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":599,\"prompt_tokens\":10697,\"total_tokens\":11296,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":0,\"reasoning_tokens\":494,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0,\"cache_write_tokens\":0,\"video_tokens\":0},\"cost\":0.00371566,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":0.00371566,\"upstream_inference_prompt_cost\":0.00278122,\"upstream_inference_completions_cost\":0.00093444}},\"provider\":\"Alibaba\"}"}, "scoring": {"f1_macro": 0.59, "f1_micro": 0.61, "cost_summary": {"total_input_tokens": 278738, "total_output_tokens": 49505, "total_tokens": 328243, "input_cost_usd": 0.07247188, "output_cost_usd": 0.0772278, "total_cost_usd": 0.14969968}}, "normalized_score": 59.0}