{"_id": {"$oid": "6a26c67c3572c06bc1455dd4"}, "test_id": "T0959", "benchmark": "business_letters", "date": "2026-04-21", "tags": {"document-type": ["letter"], "writing": ["typed", "handwritten"], "century": [20], "language": ["de"], "layout": ["prose"], "entry-type": ["person", "location"], "task": ["information-extraction"]}, "contributors": ["eric_decker", "maximilian_hindermann", "lea_kasper", "anthea_alberto", "pema_frick", "jose_luis_losada_palenzuela", "sorin_marti", "elena_spadini"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3.5-plus-02-15", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge and an expert in the field of 20th century Swiss history", "prompt_file": "prompt.txt", "rules": "{\"skip_signatures\": true, \"skip_non_signatures\": false}", "legacy_test": false}, "prompt": "            IDENTITY and PURPOSE:\n\n            You are presented with a series of images constituting a historical letter. Your task is to extract the\n            values of the following keys from the letter and return them in a JSON file where the values\n            corresponding to each key should be stored as a list, even if there is only a single value for a\n            key:\n\n            - letter_title: Title of the letter.\n            - sender_persons: Name(s) of the person(s) who wrote the letter.\n            - send_date: The exact or approximate date the letter was written.\n            - receiver_persons: Name(s) of the person(s) who received the letter.\n\n            Take a deep breath and think step by step about how to best accomplish this goal. Map out all the claims\n            and implications on a virtual whiteboard in your mind. Do not use OCR. Use the ISO format YYYY-MM-DD for\n            dates. If a piece of information is not included in the letter, set the value for the corresponding key\n            to \"null\". Do not return anything except the JSON file.\n\n            EXAMPLE:\n\n            {\n                \"letter_title\": [\"Petition for Environmental Protection\"],\n                \"send_date\": [\"1993-03-12\"],\n                \"sender_persons\": [\"Lisa Simpson\"],\n                \"receiver_persons\": [\"Mayor Joe Quimby\", \"Seymour Skinner\"],\n            }\n\n            OUTPUT:", "results": {"text": "{\"metadata\":{\"send_date\":[\"1928-05-24\"],\"letter_title\":[\"null\"],\"sender_persons\":[\"null\"],\"receiver_persons\":[\"Werner Stauffacher\"]}}", "model": "qwen/qwen3.5-plus-20260216", "provider": "openrouter", "finish_reason": "stop", "usage": {"input_tokens": 5653, "output_tokens": 439, "total_tokens": 6092, "input_cost_usd": 0.00146978, "output_cost_usd": 0.00068484, "estimated_cost_usd": 0.00215462}, "duration": 11.120447635650635, "timestamp": "2026-04-21T13:46:03.386565", "parsed": {"metadata": {"send_date": ["1928-05-24"], "letter_title": ["null"], "sender_persons": ["null"], "receiver_persons": ["Werner Stauffacher"], "document_number": "letter38"}}, "conversation_id": "5d559840-3a07-485a-8d40-3fa49c6ab7e3", "score": {"send_date_tp": 1, "send_date_fp": 0, "send_date_fn": 0, "sender_persons_tp": 0, "sender_persons_fp": 0, "sender_persons_fn": 0, "receiver_persons_tp": 1, "receiver_persons_fp": 0, "receiver_persons_fn": 0}, "raw_response": "{\"id\":\"gen-1776771952-6Prnke8uw6A7O7VCKxWF\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"metadata\\\": {\\n    \\\"send_date\\\": [\\\"1928-05-24\\\"],\\n    \\\"letter_title\\\": [\\n      \\\"null\\\"\\n    ],\\n    \\\"sender_persons\\\": [\\n      \\\"null\\\"\\n    ],\\n    \\\"receiver_persons\\\": [\\n      \\\"Werner Stauffacher\\\"\\n    ]\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"metadata\":{\"send_date\":[\"1928-05-24\"],\"letter_title\":[\"null\"],\"sender_persons\":[\"null\"],\"receiver_persons\":[\"Werner Stauffacher\"]}},\"reasoning\":\"\uff0c\u6211\u73b0\u5728\u9700\u8981\u5904\u7406\u7528\u6237\u63d0\u4f9b\u7684\u4e24\u9875\u5386\u53f2\u4fe1\u4ef6\u56fe\u7247\uff0c\u5e76\u63d0\u53d6\u6307\u5b9a\u7684\u4fe1\u606f\u5230JSON\u683c\u5f0f\u4e2d\u3002\u9996\u5148\uff0c\u6211\u8981\u4ed4\u7ec6\u9605\u8bfb\u7528\u6237\u7684\u8981\u6c42\uff0c\u786e\u4fdd\u7406\u89e3\u6bcf\u4e2a\u6b65\u9aa4\u3002\\n\\n\u9996\u5148\uff0c\u7528\u6237\u8981\u6c42\u63d0\u53d6\u56db\u4e2a\u5b57\u6bb5\uff1aletter_title\u3001sender_persons\u3001send_date\u548creceiver_persons\u3002\u6bcf\u4e2a\u5b57\u6bb5\u7684\u503c\u90fd\u5fc5\u987b\u662f\u5217\u8868\uff0c\u5373\u4f7f\u53ea\u6709\u4e00\u4e2a\u503c\u3002\u5982\u679c\u4fe1\u606f\u4e0d\u5b58\u5728\uff0c\u5219\u8bbe\u4e3anull\u3002\u65e5\u671f\u8981\u7528ISO\u683c\u5f0fYYYY-MM-DD\u3002\\n\\n\u63a5\u4e0b\u6765\uff0c\u6211\u9700\u8981\u5206\u6790\u63d0\u4f9b\u7684\u4e24\u9875\u4fe1\u4ef6\u5185\u5bb9\u3002\u7b2c\u4e00\u9875\u9876\u90e8\u6709\u65e5\u671f\u201cBasel, den 24. Mai 1928.\u201d\uff0c\u6240\u4ee5send_date\u5e94\u8be5\u662f[\\\"1928-05-24\\\"]\u3002\u7136\u540e\uff0c\u6536\u4fe1\u4eba\u662f\u201cDirektor Werner Stauffacher\u201d\uff0c\u6240\u4ee5receiver_persons\u662f[\\\"Werner Stauffacher\\\"]\u3002\u53d1\u4fe1\u4eba\u53ef\u80fd\u5728\u4fe1\u672b\uff0c\u4f46\u7b2c\u4e8c\u9875\u7ed3\u5c3e\u662f\u201cIhr ergebener\u201d\uff0c\u6ca1\u6709\u7b7e\u540d\uff0c\u6240\u4ee5sender_persons\u53ef\u80fd\u65e0\u6cd5\u786e\u5b9a\uff0c\u8bbe\u4e3anull\u3002\u4fe1\u4ef6\u6807\u9898\u901a\u5e38\u4f1a\u5728\u5f00\u5934\uff0c\u4f46\u8fd9\u91cc\u6ca1\u6709\u660e\u786e\u7684\u6807\u9898\uff0c\u6240\u4ee5letter_title\u4e5f\u662fnull\u3002\\n\\n\u9700\u8981\u786e\u8ba4\u662f\u5426\u6709\u5176\u4ed6\u53ef\u80fd\u7684\u53d1\u4fe1\u4eba\u4fe1\u606f\u3002\u7b2c\u4e00\u9875\u63d0\u5230\u201cHerr Dr. Speich\u201d\uff0c\u4f46\u4ed6\u662f\u88ab\u63d0\u53ca\u7684\u7b2c\u4e09\u65b9\uff0c\u4e0d\u662f\u53d1\u4fe1\u4eba\u3002\u7b2c\u4e8c\u9875\u63d0\u5230\u201ceine Abschrift meines heutigen Schreibens an Herrn Dr. Speich\u201d\uff0c\u8bf4\u660e\u53d1\u4fe1\u4eba\u53ef\u80fd\u7ed9Dr. Speich\u4e5f\u5199\u4e86\u4fe1\uff0c\u4f46\u5f53\u524d\u4fe1\u4ef6\u7684\u53d1\u4fe1\u4eba\u4ecd\u672a\u660e\u786e\u3002\u56e0\u6b64\uff0csender_persons\u4fdd\u6301null\u3002\\n\\n\u68c0\u67e5\u65e5\u671f\u683c\u5f0f\u662f\u5426\u6b63\u786e\uff0c24. Mai 1928\u8f6c\u6362\u4e3a1928-05-24\u3002\u6536\u4fe1\u4eba\u5168\u540d\u662fWerner Stauffacher\uff0c\u804c\u4f4d\u662fDirektor\uff0c\u4f46\u53ea\u9700\u540d\u5b57\u3002\u786e\u8ba4\u6240\u6709\u5b57\u6bb5\u90fd\u5904\u7406\u5b8c\u6bd5\uff0c\u6ca1\u6709\u9057\u6f0f\u3002\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"\uff0c\u6211\u73b0\u5728\u9700\u8981\u5904\u7406\u7528\u6237\u63d0\u4f9b\u7684\u4e24\u9875\u5386\u53f2\u4fe1\u4ef6\u56fe\u7247\uff0c\u5e76\u63d0\u53d6\u6307\u5b9a\u7684\u4fe1\u606f\u5230JSON\u683c\u5f0f\u4e2d\u3002\u9996\u5148\uff0c\u6211\u8981\u4ed4\u7ec6\u9605\u8bfb\u7528\u6237\u7684\u8981\u6c42\uff0c\u786e\u4fdd\u7406\u89e3\u6bcf\u4e2a\u6b65\u9aa4\u3002\\n\\n\u9996\u5148\uff0c\u7528\u6237\u8981\u6c42\u63d0\u53d6\u56db\u4e2a\u5b57\u6bb5\uff1aletter_title\u3001sender_persons\u3001send_date\u548creceiver_persons\u3002\u6bcf\u4e2a\u5b57\u6bb5\u7684\u503c\u90fd\u5fc5\u987b\u662f\u5217\u8868\uff0c\u5373\u4f7f\u53ea\u6709\u4e00\u4e2a\u503c\u3002\u5982\u679c\u4fe1\u606f\u4e0d\u5b58\u5728\uff0c\u5219\u8bbe\u4e3anull\u3002\u65e5\u671f\u8981\u7528ISO\u683c\u5f0fYYYY-MM-DD\u3002\\n\\n\u63a5\u4e0b\u6765\uff0c\u6211\u9700\u8981\u5206\u6790\u63d0\u4f9b\u7684\u4e24\u9875\u4fe1\u4ef6\u5185\u5bb9\u3002\u7b2c\u4e00\u9875\u9876\u90e8\u6709\u65e5\u671f\u201cBasel, den 24. Mai 1928.\u201d\uff0c\u6240\u4ee5send_date\u5e94\u8be5\u662f[\\\"1928-05-24\\\"]\u3002\u7136\u540e\uff0c\u6536\u4fe1\u4eba\u662f\u201cDirektor Werner Stauffacher\u201d\uff0c\u6240\u4ee5receiver_persons\u662f[\\\"Werner Stauffacher\\\"]\u3002\u53d1\u4fe1\u4eba\u53ef\u80fd\u5728\u4fe1\u672b\uff0c\u4f46\u7b2c\u4e8c\u9875\u7ed3\u5c3e\u662f\u201cIhr ergebener\u201d\uff0c\u6ca1\u6709\u7b7e\u540d\uff0c\u6240\u4ee5sender_persons\u53ef\u80fd\u65e0\u6cd5\u786e\u5b9a\uff0c\u8bbe\u4e3anull\u3002\u4fe1\u4ef6\u6807\u9898\u901a\u5e38\u4f1a\u5728\u5f00\u5934\uff0c\u4f46\u8fd9\u91cc\u6ca1\u6709\u660e\u786e\u7684\u6807\u9898\uff0c\u6240\u4ee5letter_title\u4e5f\u662fnull\u3002\\n\\n\u9700\u8981\u786e\u8ba4\u662f\u5426\u6709\u5176\u4ed6\u53ef\u80fd\u7684\u53d1\u4fe1\u4eba\u4fe1\u606f\u3002\u7b2c\u4e00\u9875\u63d0\u5230\u201cHerr Dr. Speich\u201d\uff0c\u4f46\u4ed6\u662f\u88ab\u63d0\u53ca\u7684\u7b2c\u4e09\u65b9\uff0c\u4e0d\u662f\u53d1\u4fe1\u4eba\u3002\u7b2c\u4e8c\u9875\u63d0\u5230\u201ceine Abschrift meines heutigen Schreibens an Herrn Dr. Speich\u201d\uff0c\u8bf4\u660e\u53d1\u4fe1\u4eba\u53ef\u80fd\u7ed9Dr. Speich\u4e5f\u5199\u4e86\u4fe1\uff0c\u4f46\u5f53\u524d\u4fe1\u4ef6\u7684\u53d1\u4fe1\u4eba\u4ecd\u672a\u660e\u786e\u3002\u56e0\u6b64\uff0csender_persons\u4fdd\u6301null\u3002\\n\\n\u68c0\u67e5\u65e5\u671f\u683c\u5f0f\u662f\u5426\u6b63\u786e\uff0c24. Mai 1928\u8f6c\u6362\u4e3a1928-05-24\u3002\u6536\u4fe1\u4eba\u5168\u540d\u662fWerner Stauffacher\uff0c\u804c\u4f4d\u662fDirektor\uff0c\u4f46\u53ea\u9700\u540d\u5b57\u3002\u786e\u8ba4\u6240\u6709\u5b57\u6bb5\u90fd\u5904\u7406\u5b8c\u6bd5\uff0c\u6ca1\u6709\u9057\u6f0f\u3002\",\"format\":\"unknown\",\"index\":0}]},\"native_finish_reason\":\"stop\"}],\"created\":1776771952,\"model\":\"qwen/qwen3.5-plus-20260216\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":439,\"prompt_tokens\":5653,\"total_tokens\":6092,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":0,\"reasoning_tokens\":355,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0,\"cache_write_tokens\":0,\"video_tokens\":0},\"cost\":0.00215462,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":0.00215462,\"upstream_inference_prompt_cost\":0.00146978,\"upstream_inference_completions_cost\":0.00068484}},\"provider\":\"Alibaba\"}"}, "scoring": {"f1_macro": 0.6, "f1_micro": 0.7, "cost_summary": {"total_input_tokens": 103495, "total_output_tokens": 19108, "total_tokens": 122603, "input_cost_usd": 0.026908699999999987, "output_cost_usd": 0.02980848, "total_cost_usd": 0.05671718000000001}}, "normalized_score": 60.0}