{"_id": {"$oid": "6a26c67c3572c06bc1455e2a"}, "test_id": "T1051", "benchmark": "business_letters", "date": "2026-05-22", "tags": {"document-type": ["letter"], "writing": ["typed", "handwritten"], "century": [20], "language": ["de"], "layout": ["prose"], "entry-type": ["person", "location"], "task": ["information-extraction"]}, "contributors": ["eric_decker", "maximilian_hindermann", "lea_kasper", "anthea_alberto", "pema_frick", "jose_luis_losada_palenzuela", "sorin_marti", "elena_spadini"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3.5-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge and an expert in the field of 20th century Swiss history", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "            IDENTITY and PURPOSE:\n\n            You are presented with a series of images constituting a historical letter. Your task is to extract the\n            values of the following keys from the letter and return them in a JSON file where the values\n            corresponding to each key should be stored as a list, even if there is only a single value for a\n            key:\n\n            - letter_title: Title of the letter.\n            - sender_persons: Name(s) of the person(s) who wrote the letter.\n            - send_date: The exact or approximate date the letter was written.\n            - receiver_persons: Name(s) of the person(s) who received the letter.\n\n            Take a deep breath and think step by step about how to best accomplish this goal. Map out all the claims\n            and implications on a virtual whiteboard in your mind. Do not use OCR. Use the ISO format YYYY-MM-DD for\n            dates. If a piece of information is not included in the letter, set the value for the corresponding key\n            to \"null\". Do not return anything except the JSON file.\n\n            EXAMPLE:\n\n            {\n                \"letter_title\": [\"Petition for Environmental Protection\"],\n                \"send_date\": [\"1993-03-12\"],\n                \"sender_persons\": [\"Lisa Simpson\"],\n                \"receiver_persons\": [\"Mayor Joe Quimby\", \"Seymour Skinner\"],\n            }\n\n            OUTPUT:", "results": {"text": "{\"metadata\":{\"send_date\":[\"1928-03-26\"],\"letter_title\":null,\"sender_persons\":null,\"receiver_persons\":null}}", "model": "gemini-3.5-flash", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1428, "output_tokens": 63, "total_tokens": 2687, "input_cost_usd": 0.002142, "output_cost_usd": 0.000567, "estimated_cost_usd": 0.0027089999999999996}, "duration": 7.47165060043335, "timestamp": "2026-05-22T09:10:12.627087", "parsed": {"metadata": {"send_date": ["1928-03-26"], "letter_title": null, "sender_persons": null, "receiver_persons": null, "document_number": "letter21"}}, "conversation_id": "64e7f8f2-f73c-4306-95ce-c172143aa7af", "score": {"send_date_tp": 1, "send_date_fp": 0, "send_date_fn": 0, "sender_persons_tp": 0, "sender_persons_fp": 0, "sender_persons_fn": 0, "receiver_persons_tp": 0, "receiver_persons_fp": 0, "receiver_persons_fn": 0}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"x-gemini-service-tier\":\"standard\",\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Fri, 22 May 2026 07:10:12 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=7331\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"media_resolution\":null,\"code_execution_result\":null,\"executable_code\":null,\"file_data\":null,\"function_call\":null,\"function_response\":null,\"inline_data\":null,\"text\":\"{\\n  \\\"metadata\\\": {\\n    \\\"send_date\\\": [\\n      \\\"1928-03-26\\\"\\n    ],\\n    \\\"letter_title\\\": null,\\n    \\\"sender_persons\\\": null,\\n    \\\"receiver_persons\\\": null\\n  }\\n}\",\"thought\":null,\"thought_signature\":\"EuIgCt8gAQw51sehVnxA_8t05x8xz7aJttpH0rVh8JpmECdMmplkkWyImPhyxHHRUn7va7G5r0CfDGr2HCKDZjkNMYqoloDr8UjW9FkhFfDtUBVRj5Y9-fTtuKR9w9K3Xu-Pr2kBJbgtuLWHA37H85ojPOZOF9Z811vM-t3ob4pVOqj_SeN2qocEo2ivoKAkUSYG35lbbZUIG_rFL3_V-wNa21hoBp7RryB_e_mK_Jk9axKEeyxJqqe-vX7m2RJpejYHMAmEChEW1kctTz0db8ik2S5PsbkjjiiArUtUwse87N79kyhRxSzAYpKqiJ-212fw5PAf4dcPxQsaa99xX1ucZHaLxT6KDML7HiaShPRQ0UjMVbq3y-4S_PTNB5tLgUuoX4IpMjU___Yi80l_rZjUVUR1-TAqzQv0Hgg56SMcJT7QpQGczXf0m8tkTpEc1AWMJC1uz_TA6nRa_jEVwxYRhjPLx8KBnXRaFJ4-WUa_T7MYtTWzVPznymFTQXImawc7jcGoiNlI8da0AJykazsS3mngZdiR_kiy_UKoJCkrMMMCuyPEGVMbB9Whnn1ckvB2JdMPd0faSikmQ9lQbeM6VNNDv6J6k7fXtX2IdRESO_NotmWaa7XWesRTBsMElPz5BjagK3rjyV5wSS1fz9_VY0Sf8EAcl_T-pIWcSWgrHdk9h-YjvxQnec-Su4WI1BJH8SEwUXMrxPl12LoLJlPrIvMT3yOfqa6ej0xtZbLZzueznyLX4zNn1asto5BFShqwRyJEyi8u35jEAxpLkmg7LD8sK0lK8yeDs7qyCiD_uEEjoCUWBr6_iMa2eSI4TQoNThgq0QySgbPB8qcEbq28OKWXDe9etZmnf53UTqYzC0qaHUKoRHOnLaN5DVdaJJuTWJuZfMEu4pVZF9KuAAJP8H1DxhfMGuEJssGadSU79WVvKrgMnkRj3HOVjgTQ1LjMPLud9vNB3R3WN26E3QbD8lD49_dmE-OUaa_7m1-3IjuenFQZB54wyv1eHbOsf8b4KEuCEgABIV1upZx9iSAMebfgyNn6eMxxSODfIuvfeNesZ1tsqLEfdh9-uQHEU3yVVpF3bd8zQ1vBy2agVFbJXlW8AjLXs5TToKb6AwlTvyGCoJ8qiq2ls4UpUj7_7tmymkTIQP8HuUJLglfhQb8pKG0vr80mJzBUg174Dqx4jcJS_sicI6eGn1f0jN64KylHDEJnSJmQEZZKyJAtG2KkRyk1Xc7u9XYbGRTcMNxQIop0w50JX9AMlDoOZo8zooPn_yCsmupa4n9C4C2xSo7yZFdvy-txqYE9dhca6bIWgCcP_uER4cE9r8P43mHtMU6Z2lvwD1Q-ihZmP6uo8_SIUpZeUEORjYa2iKYtOO24j3pnCXnJa0troFpIwxq3ho_i8PnjE24voeiKZzGuuT_vNBQsnoKTC0D2EmmG4YXRP-n3UNjjDFb7vPO-7XnMUUZUCIFHHk8C3L-76JJd2WjMbq802BXAqnHOlcr8hqH7fwCCPfIbMK6GkiMV7ertr7jDiOYH_bWOw37Psp320zb9Pd-Igpvxwb02p37j4heJG5OjkpAYupvspOth5jnsOyRCw8iPRKcovUaadvBrEFF34IlkMllNKAoyPDo0aNPMF8IaxVdjzpU5hR3TB-ynnNfNcmN2QcbxfonnCQwx7LnZwqaIBAfDB5LXnt2zSIpw9tj99H2lcQI9__pd2n9Z4v_cI9KPz73UXJ0ukLgHfsAziwfR4kxxcLGTzADz3KHeQrEnnCrDq99TbmbP3LVU6YQ0jPIIW6shFycWhcke5uLGwvVF28tV97wO9UVMCNisOIG1hAzHbbikrs9QUfBfpomBubHLgXFfutXzUo8VdrBRrSipd2NHWLtKmsFODWJesvlSV_jOb5UC8la44Wk6yIxPFQff7aPNBD1geHAdUiMbxrSrt9849sTo_bBzlpRVhRWjnQar4mEYqEKOz_4psJugnfx0nVtI6dIT_SgOvqEMt4Ntykla5zOpmNVQd98aQdJyH_2J0WiXwBaid8WJMMdkgTOU4r2_rzqxPQC91Ah3Wby9zsehLc8Khfx-4kQj7ySWt_QCjgVBsxv4E-x3Tccz40FR7k4APgyo_oD0oFQpgwTOKS72BXCcPmuNtSE18_uvu6hT1ayRGsA_rvm8FnqWdRyBn7oVMMI4PsumW7fVU61-Hxue6CXsAgBpUFN-DXHBnQ3uSxJdqWLLdxIRKqBwQvyf4PalZQEx4nDEs3zr7dVzCE8017nudv7V8iHfjoJ1N19ZyD4b4PpzjUy4CB3Wp9KTV3nlIyDoM96z__WQnUM5rACAepyF3UdZYWA2ylAqV7H2at6svdr3lCEC3DDqsOuy0LBC7EmaZcZ6j1-jNB0etQC2mGDybT9XWR30D4ItuX_gD7r5xW4kdqTLHYrMOs7Jk2I-ZMlSqlVXZyKAl9Two-qAH4QPycosjB96Cz25fdaM8KzE8zeZAHeB6eu_wbrz8O55lcFwODTnR0YtYl2grKycg_rRJJfpVsHDPw4zXDQMaCSQ2yTn0rwYCYDM__tY7eFbLgeO89lcvMr7ejFphpGP91RVny43CCSh7-n2uh-qWGNCaONc7zJPiRDgM8CJYA4HGPfBzjyR0xHIVVlwrg6E8FJ_2bUWYjmnUUTCeh_H5vvlxh2VZ4cQsH0YEX6YBdttbJxNwCHr0YDMU9JXQVkqcBUXmZQ1lFwd0-KIH1_j2KIkQa3yJ140jrN6WP8mrxaUO6J0EZSeAkFxiNcHoq3-l55uIPQC9At8G3TIZ75p1OuJqMo2k_bZKsklZkbdpK3qlc-0Rm6x5stK4wCVxe9ThhXpAOw8VKiZdOh2EDjToMYsXlf4Btxh-gJZf4ScGgAc_A3nARMHEfvKzD_F6mly7Het15-TiFrDWZ3-xmXzivq495Me0aQmlBgYMCrCq4UWL3mQ_3AH-VyAxe0bxLSPpEfrjM7OG2zj3Qj-6TpptbNhhggGd2q1ASRKMguNuBW91wrUasuWMXGzSl8oKxTBnFiGmcrdWL_co7zHyg8eRngPA-Hrou8m2-MsfxWt-iu6csR9pdWxfnSx6h4fmClMmglyxa64iTZbe3YBv5XU43Ud5TH17ecbIvADwTSB6xJOTwmI0vbpLgWitGS0ifYOBTxuNxy3yoAqFETg1nrorgTg-axMKtikb9ngHC08Iv6-DcfSF6553xy0ANlJ_whCfRS5WGVTOhaPWtVpkQtnwm2jLlatOWFHE4Mj61yfNQx8J8jDiGp2WXQERZKI9PBHt00b-EHCPpk-M27Vx5qs9PuQAgIXc1N1lD9UBehSzdQ_6w-JpiQiCMX_A1vb3O0yGlycOZ1tLwrVvX2cNn09ItoZXFLWhA7w0JSuc0KZdME05yQuYWTb6bkuPk_h-c-35ta26UCIE-R_bvV_dT4H2VqVeejyu_7NMILAp2S1PnPgFuC-dQiDfZ8BzPDfk8r3RKpUNtNxS_BSU-3cfSSm86WP6jmaXEKwv9_SISGKbASQHfoqiDMSBrhSnY01_6_jqOXK637elDxYnwWW-5UUyuETCXmUpw5zE3dR1wOQaZiP4VnTBATQ7_O-E6qUDfu2dUsA_5K-OC4GHUVLz2fWCHH-EWlXv-6VkHRfWNFzIXGpcBDpnBAiX52qN2fbzlzSZEijwdxL5A_VvdMx-NJTMyfJDlXnABZoQTG-4ogeDl_P7Aw35kuRhiuhsa2dsXuPS-cXwvGyA-Ig9dst-MIU6l5-sARoMC7nQi0_h1CxcSHGLNHFN5Mwitkpq4iPyUKO1G4unm-ru0s-4ZBkEpc2FTUNy5QSPQ1338y04vqMphTUvumNP_NgtAHp-0ywyqR2k858Mw7NVM1S8LLu3luXQMB3W_Bsez3m61-JK4tlxmDo-eXC2NANccxFsSi2xSJOUlwO6weDRixux5U3_SRr2cx3PfXLw9NuYmbNkjMMCy67HLzjq9Y7btvnOKeoOB7p_gwMe4RQEBAdfP1ctWFtyKRJ7lkeK9x7HUtEN_cZmjWFrOgZMCFc31WjqzbTfFGnPeoqyw5IY-FUi1lxiomlOasvanFSBudQ0v0B3NkUqjJ8y-Y01FBG_2dMGsvbozfxSloHNVdWQMH-TvBCvQJSrN-5h0rs0ljm9bFOwUs75b3yudzBncGTxPQojNRHWYGQuPid2_eL8r-HnTBbLPFhnJArkXEBdB70ZzNH7orDUxtp7fh1US9qRb5J01wf3Y2FGwy6VhpNrmj-uDv0maK8yN4VVW3cyK0DRPzCfdI59eRf8XpTc5TMc6K-uG__Kos5FtBr3YxLG3rRit3CLbBpudxh6i-qSpQ4TYw1tqpEt7cXMnfjCM8b3WI7MGZSMObXlwVhXQkbgpTL3fWC5RG_wRoLIeN-zVKvfLXjuBvteNLznTedioPFcPCW4kaTq6iieQU6QVlqFroadhZAFfd8-cLw0d7lFN_7CVsQkSkqCKoh8CEBvbX4GdNrwRZk1JLK_NIfmXW2lOXQbt-oTd6CPs7oIKIQPM3ZyJsrSS7I0JdTBHYD2a0heAz_Bi9aavhgnHatKY17kVeEZWCp9HC4l-gPYMnJTNc6VKWcT0yYLcruMSJAh1NYdM_iEevmaaHaW078PagpeOVXowEJfe1ibhLWZ0uFOSPw3vDlqbED4pVoKFxHYDNtgD24vFenyBJpdawfxGPSuV4gsa0QHapSQRuAecX4o0TFxaJb7B6L4v1Lgllf2JEBlPiQln0g16qdCCrE2PRdlLLwO03bBfwF3fE6QlmDPzTQA7u_gy30fpDi4ezECOWT8YBIIgWOyE-ZtgpSH7-euzi1E6Dy4QuajVs82vwZqgeU_ZLlgefQM7dx3DWu-8x_uyGFmo_yidVrxDY7oD1oZ6bn2BT71oTEes8p-ErBYhdrwslJnlWNAZKETj9sb6Pymg96qmE6VRsqdA2L05ph8I_DHYFAeZeYRZyuyXhbTd6OFRS6Vlv6ypUho84-B7Cu4Ni9Sp0D6Rtb5mxMYgt0MR-xBaqJl_KcDBCyNReEHJcVT6Zg0gMCbnEGnwcZSHHnqeCXUGbeXgYNc1bOtaMmAna_BbIFWDeP3ruJx_BywRXkt7yiwhju5RNdMdJFiQ7q01cZUNLn6BV5_fL8BKeA-iiC7-B_IrW0sDQcZftivjw_dIZcodD1kHLwpracE50-bc64aynmZhS-uk4OnZhsooPBdLJV_Uxs9fMddFgplLFtNuWai5pEj_vbelSL9KoF6J--TjPG31iaL1YS-c1ueRCL8EDdk05IEC_kzr38aSrzxT5-_6X7NJ75z9tJy3rQMXqGKFOAetCtI8J3CH74kJmgxg5-e2PBQRq9_60h2PZYQRecMQI40Ba-Vop6quZ7j_IokpVdNlmBU6ceRONwZ8Kc8tk1LvnVDuyqPnXb0KGuBR8zXokUtwl0pCljr2uIQkFpaZfWJyp3JEMRuKX2VZB1zRkvxeA2V3Jnboi4Laxf_6X3sW1hMmN7vgKIaC2APHDoNNdJnE9R18_YMMV14X0nenZtLsJB3r7hLhVUqEH1dIB5-0DnSFMhegrh946m3ZWMvXCK\",\"video_metadata\":null}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"grounding_metadata\":null,\"avg_logprobs\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null,\"url_context_metadata\":null}],\"create_time\":null,\"model_version\":\"gemini-3.5-flash\",\"prompt_feedback\":null,\"response_id\":\"TQEQauSEGqepkdUP2YCsoAw\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":63,\"candidates_tokens_details\":null,\"prompt_token_count\":1428,\"prompt_tokens_details\":[{\"modality\":\"IMAGE\",\"token_count\":1092},{\"modality\":\"TEXT\",\"token_count\":336}],\"thoughts_token_count\":1196,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2687,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"metadata\":{\"send_date\":[\"1928-03-26\"],\"letter_title\":null,\"sender_persons\":null,\"receiver_persons\":null}}}"}, "scoring": {"f1_macro": 0.66, "f1_micro": 0.69, "cost_summary": {"total_input_tokens": 126168, "total_output_tokens": 4235, "total_tokens": 130403, "input_cost_usd": 0.1892520000000002, "output_cost_usd": 0.038114999999999996, "total_cost_usd": 0.22736700000000007}}, "normalized_score": 66.0}