{"_id": {"$oid": "69d4eb12b877594bc5718bfe"}, "test_id": "T0501", "benchmark": "business_letters", "date": "2026-01-25", "tags": {"document-type": ["letter"], "writing": ["typed", "handwritten"], "century": [20], "language": ["de"], "layout": ["prose"], "entry-type": ["person", "location"], "task": ["information-extraction"]}, "contributors": ["eric_decker", "maximilian_hindermann", "lea_kasper", "anthea_alberto", "pema_frick", "jose_luis_losada_palenzuela", "sorin_marti", "elena_spadini"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3-flash-preview", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge and an expert in the field of 20th century Swiss history", "prompt_file": "prompt.txt", "rules": "{\"skip_signatures\": true, \"skip_non_signatures\": false}", "legacy_test": false}, "prompt": "            IDENTITY and PURPOSE:\n\n            You are presented with a series of images constituting a historical letter. Your task is to extract the\n            values of the following keys from the letter and return them in a JSON file where the values\n            corresponding to each key should be stored as a list, even if there is only a single value for a\n            key:\n\n            - letter_title: Title of the letter.\n            - sender_persons: Name(s) of the person(s) who wrote the letter.\n            - send_date: The exact or approximate date the letter was written.\n            - receiver_persons: Name(s) of the person(s) who received the letter.\n\n            Take a deep breath and think step by step about how to best accomplish this goal. Map out all the claims\n            and implications on a virtual whiteboard in your mind. Do not use OCR. Use the ISO format YYYY-MM-DD for\n            dates. If a piece of information is not included in the letter, set the value for the corresponding key\n            to \"null\". Do not return anything except the JSON file.\n\n            EXAMPLE:\n\n            {\n                \"letter_title\": [\"Petition for Environmental Protection\"],\n                \"send_date\": [\"1993-03-12\"],\n                \"sender_persons\": [\"Lisa Simpson\"],\n                \"receiver_persons\": [\"Mayor Joe Quimby\", \"Seymour Skinner\"],\n            }\n\n            OUTPUT:", "results": {"text": "{\"metadata\":{\"send_date\":[\"1928-05-03\"],\"letter_title\":null,\"sender_persons\":[\"Ritter\",\"Groschupf\"],\"receiver_persons\":[\"W. Stauffacher\"]}}", "model": "gemini-3-flash-preview", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1428, "output_tokens": 48, "total_tokens": 2604, "input_cost_usd": 0.000714, "output_cost_usd": 0.000144, "estimated_cost_usd": 0.000858}, "duration": 8.81415319442749, "timestamp": "2026-01-25T01:23:02.748632", "parsed": {"metadata": {"send_date": ["1928-05-03"], "letter_title": null, "sender_persons": ["Ritter", "Groschupf"], "receiver_persons": ["W. Stauffacher"], "document_number": "letter32"}}, "conversation_id": "02470d6d-ddd0-4928-9424-5a6a61043bb6", "score": {"send_date_tp": 1, "send_date_fp": 0, "send_date_fn": 0, "sender_persons_tp": 2, "sender_persons_fp": 0, "sender_persons_fn": 0, "receiver_persons_tp": 1, "receiver_persons_fp": 0, "receiver_persons_fn": 0}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sun, 25 Jan 2026 00:23:02 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=8670\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":\"ErIbCq8bAXLI2nwLF-CJk2tRd-ZapvqDEuhIvxMvxCKEHLlpSeXVKHsm3SGTaabNgFCybLyIaEixKhhgc5WNAd0vDMWh8PyW_C1eVUVDIpJxTEmC_uDP5Xe-zs0hp0l9o2rFeRoakaGvquoHGOzm7gfjIrfD8ooMyLmYXxKjPZSIqncMhAw9aIhzAkaOLCNwuhFXTaaZYHTiAaOrGbbBuOdJz0L0crEFD8NWQi2Zz2_T7VIM6VUamtQDiJPDJVFjW5-UdocTcgUfyv5TZvjtbSjwVfyms7vENfnfmiqycfS-_cUkrpD_YemJ2CdGLA6cdiF8tROWubnVHRDHABxZs_7WykFPDbkM-Ablg1kLOftY2qyR0FWw1G9AUpxukl6wClAitI6WymmXQp1fUoQGIK-RqNhLYzWHKUj8HlQcNMMpQeIziWoiAIl0brDNRoOd_r-A1j0dzn_eVptHscbuU8g83q81st0RKpkyxTVtDW9gOGdh1W09zwPrSeckGnmlAydYEMMNc-fBfg6ldUq7TwBAzfhKdA4pLEFyaMeIJFxHq6S-AkMa328OqijeLDGkIM-Oxw6aRu_D5uu8uwqwxZ7HnSRdt8nXVtab_FcvfKiUBx62_aRGpZRvqOKrXbgb-MJfAA1hHl9Jwinr4-VCeEv3FrlvA-KBOgLY0xEPG1JD__nFZQuA8eJE4KZGqnLeZ3cKtxvy-ROrLbPZjKvWJakNITYdboP3Jj615OA80YiKHd0w8HG4pf4U2WPqXuueKZECaYzX9vzEAK4qgkVf4uJ7G_KBqIVEfYhMcC14KJpySO0FA5fAjqVU0u51RCtPpjlqnira3MW6S5sBWTBVB3u-qQJVbLElqRwNHRPfAScz4ppulGinc0sHjgUqfrP5QtbENW9ufHRbvj9LEXLHBbodoNUtwREGW9h7R7CIt3ZkS66onwyKOV1pmSbq-VXd8VaOktCj23LQvxyfKqrGxDqvIm_PGQlzSQCiKtJbD-056Wil8C1jxXex6AXnvS0ZzB75K_NEtlmBMis7LnF0yZ5zHvAzgQU31t_iV2-AXriUf7gfMOtOrlyiIUIHJ9FMprqVq6ejuRNlxOCrUAja2fCAg15PXwx4Lp5yyX2xoXjufHN32sSdeBVcWHj9eirxBnan0T0cpXQQXoErYcMXGGg4dadTvzaFX0Bzh9Ub-XUP02w0vWN9XSq28EA9weJKipsJyH8Yi6mhj2Imfg5MTXLWtTS3BIO-LoKO3xj4VTOeeDoo9Nhsvu1RiCngFy8hX39aHL5RgCKsnXagfL2TwipKD6Cy15bhBZQcYfofsZS-WTWwtfrIX5FXEws3MmHk-UYtJ0vJr4474Icb55HmIYOTz7o7EZN-XxDabsFTSQFFXcJ3WGf3V3iYXWK8w1u0ItIokf9CahLsKh0OilGijHhaEwuRuhwOSMwdhSF15nFLbPx1aVgT2GPQZRn6T6OdaQzZEd9AeMaeBQcrJqv5mqv2i6_ArLiHZBBdrqGa2V1qpSRdfOPndjzoF3-zIFX8MDRVbuze-iw5MR-oMDpJA_jTDsngzIuvuhEWMa2izkdUoYnepsVxUH3CyzrHqbyWNE3rNqD5O4fWYQnYaIi3jqPBkUFZtvarsd-Fj0yZcJkxnyr0ukSrEXVZhvNINchJRLNdJs-bW7V_s9oc20aISU1lHr4LrL7zZJd1KUSGdVs8ujSKepP4wgr44vL3ujozd_E4AL_IL9mNtStu5XS2qhQaCjGAtiiPXg0unDGpBjOY4X5VgByyeY43V8dZkseF8ZGd13vJvkI1JTWf2ZIK4cNQzDjyEPa9i3RLHPWSFNP0IW8ln6c7pRflLrDsQdJFNtCTowh3Qrx79uNd9mKqJPXI-BcyhOZ2qyqeixmrzIth0BZMretImAvbmsatFdcLsmpEqmwQxJSYU9Kc1np7Dlr-D6iQyjev8xJIg8VKs2FRLe9k6IJI_gf6MVRkA3C20jGt_0zCbcmX5myIQRjRefwDR280rrb2nvwRVGkSJ8gZTQa8uq9vo51ZBEIamHOUbEp4oDAdJ2lvIueZfU2ljie-_7-HZsvoJA_sI-Gx4ZzySeUEoBzXaOYXsN5Q_fBZKhyf92loEHM-9pT3TysOUldTN86Dmc3OgJ2hJWYaI31XU5pbW6oJc9XWL6c9yL5l3-8fU6btgROrF5y01AeHLh7w5RfukoNqMmujDN3-xqt9YaNHua6q2WIMp1rKUIlJmYICZ8rBiKChoq4viT92cA2v9T_jiojIYxYNJZCO2t-RGP9k26yqCYoZAGW7BCqIH2KBd_yuhuZbzIVf0gzWqNaAb_oVfNK4mEe9eS-W512BqEvLbtzyoeTSjFSv_n41pb8KZug_M0GUtXnABACAc9mdnozGps3MTzvoErkm52cZ0QhuINqvRLKnOjDPhRs5wEtE0e0pvC9g2j84MvlX2hj3X6aiAOIE54uaIKNvk8vESFzEoTQgIJdvR-w-8b06c8IMGjwePUIiIChnQwqdD3hv4f7PK7Z49g1d6h0rtU76TMei8HcFhGkvBKiLuyVY0pMpuE_YPne7TQzwr1inbvfjVf0yJMP1pLmkucDUP_dULfXyEb9PjIvMLXf5ErXL-ZnVRkRqIqLoiWe47hNWL3UxXWOJ37fKt6crJgWg6MRwICi-NUMg_XZAyNAwv-AbVbSWnzug61Nult2hhgTtXtIV4PU6bXW9apLCRhi_u-DVRNk-Va_BbQk6AEQt0Hmk30-oTBqYqBbzMLrYRmUJ1ifDSAipMA1zW_sYQCfTF_jCF6YjDUJ__3tsGr1sBJJzncIjK2vO5HZx-Ze3W0bC125Yh-ReMdLkBQeHPiptSoxpcBU00B0AbVg2DikjRXr6sJbIHt9Zuhhxk72dE8Vw0L9SwlD2Zu9HovQRucugXc6rz2ZVPkicjBXYWGu_27gNj31nPJ5TkVRk7HYSbYiWKTkKqAonGgh1NBrUQiF5ZnCA4atkCjaAD5GF1xYkHRYa0Stg35UjUf33Fiy3JpjhEIbLQvStRV-0YGTzdPkPwmrk3mKAQ5ojhpn5ScbX-miMfTg9PuZv7DLKjH3Z4eM_4X1NTKWQB0tmTytjbetvKPJ5TppDEGdtl2pApPP4Q955J2MisjFZD_mNSOu8gZUzZSTR_zaIRS9_5rhMPPVqM2UCFjcC8fFf10YkuJJYY4Z4wQwvxYkKbG9qmU2a6DEJAkiVe32HLSSH3lU6pG7OmPVBKcWzU-3ffNXUPJtBpbMn9EMLF8G7IgGHNoRc4Kb7_PITSTZ9-4OH22NsrthuX5qn3jPaZg8XCbya4DKKaY2PBMyZEgdEviPsEp8jMSsbWZ5P1-uSG36LaXKq_tgZyKnE2tD7cESNdbzHYFIUwYfZI27XhlrAZQDkJywS05-XkM_yR0bFw2DpfaYfzP1q1T71AQy0xqE8htDg8pHHNFkiUbbePEgdBzvbSDOwr2SiDkgMeJKnARwdpcG0RNmv9zvPJWJE0-kyhsA5w3Tt1TU0fNoGAOPxhzBE9V7HVJ0TOdIPEoN1ihb6MkPYQeNNpnn0orSDuFGGdKBGwjq4TkmqAVpJp98L5ueOiSQUud05zxvYBisx0ewaCNf1EzMBnGGrB83tUelgzCcSdNrZrh7d3UYFf9lyfXHDCuju81_j6Rycujw69L84NpMxsmn4HHV3b_K5NgFq2ncXoI4XECF0gjiVpQfFJubYKewK_JN-HTP4rnHgPtfYyRRuXjkT3l3wFisDxNXwD3i_d6ccyC6UoZOHOePClSIkHrwTMArjErWeHFMVrs5vncEgXI03HSvePqZKxf8JTXUv9dvZaAanLNfC2GgkAPW9isjEADtqf3JApUtSPvsHPWUURjV8zwSqDWouScMnE_3oTOXOzD37xTtvb0isOMEDjQqaRwqLMbigopyqMZuVsifTTfpsKlXLRWdakCIewvnNjfddxqIgKB6L7C-8z6fZLCecFoahJd96WmA2i0RGIMwAsbFzduzHreAI7f4Fwb7YWBWpmbJdyGPHYM9Q0-9AdKqO9ATDHzgjGIOv36Rs5XZF67-IjmXVWtIDQ1PMBFDJQmST2ollIyujnEd-YHCwJl1pWmT5LfnzAewKBFpchBMBg11nacCZiyAZpHJvv_YrSUcaAQ6BBo5MK0iWuPSse5MMHj6F-Kfcp2PSha-3tjTfDhOAG7uslva5XHv_qSxFrEPLR_vPpqQP3Ec16sSS9tnRdGhJ1myjp3CmiraWk5f9sosnqZIU0zmjKyRPv-yBrAD7_nC8wVlpE0CcgAvWGwHfgIaxTrgLRgBdHNMbzmQWnoLorVRXnnmHl86MHCRqMNS4gnaVbaorUXVBmLT3HTkIn3cDPcIO1lkcnohAeWNo-ueVyOyeRrw6-jgSLWC63QjrkgIB5D29YxxTjzN6AXHP34oG_o8H6LFGv_FWKX0eOiwJ4EE18KV_pyQZKTFuUnfNKGYnr2y0S-KlyzW8luhjsSNHgAN3pBzi7uwR-7w1T8ZTsGgwmuvGBVIpzDZ7Qw9YhFMVKw3pjGZLs4RnFZRpZIKF5T_Ujca12BLTRjxZufFOIrmvx9UjbOdo0nd6GaGJg6baHn_ckvx6Q_UrxeKlc30hOygL2HQetY9eTypjLTNnFpkMC3_FCLUvMQeWWfmFoa26kfKAaeE=\",\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\\"metadata\\\":{\\\"send_date\\\":[\\\"1928-05-03\\\"],\\\"letter_title\\\":null,\\\"sender_persons\\\":[\\\"Ritter\\\",\\\"Groschupf\\\"],\\\"receiver_persons\\\":[\\\"W. Stauffacher\\\"]}}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-3-flash-preview\",\"prompt_feedback\":null,\"response_id\":\"ZmJ1aeyHKrPk7M8Phpv-4AI\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":48,\"candidates_tokens_details\":null,\"prompt_token_count\":1428,\"prompt_tokens_details\":[{\"modality\":\"IMAGE\",\"token_count\":1092},{\"modality\":\"TEXT\",\"token_count\":336}],\"thoughts_token_count\":1128,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2604,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"metadata\":{\"send_date\":[\"1928-05-03\"],\"letter_title\":null,\"sender_persons\":[\"Ritter\",\"Groschupf\"],\"receiver_persons\":[\"W. Stauffacher\"]}}}"}, "scoring": {"f1_macro": 0.72, "f1_micro": 0.84, "cost_summary": {"total_input_tokens": 46620, "total_output_tokens": 1157, "total_tokens": 47777, "input_cost_usd": 0.023309999999999997, "output_cost_usd": 0.003470999999999999, "total_cost_usd": 0.026781000000000003}}, "normalized_score": 72.0}