{"_id": {"$oid": "6a8456c1ef0a19c343227d7d"}, "test_id": "T1593", "benchmark": "company_lists", "date": "2026-08-18", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en", "de"], "layout": ["list"], "entry-type": ["company"], "task": ["information-extraction"]}, "contributors": ["lea_kasper", "sorin_marti"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3.7-flash", "dataclass": "ListPage", "temperature": 0.5, "role_description": "You are a Historian", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "The image you are presented with stems from a digitized book containing lists of companies.\nYour task is to extract structured information about each company listed on the page.\n\nAbout the source:\n- The image stems from a trade index of the British Swiss Chamber of Commerce.\n- The image can show an alphabetical or a thematic list of companies.\n- The companies are mostly located in Switzerland and the UK.\n- The image stems from a trade index between 1925 and 1958.\n- Most pages have one column but some years have two columns.\n- The source itself is in English and German but the company names can be in English, German, French or Italian.\n\nAbout the entries:\n- Each entry describes a single company or person.\n- Alphabetical entries have filling dots between the company name and the page number. Dots and page numbers are not part of the data and should be ignored.\n- Alphabetical entries seldom to never have locations.\n- Thematic entries often have locations.\n- Thematic entries are listed under headings that describe the type of business.\n- Some thematic headings are only references to other headings, e.g. \"X, s. Y\".\n\nAbout the output:\n- Answer in valid JSON. The JSON should be an array of objects with the following fields:\n- The page ID is given as {page_id}.\n- Do not add country information, if it is not directly written with the location.\n\n{\n  \"entry_id\": \"A unique identifier for the entry, e.g. '{page_id}-1'\",\n  \"company_name\": \"The name of the company or person\",\n  \"location\": \"The location of the company, e.g. 'Zurich' or 'London, UK'. If no location is given, set to null.\"\n  ]\n}\n", "results": {"text": "{\"page_id\":\"page_89\",\"entries\":[{\"entry_id\":\"page_89-1\",\"company_name\":\"C. Troeber-Ruesch\",\"location\":\"K\u00fcsnacht (Zurich)\"},{\"entry_id\":\"page_89-2\",\"company_name\":\"Unitraco Ltd.\",\"location\":\"Zurich\"},{\"entry_id\":\"page_89-3\",\"company_name\":\"H. Ursprung\",\"location\":\"Zurich\"},{\"entry_id\":\"page_89-4\",\"company_name\":\"Wettler & Frey\",\"location\":\"Zurich 6\"},{\"entry_id\":\"page_89-5\",\"company_name\":\"Paul W\u00fcthrich\",\"location\":\"Lausanne\"},{\"entry_id\":\"page_89-6\",\"company_name\":\"Hultons Press Ltd.\",\"location\":\"London E. C. 4\"},{\"entry_id\":\"page_89-7\",\"company_name\":\"George Fischer Ltd.\",\"location\":\"Schaffhausen\"},{\"entry_id\":\"page_89-8\",\"company_name\":\"Louis Martin\",\"location\":\"Lausanne\"},{\"entry_id\":\"page_89-9\",\"company_name\":\"Techag Ltd.\",\"location\":\"Zurich\"},{\"entry_id\":\"page_89-10\",\"company_name\":\"Usine Fega\",\"location\":\"Zurich\"},{\"entry_id\":\"page_89-11\",\"company_name\":\"R. Amiguet-Peter\",\"location\":\"Lausanne\"}]}", "model": "gemini-3.7-flash", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1479, "output_tokens": 521, "total_tokens": 2888, "input_cost_usd": 0.00110925, "output_cost_usd": 0.0019537499999999998, "estimated_cost_usd": 0.0030629999999999998}, "duration": 6.1825103759765625, "timestamp": "2026-08-18T09:13:37.062867", "parsed": {"page_id": "page_89", "entries": [{"entry_id": "page_89-1", "company_name": "C. Troeber-Ruesch", "location": "K\u00fcsnacht (Zurich)"}, {"entry_id": "page_89-2", "company_name": "Unitraco Ltd.", "location": "Zurich"}, {"entry_id": "page_89-3", "company_name": "H. Ursprung", "location": "Zurich"}, {"entry_id": "page_89-4", "company_name": "Wettler & Frey", "location": "Zurich 6"}, {"entry_id": "page_89-5", "company_name": "Paul W\u00fcthrich", "location": "Lausanne"}, {"entry_id": "page_89-6", "company_name": "Hultons Press Ltd.", "location": "London E. C. 4"}, {"entry_id": "page_89-7", "company_name": "George Fischer Ltd.", "location": "Schaffhausen"}, {"entry_id": "page_89-8", "company_name": "Louis Martin", "location": "Lausanne"}, {"entry_id": "page_89-9", "company_name": "Techag Ltd.", "location": "Zurich"}, {"entry_id": "page_89-10", "company_name": "Usine Fega", "location": "Zurich"}, {"entry_id": "page_89-11", "company_name": "R. Amiguet-Peter", "location": "Lausanne"}]}, "conversation_id": "4c777193-c4dc-4b38-9703-8b1b236714c1", "score": {"f1_score": 0.65, "precision": 0.6470588235294118, "recall": 0.6470588235294118, "true_positives": 22, "false_positives": 12, "false_negatives": 12, "field_scores": {"entries[4].company_name": {"response": "Paul W\u00fcthrich", "ground_truth": "Paul W\u00fcthrich", "score": 1.0}, "entries[10].entry_id": {"response": "page_89-11", "ground_truth": "156089_1321097_125-11", "score": 0.32258064516129037}, "entries[4].entry_id": {"response": "page_89-5", "ground_truth": "156089_1321097_125-5", "score": 0.27586206896551724}, "entries[10].location": {"response": "Lausanne", "ground_truth": "Lausanne", "score": 1.0}, "page_id": {"response": "page_89", "ground_truth": "156089_1321097_125", "score": 0.16000000000000003}, "entries[0].company_name": {"response": "C. Troeber-Ruesch", "ground_truth": "C. Troeber-Ruesch", "score": 1.0}, "entries[8].entry_id": {"response": "page_89-9", "ground_truth": "156089_1321097_125-9", "score": 0.27586206896551724}, "entries[2].entry_id": {"response": "page_89-3", "ground_truth": "156089_1321097_125-3", "score": 0.27586206896551724}, "entries[6].location": {"response": "Schaffhausen", "ground_truth": "Schaffhausen", "score": 1.0}, "entries[7].entry_id": {"response": "page_89-8", "ground_truth": "156089_1321097_125-8", "score": 0.27586206896551724}, "entries[6].entry_id": {"response": "page_89-7", "ground_truth": "156089_1321097_125-7", "score": 0.27586206896551724}, "entries[3].entry_id": {"response": "page_89-4", "ground_truth": "156089_1321097_125-4", "score": 0.27586206896551724}, "entries[1].company_name": {"response": "Unitraco Ltd.", "ground_truth": "Unitraco Ltd.", "score": 1.0}, "entries[5].company_name": {"response": "Hultons Press Ltd.", "ground_truth": "Hultons Press Ltd.", "score": 1.0}, "entries[3].company_name": {"response": "Wettler & Frey", "ground_truth": "Wettler & Frey", "score": 1.0}, "entries[0].entry_id": {"response": "page_89-1", "ground_truth": "156089_1321097_125-1", "score": 0.27586206896551724}, "entries[7].location": {"response": "Lausanne", "ground_truth": "Lausanne", "score": 1.0}, "entries[7].company_name": {"response": "Louis Martin", "ground_truth": "Louis Martin", "score": 1.0}, "entries[3].location": {"response": "Zurich 6", "ground_truth": "Zurich 6", "score": 1.0}, "entries[5].location": {"response": "London E. C. 4", "ground_truth": "London E. C. 4", "score": 1.0}, "entries[4].location": {"response": "Lausanne", "ground_truth": "Lausanne", "score": 1.0}, "entries[2].location": {"response": "Zurich", "ground_truth": "Zurich", "score": 1.0}, "entries[9].location": {"response": "Zurich", "ground_truth": "Zurich", "score": 1.0}, "entries[9].company_name": {"response": "Usine Fega", "ground_truth": "Usine Fega", "score": 1.0}, "entries[2].company_name": {"response": "H. Ursprung", "ground_truth": "H. Ursprung", "score": 1.0}, "entries[1].entry_id": {"response": "page_89-2", "ground_truth": "156089_1321097_125-2", "score": 0.27586206896551724}, "entries[5].entry_id": {"response": "page_89-6", "ground_truth": "156089_1321097_125-6", "score": 0.27586206896551724}, "entries[0].location": {"response": "K\u00fcsnacht (Zurich)", "ground_truth": "K\u00fcsnacht (Zurich)", "score": 1.0}, "entries[8].location": {"response": "Zurich", "ground_truth": "Zurich", "score": 1.0}, "entries[6].company_name": {"response": "George Fischer Ltd.", "ground_truth": "George Fischer Ltd.", "score": 1.0}, "entries[1].location": {"response": "Zurich", "ground_truth": "Zurich", "score": 1.0}, "entries[10].company_name": {"response": "R. Amiguet-Peter", "ground_truth": "R. Amiguet-Peter", "score": 1.0}, "entries[9].entry_id": {"response": "page_89-10", "ground_truth": "156089_1321097_125-10", "score": 0.32258064516129037}, "entries[8].company_name": {"response": "Techag Ltd.", "ground_truth": "Techag Ltd.", "score": 1.0}}, "total_fields": 34}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"x-gemini-service-tier\":\"standard\",\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Tue, 18 Aug 2026 07:13:37 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=5885\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"media_resolution\":null,\"code_execution_result\":null,\"executable_code\":null,\"file_data\":null,\"function_call\":null,\"function_response\":null,\"inline_data\":null,\"text\":\"{\\n  \\\"page_id\\\": \\\"page_89\\\",\\n  \\\"entries\\\": [\\n    {\\n      \\\"entry_id\\\": \\\"page_89-1\\\",\\n      \\\"company_name\\\": \\\"C. Troeber-Ruesch\\\",\\n      \\\"location\\\": \\\"K\u00fcsnacht (Zurich)\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"page_89-2\\\",\\n      \\\"company_name\\\": \\\"Unitraco Ltd.\\\",\\n      \\\"location\\\": \\\"Zurich\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"page_89-3\\\",\\n      \\\"company_name\\\": \\\"H. Ursprung\\\",\\n      \\\"location\\\": \\\"Zurich\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"page_89-4\\\",\\n      \\\"company_name\\\": \\\"Wettler & Frey\\\",\\n      \\\"location\\\": \\\"Zurich 6\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"page_89-5\\\",\\n      \\\"company_name\\\": \\\"Paul W\u00fcthrich\\\",\\n      \\\"location\\\": \\\"Lausanne\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"page_89-6\\\",\\n      \\\"company_name\\\": \\\"Hultons Press Ltd.\\\",\\n      \\\"location\\\": \\\"London E. C. 4\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"page_89-7\\\",\\n      \\\"company_name\\\": \\\"George Fischer Ltd.\\\",\\n      \\\"location\\\": \\\"Schaffhausen\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"page_89-8\\\",\\n      \\\"company_name\\\": \\\"Louis Martin\\\",\\n      \\\"location\\\": \\\"Lausanne\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"page_89-9\\\",\\n      \\\"company_name\\\": \\\"Techag Ltd.\\\",\\n      \\\"location\\\": \\\"Zurich\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"page_89-10\\\",\\n      \\\"company_name\\\": \\\"Usine Fega\\\",\\n      \\\"location\\\": \\\"Zurich\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"page_89-11\\\",\\n      \\\"company_name\\\": \\\"R. Amiguet-Peter\\\",\\n      \\\"location\\\": \\\"Lausanne\\\"\\n    }\\n  ]\\n}\",\"thought\":null,\"thought_signature\":\"EvQTCvETARFNMg_MJPoGEINk_VWrSDgeLh2PrbYps3OKMAY7lZEdEoQlgodAtVIbOS8mpvsNchPT4uqCkAZBtQTfcS6m21CWJxg7E3Ds9uqAV1dN0YHOxXX0gQbOl9bsc06tCSb9nrTlymCx5vOwYvdA13yR5JzPHjiFAWBHPpFu-jThvZGxrgDZlH_91h81NFjinFvUeEOvlUEIzd_DHYRbNxwgAVdTi_-IuYsQqbUXtRg6N9s8ZN6IyuHHRF21eBsIPfbErx65InNCZy_RKrkSqkhSepIDYS2_ckEN8s2yjD4sPiw63XVcqkpJgG4lqZjLbXX62nUmm9SF709ichpTJxkX3KcuLk7qbsDuVhKCQqUoHNkCMNFNQ901gBm3U0aohCHJJHCwGQH1xLH1I5kiHcm8yQ71wgsD17LgNfUX6SOxsshSzHpvAowK4ke64o4nGTjWBmlhOkY4MCaYhlpWgxFmd4PE_vh9zJSWnder-99jj6aCHSAORHV4IzRgOkH8FLI7th-qm83AdK_-W_Z-oNmUoQDnrkx2UbE_XZgf0a3FQ93qXT_URzqdEcfdWBD6sxt3FvP4g2t_RjBpFUPzxp1JjepnGcxyxQH1iX82kQroyGfMub3ptQNf1SLCwm3tl3mga5YCVgE-gvSUzCciPWhN-V8ZuIrQ101FPPFu7AF36qmM9BjGgTo4LYZZO9Er6nNc1Dw6jr9Hb506gZPInbAtf5IwcuaLhX6enXhMizR0elCtL7oyUcj2Q_ned7ndDjJ3reJ8bf3-FC7P7B0CYYpGUeA5xy3SUZ5IBrMey1HJeztJlRCR_LHVLmOqZKg_2buSu8AuLLj_KqkoB8oJQ14NPTqBuZHFaOhO7PIHifsfkwVxPpRFKc8dAgC59gotS1l4mYnh_m2plzMl9HKZ5tW83VPG8aGTCcNOLJRb9m89IVHBx5HKLCGvyleNAq3tP0KTcLHu7IGEsBuWmpc5Kc98yAvjdk1U7ckts6lq-gcJcNl-8ZT2TTTKpD3GDKwOhCvBMKktxsZrJnMeen56r-qyigaKo5IYF3a6RT-T95XBu5mvvByZYZo83FJ-WtfJ_xVnUHKe6aZyUchndQ2JHWRDTIdmrwf7gfMBBE9qZouL4TcC6luukwVc4Z_zaqLEx0Ne3SNC6ONuZbCFWcZhDRbQOH_l_bwSAksXYbZZ7N0AiClQhOTKZ160Fuv-BOhpUn_N6UZT9iw4DQOWaCwBo3SoMqCoYwen3qfluToXRjOb_ULJ1VHsTQiMOyEV3qPgfqA8oQGz6iHOetPaLKQDEjKVGhmag0Zl0sNgD3G2jOGEFwraFhqmDZjhgeUyVP3Msj-RA2gLhf2geo6wcARQm6Ve9scxD8VMxjZmor6lqDczeJ1WOLejTsOf0GFT3e7Dn0MlTowVfPHyTFBq5sSE8EwJNSZVyIu4T0dw0QTCOlekl4DAI2blxJpKdfK79CSS7CegxD9ZwPzskCTXEio0xsFUtaFeaNA-4qRS3YfuHULXFaHBkz7q--rM5tZYY1D0L5iVnzA6Ivvsl0SzIrEr_WGllhtpbtrYTPVbCC9VZPG-WoZWpHNi0mL7kuizALLE5QbtCEssFN7Zjhg_uyZQW6hcARUVyaGlvhbOelTNah1TIrN6Rd2_xt9Fym6CAge0s2oZV7xeqn1LWSGAuBX1iLS0kxlImFi-gyIPVcD62V-su0Ct97RfcB3sllRiZbpGCYLWJosAbr2hyoKFzIwcnJAwqs6susA-kOlQcHnAM5rxo7-Yfx-DpOhgskJN3-IibAzg5FVJ_97EdLe3sa_8ED86NLeJTZTZRkv-DAwa8XOV-Lxd7YDtWU65_cSxNi1705tzxVbi60kctBLo5qGSl-t5mKapLCh4yNbbWkpplJd5B1-1SQnzHp9CyNCkWEaDcKT96X8CF2ZLrqgXMEKWupScHCVUx-o_tQUfZaj4X_rYnTw_KIFazIbVsJBLsGJmADLQ8joSCjy2II8FHGSmUst8C5_rFzx_k_DaI3pOcpUUtAtTzMSM1IOzEku0K7FTjwh1ylG82vj5WqukhE32vsc_wtcvgiqrxz5dMIsCr0-AznZHkMTTCygBAqfvNOaE8nSTWMXDrqjLRcK6jkt8OfCQ8FPxgtP9uAjJg1lX8UdM48h-ndGA2tcg9bZF_l2xzAl0-3-AckYrNYgyK-Syep2Ipm61BIDNsg2ZPBneFXRUsmyxGHg5V28a8NLWo7ho-UOsEYlRnQ3JyTkTNiVeHFN9a0zOZRrwzBVrxctYS2DmdpnGk71ZD6WcnMCxF7BnXrqY7azunUiX8CFnIXxGoz7o9oRJmFfDA5Zg-DG0_gzo2l0XWhzUV3gWz3pEpJ7XvouzAImuJojbd3tK3ECTp0cN2rSglZ-yKTvb_OFTMRsRSfiUEr5QYotrDGggwFgPhB5nUYdVf15WUpnSXKqKYn859p-t7njXgJCWe6TyhvV41uzc_TZ49RhpLWEObb-DVf-05b8Q29wFoDVkS7dc0D4Km2bjHy6sGH3zdIF-Oa3HYo1E_Lk-p5ERynjxI5T8meY27aMslKoNstyILL3A24qQtEzXQs2UO7EWzWV9VfQVsORaDg4eX3Bil4MjxaCDtfnfOJck2GdD9CtrcTZlUeH3dK_aXfUlVyGHq9ZH1_gp2S0X5KavcDhssGaNYeHMEGYJbAHKOUpe4pvDP8v57q6LjUvgmzKS6R1TNEcmIkvD5xWTqH-iAVmE84P1x_g_93kskRJtl-GGlCn0VTCYShRboZlGd7SLWMJAWWkCpa6JrIJEycY63yGyQ5mZqlPYy-zirs8ADT8kFOTtkzEjgoO7eueWEgQrKiQnSsCjhUmBdODGTgjDXVY04WG4_uRdGNpO9jOnZyZXnnMEBj0KICderztxCdAsHNcFIBlSfcm-Shnj_zOGw5E0-G1LYvi4ak_eIpk3-QREUaX8wbjAi5YiBGuXTdI-UKErAcMdXMMILCZFODAQjRFHBXDhq5FCAXGoU3l9ltgA6qYaDXY4nqfGdB1RvvwM7YPONCk5JYzbMFyfcwF5mSdRPVOfIF2zeqvTJXdYOFf-ut1R2JkhIUDj-MTiwmYQwMnTZcgfq2PXuxVx-n8Al502KWHWV8bwhqxQZ3WNfh_Nk0va_0_Ounu6dD6YjhcXLGLRvo7lX_c-LcBTY77WH8YoYKqZ3lZ2UvK5Twd1vMSfru4Yb-70ICIbVU_5_GBsu0TSvd4rt7Za_jdQWmJqLa0JrnTjscqIYMCjnZePtYFnA7NQrcwz3D1fSkledN289_a-ztoKy0OwCrtp-qphhHr8f-3UKjwc0P1w-LkBKn0Gj4492HYVjHZshJEBAK5x7CzXV2vn_5TNHxjJFq95yWesE0IiIgyFmrYGqQ==\",\"video_metadata\":null}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"grounding_metadata\":null,\"avg_logprobs\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null,\"url_context_metadata\":null}],\"create_time\":null,\"model_version\":\"gemini-3.7-flash\",\"prompt_feedback\":null,\"response_id\":\"GwaEasbYF8vqnsEPjNGbiAg\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":521,\"candidates_tokens_details\":null,\"prompt_token_count\":1479,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":399},{\"modality\":\"IMAGE\",\"token_count\":1080}],\"thoughts_token_count\":888,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2888,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"page_id\":\"page_89\",\"entries\":[{\"entry_id\":\"page_89-1\",\"company_name\":\"C. Troeber-Ruesch\",\"location\":\"K\u00fcsnacht (Zurich)\"},{\"entry_id\":\"page_89-2\",\"company_name\":\"Unitraco Ltd.\",\"location\":\"Zurich\"},{\"entry_id\":\"page_89-3\",\"company_name\":\"H. Ursprung\",\"location\":\"Zurich\"},{\"entry_id\":\"page_89-4\",\"company_name\":\"Wettler & Frey\",\"location\":\"Zurich 6\"},{\"entry_id\":\"page_89-5\",\"company_name\":\"Paul W\u00fcthrich\",\"location\":\"Lausanne\"},{\"entry_id\":\"page_89-6\",\"company_name\":\"Hultons Press Ltd.\",\"location\":\"London E. C. 4\"},{\"entry_id\":\"page_89-7\",\"company_name\":\"George Fischer Ltd.\",\"location\":\"Schaffhausen\"},{\"entry_id\":\"page_89-8\",\"company_name\":\"Louis Martin\",\"location\":\"Lausanne\"},{\"entry_id\":\"page_89-9\",\"company_name\":\"Techag Ltd.\",\"location\":\"Zurich\"},{\"entry_id\":\"page_89-10\",\"company_name\":\"Usine Fega\",\"location\":\"Zurich\"},{\"entry_id\":\"page_89-11\",\"company_name\":\"R. Amiguet-Peter\",\"location\":\"Lausanne\"}]}}"}, "scoring": {"f1_micro": 0.49570055639858374, "f1_macro": 0.49133333333333334, "micro_precision": 0.49746192893401014, "micro_recall": 0.4939516129032258, "total_instances": 15, "total_tp": 490, "total_fp": 495, "total_fn": 502, "cost_summary": {"total_input_tokens": 22303, "total_output_tokens": 16955, "total_tokens": 39258, "input_cost_usd": 0.016727250000000003, "output_cost_usd": 0.06358125, "total_cost_usd": 0.08030849999999999}}, "normalized_score": 49.13333333333333}