{"_id": {"$oid": "69d4eb12b877594bc5718a58"}, "test_id": "T0412", "benchmark": "company_lists", "date": "2025-11-25", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en", "de"], "layout": ["list"], "entry-type": ["company"], "task": ["information-extraction"]}, "contributors": ["lea_kasper", "sorin_marti"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.1-2025-11-13", "dataclass": "ListPage", "temperature": 0.5, "role_description": "You are a Historian", "prompt_file": "prompt_min.txt", "legacy_test": false}, "prompt": "- Answer in valid JSON.\n- The page ID is given as {page_id}.\n", "results": {"text": "{\"page_id\":\"{page_id}\",\"entries\":[{\"entry_id\":\"dyers_farbereien_schetty_ag_basle\",\"company_name\":\"F\u00e4rbereien Schetty, A.-G.\",\"location\":\"Basle\"},{\"entry_id\":\"elastics_g_kappeler_ag_zofingen\",\"company_name\":\"G. Kappeler A.-G.\",\"location\":\"Zofingen\"},{\"entry_id\":\"electrical_accessories_j_a_crabtree_co_ltd_walsall\",\"company_name\":\"J. A. Crabtree & Co., Ltd.\",\"location\":\"Walsall, England\"},{\"entry_id\":\"electric_furnaces_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_generators_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_lamps_magnetos_lucifer_sa_geneva\",\"company_name\":\"Magn\u00e9tos Lucifer S.A.\",\"location\":\"Carouge-Gen\u00e8ve, Geneva\"},{\"entry_id\":\"electric_locomotives_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_machinery_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_motors_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_switchgear_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_welding_plants_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electro_thermic_plants_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"}]}", "model": "gpt-5.1-2025-11-13", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1055, "output_tokens": 495, "total_tokens": 1550}, "duration": 6.966284990310669, "timestamp": "2025-11-25T15:34:20.817743", "parsed": {"page_id": "{page_id}", "entries": [{"entry_id": "dyers_farbereien_schetty_ag_basle", "company_name": "F\u00e4rbereien Schetty, A.-G.", "location": "Basle"}, {"entry_id": "elastics_g_kappeler_ag_zofingen", "company_name": "G. Kappeler A.-G.", "location": "Zofingen"}, {"entry_id": "electrical_accessories_j_a_crabtree_co_ltd_walsall", "company_name": "J. A. Crabtree & Co., Ltd.", "location": "Walsall, England"}, {"entry_id": "electric_furnaces_brown_boveri_cie_ag_baden", "company_name": "Brown Boveri & Cie., A.-G.", "location": "Baden, Switzerland"}, {"entry_id": "electric_generators_brown_boveri_cie_ag_baden", "company_name": "Brown Boveri & Cie., A.-G.", "location": "Baden, Switzerland"}, {"entry_id": "electric_lamps_magnetos_lucifer_sa_geneva", "company_name": "Magn\u00e9tos Lucifer S.A.", "location": "Carouge-Gen\u00e8ve, Geneva"}, {"entry_id": "electric_locomotives_brown_boveri_cie_ag_baden", "company_name": "Brown Boveri & Cie., A.-G.", "location": "Baden, Switzerland"}, {"entry_id": "electric_machinery_brown_boveri_cie_ag_baden", "company_name": "Brown Boveri & Cie., A.-G.", "location": "Baden, Switzerland"}, {"entry_id": "electric_motors_brown_boveri_cie_ag_baden", "company_name": "Brown Boveri & Cie., A.-G.", "location": "Baden, Switzerland"}, {"entry_id": "electric_switchgear_brown_boveri_cie_ag_baden", "company_name": "Brown Boveri & Cie., A.-G.", "location": "Baden, Switzerland"}, {"entry_id": "electric_welding_plants_brown_boveri_cie_ag_baden", "company_name": "Brown Boveri & Cie., A.-G.", "location": "Baden, Switzerland"}, {"entry_id": "electro_thermic_plants_brown_boveri_cie_ag_baden", "company_name": "Brown Boveri & Cie., A.-G.", "location": "Baden, Switzerland"}]}, "score": {"f1_score": 0.41, "precision": 0.40540540540540543, "recall": 0.40540540540540543, "true_positives": 15, "false_positives": 22, "false_negatives": 22, "field_scores": {"entries[4].entry_id": {"response": "electric_generators_brown_boveri_cie_ag_baden", "ground_truth": "156089_1321092_18-5", "score": 0.0625}, "page_id": {"response": "{page_id}", "ground_truth": "156089_1321092_18", "score": 0.07692307692307687}, "entries[1].company_name": {"response": "G. Kappeler A.-G.", "ground_truth": "G. Kappeler A.-G.", "score": 1.0}, "entries[8].entry_id": {"response": "electric_motors_brown_boveri_cie_ag_baden", "ground_truth": "156089_1321092_18-9", "score": 0.06666666666666665}, "entries[10].entry_id": {"response": "electric_welding_plants_brown_boveri_cie_ag_baden", "ground_truth": "156089_1321092_18-11", "score": 0.05797101449275366}, "entries[0].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[2].location": {"response": "Walsall, England", "ground_truth": "Walsall, England", "score": 1.0}, "entries[7].location": {"response": "Baden, Switzerland", "ground_truth": "Baden (Switzerland)", "score": 0.9189189189189189}, "entries[5].entry_id": {"response": "electric_lamps_magnetos_lucifer_sa_geneva", "ground_truth": "156089_1321092_18-6", "score": 0.06666666666666665}, "entries[0].company_name": {"response": "F\u00e4rbereien Schetty, A.-G.", "ground_truth": "F\u00e4rbereien Schetty, A.-G.", "score": 1.0}, "entries[1].entry_id": {"response": "elastics_g_kappeler_ag_zofingen", "ground_truth": "156089_1321092_18-2", "score": 0.07999999999999996}, "entries[4].location": {"response": "Baden, Switzerland", "ground_truth": "Baden (Switzerland)", "score": 0.9189189189189189}, "entries[0].entry_id": {"response": "dyers_farbereien_schetty_ag_basle", "ground_truth": "156089_1321092_18-1", "score": 0.07692307692307687}, "entries[5].company_name": {"response": "Magn\u00e9tos Lucifer S.A.", "ground_truth": "Magn\u00e9tos Lucifer S. A.", "score": 0.9767441860465115}, "entries[4].company_name": {"response": "Brown Boveri & Cie., A.-G.", "ground_truth": "Brown Boveri & Cie., A.-G.", "score": 1.0}, "entries[10].location": {"response": "Baden, Switzerland", "ground_truth": "Baden (Switzerland)", "score": 0.9189189189189189}, "entries[7].company_name": {"response": "Brown Boveri & Cie., A.-G.", "ground_truth": "Brown Boveri & Cie., A.-G.", "score": 1.0}, "entries[8].company_name": {"response": "Brown Boveri & Cie., A.-G.", "ground_truth": "Brown Boveri & Cie., A.-G.", "score": 1.0}, "entries[9].entry_id": {"response": "electric_switchgear_brown_boveri_cie_ag_baden", "ground_truth": "156089_1321092_18-10", "score": 0.06153846153846154}, "entries[10].company_name": {"response": "Brown Boveri & Cie., A.-G.", "ground_truth": "Brown Boveri & Cie., A.-G.", "score": 1.0}, "entries[8].location": {"response": "Baden, Switzerland", "ground_truth": "Baden (Switzerland)", "score": 0.9189189189189189}, "entries[6].company_name": {"response": "Brown Boveri & Cie., A.-G.", "ground_truth": "Brown Boveri & Cie., A.-G.", "score": 1.0}, "entries[2].entry_id": {"response": "electrical_accessories_j_a_crabtree_co_ltd_walsall", "ground_truth": "156089_1321092_18-3", "score": 0.05797101449275366}, "entries[5].location": {"response": "Carouge-Gen\u00e8ve, Geneva", "ground_truth": "Carouge-Gen\u00e8ve", "score": 0.7777777777777779}, "entries[11].entry_id": {"response": "electro_thermic_plants_brown_boveri_cie_ag_baden", "ground_truth": "156089_1321092_18-12", "score": 0.05882352941176472}, "entries[6].location": {"response": "Baden, Switzerland", "ground_truth": "Baden (Switzerland)", "score": 0.9189189189189189}, "entries[9].company_name": {"response": "Brown Boveri & Cie., A.-G.", "ground_truth": "Brown Boveri & Cie., A.-G.", "score": 1.0}, "entries[9].location": {"response": "Baden, Switzerland", "ground_truth": "Baden (Switzerland)", "score": 0.9189189189189189}, "entries[3].entry_id": {"response": "electric_furnaces_brown_boveri_cie_ag_baden", "ground_truth": "156089_1321092_18-4", "score": 0.06451612903225812}, "entries[11].company_name": {"response": "Brown Boveri & Cie., A.-G.", "ground_truth": "Brown Boveri & Cie., A.-G.", "score": 1.0}, "entries[7].entry_id": {"response": "electric_machinery_brown_boveri_cie_ag_baden", "ground_truth": "156089_1321092_18-8", "score": 0.06349206349206349}, "entries[11].location": {"response": "Baden, Switzerland", "ground_truth": "Baden (Switzerland)", "score": 0.9189189189189189}, "entries[3].company_name": {"response": "Brown Boveri & Cie., A.-G.", "ground_truth": "Brown Boveri & Cie., A.-G.", "score": 1.0}, "entries[3].location": {"response": "Baden, Switzerland", "ground_truth": "Baden (Switzerland)", "score": 0.9189189189189189}, "entries[6].entry_id": {"response": "electric_locomotives_brown_boveri_cie_ag_baden", "ground_truth": "156089_1321092_18-7", "score": 0.06153846153846154}, "entries[2].company_name": {"response": "J. A. Crabtree & Co., Ltd.", "ground_truth": "J. A. Crabtree & Co., Ltd.", "score": 1.0}, "entries[1].location": {"response": "Zofingen", "ground_truth": "Zofingen", "score": 1.0}}, "total_fields": 37}, "raw_response": "{\"id\":\"chatcmpl-CfoPPWu6FYRiQKN6rY4TJ1y9w4944\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"page_id\\\":\\\"{page_id}\\\",\\\"entries\\\":[{\\\"entry_id\\\":\\\"dyers_farbereien_schetty_ag_basle\\\",\\\"company_name\\\":\\\"F\u00e4rbereien Schetty, A.-G.\\\",\\\"location\\\":\\\"Basle\\\"},{\\\"entry_id\\\":\\\"elastics_g_kappeler_ag_zofingen\\\",\\\"company_name\\\":\\\"G. Kappeler A.-G.\\\",\\\"location\\\":\\\"Zofingen\\\"},{\\\"entry_id\\\":\\\"electrical_accessories_j_a_crabtree_co_ltd_walsall\\\",\\\"company_name\\\":\\\"J. A. Crabtree & Co., Ltd.\\\",\\\"location\\\":\\\"Walsall, England\\\"},{\\\"entry_id\\\":\\\"electric_furnaces_brown_boveri_cie_ag_baden\\\",\\\"company_name\\\":\\\"Brown Boveri & Cie., A.-G.\\\",\\\"location\\\":\\\"Baden, Switzerland\\\"},{\\\"entry_id\\\":\\\"electric_generators_brown_boveri_cie_ag_baden\\\",\\\"company_name\\\":\\\"Brown Boveri & Cie., A.-G.\\\",\\\"location\\\":\\\"Baden, Switzerland\\\"},{\\\"entry_id\\\":\\\"electric_lamps_magnetos_lucifer_sa_geneva\\\",\\\"company_name\\\":\\\"Magn\u00e9tos Lucifer S.A.\\\",\\\"location\\\":\\\"Carouge-Gen\u00e8ve, Geneva\\\"},{\\\"entry_id\\\":\\\"electric_locomotives_brown_boveri_cie_ag_baden\\\",\\\"company_name\\\":\\\"Brown Boveri & Cie., A.-G.\\\",\\\"location\\\":\\\"Baden, Switzerland\\\"},{\\\"entry_id\\\":\\\"electric_machinery_brown_boveri_cie_ag_baden\\\",\\\"company_name\\\":\\\"Brown Boveri & Cie., A.-G.\\\",\\\"location\\\":\\\"Baden, Switzerland\\\"},{\\\"entry_id\\\":\\\"electric_motors_brown_boveri_cie_ag_baden\\\",\\\"company_name\\\":\\\"Brown Boveri & Cie., A.-G.\\\",\\\"location\\\":\\\"Baden, Switzerland\\\"},{\\\"entry_id\\\":\\\"electric_switchgear_brown_boveri_cie_ag_baden\\\",\\\"company_name\\\":\\\"Brown Boveri & Cie., A.-G.\\\",\\\"location\\\":\\\"Baden, Switzerland\\\"},{\\\"entry_id\\\":\\\"electric_welding_plants_brown_boveri_cie_ag_baden\\\",\\\"company_name\\\":\\\"Brown Boveri & Cie., A.-G.\\\",\\\"location\\\":\\\"Baden, Switzerland\\\"},{\\\"entry_id\\\":\\\"electro_thermic_plants_brown_boveri_cie_ag_baden\\\",\\\"company_name\\\":\\\"Brown Boveri & Cie., A.-G.\\\",\\\"location\\\":\\\"Baden, Switzerland\\\"}]}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"page_id\":\"{page_id}\",\"entries\":[{\"entry_id\":\"dyers_farbereien_schetty_ag_basle\",\"company_name\":\"F\u00e4rbereien Schetty, A.-G.\",\"location\":\"Basle\"},{\"entry_id\":\"elastics_g_kappeler_ag_zofingen\",\"company_name\":\"G. Kappeler A.-G.\",\"location\":\"Zofingen\"},{\"entry_id\":\"electrical_accessories_j_a_crabtree_co_ltd_walsall\",\"company_name\":\"J. A. Crabtree & Co., Ltd.\",\"location\":\"Walsall, England\"},{\"entry_id\":\"electric_furnaces_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_generators_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_lamps_magnetos_lucifer_sa_geneva\",\"company_name\":\"Magn\u00e9tos Lucifer S.A.\",\"location\":\"Carouge-Gen\u00e8ve, Geneva\"},{\"entry_id\":\"electric_locomotives_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_machinery_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_motors_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_switchgear_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electric_welding_plants_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"},{\"entry_id\":\"electro_thermic_plants_brown_boveri_cie_ag_baden\",\"company_name\":\"Brown Boveri & Cie., A.-G.\",\"location\":\"Baden, Switzerland\"}]}}}],\"created\":1764081255,\"model\":\"gpt-5.1-2025-11-13\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":495,\"prompt_tokens\":1055,\"total_tokens\":1550,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.43254168772107127, "f1_macro": 0.4186666666666667, "micro_precision": 0.4336372847011145, "micro_recall": 0.4314516129032258, "total_instances": 15, "total_tp": 428, "total_fp": 559, "total_fn": 564, "cost_summary": {"total_input_tokens": 15825, "total_output_tokens": 10187, "total_tokens": 26012, "input_cost_usd": 0.019781250000000004, "output_cost_usd": 0.10187, "total_cost_usd": 0.12165125000000002}}, "normalized_score": 41.86666666666667}