{"_id": {"$oid": "6a671d25338dadb0be4e84a1"}, "test_id": "T1353", "benchmark": "company_lists", "date": "2026-07-25", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en", "de"], "layout": ["list"], "entry-type": ["company"], "task": ["information-extraction"]}, "contributors": ["lea_kasper", "sorin_marti"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.6-sol", "dataclass": "ListPage", "temperature": 0.5, "role_description": "You are a Historian", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "The image you are presented with stems from a digitized book containing lists of companies.\nYour task is to extract structured information about each company listed on the page.\n\nAbout the source:\n- The image stems from a trade index of the British Swiss Chamber of Commerce.\n- The image can show an alphabetical or a thematic list of companies.\n- The companies are mostly located in Switzerland and the UK.\n- The image stems from a trade index between 1925 and 1958.\n- Most pages have one column but some years have two columns.\n- The source itself is in English and German but the company names can be in English, German, French or Italian.\n\nAbout the entries:\n- Each entry describes a single company or person.\n- Alphabetical entries have filling dots between the company name and the page number. Dots and page numbers are not part of the data and should be ignored.\n- Alphabetical entries seldom to never have locations.\n- Thematic entries often have locations.\n- Thematic entries are listed under headings that describe the type of business.\n- Some thematic headings are only references to other headings, e.g. \"X, s. Y\".\n\nAbout the output:\n- Answer in valid JSON. The JSON should be an array of objects with the following fields:\n- The page ID is given as {page_id}.\n- Do not add country information, if it is not directly written with the location.\n\n{\n  \"entry_id\": \"A unique identifier for the entry, e.g. '{page_id}-1'\",\n  \"company_name\": \"The name of the company or person\",\n  \"location\": \"The location of the company, e.g. 'Zurich' or 'London, UK'. If no location is given, set to null.\"\n  ]\n}\n", "results": {"text": "{\"page_id\":\"{page_id}\",\"entries\":[{\"entry_id\":\"{page_id}-1\",\"company_name\":\"Crowe & Co., Ltd.\",\"location\":\"67, Nauenstrasse, Basle\"},{\"entry_id\":\"{page_id}-2\",\"company_name\":\"Imperial Airways Ltd.\",\"location\":\"Airport of London, Croydon\"},{\"entry_id\":\"{page_id}-3\",\"company_name\":\"Metallwerke A.G.\",\"location\":\"Dornach (Solothurn)\"},{\"entry_id\":\"{page_id}-4\",\"company_name\":\"Aluminium-Industrie-Aktien-Gesellschaft\",\"location\":\"Neuhausen\"},{\"entry_id\":\"{page_id}-5\",\"company_name\":\"Chemische Fabrik vorm. Sandoz\",\"location\":\"Basle\"},{\"entry_id\":\"{page_id}-6\",\"company_name\":\"Clayton Aniline Co., Ltd.\",\"location\":\"Ashton New Road, Clayton, Manchester\"},{\"entry_id\":\"{page_id}-7\",\"company_name\":\"J. R. Geigy, A.-G.\",\"location\":\"Basle\"},{\"entry_id\":\"{page_id}-8\",\"company_name\":\"Society of Chemical Industry in Basle\",\"location\":\"Basle\"},{\"entry_id\":\"{page_id}-9\",\"company_name\":\"J. Hopf & Co. (Merchants)\",\"location\":\"12, Aeschengraben, Basle\"},{\"entry_id\":\"{page_id}-10\",\"company_name\":\"Rayon A.-G. (Merchants)\",\"location\":\"14, Stollenrain, Arlesheim\"},{\"entry_id\":\"{page_id}-11\",\"company_name\":\"Banque F\u00e9d\u00e9rale S. A.\",\"location\":\"Vevey\"},{\"entry_id\":\"{page_id}-12\",\"company_name\":\"Barclays Bank Ltd. (Foreign Branch)\",\"location\":\"168, Fenchurch Street, London E.C. 3\"},{\"entry_id\":\"{page_id}-13\",\"company_name\":\"Galland & Co.\",\"location\":\"3, Place St-Fran\u00e7ois, Lausanne\"},{\"entry_id\":\"{page_id}-14\",\"company_name\":\"Lloyds and National Provincial Foreign Bank, Ltd.\",\"location\":\"1, Place Bel-Air, Geneva\"},{\"entry_id\":\"{page_id}-15\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"1, Aeschenvorstadt, Basle\"},{\"entry_id\":\"{page_id}-16\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"Lausanne\"},{\"entry_id\":\"{page_id}-17\",\"company_name\":\"Wagons-Lits/Cook S.A.\",\"location\":\"4, Rue du Mont Blanc, Geneva\"}]}", "model": "gpt-5.6-sol", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 3538, "output_tokens": 915, "total_tokens": 4453, "input_cost_usd": 0.01769, "output_cost_usd": 0.027450000000000002, "estimated_cost_usd": 0.04514}, "duration": 11.088995456695557, "timestamp": "2026-07-25T20:43:34.199350", "parsed": {"page_id": "{page_id}", "entries": [{"entry_id": "{page_id}-1", "company_name": "Crowe & Co., Ltd.", "location": "67, Nauenstrasse, Basle"}, {"entry_id": "{page_id}-2", "company_name": "Imperial Airways Ltd.", "location": "Airport of London, Croydon"}, {"entry_id": "{page_id}-3", "company_name": "Metallwerke A.G.", "location": "Dornach (Solothurn)"}, {"entry_id": "{page_id}-4", "company_name": "Aluminium-Industrie-Aktien-Gesellschaft", "location": "Neuhausen"}, {"entry_id": "{page_id}-5", "company_name": "Chemische Fabrik vorm. Sandoz", "location": "Basle"}, {"entry_id": "{page_id}-6", "company_name": "Clayton Aniline Co., Ltd.", "location": "Ashton New Road, Clayton, Manchester"}, {"entry_id": "{page_id}-7", "company_name": "J. R. Geigy, A.-G.", "location": "Basle"}, {"entry_id": "{page_id}-8", "company_name": "Society of Chemical Industry in Basle", "location": "Basle"}, {"entry_id": "{page_id}-9", "company_name": "J. Hopf & Co. (Merchants)", "location": "12, Aeschengraben, Basle"}, {"entry_id": "{page_id}-10", "company_name": "Rayon A.-G. (Merchants)", "location": "14, Stollenrain, Arlesheim"}, {"entry_id": "{page_id}-11", "company_name": "Banque F\u00e9d\u00e9rale S. A.", "location": "Vevey"}, {"entry_id": "{page_id}-12", "company_name": "Barclays Bank Ltd. (Foreign Branch)", "location": "168, Fenchurch Street, London E.C. 3"}, {"entry_id": "{page_id}-13", "company_name": "Galland & Co.", "location": "3, Place St-Fran\u00e7ois, Lausanne"}, {"entry_id": "{page_id}-14", "company_name": "Lloyds and National Provincial Foreign Bank, Ltd.", "location": "1, Place Bel-Air, Geneva"}, {"entry_id": "{page_id}-15", "company_name": "Swiss Bank Corporation", "location": "1, Aeschenvorstadt, Basle"}, {"entry_id": "{page_id}-16", "company_name": "Swiss Bank Corporation", "location": "Lausanne"}, {"entry_id": "{page_id}-17", "company_name": "Wagons-Lits/Cook S.A.", "location": "4, Rue du Mont Blanc, Geneva"}]}, "conversation_id": "521c8f47-e44f-4caa-becf-867999e8fbd9", "score": {"f1_score": 0.46, "precision": 0.46153846153846156, "recall": 0.46153846153846156, "true_positives": 24, "false_positives": 28, "false_negatives": 28, "field_scores": {"entries[12].entry_id": {"response": "{page_id}-13", "ground_truth": "156089_1321093_11-13", "score": 0.25}, "entries[1].entry_id": {"response": "{page_id}-2", "ground_truth": "156089_1321093_11-2", "score": 0.19999999999999996}, "entries[13].entry_id": {"response": "{page_id}-14", "ground_truth": "156089_1321093_11-14", "score": 0.25}, "entries[11].company_name": {"response": "Barclays Bank Ltd. (Foreign Branch)", "ground_truth": "Barclays Bank Ltd. (Foreign Branch)", "score": 1.0}, "entries[14].entry_id": {"response": "{page_id}-15", "ground_truth": "156089_1321093_11-15", "score": 0.25}, "entries[8].company_name": {"response": "J. Hopf & Co. (Merchants)", "ground_truth": "J. Hopf & Co. (Merchants)", "score": 1.0}, "entries[12].location": {"response": "3, Place St-Fran\u00e7ois, Lausanne", "ground_truth": "Lausanne", "score": 0.42105263157894735}, "entries[9].location": {"response": "14, Stollenrain, Arlesheim", "ground_truth": "Arlesheim", "score": 0.5142857142857142}, "entries[3].location": {"response": "Neuhausen", "ground_truth": "Neuhausen", "score": 1.0}, "entries[16].entry_id": {"response": "{page_id}-17", "ground_truth": "156089_1321093_11-17", "score": 0.25}, "entries[9].entry_id": {"response": "{page_id}-10", "ground_truth": "156089_1321093_11-10", "score": 0.25}, "entries[6].company_name": {"response": "J. R. Geigy, A.-G.", "ground_truth": "J. R. Geigy, A.-G.", "score": 1.0}, "entries[13].location": {"response": "1, Place Bel-Air, Geneva", "ground_truth": "Geneva", "score": 0.4}, "entries[16].company_name": {"response": "Wagons-Lits/Cook S.A.", "ground_truth": "Wagons-Lits/Cook S.A.", "score": 1.0}, "entries[4].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[5].location": {"response": "Ashton New Road, Clayton, Manchester", "ground_truth": "Manchester", "score": 0.43478260869565216}, "entries[0].entry_id": {"response": "{page_id}-1", "ground_truth": "156089_1321093_11-1", "score": 0.19999999999999996}, "entries[10].entry_id": {"response": "{page_id}-11", "ground_truth": "156089_1321093_11-11", "score": 0.25}, "entries[14].company_name": {"response": "Swiss Bank Corporation", "ground_truth": "Swiss Bank Corporation", "score": 1.0}, "entries[12].company_name": {"response": "Galland & Co.", "ground_truth": "Galland & Co.", "score": 1.0}, "entries[4].entry_id": {"response": "{page_id}-5", "ground_truth": "156089_1321093_11-5", "score": 0.19999999999999996}, "entries[7].entry_id": {"response": "{page_id}-8", "ground_truth": "156089_1321093_11-8", "score": 0.19999999999999996}, "entries[15].location": {"response": "Lausanne", "ground_truth": "Lausanne", "score": 1.0}, "entries[15].entry_id": {"response": "{page_id}-16", "ground_truth": "156089_1321093_11-16", "score": 0.25}, "entries[2].company_name": {"response": "Metallwerke A.G.", "ground_truth": "Metallwerke A.G.", "score": 1.0}, "entries[0].location": {"response": "67, Nauenstrasse, Basle", "ground_truth": "Basle", "score": 0.3571428571428571}, "entries[1].company_name": {"response": "Imperial Airways Ltd.", "ground_truth": "Imperial Airways Ltd.", "score": 1.0}, "entries[10].company_name": {"response": "Banque F\u00e9d\u00e9rale S. A.", "ground_truth": "Banque F\u00e9d\u00e9rale S.A.", "score": 0.975609756097561}, "page_id": {"response": "{page_id}", "ground_truth": "156089_1321093_11", "score": 0.07692307692307687}, "entries[11].entry_id": {"response": "{page_id}-12", "ground_truth": "156089_1321093_11-12", "score": 0.25}, "entries[3].company_name": {"response": "Aluminium-Industrie-Aktien-Gesellschaft", "ground_truth": "Aluminium-Industrie-Aktien-Gesellschaft", "score": 1.0}, "entries[6].entry_id": {"response": "{page_id}-7", "ground_truth": "156089_1321093_11-7", "score": 0.19999999999999996}, "entries[7].company_name": {"response": "Society of Chemical Industry in Basle", "ground_truth": "Society of Chemical Industry in Basle", "score": 1.0}, "entries[7].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[8].entry_id": {"response": "{page_id}-9", "ground_truth": "156089_1321093_11-9", "score": 0.19999999999999996}, "entries[13].company_name": {"response": "Lloyds and National Provincial Foreign Bank, Ltd.", "ground_truth": "Lloyds and National Provincial Foreign Bank, Ltd.", "score": 1.0}, "entries[3].entry_id": {"response": "{page_id}-4", "ground_truth": "156089_1321093_11-4", "score": 0.19999999999999996}, "entries[6].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[2].entry_id": {"response": "{page_id}-3", "ground_truth": "156089_1321093_11-3", "score": 0.19999999999999996}, "entries[15].company_name": {"response": "Swiss Bank Corporation", "ground_truth": "Swiss Bank Corporation", "score": 1.0}, "entries[8].location": {"response": "12, Aeschengraben, Basle", "ground_truth": "Basle", "score": 0.3448275862068966}, "entries[0].company_name": {"response": "Crowe & Co., Ltd.", "ground_truth": "Crowe & Co., Ltd.", "score": 1.0}, "entries[1].location": {"response": "Airport of London, Croydon", "ground_truth": "Croydon", "score": 0.4242424242424242}, "entries[14].location": {"response": "1, Aeschenvorstadt, Basle", "ground_truth": "Basle", "score": 0.33333333333333337}, "entries[16].location": {"response": "4, Rue du Mont Blanc, Geneva", "ground_truth": "Geneva", "score": 0.3529411764705882}, "entries[9].company_name": {"response": "Rayon A.-G. (Merchants)", "ground_truth": "Rayon A.-G. (Merchants)", "score": 1.0}, "entries[10].location": {"response": "Vevey", "ground_truth": "Vevey", "score": 1.0}, "entries[5].entry_id": {"response": "{page_id}-6", "ground_truth": "156089_1321093_11-6", "score": 0.19999999999999996}, "entries[2].location": {"response": "Dornach (Solothurn)", "ground_truth": "Dornach (Solothurn)", "score": 1.0}, "entries[5].company_name": {"response": "Clayton Aniline Co., Ltd.", "ground_truth": "Clayton Aniline Co., Ltd.", "score": 1.0}, "entries[11].location": {"response": "168, Fenchurch Street, London E.C. 3", "ground_truth": "London E.C. 3.", "score": 0.52}, "entries[4].company_name": {"response": "Chemische Fabrik vorm. Sandoz", "ground_truth": "Chemische Fabrik vorm. Sandoz", "score": 1.0}}, "total_fields": 52}, "raw_response": "{\"id\":\"chatcmpl-E5bdFYrUIarZ0KGcJ6oIwK20GafAb\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"entries\\\":[{\\\"company_name\\\":\\\"Crowe & Co., Ltd.\\\",\\\"entry_id\\\":\\\"{page_id}-1\\\",\\\"location\\\":\\\"67, Nauenstrasse, Basle\\\"},{\\\"company_name\\\":\\\"Imperial Airways Ltd.\\\",\\\"entry_id\\\":\\\"{page_id}-2\\\",\\\"location\\\":\\\"Airport of London, Croydon\\\"},{\\\"company_name\\\":\\\"Metallwerke A.G.\\\",\\\"entry_id\\\":\\\"{page_id}-3\\\",\\\"location\\\":\\\"Dornach (Solothurn)\\\"},{\\\"company_name\\\":\\\"Aluminium-Industrie-Aktien-Gesellschaft\\\",\\\"entry_id\\\":\\\"{page_id}-4\\\",\\\"location\\\":\\\"Neuhausen\\\"},{\\\"company_name\\\":\\\"Chemische Fabrik vorm. Sandoz\\\",\\\"entry_id\\\":\\\"{page_id}-5\\\",\\\"location\\\":\\\"Basle\\\"},{\\\"company_name\\\":\\\"Clayton Aniline Co., Ltd.\\\",\\\"entry_id\\\":\\\"{page_id}-6\\\",\\\"location\\\":\\\"Ashton New Road, Clayton, Manchester\\\"},{\\\"company_name\\\":\\\"J. R. Geigy, A.-G.\\\",\\\"entry_id\\\":\\\"{page_id}-7\\\",\\\"location\\\":\\\"Basle\\\"},{\\\"company_name\\\":\\\"Society of Chemical Industry in Basle\\\",\\\"entry_id\\\":\\\"{page_id}-8\\\",\\\"location\\\":\\\"Basle\\\"},{\\\"company_name\\\":\\\"J. Hopf & Co. (Merchants)\\\",\\\"entry_id\\\":\\\"{page_id}-9\\\",\\\"location\\\":\\\"12, Aeschengraben, Basle\\\"},{\\\"company_name\\\":\\\"Rayon A.-G. (Merchants)\\\",\\\"entry_id\\\":\\\"{page_id}-10\\\",\\\"location\\\":\\\"14, Stollenrain, Arlesheim\\\"},{\\\"company_name\\\":\\\"Banque F\u00e9d\u00e9rale S. A.\\\",\\\"entry_id\\\":\\\"{page_id}-11\\\",\\\"location\\\":\\\"Vevey\\\"},{\\\"company_name\\\":\\\"Barclays Bank Ltd. (Foreign Branch)\\\",\\\"entry_id\\\":\\\"{page_id}-12\\\",\\\"location\\\":\\\"168, Fenchurch Street, London E.C. 3\\\"},{\\\"company_name\\\":\\\"Galland & Co.\\\",\\\"entry_id\\\":\\\"{page_id}-13\\\",\\\"location\\\":\\\"3, Place St-Fran\u00e7ois, Lausanne\\\"},{\\\"company_name\\\":\\\"Lloyds and National Provincial Foreign Bank, Ltd.\\\",\\\"entry_id\\\":\\\"{page_id}-14\\\",\\\"location\\\":\\\"1, Place Bel-Air, Geneva\\\"},{\\\"company_name\\\":\\\"Swiss Bank Corporation\\\",\\\"entry_id\\\":\\\"{page_id}-15\\\",\\\"location\\\":\\\"1, Aeschenvorstadt, Basle\\\"},{\\\"company_name\\\":\\\"Swiss Bank Corporation\\\",\\\"entry_id\\\":\\\"{page_id}-16\\\",\\\"location\\\":\\\"Lausanne\\\"},{\\\"company_name\\\":\\\"Wagons-Lits/Cook S.A.\\\",\\\"entry_id\\\":\\\"{page_id}-17\\\",\\\"location\\\":\\\"4, Rue du Mont Blanc, Geneva\\\"}],\\\"page_id\\\":\\\"{page_id}\\\"}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"page_id\":\"{page_id}\",\"entries\":[{\"entry_id\":\"{page_id}-1\",\"company_name\":\"Crowe & Co., Ltd.\",\"location\":\"67, Nauenstrasse, Basle\"},{\"entry_id\":\"{page_id}-2\",\"company_name\":\"Imperial Airways Ltd.\",\"location\":\"Airport of London, Croydon\"},{\"entry_id\":\"{page_id}-3\",\"company_name\":\"Metallwerke A.G.\",\"location\":\"Dornach (Solothurn)\"},{\"entry_id\":\"{page_id}-4\",\"company_name\":\"Aluminium-Industrie-Aktien-Gesellschaft\",\"location\":\"Neuhausen\"},{\"entry_id\":\"{page_id}-5\",\"company_name\":\"Chemische Fabrik vorm. Sandoz\",\"location\":\"Basle\"},{\"entry_id\":\"{page_id}-6\",\"company_name\":\"Clayton Aniline Co., Ltd.\",\"location\":\"Ashton New Road, Clayton, Manchester\"},{\"entry_id\":\"{page_id}-7\",\"company_name\":\"J. R. Geigy, A.-G.\",\"location\":\"Basle\"},{\"entry_id\":\"{page_id}-8\",\"company_name\":\"Society of Chemical Industry in Basle\",\"location\":\"Basle\"},{\"entry_id\":\"{page_id}-9\",\"company_name\":\"J. Hopf & Co. (Merchants)\",\"location\":\"12, Aeschengraben, Basle\"},{\"entry_id\":\"{page_id}-10\",\"company_name\":\"Rayon A.-G. (Merchants)\",\"location\":\"14, Stollenrain, Arlesheim\"},{\"entry_id\":\"{page_id}-11\",\"company_name\":\"Banque F\u00e9d\u00e9rale S. A.\",\"location\":\"Vevey\"},{\"entry_id\":\"{page_id}-12\",\"company_name\":\"Barclays Bank Ltd. (Foreign Branch)\",\"location\":\"168, Fenchurch Street, London E.C. 3\"},{\"entry_id\":\"{page_id}-13\",\"company_name\":\"Galland & Co.\",\"location\":\"3, Place St-Fran\u00e7ois, Lausanne\"},{\"entry_id\":\"{page_id}-14\",\"company_name\":\"Lloyds and National Provincial Foreign Bank, Ltd.\",\"location\":\"1, Place Bel-Air, Geneva\"},{\"entry_id\":\"{page_id}-15\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"1, Aeschenvorstadt, Basle\"},{\"entry_id\":\"{page_id}-16\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"Lausanne\"},{\"entry_id\":\"{page_id}-17\",\"company_name\":\"Wagons-Lits/Cook S.A.\",\"location\":\"4, Rue du Mont Blanc, Geneva\"}]}}}],\"created\":1785005005,\"model\":\"gpt-5.6-sol\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":915,\"prompt_tokens\":3538,\"total_tokens\":4453,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":379,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.5240263024785028, "f1_macro": 0.5213333333333333, "micro_precision": 0.5258883248730964, "micro_recall": 0.5221774193548387, "total_instances": 15, "total_tp": 518, "total_fp": 467, "total_fn": 474, "cost_summary": {"total_input_tokens": 53418, "total_output_tokens": 17413, "total_tokens": 70831, "input_cost_usd": 0.26709, "output_cost_usd": 0.52239, "total_cost_usd": 0.78948}}, "normalized_score": 52.13333333333333}