{"_id": {"$oid": "69d4eb12b877594bc5718b20"}, "test_id": "T0529", "benchmark": "company_lists", "date": "2026-01-24", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en", "de"], "layout": ["list"], "entry-type": ["company"], "task": ["information-extraction"]}, "contributors": ["lea_kasper", "sorin_marti"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-haiku-4-5-20251001", "dataclass": "ListPage", "temperature": 0.5, "role_description": "You are a Historian", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "The image you are presented with stems from a digitized book containing lists of companies.\nYour task is to extract structured information about each company listed on the page.\n\nAbout the source:\n- The image stems from a trade index of the British Swiss Chamber of Commerce.\n- The image can show an alphabetical or a thematic list of companies.\n- The companies are mostly located in Switzerland and the UK.\n- The image stems from a trade index between 1925 and 1958.\n- Most pages have one column but some years have two columns.\n- The source itself is in English and German but the company names can be in English, German, French or Italian.\n\nAbout the entries:\n- Each entry describes a single company or person.\n- Alphabetical entries have filling dots between the company name and the page number. Dots and page numbers are not part of the data and should be ignored.\n- Alphabetical entries seldom to never have locations.\n- Thematic entries often have locations.\n- Thematic entries are listed under headings that describe the type of business.\n- Some thematic headings are only references to other headings, e.g. \"X, s. Y\".\n\nAbout the output:\n- Answer in valid JSON. The JSON should be an array of objects with the following fields:\n- The page ID is given as {page_id}.\n- Do not add country information, if it is not directly written with the location.\n\n{\n  \"entry_id\": \"A unique identifier for the entry, e.g. '{page_id}-1'\",\n  \"company_name\": \"The name of the company or person\",\n  \"location\": \"The location of the company, e.g. 'Zurich' or 'London, UK'. If no location is given, set to null.\"\n  ]\n}\n", "results": {"text": "{\"page_id\":\"12\",\"entries\":[{\"entry_id\":\"12-1\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"Lausanne\"},{\"entry_id\":\"12-2\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"Neuch\u00e2tel\"},{\"entry_id\":\"12-3\",\"company_name\":\"Union Bank of Switzerland\",\"location\":\"Z\u00fcrich\"},{\"entry_id\":\"12-4\",\"company_name\":\"Barcham Green & Sons\",\"location\":\"Hayle Mill, Maidstone, Kent\"},{\"entry_id\":\"12-5\",\"company_name\":\"Herren & Guerchet\",\"location\":\"Rue Jean-Petitot 10, Gen\u00e8ve\"},{\"entry_id\":\"12-6\",\"company_name\":\"Louis Kohler\",\"location\":\"11, St. Pierre, Lausanne\"},{\"entry_id\":\"12-7\",\"company_name\":\"Giuliniwerke A. G.\",\"location\":\"Basel\"},{\"entry_id\":\"12-8\",\"company_name\":\"Georges Michoud\",\"location\":\"Lausanne\"},{\"entry_id\":\"12-9\",\"company_name\":\"August Fr\u00f6hlich\",\"location\":\"Bahnhofstrasse 110, Z\u00fcrich\"},{\"entry_id\":\"12-10\",\"company_name\":\"Altherr & Guex\",\"location\":\"Flawil\"},{\"entry_id\":\"12-11\",\"company_name\":\"Arbon Embroidery Works Ltd.\",\"location\":\"Arbon\"},{\"entry_id\":\"12-12\",\"company_name\":\"Grieder & Co.\",\"location\":\"Paradeplatz, Z\u00fcrich\"},{\"entry_id\":\"12-13\",\"company_name\":\"A. C. Pearse Ltd.\",\"location\":\"14, Pilcher Gate, Nottingham\"},{\"entry_id\":\"12-14\",\"company_name\":\"Zuberb\u00fchler & Co., A. G.\",\"location\":\"Zurzach\"},{\"entry_id\":\"12-15\",\"company_name\":\"Sulzer Fr\u00e8res S. A.\",\"location\":\"Winterthur\"},{\"entry_id\":\"12-16\",\"company_name\":\"Schw. Seidengazefabrik A. G.\",\"location\":\"Thal, bei Rheineck\"},{\"entry_id\":\"12-17\",\"company_name\":\"Schw. Seidengazefabrik A. G.\",\"location\":\"Z\u00fcrich\"}]}", "model": "claude-haiku-4-5-20251001", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2757, "output_tokens": 867, "total_tokens": 3624, "input_cost_usd": 0.002757, "output_cost_usd": 0.004335, "estimated_cost_usd": 0.007092}, "duration": 6.278717279434204, "timestamp": "2026-01-24T18:34:17.335413", "parsed": {"page_id": "12", "entries": [{"entry_id": "12-1", "company_name": "Swiss Bank Corporation", "location": "Lausanne"}, {"entry_id": "12-2", "company_name": "Swiss Bank Corporation", "location": "Neuch\u00e2tel"}, {"entry_id": "12-3", "company_name": "Union Bank of Switzerland", "location": "Z\u00fcrich"}, {"entry_id": "12-4", "company_name": "Barcham Green & Sons", "location": "Hayle Mill, Maidstone, Kent"}, {"entry_id": "12-5", "company_name": "Herren & Guerchet", "location": "Rue Jean-Petitot 10, Gen\u00e8ve"}, {"entry_id": "12-6", "company_name": "Louis Kohler", "location": "11, St. Pierre, Lausanne"}, {"entry_id": "12-7", "company_name": "Giuliniwerke A. G.", "location": "Basel"}, {"entry_id": "12-8", "company_name": "Georges Michoud", "location": "Lausanne"}, {"entry_id": "12-9", "company_name": "August Fr\u00f6hlich", "location": "Bahnhofstrasse 110, Z\u00fcrich"}, {"entry_id": "12-10", "company_name": "Altherr & Guex", "location": "Flawil"}, {"entry_id": "12-11", "company_name": "Arbon Embroidery Works Ltd.", "location": "Arbon"}, {"entry_id": "12-12", "company_name": "Grieder & Co.", "location": "Paradeplatz, Z\u00fcrich"}, {"entry_id": "12-13", "company_name": "A. C. Pearse Ltd.", "location": "14, Pilcher Gate, Nottingham"}, {"entry_id": "12-14", "company_name": "Zuberb\u00fchler & Co., A. G.", "location": "Zurzach"}, {"entry_id": "12-15", "company_name": "Sulzer Fr\u00e8res S. A.", "location": "Winterthur"}, {"entry_id": "12-16", "company_name": "Schw. Seidengazefabrik A. G.", "location": "Thal, bei Rheineck"}, {"entry_id": "12-17", "company_name": "Schw. Seidengazefabrik A. G.", "location": "Z\u00fcrich"}]}, "conversation_id": "e4caa569-2241-43a9-9c18-3af69197d298", "score": {"f1_score": 0.54, "precision": 0.5384615384615384, "recall": 0.5384615384615384, "true_positives": 28, "false_positives": 24, "false_negatives": 24, "field_scores": {"entries[16].location": {"response": "Z\u00fcrich", "ground_truth": "Z\u00fcrich", "score": 1.0}, "entries[2].company_name": {"response": "Union Bank of Switzerland", "ground_truth": "Union Bank of Switzerland", "score": 1.0}, "entries[3].company_name": {"response": "Barcham Green & Sons", "ground_truth": "Barcham Green & Sons", "score": 1.0}, "entries[1].company_name": {"response": "Swiss Bank Corporation", "ground_truth": "Swiss Bank Corporation", "score": 1.0}, "entries[6].company_name": {"response": "Giuliniwerke A. G.", "ground_truth": "Giuliniwerke A. G.", "score": 1.0}, "entries[10].location": {"response": "Arbon", "ground_truth": "Arbon", "score": 1.0}, "entries[16].company_name": {"response": "Schw. Seidengazefabrik A. G.", "ground_truth": "Schw. Seidengazefabrik A. G.", "score": 1.0}, "entries[13].location": {"response": "Zurzach", "ground_truth": "Zurzach", "score": 1.0}, "entries[14].entry_id": {"response": "12-15", "ground_truth": "156089_1321081_14-15", "score": 0.4}, "entries[5].location": {"response": "11, St. Pierre, Lausanne", "ground_truth": "Lausanne", "score": 0.5}, "entries[6].location": {"response": "Basel", "ground_truth": "Basel", "score": 1.0}, "entries[13].company_name": {"response": "Zuberb\u00fchler & Co., A. G.", "ground_truth": "Zuberb\u00fchler & Co., A. G.", "score": 1.0}, "entries[13].entry_id": {"response": "12-14", "ground_truth": "156089_1321081_14-14", "score": 0.4}, "entries[11].location": {"response": "Paradeplatz, Z\u00fcrich", "ground_truth": "Z\u00fcrich", "score": 0.48}, "entries[1].entry_id": {"response": "12-2", "ground_truth": "156089_1321081_14-2", "score": 0.34782608695652173}, "entries[16].entry_id": {"response": "12-17", "ground_truth": "156089_1321081_14-17", "score": 0.4}, "entries[8].company_name": {"response": "August Fr\u00f6hlich", "ground_truth": "August Fr\u00f6hlich", "score": 1.0}, "entries[8].entry_id": {"response": "12-9", "ground_truth": "156089_1321081_14-9", "score": 0.34782608695652173}, "entries[12].entry_id": {"response": "12-13", "ground_truth": "156089_1321081_14-13", "score": 0.4}, "entries[0].entry_id": {"response": "12-1", "ground_truth": "156089_1321081_14-1", "score": 0.34782608695652173}, "entries[5].company_name": {"response": "Louis Kohler", "ground_truth": "Louis Kohler", "score": 1.0}, "entries[4].company_name": {"response": "Herren & Guerchet", "ground_truth": "Herren & Guerchet", "score": 1.0}, "entries[6].entry_id": {"response": "12-7", "ground_truth": "156089_1321081_14-7", "score": 0.34782608695652173}, "entries[0].location": {"response": "Lausanne", "ground_truth": "Lausanne", "score": 1.0}, "entries[0].company_name": {"response": "Swiss Bank Corporation", "ground_truth": "Swiss Bank Corporation", "score": 1.0}, "entries[3].entry_id": {"response": "12-4", "ground_truth": "156089_1321081_14-4", "score": 0.34782608695652173}, "entries[14].company_name": {"response": "Sulzer Fr\u00e8res S. A.", "ground_truth": "Sulzer Fr\u00e8res S. A.", "score": 1.0}, "entries[14].location": {"response": "Winterthur", "ground_truth": "Winterthur", "score": 1.0}, "entries[9].entry_id": {"response": "12-10", "ground_truth": "156089_1321081_14-10", "score": 0.4}, "entries[4].location": {"response": "Rue Jean-Petitot 10, Gen\u00e8ve", "ground_truth": "Gen\u00e8ve", "score": 0.36363636363636365}, "entries[2].location": {"response": "Z\u00fcrich", "ground_truth": "Z\u00fcrich", "score": 1.0}, "entries[15].location": {"response": "Thal, bei Rheineck", "ground_truth": "Thal, bei Rheineck", "score": 1.0}, "entries[10].entry_id": {"response": "12-11", "ground_truth": "156089_1321081_14-11", "score": 0.4}, "entries[7].company_name": {"response": "Georges Michoud", "ground_truth": "Georges Michoud", "score": 1.0}, "entries[4].entry_id": {"response": "12-5", "ground_truth": "156089_1321081_14-5", "score": 0.34782608695652173}, "entries[11].entry_id": {"response": "12-12", "ground_truth": "156089_1321081_14-12", "score": 0.4}, "entries[2].entry_id": {"response": "12-3", "ground_truth": "156089_1321081_14-3", "score": 0.34782608695652173}, "entries[7].entry_id": {"response": "12-8", "ground_truth": "156089_1321081_14-8", "score": 0.34782608695652173}, "entries[10].company_name": {"response": "Arbon Embroidery Works Ltd.", "ground_truth": "Arbon Embroidery Works Ltd.", "score": 1.0}, "entries[15].entry_id": {"response": "12-16", "ground_truth": "156089_1321081_14-16", "score": 0.4}, "entries[15].company_name": {"response": "Schw. Seidengazefabrik A. G.", "ground_truth": "Schw. Seidengazefabrik A. G.", "score": 1.0}, "entries[12].company_name": {"response": "A. C. Pearse Ltd.", "ground_truth": "A. C. Pearse Ltd.", "score": 1.0}, "page_id": {"response": "12", "ground_truth": "156089_1321081_14", "score": 0.21052631578947367}, "entries[3].location": {"response": "Hayle Mill, Maidstone, Kent", "ground_truth": "Maidstone, Kent", "score": 0.7142857142857143}, "entries[11].company_name": {"response": "Grieder & Co.", "ground_truth": "Grieder & Co.", "score": 1.0}, "entries[5].entry_id": {"response": "12-6", "ground_truth": "156089_1321081_14-6", "score": 0.34782608695652173}, "entries[12].location": {"response": "14, Pilcher Gate, Nottingham", "ground_truth": "Nottingham", "score": 0.5263157894736843}, "entries[8].location": {"response": "Bahnhofstrasse 110, Z\u00fcrich", "ground_truth": "Z\u00fcrich", "score": 0.375}, "entries[9].company_name": {"response": "Altherr & Guex", "ground_truth": "Altherr & Guex", "score": 1.0}, "entries[7].location": {"response": "Lausanne", "ground_truth": "Lausanne", "score": 1.0}, "entries[9].location": {"response": "Flawil", "ground_truth": "Flawil", "score": 1.0}, "entries[1].location": {"response": "Neuch\u00e2tel", "ground_truth": "Neuch\u00e2tel", "score": 1.0}}, "total_fields": 52}, "raw_response": "{\"id\":\"msg_01QxbTt6jZ72JsknmthyVm7C\",\"content\":[{\"id\":\"toolu_01SHbzujL8sfL3F4gn8ed4ds\",\"input\":{\"page_id\":\"12\",\"entries\":[{\"entry_id\":\"12-1\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"Lausanne\"},{\"entry_id\":\"12-2\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"Neuch\u00e2tel\"},{\"entry_id\":\"12-3\",\"company_name\":\"Union Bank of Switzerland\",\"location\":\"Z\u00fcrich\"},{\"entry_id\":\"12-4\",\"company_name\":\"Barcham Green & Sons\",\"location\":\"Hayle Mill, Maidstone, Kent\"},{\"entry_id\":\"12-5\",\"company_name\":\"Herren & Guerchet\",\"location\":\"Rue Jean-Petitot 10, Gen\u00e8ve\"},{\"entry_id\":\"12-6\",\"company_name\":\"Louis Kohler\",\"location\":\"11, St. Pierre, Lausanne\"},{\"entry_id\":\"12-7\",\"company_name\":\"Giuliniwerke A. G.\",\"location\":\"Basel\"},{\"entry_id\":\"12-8\",\"company_name\":\"Georges Michoud\",\"location\":\"Lausanne\"},{\"entry_id\":\"12-9\",\"company_name\":\"August Fr\u00f6hlich\",\"location\":\"Bahnhofstrasse 110, Z\u00fcrich\"},{\"entry_id\":\"12-10\",\"company_name\":\"Altherr & Guex\",\"location\":\"Flawil\"},{\"entry_id\":\"12-11\",\"company_name\":\"Arbon Embroidery Works Ltd.\",\"location\":\"Arbon\"},{\"entry_id\":\"12-12\",\"company_name\":\"Grieder & Co.\",\"location\":\"Paradeplatz, Z\u00fcrich\"},{\"entry_id\":\"12-13\",\"company_name\":\"A. C. Pearse Ltd.\",\"location\":\"14, Pilcher Gate, Nottingham\"},{\"entry_id\":\"12-14\",\"company_name\":\"Zuberb\u00fchler & Co., A. G.\",\"location\":\"Zurzach\"},{\"entry_id\":\"12-15\",\"company_name\":\"Sulzer Fr\u00e8res S. A.\",\"location\":\"Winterthur\"},{\"entry_id\":\"12-16\",\"company_name\":\"Schw. Seidengazefabrik A. G.\",\"location\":\"Thal, bei Rheineck\"},{\"entry_id\":\"12-17\",\"company_name\":\"Schw. Seidengazefabrik A. G.\",\"location\":\"Z\u00fcrich\"}]},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-haiku-4-5-20251001\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2757,\"output_tokens\":867,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"f1_micro": 0.48689516129032256, "f1_macro": 0.4906666666666667, "micro_precision": 0.48689516129032256, "micro_recall": 0.48689516129032256, "total_instances": 15, "total_tp": 483, "total_fp": 509, "total_fn": 509, "cost_summary": {"total_input_tokens": 41604, "total_output_tokens": 17305, "total_tokens": 58909, "input_cost_usd": 0.04160399999999999, "output_cost_usd": 0.08652499999999999, "total_cost_usd": 0.128129}}, "normalized_score": 49.06666666666667}