{"_id": {"$oid": "6a26c67c3572c06bc1455ca9"}, "test_id": "T0677", "benchmark": "company_lists", "date": "2026-03-16", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en", "de"], "layout": ["list"], "entry-type": ["company"], "task": ["information-extraction"]}, "contributors": ["lea_kasper", "sorin_marti"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3.1-flash-lite-preview", "dataclass": "ListPage", "temperature": 0.5, "role_description": "You are a Historian", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "The image you are presented with stems from a digitized book containing lists of companies.\nYour task is to extract structured information about each company listed on the page.\n\nAbout the source:\n- The image stems from a trade index of the British Swiss Chamber of Commerce.\n- The image can show an alphabetical or a thematic list of companies.\n- The companies are mostly located in Switzerland and the UK.\n- The image stems from a trade index between 1925 and 1958.\n- Most pages have one column but some years have two columns.\n- The source itself is in English and German but the company names can be in English, German, French or Italian.\n\nAbout the entries:\n- Each entry describes a single company or person.\n- Alphabetical entries have filling dots between the company name and the page number. Dots and page numbers are not part of the data and should be ignored.\n- Alphabetical entries seldom to never have locations.\n- Thematic entries often have locations.\n- Thematic entries are listed under headings that describe the type of business.\n- Some thematic headings are only references to other headings, e.g. \"X, s. Y\".\n\nAbout the output:\n- Answer in valid JSON. The JSON should be an array of objects with the following fields:\n- The page ID is given as {page_id}.\n- Do not add country information, if it is not directly written with the location.\n\n{\n  \"entry_id\": \"A unique identifier for the entry, e.g. '{page_id}-1'\",\n  \"company_name\": \"The name of the company or person\",\n  \"location\": \"The location of the company, e.g. 'Zurich' or 'London, UK'. If no location is given, set to null.\"\n  ]\n}\n", "results": {"text": "{\"page_id\":\"12\",\"entries\":[{\"entry_id\":\"12-1\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"Lausanne\"},{\"entry_id\":\"12-2\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"Neuch\u00e2tel\"},{\"entry_id\":\"12-3\",\"company_name\":\"Union Bank of Switzerland\",\"location\":\"Z\u00fcrich\"},{\"entry_id\":\"12-4\",\"company_name\":\"Barcham Green & Sons\",\"location\":\"Hayle Mill, Maidstone, Kent\"},{\"entry_id\":\"12-5\",\"company_name\":\"Herren & Guerchet\",\"location\":\"Rue Jean-Petitot 10, Gen\u00e8ve\"},{\"entry_id\":\"12-6\",\"company_name\":\"Louis Kohler\",\"location\":\"11, St. Pierre, Lausanne\"},{\"entry_id\":\"12-7\",\"company_name\":\"Giuliniwerke A. G.\",\"location\":\"Basel\"},{\"entry_id\":\"12-8\",\"company_name\":\"Georges Michoud\",\"location\":\"Lausanne\"},{\"entry_id\":\"12-9\",\"company_name\":\"August Fr\u00f6hlich\",\"location\":\"Bahnhofstrasse 110, Z\u00fcrich\"},{\"entry_id\":\"12-10\",\"company_name\":\"Altherr & Guex\",\"location\":\"Flawil\"},{\"entry_id\":\"12-11\",\"company_name\":\"Arbon Embroidery Works Ltd.\",\"location\":\"Arbon\"},{\"entry_id\":\"12-12\",\"company_name\":\"Grieder & Co.\",\"location\":\"Paradeplatz, Z\u00fcrich\"},{\"entry_id\":\"12-13\",\"company_name\":\"A. C. Pearse Ltd.\",\"location\":\"14, Pilcher Gate, Nottingham\"},{\"entry_id\":\"12-14\",\"company_name\":\"Zuberb\u00fchler & Co., A. G.\",\"location\":\"Zurzach\"},{\"entry_id\":\"12-15\",\"company_name\":\"Sulzer Fr\u00e8res S. A.\",\"location\":\"Winterthur\"},{\"entry_id\":\"12-16\",\"company_name\":\"Schw. Seidengazefabrik A. G.\",\"location\":\"Thal, bei Rheineck\"},{\"entry_id\":\"12-17\",\"company_name\":\"Schw. Seidengazefabrik A. G.\",\"location\":\"Z\u00fcrich\"}]}", "model": "gemini-3.1-flash-lite-preview", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1479, "output_tokens": 806, "total_tokens": 2285, "input_cost_usd": 0.00036975, "output_cost_usd": 0.001209, "estimated_cost_usd": 0.00157875}, "duration": 3.923034906387329, "timestamp": "2026-03-16T19:09:44.722385", "parsed": {"page_id": "12", "entries": [{"entry_id": "12-1", "company_name": "Swiss Bank Corporation", "location": "Lausanne"}, {"entry_id": "12-2", "company_name": "Swiss Bank Corporation", "location": "Neuch\u00e2tel"}, {"entry_id": "12-3", "company_name": "Union Bank of Switzerland", "location": "Z\u00fcrich"}, {"entry_id": "12-4", "company_name": "Barcham Green & Sons", "location": "Hayle Mill, Maidstone, Kent"}, {"entry_id": "12-5", "company_name": "Herren & Guerchet", "location": "Rue Jean-Petitot 10, Gen\u00e8ve"}, {"entry_id": "12-6", "company_name": "Louis Kohler", "location": "11, St. Pierre, Lausanne"}, {"entry_id": "12-7", "company_name": "Giuliniwerke A. G.", "location": "Basel"}, {"entry_id": "12-8", "company_name": "Georges Michoud", "location": "Lausanne"}, {"entry_id": "12-9", "company_name": "August Fr\u00f6hlich", "location": "Bahnhofstrasse 110, Z\u00fcrich"}, {"entry_id": "12-10", "company_name": "Altherr & Guex", "location": "Flawil"}, {"entry_id": "12-11", "company_name": "Arbon Embroidery Works Ltd.", "location": "Arbon"}, {"entry_id": "12-12", "company_name": "Grieder & Co.", "location": "Paradeplatz, Z\u00fcrich"}, {"entry_id": "12-13", "company_name": "A. C. Pearse Ltd.", "location": "14, Pilcher Gate, Nottingham"}, {"entry_id": "12-14", "company_name": "Zuberb\u00fchler & Co., A. G.", "location": "Zurzach"}, {"entry_id": "12-15", "company_name": "Sulzer Fr\u00e8res S. A.", "location": "Winterthur"}, {"entry_id": "12-16", "company_name": "Schw. Seidengazefabrik A. G.", "location": "Thal, bei Rheineck"}, {"entry_id": "12-17", "company_name": "Schw. Seidengazefabrik A. G.", "location": "Z\u00fcrich"}]}, "conversation_id": "03151cfa-ff6f-41e9-8d02-6e79484da899", "score": {"f1_score": 0.54, "precision": 0.5384615384615384, "recall": 0.5384615384615384, "true_positives": 28, "false_positives": 24, "false_negatives": 24, "field_scores": {"entries[2].location": {"response": "Z\u00fcrich", "ground_truth": "Z\u00fcrich", "score": 1.0}, "entries[1].entry_id": {"response": "12-2", "ground_truth": "156089_1321081_14-2", "score": 0.34782608695652173}, "entries[1].company_name": {"response": "Swiss Bank Corporation", "ground_truth": "Swiss Bank Corporation", "score": 1.0}, "entries[15].company_name": {"response": "Schw. Seidengazefabrik A. G.", "ground_truth": "Schw. Seidengazefabrik A. G.", "score": 1.0}, "entries[14].location": {"response": "Winterthur", "ground_truth": "Winterthur", "score": 1.0}, "entries[0].entry_id": {"response": "12-1", "ground_truth": "156089_1321081_14-1", "score": 0.34782608695652173}, "entries[10].company_name": {"response": "Arbon Embroidery Works Ltd.", "ground_truth": "Arbon Embroidery Works Ltd.", "score": 1.0}, "page_id": {"response": "12", "ground_truth": "156089_1321081_14", "score": 0.21052631578947367}, "entries[2].entry_id": {"response": "12-3", "ground_truth": "156089_1321081_14-3", "score": 0.34782608695652173}, "entries[14].company_name": {"response": "Sulzer Fr\u00e8res S. A.", "ground_truth": "Sulzer Fr\u00e8res S. A.", "score": 1.0}, "entries[12].entry_id": {"response": "12-13", "ground_truth": "156089_1321081_14-13", "score": 0.4}, "entries[7].entry_id": {"response": "12-8", "ground_truth": "156089_1321081_14-8", "score": 0.34782608695652173}, "entries[5].entry_id": {"response": "12-6", "ground_truth": "156089_1321081_14-6", "score": 0.34782608695652173}, "entries[7].company_name": {"response": "Georges Michoud", "ground_truth": "Georges Michoud", "score": 1.0}, "entries[16].location": {"response": "Z\u00fcrich", "ground_truth": "Z\u00fcrich", "score": 1.0}, "entries[13].entry_id": {"response": "12-14", "ground_truth": "156089_1321081_14-14", "score": 0.4}, "entries[9].location": {"response": "Flawil", "ground_truth": "Flawil", "score": 1.0}, "entries[0].company_name": {"response": "Swiss Bank Corporation", "ground_truth": "Swiss Bank Corporation", "score": 1.0}, "entries[5].company_name": {"response": "Louis Kohler", "ground_truth": "Louis Kohler", "score": 1.0}, "entries[16].company_name": {"response": "Schw. Seidengazefabrik A. G.", "ground_truth": "Schw. Seidengazefabrik A. G.", "score": 1.0}, "entries[11].location": {"response": "Paradeplatz, Z\u00fcrich", "ground_truth": "Z\u00fcrich", "score": 0.48}, "entries[2].company_name": {"response": "Union Bank of Switzerland", "ground_truth": "Union Bank of Switzerland", "score": 1.0}, "entries[11].entry_id": {"response": "12-12", "ground_truth": "156089_1321081_14-12", "score": 0.4}, "entries[6].location": {"response": "Basel", "ground_truth": "Basel", "score": 1.0}, "entries[8].location": {"response": "Bahnhofstrasse 110, Z\u00fcrich", "ground_truth": "Z\u00fcrich", "score": 0.375}, "entries[4].entry_id": {"response": "12-5", "ground_truth": "156089_1321081_14-5", "score": 0.34782608695652173}, "entries[11].company_name": {"response": "Grieder & Co.", "ground_truth": "Grieder & Co.", "score": 1.0}, "entries[3].company_name": {"response": "Barcham Green & Sons", "ground_truth": "Barcham Green & Sons", "score": 1.0}, "entries[3].location": {"response": "Hayle Mill, Maidstone, Kent", "ground_truth": "Maidstone, Kent", "score": 0.7142857142857143}, "entries[4].location": {"response": "Rue Jean-Petitot 10, Gen\u00e8ve", "ground_truth": "Gen\u00e8ve", "score": 0.36363636363636365}, "entries[4].company_name": {"response": "Herren & Guerchet", "ground_truth": "Herren & Guerchet", "score": 1.0}, "entries[0].location": {"response": "Lausanne", "ground_truth": "Lausanne", "score": 1.0}, "entries[9].company_name": {"response": "Altherr & Guex", "ground_truth": "Altherr & Guex", "score": 1.0}, "entries[3].entry_id": {"response": "12-4", "ground_truth": "156089_1321081_14-4", "score": 0.34782608695652173}, "entries[8].entry_id": {"response": "12-9", "ground_truth": "156089_1321081_14-9", "score": 0.34782608695652173}, "entries[10].location": {"response": "Arbon", "ground_truth": "Arbon", "score": 1.0}, "entries[9].entry_id": {"response": "12-10", "ground_truth": "156089_1321081_14-10", "score": 0.4}, "entries[1].location": {"response": "Neuch\u00e2tel", "ground_truth": "Neuch\u00e2tel", "score": 1.0}, "entries[13].company_name": {"response": "Zuberb\u00fchler & Co., A. G.", "ground_truth": "Zuberb\u00fchler & Co., A. G.", "score": 1.0}, "entries[12].company_name": {"response": "A. C. Pearse Ltd.", "ground_truth": "A. C. Pearse Ltd.", "score": 1.0}, "entries[6].entry_id": {"response": "12-7", "ground_truth": "156089_1321081_14-7", "score": 0.34782608695652173}, "entries[10].entry_id": {"response": "12-11", "ground_truth": "156089_1321081_14-11", "score": 0.4}, "entries[15].entry_id": {"response": "12-16", "ground_truth": "156089_1321081_14-16", "score": 0.4}, "entries[13].location": {"response": "Zurzach", "ground_truth": "Zurzach", "score": 1.0}, "entries[16].entry_id": {"response": "12-17", "ground_truth": "156089_1321081_14-17", "score": 0.4}, "entries[8].company_name": {"response": "August Fr\u00f6hlich", "ground_truth": "August Fr\u00f6hlich", "score": 1.0}, "entries[12].location": {"response": "14, Pilcher Gate, Nottingham", "ground_truth": "Nottingham", "score": 0.5263157894736843}, "entries[5].location": {"response": "11, St. Pierre, Lausanne", "ground_truth": "Lausanne", "score": 0.5}, "entries[15].location": {"response": "Thal, bei Rheineck", "ground_truth": "Thal, bei Rheineck", "score": 1.0}, "entries[14].entry_id": {"response": "12-15", "ground_truth": "156089_1321081_14-15", "score": 0.4}, "entries[7].location": {"response": "Lausanne", "ground_truth": "Lausanne", "score": 1.0}, "entries[6].company_name": {"response": "Giuliniwerke A. G.", "ground_truth": "Giuliniwerke A. G.", "score": 1.0}}, "total_fields": 52}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Mon, 16 Mar 2026 18:09:51 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=3679\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"media_resolution\":null,\"code_execution_result\":null,\"executable_code\":null,\"file_data\":null,\"function_call\":null,\"function_response\":null,\"inline_data\":null,\"text\":\"{\\n  \\\"page_id\\\": \\\"12\\\",\\n  \\\"entries\\\": [\\n    {\\n      \\\"entry_id\\\": \\\"12-1\\\",\\n      \\\"company_name\\\": \\\"Swiss Bank Corporation\\\",\\n      \\\"location\\\": \\\"Lausanne\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-2\\\",\\n      \\\"company_name\\\": \\\"Swiss Bank Corporation\\\",\\n      \\\"location\\\": \\\"Neuch\u00e2tel\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-3\\\",\\n      \\\"company_name\\\": \\\"Union Bank of Switzerland\\\",\\n      \\\"location\\\": \\\"Z\u00fcrich\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-4\\\",\\n      \\\"company_name\\\": \\\"Barcham Green & Sons\\\",\\n      \\\"location\\\": \\\"Hayle Mill, Maidstone, Kent\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-5\\\",\\n      \\\"company_name\\\": \\\"Herren & Guerchet\\\",\\n      \\\"location\\\": \\\"Rue Jean-Petitot 10, Gen\u00e8ve\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-6\\\",\\n      \\\"company_name\\\": \\\"Louis Kohler\\\",\\n      \\\"location\\\": \\\"11, St. Pierre, Lausanne\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-7\\\",\\n      \\\"company_name\\\": \\\"Giuliniwerke A. G.\\\",\\n      \\\"location\\\": \\\"Basel\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-8\\\",\\n      \\\"company_name\\\": \\\"Georges Michoud\\\",\\n      \\\"location\\\": \\\"Lausanne\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-9\\\",\\n      \\\"company_name\\\": \\\"August Fr\u00f6hlich\\\",\\n      \\\"location\\\": \\\"Bahnhofstrasse 110, Z\u00fcrich\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-10\\\",\\n      \\\"company_name\\\": \\\"Altherr & Guex\\\",\\n      \\\"location\\\": \\\"Flawil\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-11\\\",\\n      \\\"company_name\\\": \\\"Arbon Embroidery Works Ltd.\\\",\\n      \\\"location\\\": \\\"Arbon\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-12\\\",\\n      \\\"company_name\\\": \\\"Grieder & Co.\\\",\\n      \\\"location\\\": \\\"Paradeplatz, Z\u00fcrich\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-13\\\",\\n      \\\"company_name\\\": \\\"A. C. Pearse Ltd.\\\",\\n      \\\"location\\\": \\\"14, Pilcher Gate, Nottingham\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-14\\\",\\n      \\\"company_name\\\": \\\"Zuberb\u00fchler & Co., A. G.\\\",\\n      \\\"location\\\": \\\"Zurzach\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-15\\\",\\n      \\\"company_name\\\": \\\"Sulzer Fr\u00e8res S. A.\\\",\\n      \\\"location\\\": \\\"Winterthur\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-16\\\",\\n      \\\"company_name\\\": \\\"Schw. Seidengazefabrik A. G.\\\",\\n      \\\"location\\\": \\\"Thal, bei Rheineck\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12-17\\\",\\n      \\\"company_name\\\": \\\"Schw. Seidengazefabrik A. G.\\\",\\n      \\\"location\\\": \\\"Z\u00fcrich\\\"\\n    }\\n  ]\\n}\",\"thought\":null,\"thought_signature\":\"EjQKMgG-Pvb7ky0VRLNt-iTZeTQFQ4cRgdBGVoJpSCE85wIKEWchwzB5JQBE3l-cWo1umJGB\",\"video_metadata\":null}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null,\"url_context_metadata\":null}],\"create_time\":null,\"model_version\":\"gemini-3.1-flash-lite-preview\",\"prompt_feedback\":null,\"response_id\":\"b0e4aafvHpXvvdIPpsau2Qs\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":806,\"candidates_tokens_details\":null,\"prompt_token_count\":1479,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":399},{\"modality\":\"IMAGE\",\"token_count\":1080}],\"thoughts_token_count\":null,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2285,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"page_id\":\"12\",\"entries\":[{\"entry_id\":\"12-1\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"Lausanne\"},{\"entry_id\":\"12-2\",\"company_name\":\"Swiss Bank Corporation\",\"location\":\"Neuch\u00e2tel\"},{\"entry_id\":\"12-3\",\"company_name\":\"Union Bank of Switzerland\",\"location\":\"Z\u00fcrich\"},{\"entry_id\":\"12-4\",\"company_name\":\"Barcham Green & Sons\",\"location\":\"Hayle Mill, Maidstone, Kent\"},{\"entry_id\":\"12-5\",\"company_name\":\"Herren & Guerchet\",\"location\":\"Rue Jean-Petitot 10, Gen\u00e8ve\"},{\"entry_id\":\"12-6\",\"company_name\":\"Louis Kohler\",\"location\":\"11, St. Pierre, Lausanne\"},{\"entry_id\":\"12-7\",\"company_name\":\"Giuliniwerke A. G.\",\"location\":\"Basel\"},{\"entry_id\":\"12-8\",\"company_name\":\"Georges Michoud\",\"location\":\"Lausanne\"},{\"entry_id\":\"12-9\",\"company_name\":\"August Fr\u00f6hlich\",\"location\":\"Bahnhofstrasse 110, Z\u00fcrich\"},{\"entry_id\":\"12-10\",\"company_name\":\"Altherr & Guex\",\"location\":\"Flawil\"},{\"entry_id\":\"12-11\",\"company_name\":\"Arbon Embroidery Works Ltd.\",\"location\":\"Arbon\"},{\"entry_id\":\"12-12\",\"company_name\":\"Grieder & Co.\",\"location\":\"Paradeplatz, Z\u00fcrich\"},{\"entry_id\":\"12-13\",\"company_name\":\"A. C. Pearse Ltd.\",\"location\":\"14, Pilcher Gate, Nottingham\"},{\"entry_id\":\"12-14\",\"company_name\":\"Zuberb\u00fchler & Co., A. G.\",\"location\":\"Zurzach\"},{\"entry_id\":\"12-15\",\"company_name\":\"Sulzer Fr\u00e8res S. A.\",\"location\":\"Winterthur\"},{\"entry_id\":\"12-16\",\"company_name\":\"Schw. Seidengazefabrik A. G.\",\"location\":\"Thal, bei Rheineck\"},{\"entry_id\":\"12-17\",\"company_name\":\"Schw. Seidengazefabrik A. G.\",\"location\":\"Z\u00fcrich\"}]}}"}, "scoring": {"f1_micro": 0.513853904282116, "f1_macro": 0.5246666666666667, "micro_precision": 0.513595166163142, "micro_recall": 0.5141129032258065, "total_instances": 15, "total_tp": 510, "total_fp": 483, "total_fn": 482, "cost_summary": {"total_input_tokens": 22303, "total_output_tokens": 16561, "total_tokens": 38864, "input_cost_usd": 0.005575750000000001, "output_cost_usd": 0.0248415, "total_cost_usd": 0.03041725}}, "normalized_score": 52.466666666666676}