{"_id": {"$oid": "69d4eb12b877594bc5718a3a"}, "test_id": "T0398", "benchmark": "company_lists", "date": "2025-10-28", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en", "de"], "layout": ["list"], "entry-type": ["company"], "task": ["information-extraction"]}, "contributors": ["lea_kasper", "sorin_marti"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-30b-a3b-instruct", "dataclass": "ListPage", "temperature": 0.5, "role_description": "You are a Historian", "prompt_file": "prompt_min.txt", "legacy_test": false}, "prompt": "- Answer in valid JSON.\n- The page ID is given as {page_id}.\n", "results": {"provider": "openrouter", "model": "qwen/qwen3-vl-30b-a3b-instruct", "duration": 8.526477336883545, "timestamp": "2025-10-28T10:59:30.779545", "parsed": {"page_id": "25", "entries": [{"entry_id": "1", "company_name": "General Superintendence Co. Ltd.", "location": "1, place des Alpes, Geneva."}, {"entry_id": "2", "company_name": "Gondrand Brothers Ltd.", "location": "8, Viaduktstrasse, Basle."}, {"entry_id": "3", "company_name": "Goth & Co. Ltd.", "location": "45, Elisabethenstrasse, Basle."}, {"entry_id": "4", "company_name": "Charles Im Obersteg & Co. Ltd.", "location": "30, Aeschengraben, Basle."}, {"entry_id": "5", "company_name": "John Im Obersteg & Co. Ltd.", "location": "28, Aeschengraben, Basle."}, {"entry_id": "6", "company_name": "Jacky, Maeder & Co.", "location": "35, Uraniastrasse Zurich 1."}, {"entry_id": "7", "company_name": "Jacky, Maeder & Co.", "location": "8, Wallstrasse, Basle."}, {"entry_id": "8", "company_name": "Lavanchy & Co. Ltd.", "location": "88, rue de Gen\u00e8ve, Lausanne."}, {"entry_id": "9", "company_name": "Marunion S.A.", "location": "21, Zielackerstrasse, Zurich 9."}, {"entry_id": "10", "company_name": "MAT Transport Ltd.", "location": "56, Peter Merian-Strasse, Basle."}, {"entry_id": "11", "company_name": "M\u00fcller-Gysin Ltd.", "location": "Zollfreilager, Basle 23."}, {"entry_id": "12", "company_name": "Natural Ltd.", "location": "67, Nauenstrasse, Basle."}, {"entry_id": "13", "company_name": "A. Natural, Le Coultre & Co. Ltd.", "location": "25, rue de Chantepoulet, Geneva."}, {"entry_id": "14", "company_name": "Roba Shipping Agency & Warehouse Ltd.", "location": "Zollfreilager, Basle 23."}, {"entry_id": "15", "company_name": "J. V\u00e9ron, Grauer & Co. Ltd.", "location": "22, rue du Mont-Blanc, Geneva."}, {"entry_id": "16", "company_name": "Weltfurrer International Transport Co. Ltd.", "location": "8, Pelikanstrasse, Zurich 1."}, {"entry_id": "17", "company_name": "A. Welti-Furrer Ltd.", "location": "16, M\u00fcllerstrasse, Zurich 4."}, {"entry_id": "18", "company_name": "World Transport Ltd.", "location": "33, Nauenstrasse, Basle."}, {"entry_id": "19", "company_name": "George Fischer Ltd.", "location": "Schaffhausen."}, {"entry_id": "20", "company_name": "Morganite Exports Ltd.", "location": "London; Zurich Office: 6, Sp\u00fclgenstrasse, Zurich 2."}, {"entry_id": "21", "company_name": "Louis de Roll Iron Works Ltd.", "location": "Gerlafingen."}, {"entry_id": "22", "company_name": "Sulzer Brothers Ltd.", "location": "Winterthur."}, {"entry_id": "23", "company_name": "Schubarth & Co. (agents)", "location": "45, Peter Merian-Strasse, Basle."}, {"entry_id": "24", "company_name": "Marunion S.A.", "location": "(for steamship services), 21, Zielackerstrasse, Zurich 9."}, {"entry_id": "25", "company_name": "W. Simon", "location": "(wholesale and retail dealers), 285-287, Hardturmstrasse, Zurich 5."}, {"entry_id": "26", "company_name": "Emil Flachsmann", "location": "(fruit extracts, concentrates and distillates), 60, Butzenstrasse, Zurich 30."}, {"entry_id": "27", "company_name": "H. Gallmann & Co.", "location": "(manufacturers of tinned tomato juice), 2, Rosenstrasse, Winterthur."}, {"entry_id": "28", "company_name": "L. E. Michel S.A.", "location": "(manufacturers of juices, concentrates, essences), pont du Ga- lieten, Malley-Lausanne."}, {"entry_id": "29", "company_name": "P.H. Stehelin & Co. Ltd.", "location": "(importers of concentrated juices), 62, Spalentorweg, Basle."}, {"entry_id": "30", "company_name": "Gustav Baader", "location": "(importers and wholesalers; also retail), 57, Aeschenvorstadt, Basle."}, {"entry_id": "31", "company_name": "Aug. Birchmeier's S\u00f6hne", "location": "(manufacturers of locks and fittings), Murgenthal."}, {"entry_id": "32", "company_name": "J. Coray", "location": "(agent and wholesaler), 20, T\u00f6distrasse, Zurich 2."}, {"entry_id": "33", "company_name": "Guhl & Scheibler Ltd.", "location": "(wooden and steel office furniture), 22, Elisabethenstrasse, Basle."}]}, "text": "", "raw_response": {"id": "gen-1761645564-SympRErCrmLpA6lMw3fF", "model": "qwen/qwen3-vl-30b-a3b-instruct", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"page_id\": \"25\", \"entries\": [{\"entry_id\": \"1\", \"company_name\": \"General Superintendence Co. Ltd.\", \"location\": \"1, place des Alpes, Geneva.\"}, {\"entry_id\": \"2\", \"company_name\": \"Gondrand Brothers Ltd.\", \"location\": \"8, Viaduktstrasse, Basle.\"}, {\"entry_id\": \"3\", \"company_name\": \"Goth & Co. Ltd.\", \"location\": \"45, Elisabethenstrasse, Basle.\"}, {\"entry_id\": \"4\", \"company_name\": \"Charles Im Obersteg & Co. Ltd.\", \"location\": \"30, Aeschengraben, Basle.\"}, {\"entry_id\": \"5\", \"company_name\": \"John Im Obersteg & Co. Ltd.\", \"location\": \"28, Aeschengraben, Basle.\"}, {\"entry_id\": \"6\", \"company_name\": \"Jacky, Maeder & Co.\", \"location\": \"35, Uraniastrasse Zurich 1.\"}, {\"entry_id\": \"7\", \"company_name\": \"Jacky, Maeder & Co.\", \"location\": \"8, Wallstrasse, Basle.\"}, {\"entry_id\": \"8\", \"company_name\": \"Lavanchy & Co. Ltd.\", \"location\": \"88, rue de Gen\u00e8ve, Lausanne.\"}, {\"entry_id\": \"9\", \"company_name\": \"Marunion S.A.\", \"location\": \"21, Zielackerstrasse, Zurich 9.\"}, {\"entry_id\": \"10\", \"company_name\": \"MAT Transport Ltd.\", \"location\": \"56, Peter Merian-Strasse, Basle.\"}, {\"entry_id\": \"11\", \"company_name\": \"M\u00fcller-Gysin Ltd.\", \"location\": \"Zollfreilager, Basle 23.\"}, {\"entry_id\": \"12\", \"company_name\": \"Natural Ltd.\", \"location\": \"67, Nauenstrasse, Basle.\"}, {\"entry_id\": \"13\", \"company_name\": \"A. Natural, Le Coultre & Co. Ltd.\", \"location\": \"25, rue de Chantepoulet, Geneva.\"}, {\"entry_id\": \"14\", \"company_name\": \"Roba Shipping Agency & Warehouse Ltd.\", \"location\": \"Zollfreilager, Basle 23.\"}, {\"entry_id\": \"15\", \"company_name\": \"J. V\u00e9ron, Grauer & Co. Ltd.\", \"location\": \"22, rue du Mont-Blanc, Geneva.\"}, {\"entry_id\": \"16\", \"company_name\": \"Weltfurrer International Transport Co. Ltd.\", \"location\": \"8, Pelikanstrasse, Zurich 1.\"}, {\"entry_id\": \"17\", \"company_name\": \"A. Welti-Furrer Ltd.\", \"location\": \"16, M\u00fcllerstrasse, Zurich 4.\"}, {\"entry_id\": \"18\", \"company_name\": \"World Transport Ltd.\", \"location\": \"33, Nauenstrasse, Basle.\"}, {\"entry_id\": \"19\", \"company_name\": \"George Fischer Ltd.\", \"location\": \"Schaffhausen.\"}, {\"entry_id\": \"20\", \"company_name\": \"Morganite Exports Ltd.\", \"location\": \"London; Zurich Office: 6, Sp\u00fclgenstrasse, Zurich 2.\"}, {\"entry_id\": \"21\", \"company_name\": \"Louis de Roll Iron Works Ltd.\", \"location\": \"Gerlafingen.\"}, {\"entry_id\": \"22\", \"company_name\": \"Sulzer Brothers Ltd.\", \"location\": \"Winterthur.\"}, {\"entry_id\": \"23\", \"company_name\": \"Schubarth & Co. (agents)\", \"location\": \"45, Peter Merian-Strasse, Basle.\"}, {\"entry_id\": \"24\", \"company_name\": \"Marunion S.A.\", \"location\": \"(for steamship services), 21, Zielackerstrasse, Zurich 9.\"}, {\"entry_id\": \"25\", \"company_name\": \"W. Simon\", \"location\": \"(wholesale and retail dealers), 285-287, Hardturmstrasse, Zurich 5.\"}, {\"entry_id\": \"26\", \"company_name\": \"Emil Flachsmann\", \"location\": \"(fruit extracts, concentrates and distillates), 60, Butzenstrasse, Zurich 30.\"}, {\"entry_id\": \"27\", \"company_name\": \"H. Gallmann & Co.\", \"location\": \"(manufacturers of tinned tomato juice), 2, Rosenstrasse, Winterthur.\"}, {\"entry_id\": \"28\", \"company_name\": \"L. E. Michel S.A.\", \"location\": \"(manufacturers of juices, concentrates, essences), pont du Ga- lieten, Malley-Lausanne.\"}, {\"entry_id\": \"29\", \"company_name\": \"P.H. Stehelin & Co. Ltd.\", \"location\": \"(importers of concentrated juices), 62, Spalentorweg, Basle.\"}, {\"entry_id\": \"30\", \"company_name\": \"Gustav Baader\", \"location\": \"(importers and wholesalers; also retail), 57, Aeschenvorstadt, Basle.\"}, {\"entry_id\": \"31\", \"company_name\": \"Aug. Birchmeier's S\u00f6hne\", \"location\": \"(manufacturers of locks and fittings), Murgenthal.\"}, {\"entry_id\": \"32\", \"company_name\": \"J. Coray\", \"location\": \"(agent and wholesaler), 20, T\u00f6distrasse, Zurich 2.\"}, {\"entry_id\": \"33\", \"company_name\": \"Guhl & Scheibler Ltd.\", \"location\": \"(wooden and steel office furniture), 22, Elisabethenstrasse, Basle.\"}]}", "role": "assistant"}}], "usage": {"prompt_tokens": 731, "completion_tokens": 1291, "total_tokens": 2022}}, "usage": {"input_tokens": 731, "output_tokens": 1291, "total_tokens": 2022}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.24346076458752516, "f1_macro": 0.256, "micro_precision": 0.2429718875502008, "micro_recall": 0.2439516129032258, "total_instances": 15, "total_tp": 242, "total_fp": 754, "total_fn": 750, "cost_summary": {"total_input_tokens": 11612, "total_output_tokens": 13486, "total_tokens": 25098, "input_cost_usd": 0.002322, "output_cost_usd": 0.00944, "total_cost_usd": 0.011763, "pricing_date": "2025-10-28", "input_price_per_million": 0.2, "output_price_per_million": 0.7}}, "normalized_score": 25.6}