{"_id": {"$oid": "6a8456c1ef0a19c343227daa"}, "test_id": "T1639", "benchmark": "company_lists", "date": "2026-08-18", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en", "de"], "layout": ["list"], "entry-type": ["company"], "task": ["information-extraction"]}, "contributors": ["lea_kasper", "sorin_marti"], "hidden": false, "config": {"provider": "openrouter", "model": "z-ai/glm-5v-turbo", "dataclass": "ListPage", "temperature": 0.5, "role_description": "You are a Historian", "prompt_file": "prompt_min.txt", "legacy_test": false}, "prompt": "- Answer in valid JSON.\n- The page ID is given as {page_id}.\n", "results": {"text": "{\n  \"page_id\": \"61\",\n  \"entries\": [\n    {\n      \"entry_id\": \"1\",\n      \"company_name\": \"Nardin, Ulysse\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"2\",\n      \"company_name\": \"Natural, Le Coultre & Cie., S. A.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"3\",\n      \"company_name\": \"Nestl\u00e9 & Anglo-Swiss Condensed Milk Co., Ltd.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"4\",\n      \"company_name\": \"Neuburger & Co.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"5\",\n      \"company_name\": \"North British Rubber Co., Ltd., The\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"6\",\n      \"company_name\": \"Obrist & Co.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"7\",\n      \"company_name\": \"Paillard, E. & Cie., S. A.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"8\",\n      \"company_name\": \"Paillard & Benoit\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"9\",\n      \"company_name\": \"Palace Hotel\",\n      \"location\": \"Montana\"\n    },\n    {\n      \"entry_id\": \"10\",\n      \"company_name\": \"Pearse, A. C. & Co., Ltd.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"11\",\n      \"company_name\": \"Perrin & Cie.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"12\",\n      \"company_name\": \"Peter, Cailler, Kohler\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"13\",\n      \"company_name\": \"Philippi & Hermann\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"14\",\n      \"company_name\": \"Planta, J. & Co.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"15\",\n      \"company_name\": \"Porteous, James & Co., Ltd.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"16\",\n      \"company_name\": \"Preisig, Walter & Cie\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"17\",\n      \"company_name\": \"Quasi-Arc Co., Ltd., The\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"18\",\n      \"company_name\": \"Record Dreadnought Watch Co., S. A.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"19\",\n      \"company_name\": \"Reichenbach & Cie.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"20\",\n      \"company_name\": \"Reinhart & Co.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"21\",\n      \"company_name\": \"Rinderknecht, J.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"22\",\n      \"company_name\": \"Rolling & Drawing Mills Ltd.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"23\",\n      \"company_name\": \"Rolls, Frank V.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"24\",\n      \"company_name\": \"Rolo & Co.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"25\",\n      \"company_name\": \"\"Roneo\" A. G.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"26\",\n      \"company_name\": \"Roussy, Th\u00e9odore\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"27\",\n      \"company_name\": \"R\u00fcegger & Co.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"28\",\n      \"company_name\": \"Ryff & Co., Ltd.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"29\",\n      \"company_name\": \"Sandoz Chemical Co., Ltd., The\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"30\",\n      \"company_name\": \"Sandreuter & Co.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"31\",\n      \"company_name\": \"Sarasin, Rodolphe & Co., Ltd.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"32\",\n      \"company_name\": \"Sarasin Sons Ltd.\",\n      \"location\": \"\"\n    },\n    {\n      \"entry_id\": \"33\",\n      \"company_name\": \"Sarasin, W. & Co.\",\n      \"location\": \"\"\n    }\n  ]\n}\n", "model": "z-ai/glm-5v-turbo", "provider": "openrouter", "finish_reason": "stop", "usage": {"input_tokens": 4006, "output_tokens": 1092, "total_tokens": 5098, "cached_tokens": 87, "estimated_cost_usd": 0.009175200000000001, "input_cost_usd": 0.0048072, "output_cost_usd": 0.004368}, "duration": 41.65847969055176, "timestamp": "2026-08-18T11:51:03.633707", "conversation_id": "7b1e9842-999b-4e3a-9c0c-9e93d713065f", "score": null, "raw_response": "{\"id\":\"gen-1787046645-YXcOTGU59rBilnMDXLSW\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"page_id\\\": \\\"61\\\",\\n  \\\"entries\\\": [\\n    {\\n      \\\"entry_id\\\": \\\"1\\\",\\n      \\\"company_name\\\": \\\"Nardin, Ulysse\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"2\\\",\\n      \\\"company_name\\\": \\\"Natural, Le Coultre & Cie., S. A.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"3\\\",\\n      \\\"company_name\\\": \\\"Nestl\u00e9 & Anglo-Swiss Condensed Milk Co., Ltd.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"4\\\",\\n      \\\"company_name\\\": \\\"Neuburger & Co.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"5\\\",\\n      \\\"company_name\\\": \\\"North British Rubber Co., Ltd., The\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"6\\\",\\n      \\\"company_name\\\": \\\"Obrist & Co.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"7\\\",\\n      \\\"company_name\\\": \\\"Paillard, E. & Cie., S. A.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"8\\\",\\n      \\\"company_name\\\": \\\"Paillard & Benoit\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"9\\\",\\n      \\\"company_name\\\": \\\"Palace Hotel\\\",\\n      \\\"location\\\": \\\"Montana\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"10\\\",\\n      \\\"company_name\\\": \\\"Pearse, A. C. & Co., Ltd.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"11\\\",\\n      \\\"company_name\\\": \\\"Perrin & Cie.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"12\\\",\\n      \\\"company_name\\\": \\\"Peter, Cailler, Kohler\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"13\\\",\\n      \\\"company_name\\\": \\\"Philippi & Hermann\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"14\\\",\\n      \\\"company_name\\\": \\\"Planta, J. & Co.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"15\\\",\\n      \\\"company_name\\\": \\\"Porteous, James & Co., Ltd.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"16\\\",\\n      \\\"company_name\\\": \\\"Preisig, Walter & Cie\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"17\\\",\\n      \\\"company_name\\\": \\\"Quasi-Arc Co., Ltd., The\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"18\\\",\\n      \\\"company_name\\\": \\\"Record Dreadnought Watch Co., S. A.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"19\\\",\\n      \\\"company_name\\\": \\\"Reichenbach & Cie.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"20\\\",\\n      \\\"company_name\\\": \\\"Reinhart & Co.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"21\\\",\\n      \\\"company_name\\\": \\\"Rinderknecht, J.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"22\\\",\\n      \\\"company_name\\\": \\\"Rolling & Drawing Mills Ltd.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"23\\\",\\n      \\\"company_name\\\": \\\"Rolls, Frank V.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"24\\\",\\n      \\\"company_name\\\": \\\"Rolo & Co.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"25\\\",\\n      \\\"company_name\\\": \\\"\\\"Roneo\\\" A. G.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"26\\\",\\n      \\\"company_name\\\": \\\"Roussy, Th\u00e9odore\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"27\\\",\\n      \\\"company_name\\\": \\\"R\u00fcegger & Co.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"28\\\",\\n      \\\"company_name\\\": \\\"Ryff & Co., Ltd.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"29\\\",\\n      \\\"company_name\\\": \\\"Sandoz Chemical Co., Ltd., The\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"30\\\",\\n      \\\"company_name\\\": \\\"Sandreuter & Co.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"31\\\",\\n      \\\"company_name\\\": \\\"Sarasin, Rodolphe & Co., Ltd.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"32\\\",\\n      \\\"company_name\\\": \\\"Sarasin Sons Ltd.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    },\\n    {\\n      \\\"entry_id\\\": \\\"33\\\",\\n      \\\"company_name\\\": \\\"Sarasin, W. & Co.\\\",\\n      \\\"location\\\": \\\"\\\"\\n    }\\n  ]\\n}\\n\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning\":\"I will extract the page ID from the top right corner of the image and then identify each company name listed on the left, along with any associated location information, to construct the JSON object as requested.\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"I will extract the page ID from the top right corner of the image and then identify each company name listed on the left, along with any associated location information, to construct the JSON object as requested.\",\"format\":\"unknown\",\"index\":0}]},\"native_finish_reason\":\"stop\"}],\"created\":1787046645,\"model\":\"z-ai/glm-5v-turbo\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":1092,\"prompt_tokens\":4006,\"total_tokens\":5098,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":0,\"reasoning_tokens\":40,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":87,\"cache_write_tokens\":0,\"video_tokens\":0},\"cost\":0.00909168,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":0.00909168,\"upstream_inference_prompt_cost\":0.00472368,\"upstream_inference_completions_cost\":0.004368}},\"provider\":\"Z.AI\"}"}, "scoring": {"f1_micro": 0.443579766536965, "f1_macro": 0.45545454545454545, "micro_precision": 0.4432348367029549, "micro_recall": 0.4439252336448598, "total_instances": 11, "total_tp": 285, "total_fp": 358, "total_fn": 357, "cost_summary": {"total_input_tokens": 56922, "total_output_tokens": 24277, "total_tokens": 81199, "input_cost_usd": 0.06830639999999998, "output_cost_usd": 0.097108, "total_cost_usd": 0.16541440000000002}}, "normalized_score": 45.54545454545455}