{"_id": {"$oid": "69d4eb12b877594bc5718a20"}, "test_id": "T0396", "benchmark": "company_lists", "date": "2025-10-28", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en", "de"], "layout": ["list"], "entry-type": ["company"], "task": ["information-extraction"]}, "contributors": ["lea_kasper", "sorin_marti"], "hidden": false, "config": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "dataclass": "ListPage", "temperature": 0.5, "role_description": "You are a Historian", "prompt_file": "prompt_min.txt", "legacy_test": false}, "prompt": "- Answer in valid JSON.\n- The page ID is given as {page_id}.\n", "results": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "duration": 18.974151611328125, "timestamp": "2025-10-28T10:54:32.907508", "parsed": {"page_id": "{page_id}", "entries": [{"entry_id": ":1", "company_name": "International Trading Co.", "location": "151/7, Fitzwilliam Street, Sheffield; Phone Sheffield 21550; Telgr. \"Diligent\" Sheffield."}, {"entry_id": ":2", "company_name": "Albert Isliker & Co.", "location": "35a. L\u00f6wenstrasse, Zurich; Phone (051) 23 56 26; 23 56 28; Telgr. \"Islikerco\" Zurich."}, {"entry_id": ":3", "company_name": "V. & W. Kaiser", "location": "(exporters), Muttens; Phone (061) 9 32 98; 7 20 25; Telgr. \"Kaiser\" Muttens."}, {"entry_id": ":4", "company_name": "Matiera Ltd.", "location": "(importers of fine and heavy), 37, Leonhardstrasse, Basle; Phone (061) 317 87; Telgr. \"Materba\" Basle."}, {"entry_id": ":5", "company_name": "A. Mathys", "location": "(for industrial purposes), Zurich F. 1. Postbox Sihlpost, Zurich; Phone (051) 28 24 91; 28 19 35; Telgr. \"Almathys\" Zurich."}, {"entry_id": ":6", "company_name": "Medo-Chemicals Ltd.", "location": "178, Pentonville Road, London N.1; Phone Terminus 6568; 7438; Telgr. \"Terminus 6568\" London."}, {"entry_id": ":7", "company_name": "Metallum Pratteln Ltd.", "location": "(importers), Pratteln, Basle; Phone (061) 6 04 55; Telgr. \"Metallum\" Pratteln."}, {"entry_id": ":8", "company_name": "Walter Moesch & Co.", "location": "135, Drusbergstrasse, Zurich 7; Phone (051) 24 74 24; Telgr. \"Walma\" Zurich."}, {"entry_id": ":9", "company_name": "Novatia Ltd.", "location": "(for industrial purposes), Parkring 53, Zurich 2; Phone (051) 25 85 49; Telgr. \"Novatia\" Zurich."}, {"entry_id": ":10", "company_name": "Pl\u00fcss-Staufer Ltd.", "location": "Oftringen; Phone (062) 7 35 44; Telgr. \"Omya\" Oftringen."}, {"entry_id": ":11", "company_name": "Prifa Ltd.", "location": "60, Goethestrasse, Zurich; Phone (051) 32 52 52; Telgr. \"Ageprifa\" Zurich."}, {"entry_id": ":12", "company_name": "Hans Rahn & Co.", "location": "30, Talacker, Zurich; Phone (051) 25 17 14; Telgr. \"Chemrahn\" Zurich."}, {"entry_id": ":13", "company_name": "Alex Ross & Co.", "location": "Lancaster House, 80, Princess Street, Manchester 1; Phone Central 7212; 7213; 7214; Telgr. \"Alacrious\" Manchester."}, {"entry_id": ":14", "company_name": "Rubber Raw Materials Ltd.", "location": "(for the rubber trade), 17, M\u00fcnsterhof, Zurich; Phone (051) 27 45 45; Telgr. \"Raw Rubber\" Zurich."}, {"entry_id": ":15", "company_name": "Emile Scheller & Co. Ltd.", "location": "(importers of all kinds), Hottingerstrasse 21, Zurich; Phone (051) 32 68 60; Telgr. \"Acidus\" Zurich."}, {"entry_id": ":16", "company_name": "A. Schubarth & Co.", "location": "85, Elisabethenstrasse, Basle; Phone (061) 2 36 50; Telgr. \"Schubarth\" Basle."}, {"entry_id": ":17", "company_name": "Schweizerhall Ltd.", "location": "(importers of fine and heavy chemicals), 229, Els\u00e4sserstrasse, Basle; Phone (061) 2 88 90; Telgr. \"Schweizerhall\" Basle."}, {"entry_id": ":18", "company_name": "Steven Brothers (Londmil) Ltd.", "location": "21, Liverpool Street, London E. C. 2.; Phone Bishopsgate 0267; Telgr. \"Stebmil\" London."}]}, "text": "", "raw_response": {"id": "gen-1761645255-i34ynZNV2SvRVvAMwOcj", "model": "meta-llama/llama-4-maverick", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n\"page_id\": \"{page_id}\",\n\"entries\": [\n{\n\"entry_id\": \":1\",\n\"company_name\": \"International Trading Co.\",\n\"location\": \"151/7, Fitzwilliam Street, Sheffield; Phone Sheffield 21550; Telgr. \\\"Diligent\\\" Sheffield.\"\n},\n{\n\"entry_id\": \":2\",\n\"company_name\": \"Albert Isliker & Co.\",\n\"location\": \"35a. L\u00f6wenstrasse, Zurich; Phone (051) 23 56 26; 23 56 28; Telgr. \\\"Islikerco\\\" Zurich.\"\n},\n{\n\"entry_id\": \":3\",\n\"company_name\": \"V. & W. Kaiser\",\n\"location\": \"(exporters), Muttens; Phone (061) 9 32 98; 7 20 25; Telgr. \\\"Kaiser\\\" Muttens.\"\n},\n{\n\"entry_id\": \":4\",\n\"company_name\": \"Matiera Ltd.\",\n\"location\": \"(importers of fine and heavy), 37, Leonhardstrasse, Basle; Phone (061) 317 87; Telgr. \\\"Materba\\\" Basle.\"\n},\n{\n\"entry_id\": \":5\",\n\"company_name\": \"A. Mathys\",\n\"location\": \"(for industrial purposes), Zurich F. 1. Postbox Sihlpost, Zurich; Phone (051) 28 24 91; 28 19 35; Telgr. \\\"Almathys\\\" Zurich.\"\n},\n{\n\"entry_id\": \":6\",\n\"company_name\": \"Medo-Chemicals Ltd.\",\n\"location\": \"178, Pentonville Road, London N.1; Phone Terminus 6568; 7438; Telgr. \\\"Terminus 6568\\\" London.\"\n},\n{\n\"entry_id\": \":7\",\n\"company_name\": \"Metallum Pratteln Ltd.\",\n\"location\": \"(importers), Pratteln, Basle; Phone (061) 6 04 55; Telgr. \\\"Metallum\\\" Pratteln.\"\n},\n{\n\"entry_id\": \":8\",\n\"company_name\": \"Walter Moesch & Co.\",\n\"location\": \"135, Drusbergstrasse, Zurich 7; Phone (051) 24 74 24; Telgr. \\\"Walma\\\" Zurich.\"\n},\n{\n\"entry_id\": \":9\",\n\"company_name\": \"Novatia Ltd.\",\n\"location\": \"(for industrial purposes), Parkring 53, Zurich 2; Phone (051) 25 85 49; Telgr. \\\"Novatia\\\" Zurich.\"\n},\n{\n\"entry_id\": \":10\",\n\"company_name\": \"Pl\u00fcss-Staufer Ltd.\",\n\"location\": \"Oftringen; Phone (062) 7 35 44; Telgr. \\\"Omya\\\" Oftringen.\"\n},\n{\n\"entry_id\": \":11\",\n\"company_name\": \"Prifa Ltd.\",\n\"location\": \"60, Goethestrasse, Zurich; Phone (051) 32 52 52; Telgr. \\\"Ageprifa\\\" Zurich.\"\n},\n{\n\"entry_id\": \":12\",\n\"company_name\": \"Hans Rahn & Co.\",\n\"location\": \"30, Talacker, Zurich; Phone (051) 25 17 14; Telgr. \\\"Chemrahn\\\" Zurich.\"\n},\n{\n\"entry_id\": \":13\",\n\"company_name\": \"Alex Ross & Co.\",\n\"location\": \"Lancaster House, 80, Princess Street, Manchester 1; Phone Central 7212; 7213; 7214; Telgr. \\\"Alacrious\\\" Manchester.\"\n},\n{\n\"entry_id\": \":14\",\n\"company_name\": \"Rubber Raw Materials Ltd.\",\n\"location\": \"(for the rubber trade), 17, M\u00fcnsterhof, Zurich; Phone (051) 27 45 45; Telgr. \\\"Raw Rubber\\\" Zurich.\"\n},\n{\n\"entry_id\": \":15\",\n\"company_name\": \"Emile Scheller & Co. Ltd.\",\n\"location\": \"(importers of all kinds), Hottingerstrasse 21, Zurich; Phone (051) 32 68 60; Telgr. \\\"Acidus\\\" Zurich.\"\n},\n{\n\"entry_id\": \":16\",\n\"company_name\": \"A. Schubarth & Co.\",\n\"location\": \"85, Elisabethenstrasse, Basle; Phone (061) 2 36 50; Telgr. \\\"Schubarth\\\" Basle.\"\n},\n{\n\"entry_id\": \":17\",\n\"company_name\": \"Schweizerhall Ltd.\",\n\"location\": \"(importers of fine and heavy chemicals), 229, Els\u00e4sserstrasse, Basle; Phone (061) 2 88 90; Telgr. \\\"Schweizerhall\\\" Basle.\"\n},\n{\n\"entry_id\": \":18\",\n\"company_name\": \"Steven Brothers (Londmil) Ltd.\",\n\"location\": \"21, Liverpool Street, London E. C. 2.; Phone Bishopsgate 0267; Telgr. \\\"Stebmil\\\" London.\"\n}\n]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1334, "completion_tokens": 1060, "total_tokens": 2394}}, "usage": {"input_tokens": 1334, "output_tokens": 1060, "total_tokens": 2394}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.2949117341640706, "f1_macro": 0.324, "micro_precision": 0.30406852248394006, "micro_recall": 0.2862903225806452, "total_instances": 15, "total_tp": 284, "total_fp": 650, "total_fn": 708, "cost_summary": {"total_input_tokens": 27387, "total_output_tokens": 15005, "total_tokens": 42392, "input_cost_usd": 0.004108, "output_cost_usd": 0.009003, "total_cost_usd": 0.013111, "pricing_date": "2025-10-28", "input_price_per_million": 0.15, "output_price_per_million": 0.6}}, "normalized_score": 32.4}