{"_id": {"$oid": "69d4eb12b877594bc5718a0b"}, "test_id": "T0389", "benchmark": "company_lists", "date": "2025-10-28", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en", "de"], "layout": ["list"], "entry-type": ["company"], "task": ["information-extraction"]}, "contributors": ["lea_kasper", "sorin_marti"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-large-2411", "dataclass": "ListPage", "temperature": 0.5, "role_description": "You are a Historian", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "The image you are presented with stems from a digitized book containing lists of companies.\nYour task is to extract structured information about each company listed on the page.\n\nAbout the source:\n- The image stems from a trade index of the British Swiss Chamber of Commerce.\n- The image can show an alphabetical or a thematic list of companies.\n- The companies are mostly located in Switzerland and the UK.\n- The image stems from a trade index between 1925 and 1958.\n- Most pages have one column but some years have two columns.\n- The source itself is in English and German but the company names can be in English, German, French or Italian.\n\nAbout the entries:\n- Each entry describes a single company or person.\n- Alphabetical entries have filling dots between the company name and the page number. Dots and page numbers are not part of the data and should be ignored.\n- Alphabetical entries seldom to never have locations.\n- Thematic entries often have locations.\n- Thematic entries are listed under headings that describe the type of business.\n- Some thematic headings are only references to other headings, e.g. \"X, s. Y\".\n\nAbout the output:\n- Answer in valid JSON. The JSON should be an array of objects with the following fields:\n- The page ID is given as {page_id}.\n- Do not add country information, if it is not directly written with the location.\n\n{\n  \"entry_id\": \"A unique identifier for the entry, e.g. '{page_id}-1'\",\n  \"company_name\": \"The name of the company or person\",\n  \"location\": \"The location of the company, e.g. 'Zurich' or 'London, UK'. If no location is given, set to null.\"\n  ]\n}\n", "results": {"provider": "mistral", "model": "mistral-large-latest", "duration": 25.66067385673523, "timestamp": "2025-10-28T14:22:22.846489", "parsed": {"page_id": "{page_id}", "entries": [{"entry_id": "{page_id}-1", "company_name": "Perrin & Co.", "location": "Place de la Gare, Lausanne"}, {"entry_id": "{page_id}-2", "company_name": "Sulzer Fr\u00e8res, S. A.", "location": "Winterthur"}, {"entry_id": "{page_id}-3", "company_name": "Huguenin, Gattiker & Co., A.-G.", "location": "39, Bahnhofstrasse, Zurich"}, {"entry_id": "{page_id}-4", "company_name": "The Cattle Food Supply Co., Ltd.", "location": "Antwerp"}, {"entry_id": "{page_id}-5", "company_name": "Basle Celluloid Manufactures, Ltd.", "location": "Therwil/Basle"}, {"entry_id": "{page_id}-6", "company_name": "London Chamber of Commerce", "location": "1, 2 & 3, Oxford Court, Cannon Street, London E. C. 4"}, {"entry_id": "{page_id}-7", "company_name": "Alpina Cheese Co., Ltd.", "location": "Burgdorf"}, {"entry_id": "{page_id}-8", "company_name": "Aktiengesellschaft vorm. B. Siegfried (synthetics)", "location": "Zofingen"}, {"entry_id": "{page_id}-9", "company_name": "Chemical Works Flora", "location": "D\u00fcbendorf, Zurich"}, {"entry_id": "{page_id}-10", "company_name": "Chemische Fabrik vorm. Sandoz", "location": "Basle"}, {"entry_id": "{page_id}-11", "company_name": "Clayton Aniline Co., Ltd.", "location": "Ashton New Road, Clayton, Manchester"}, {"entry_id": "{page_id}-12", "company_name": "Durand & Huguenin, A.-G.", "location": "Basle"}, {"entry_id": "{page_id}-13", "company_name": "Herbert Green & Co., Ltd.", "location": "Ruskin Chambers, Scale Lane, Hull"}, {"entry_id": "{page_id}-14", "company_name": "Hoffmann-La Roche & Co.", "location": "Basle"}, {"entry_id": "{page_id}-15", "company_name": "Th. M\u00fchlethaler, S. A", "location": "Nyon"}, {"entry_id": "{page_id}-16", "company_name": "Society of Chemical Industry in Basle", "location": "Basle"}, {"entry_id": "{page_id}-17", "company_name": "Usines de l'Allondon", "location": "La Plaine, Geneva"}, {"entry_id": "{page_id}-18", "company_name": "Chocolats de Villars, S. A.", "location": "Fribourg"}, {"entry_id": "{page_id}-19", "company_name": "Nestl\u00e9 & Anglo-Swiss Condensed Milk Co.", "location": "Vevey"}, {"entry_id": "{page_id}-20", "company_name": "Peter, Cailler, Kohler", "location": "La Tour-de-Peilz"}, {"entry_id": "{page_id}-21", "company_name": "Suchard, S. A.", "location": "Neuch\u00e2tel"}, {"entry_id": "{page_id}-22", "company_name": "Audemars, Piguet & Co., Ltd.", "location": "Les Brassus"}, {"entry_id": "{page_id}-23", "company_name": "Ulysse Nardin", "location": "Le Locle and Geneva"}]}, "text": "", "raw_response": {"id": "771c18edfc36469eb12a4ee18f726f3b", "model": "mistral-large-latest", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"page_id\": \"{page_id}\",\n  \"entries\": [\n    {\n      \"entry_id\": \"{page_id}-1\",\n      \"company_name\": \"Perrin & Co.\",\n      \"location\": \"Place de la Gare, Lausanne\"\n    },\n    {\n      \"entry_id\": \"{page_id}-2\",\n      \"company_name\": \"Sulzer Fr\u00e8res, S. A.\",\n      \"location\": \"Winterthur\"\n    },\n    {\n      \"entry_id\": \"{page_id}-3\",\n      \"company_name\": \"Huguenin, Gattiker & Co., A.-G.\",\n      \"location\": \"39, Bahnhofstrasse, Zurich\"\n    },\n    {\n      \"entry_id\": \"{page_id}-4\",\n      \"company_name\": \"The Cattle Food Supply Co., Ltd.\",\n      \"location\": \"Antwerp\"\n    },\n    {\n      \"entry_id\": \"{page_id}-5\",\n      \"company_name\": \"Basle Celluloid Manufactures, Ltd.\",\n      \"location\": \"Therwil/Basle\"\n    },\n    {\n      \"entry_id\": \"{page_id}-6\",\n      \"company_name\": \"London Chamber of Commerce\",\n      \"location\": \"1, 2 & 3, Oxford Court, Cannon Street, London E. C. 4\"\n    },\n    {\n      \"entry_id\": \"{page_id}-7\",\n      \"company_name\": \"Alpina Cheese Co., Ltd.\",\n      \"location\": \"Burgdorf\"\n    },\n    {\n      \"entry_id\": \"{page_id}-8\",\n      \"company_name\": \"Aktiengesellschaft vorm. B. Siegfried (synthetics)\",\n      \"location\": \"Zofingen\"\n    },\n    {\n      \"entry_id\": \"{page_id}-9\",\n      \"company_name\": \"Chemical Works Flora\",\n      \"location\": \"D\u00fcbendorf, Zurich\"\n    },\n    {\n      \"entry_id\": \"{page_id}-10\",\n      \"company_name\": \"Chemische Fabrik vorm. Sandoz\",\n      \"location\": \"Basle\"\n    },\n    {\n      \"entry_id\": \"{page_id}-11\",\n      \"company_name\": \"Clayton Aniline Co., Ltd.\",\n      \"location\": \"Ashton New Road, Clayton, Manchester\"\n    },\n    {\n      \"entry_id\": \"{page_id}-12\",\n      \"company_name\": \"Durand & Huguenin, A.-G.\",\n      \"location\": \"Basle\"\n    },\n    {\n      \"entry_id\": \"{page_id}-13\",\n      \"company_name\": \"Herbert Green & Co., Ltd.\",\n      \"location\": \"Ruskin Chambers, Scale Lane, Hull\"\n    },\n    {\n      \"entry_id\": \"{page_id}-14\",\n      \"company_name\": \"Hoffmann-La Roche & Co.\",\n      \"location\": \"Basle\"\n    },\n    {\n      \"entry_id\": \"{page_id}-15\",\n      \"company_name\": \"Th. M\u00fchlethaler, S. A\",\n      \"location\": \"Nyon\"\n    },\n    {\n      \"entry_id\": \"{page_id}-16\",\n      \"company_name\": \"Society of Chemical Industry in Basle\",\n      \"location\": \"Basle\"\n    },\n    {\n      \"entry_id\": \"{page_id}-17\",\n      \"company_name\": \"Usines de l'Allondon\",\n      \"location\": \"La Plaine, Geneva\"\n    },\n    {\n      \"entry_id\": \"{page_id}-18\",\n      \"company_name\": \"Chocolats de Villars, S. A.\",\n      \"location\": \"Fribourg\"\n    },\n    {\n      \"entry_id\": \"{page_id}-19\",\n      \"company_name\": \"Nestl\u00e9 & Anglo-Swiss Condensed Milk Co.\",\n      \"location\": \"Vevey\"\n    },\n    {\n      \"entry_id\": \"{page_id}-20\",\n      \"company_name\": \"Peter, Cailler, Kohler\",\n      \"location\": \"La Tour-de-Peilz\"\n    },\n    {\n      \"entry_id\": \"{page_id}-21\",\n      \"company_name\": \"Suchard, S. A.\",\n      \"location\": \"Neuch\u00e2tel\"\n    },\n    {\n      \"entry_id\": \"{page_id}-22\",\n      \"company_name\": \"Audemars, Piguet & Co., Ltd.\",\n      \"location\": \"Les Brassus\"\n    },\n    {\n      \"entry_id\": \"{page_id}-23\",\n      \"company_name\": \"Ulysse Nardin\",\n      \"location\": \"Le Locle and Geneva\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1505, "completion_tokens": 972, "total_tokens": 2477}}, "usage": {"input_tokens": 1505, "output_tokens": 972, "total_tokens": 2477}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.437941473259334, "f1_macro": 0.4373333333333333, "micro_precision": 0.4383838383838384, "micro_recall": 0.4375, "total_instances": 15, "total_tp": 434, "total_fp": 556, "total_fn": 558, "cost_summary": {"total_input_tokens": 22612, "total_output_tokens": 14199, "total_tokens": 36811, "input_cost_usd": 0.045224, "output_cost_usd": 0.085194, "total_cost_usd": 0.130418, "pricing_date": "2025-10-28", "input_price_per_million": 2.0, "output_price_per_million": 6.0}}, "normalized_score": 43.73333333333333}