{"_id": {"$oid": "6a478c6124b401ab26c43da0"}, "test_id": "T1198", "benchmark": "company_lists", "date": "2026-07-02", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en", "de"], "layout": ["list"], "entry-type": ["company"], "task": ["information-extraction"]}, "contributors": ["lea_kasper", "sorin_marti"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-fable-5", "dataclass": "ListPage", "temperature": 0.5, "role_description": "You are a Historian", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "The image you are presented with stems from a digitized book containing lists of companies.\nYour task is to extract structured information about each company listed on the page.\n\nAbout the source:\n- The image stems from a trade index of the British Swiss Chamber of Commerce.\n- The image can show an alphabetical or a thematic list of companies.\n- The companies are mostly located in Switzerland and the UK.\n- The image stems from a trade index between 1925 and 1958.\n- Most pages have one column but some years have two columns.\n- The source itself is in English and German but the company names can be in English, German, French or Italian.\n\nAbout the entries:\n- Each entry describes a single company or person.\n- Alphabetical entries have filling dots between the company name and the page number. Dots and page numbers are not part of the data and should be ignored.\n- Alphabetical entries seldom to never have locations.\n- Thematic entries often have locations.\n- Thematic entries are listed under headings that describe the type of business.\n- Some thematic headings are only references to other headings, e.g. \"X, s. Y\".\n\nAbout the output:\n- Answer in valid JSON. The JSON should be an array of objects with the following fields:\n- The page ID is given as {page_id}.\n- Do not add country information, if it is not directly written with the location.\n\n{\n  \"entry_id\": \"A unique identifier for the entry, e.g. '{page_id}-1'\",\n  \"company_name\": \"The name of the company or person\",\n  \"location\": \"The location of the company, e.g. 'Zurich' or 'London, UK'. If no location is given, set to null.\"\n  ]\n}\n", "results": {"text": "{\"page_id\":\"74\",\"entries\":[{\"entry_id\":\"74-1\",\"company_name\":\"F. Blumer & Co.\",\"location\":\"Schwanden (Glarus)\"},{\"entry_id\":\"74-2\",\"company_name\":\"Inglo Louis Jablonsky\",\"location\":\"Zurich 2\"},{\"entry_id\":\"74-3\",\"company_name\":\"J. Spiro\",\"location\":\"Zurich 6\"},{\"entry_id\":\"74-4\",\"company_name\":\"Vischer & Co.\",\"location\":\"Basle\"},{\"entry_id\":\"74-5\",\"company_name\":\"Swiss Silk Bolting Cloth Manufacturing Co. Ltd.\",\"location\":\"Thal near Rheineck (St. Gallen)\"},{\"entry_id\":\"74-6\",\"company_name\":\"Swiss Silk Bolting Cloth Manufacturing Co. Ltd.\",\"location\":\"Zurich 2\"},{\"entry_id\":\"74-7\",\"company_name\":\"Tavaro Ltd.\",\"location\":\"Geneva\"},{\"entry_id\":\"74-8\",\"company_name\":\"Beltex Ltd.\",\"location\":\"Arzo (Tessin)\"},{\"entry_id\":\"74-9\",\"company_name\":\"Gust. Metzger Ltd.\",\"location\":\"Basle\"},{\"entry_id\":\"74-10\",\"company_name\":\"Schellenberg, Sax & Co. Ltd.\",\"location\":\"Tr\u00fcbbach (St. Gallen)\"},{\"entry_id\":\"74-11\",\"company_name\":\"Swiss Silk Bolting Cloth Manufacturing Co. Ltd.\",\"location\":\"Thal near Rheineck (St. Gallen)\"},{\"entry_id\":\"74-12\",\"company_name\":\"Swiss Silk Bolting Cloth Manufacturing Co. Ltd.\",\"location\":\"Zurich 2\"},{\"entry_id\":\"74-13\",\"company_name\":\"Abegg & Co. Ltd.\",\"location\":\"Zurich 1\"},{\"entry_id\":\"74-14\",\"company_name\":\"Heer & Co. Ltd.\",\"location\":\"Thalwil\"},{\"entry_id\":\"74-15\",\"company_name\":\"J. Hopf & Co.\",\"location\":\"Basle\"},{\"entry_id\":\"74-16\",\"company_name\":\"P. Marchal & Co.\",\"location\":\"Basle\"},{\"entry_id\":\"74-17\",\"company_name\":\"Saweco Ltd.\",\"location\":\"Basle\"},{\"entry_id\":\"74-18\",\"company_name\":\"Robert Schwarzenbach & Co.\",\"location\":\"Thalwil\"},{\"entry_id\":\"74-19\",\"company_name\":\"Siber, Hegner & Co. Ltd.\",\"location\":\"Zurich 1\"},{\"entry_id\":\"74-20\",\"company_name\":\"Stehli & Co.\",\"location\":\"Zurich 2\"},{\"entry_id\":\"74-21\",\"company_name\":\"Soci\u00e9t\u00e9 Industrielle pour la Schappe\",\"location\":\"Basle\"},{\"entry_id\":\"74-22\",\"company_name\":\"P. Marchal & Co.\",\"location\":\"Basle\"},{\"entry_id\":\"74-23\",\"company_name\":\"Schmid Brothers Ltd.\",\"location\":\"Basle\"},{\"entry_id\":\"74-24\",\"company_name\":\"Siber, Hegner & Co. Ltd.\",\"location\":\"Zurich 1\"}]}", "model": "claude-fable-5", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 4979, "output_tokens": 1212, "total_tokens": 6191, "input_cost_usd": 0.04979, "output_cost_usd": 0.0606, "estimated_cost_usd": 0.11039}, "duration": 17.720471620559692, "timestamp": "2026-07-02T22:56:55.505338", "parsed": {"page_id": "74", "entries": [{"entry_id": "74-1", "company_name": "F. Blumer & Co.", "location": "Schwanden (Glarus)"}, {"entry_id": "74-2", "company_name": "Inglo Louis Jablonsky", "location": "Zurich 2"}, {"entry_id": "74-3", "company_name": "J. Spiro", "location": "Zurich 6"}, {"entry_id": "74-4", "company_name": "Vischer & Co.", "location": "Basle"}, {"entry_id": "74-5", "company_name": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "location": "Thal near Rheineck (St. Gallen)"}, {"entry_id": "74-6", "company_name": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "location": "Zurich 2"}, {"entry_id": "74-7", "company_name": "Tavaro Ltd.", "location": "Geneva"}, {"entry_id": "74-8", "company_name": "Beltex Ltd.", "location": "Arzo (Tessin)"}, {"entry_id": "74-9", "company_name": "Gust. Metzger Ltd.", "location": "Basle"}, {"entry_id": "74-10", "company_name": "Schellenberg, Sax & Co. Ltd.", "location": "Tr\u00fcbbach (St. Gallen)"}, {"entry_id": "74-11", "company_name": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "location": "Thal near Rheineck (St. Gallen)"}, {"entry_id": "74-12", "company_name": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "location": "Zurich 2"}, {"entry_id": "74-13", "company_name": "Abegg & Co. Ltd.", "location": "Zurich 1"}, {"entry_id": "74-14", "company_name": "Heer & Co. Ltd.", "location": "Thalwil"}, {"entry_id": "74-15", "company_name": "J. Hopf & Co.", "location": "Basle"}, {"entry_id": "74-16", "company_name": "P. Marchal & Co.", "location": "Basle"}, {"entry_id": "74-17", "company_name": "Saweco Ltd.", "location": "Basle"}, {"entry_id": "74-18", "company_name": "Robert Schwarzenbach & Co.", "location": "Thalwil"}, {"entry_id": "74-19", "company_name": "Siber, Hegner & Co. Ltd.", "location": "Zurich 1"}, {"entry_id": "74-20", "company_name": "Stehli & Co.", "location": "Zurich 2"}, {"entry_id": "74-21", "company_name": "Soci\u00e9t\u00e9 Industrielle pour la Schappe", "location": "Basle"}, {"entry_id": "74-22", "company_name": "P. Marchal & Co.", "location": "Basle"}, {"entry_id": "74-23", "company_name": "Schmid Brothers Ltd.", "location": "Basle"}, {"entry_id": "74-24", "company_name": "Siber, Hegner & Co. Ltd.", "location": "Zurich 1"}]}, "conversation_id": "ed7d8b61-7f9b-4d45-85f9-eec3a1c3edf0", "score": {"f1_score": 0.66, "precision": 0.6575342465753424, "recall": 0.6575342465753424, "true_positives": 48, "false_positives": 25, "false_negatives": 25, "field_scores": {"entries[19].entry_id": {"response": "74-20", "ground_truth": "156089_1321099_102-20", "score": 0.23076923076923073}, "entries[5].entry_id": {"response": "74-6", "ground_truth": "156089_1321099_102-6", "score": 0.16666666666666663}, "entries[8].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[11].company_name": {"response": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "ground_truth": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "score": 1.0}, "entries[20].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[12].entry_id": {"response": "74-13", "ground_truth": "156089_1321099_102-13", "score": 0.23076923076923073}, "entries[22].company_name": {"response": "Schmid Brothers Ltd.", "ground_truth": "Schmid Brothers Ltd.", "score": 1.0}, "entries[5].company_name": {"response": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "ground_truth": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "score": 1.0}, "entries[6].company_name": {"response": "Tavaro Ltd.", "ground_truth": "Tavaro Ltd.", "score": 1.0}, "entries[7].location": {"response": "Arzo (Tessin)", "ground_truth": "Arzo (Tessin)", "score": 1.0}, "entries[23].entry_id": {"response": "74-24", "ground_truth": "156089_1321099_102-24", "score": 0.23076923076923073}, "entries[8].company_name": {"response": "Gust. Metzger Ltd.", "ground_truth": "Gust. Metzger Ltd.", "score": 1.0}, "entries[21].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[10].location": {"response": "Thal near Rheineck (St. Gallen)", "ground_truth": "Thal near Rheineck (St. Gallen)", "score": 1.0}, "entries[9].company_name": {"response": "Schellenberg, Sax & Co. Ltd.", "ground_truth": "Schellenberg, Sax & Co. Ltd.", "score": 1.0}, "entries[4].company_name": {"response": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "ground_truth": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "score": 1.0}, "entries[22].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[4].entry_id": {"response": "74-5", "ground_truth": "156089_1321099_102-5", "score": 0.16666666666666663}, "entries[23].company_name": {"response": "Siber, Hegner & Co. Ltd.", "ground_truth": "Siber, Hegner & Co. Ltd.", "score": 1.0}, "entries[2].entry_id": {"response": "74-3", "ground_truth": "156089_1321099_102-3", "score": 0.16666666666666663}, "entries[17].location": {"response": "Thalwil", "ground_truth": "Thalwil", "score": 1.0}, "entries[9].location": {"response": "Tr\u00fcbbach (St. Gallen)", "ground_truth": "Tr\u00fcbbach (St. Gallen)", "score": 1.0}, "entries[19].company_name": {"response": "Stehli & Co.", "ground_truth": "Stehli & Co.", "score": 1.0}, "entries[21].company_name": {"response": "P. Marchal & Co.", "ground_truth": "P. Marchal & Co.", "score": 1.0}, "entries[2].location": {"response": "Zurich 6", "ground_truth": "Zurich 6", "score": 1.0}, "entries[21].entry_id": {"response": "74-22", "ground_truth": "156089_1321099_102-22", "score": 0.23076923076923073}, "entries[18].entry_id": {"response": "74-19", "ground_truth": "156089_1321099_102-19", "score": 0.23076923076923073}, "entries[13].entry_id": {"response": "74-14", "ground_truth": "156089_1321099_102-14", "score": 0.23076923076923073}, "entries[16].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[11].entry_id": {"response": "74-12", "ground_truth": "156089_1321099_102-12", "score": 0.23076923076923073}, "entries[5].location": {"response": "Zurich 2", "ground_truth": "Zurich 2", "score": 1.0}, "entries[6].location": {"response": "Geneva", "ground_truth": "Geneva", "score": 1.0}, "entries[1].company_name": {"response": "Inglo Louis Jablonsky", "ground_truth": "Inglo Louis Jablonsky", "score": 1.0}, "entries[10].company_name": {"response": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "ground_truth": "Swiss Silk Bolting Cloth Manufacturing Co. Ltd.", "score": 1.0}, "entries[20].entry_id": {"response": "74-21", "ground_truth": "156089_1321099_102-21", "score": 0.23076923076923073}, "entries[18].company_name": {"response": "Siber, Hegner & Co. Ltd.", "ground_truth": "Siber, Hegner & Co. Ltd.", "score": 1.0}, "entries[22].entry_id": {"response": "74-23", "ground_truth": "156089_1321099_102-23", "score": 0.23076923076923073}, "entries[4].location": {"response": "Thal near Rheineck (St. Gallen)", "ground_truth": "Thal near Rheineck (St. Gallen)", "score": 1.0}, "entries[0].location": {"response": "Schwanden (Glarus)", "ground_truth": "Schwanden (Glarus)", "score": 1.0}, "entries[3].entry_id": {"response": "74-4", "ground_truth": "156089_1321099_102-4", "score": 0.16666666666666663}, "entries[18].location": {"response": "Zurich 1", "ground_truth": "Zurich 1", "score": 1.0}, "entries[11].location": {"response": "Zurich 2", "ground_truth": "Zurich 2", "score": 1.0}, "entries[15].entry_id": {"response": "74-16", "ground_truth": "156089_1321099_102-16", "score": 0.23076923076923073}, "entries[7].entry_id": {"response": "74-8", "ground_truth": "156089_1321099_102-8", "score": 0.16666666666666663}, "entries[10].entry_id": {"response": "74-11", "ground_truth": "156089_1321099_102-11", "score": 0.23076923076923073}, "entries[19].location": {"response": "Zurich 2", "ground_truth": "Zurich 2", "score": 1.0}, "entries[14].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[20].company_name": {"response": "Soci\u00e9t\u00e9 Industrielle pour la Schappe", "ground_truth": "Soci\u00e9t\u00e9 Industrielle pour la Schappe", "score": 1.0}, "entries[17].company_name": {"response": "Robert Schwarzenbach & Co.", "ground_truth": "Robert Schwarzenbach & Co.", "score": 1.0}, "entries[23].location": {"response": "Zurich 1", "ground_truth": "Zurich 1", "score": 1.0}, "entries[2].company_name": {"response": "J. Spiro", "ground_truth": "J. Spiro", "score": 1.0}, "entries[1].location": {"response": "Zurich 2", "ground_truth": "Zurich 2", "score": 1.0}, "entries[17].entry_id": {"response": "74-18", "ground_truth": "156089_1321099_102-18", "score": 0.23076923076923073}, "entries[15].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[13].company_name": {"response": "Heer & Co. Ltd.", "ground_truth": "Heer & Co. Ltd.", "score": 1.0}, "entries[9].entry_id": {"response": "74-10", "ground_truth": "156089_1321099_102-10", "score": 0.23076923076923073}, "entries[3].company_name": {"response": "Vischer & Co.", "ground_truth": "Vischer & Co.", "score": 1.0}, "entries[12].company_name": {"response": "Abegg & Co. Ltd.", "ground_truth": "Abegg & Co. Ltd.", "score": 1.0}, "entries[15].company_name": {"response": "P. Marchal & Co.", "ground_truth": "P. Marchal & Co.", "score": 1.0}, "entries[14].company_name": {"response": "J. Hopf & Co.", "ground_truth": "J. Hopf & Co.", "score": 1.0}, "entries[16].company_name": {"response": "Saweco Ltd.", "ground_truth": "Saweco Ltd.", "score": 1.0}, "entries[1].entry_id": {"response": "74-2", "ground_truth": "156089_1321099_102-2", "score": 0.16666666666666663}, "entries[16].entry_id": {"response": "74-17", "ground_truth": "156089_1321099_102-17", "score": 0.23076923076923073}, "entries[6].entry_id": {"response": "74-7", "ground_truth": "156089_1321099_102-7", "score": 0.16666666666666663}, "entries[0].company_name": {"response": "F. Blumer & Co.", "ground_truth": "F. Blumer & Co.", "score": 1.0}, "entries[12].location": {"response": "Zurich 1", "ground_truth": "Zurich 1", "score": 1.0}, "entries[14].entry_id": {"response": "74-15", "ground_truth": "156089_1321099_102-15", "score": 0.23076923076923073}, "page_id": {"response": "74", "ground_truth": "156089_1321099_102", "score": 0.0}, "entries[13].location": {"response": "Thalwil", "ground_truth": "Thalwil", "score": 1.0}, "entries[8].entry_id": {"response": "74-9", "ground_truth": "156089_1321099_102-9", "score": 0.16666666666666663}, "entries[0].entry_id": {"response": "74-1", "ground_truth": "156089_1321099_102-1", "score": 0.16666666666666663}, "entries[3].location": {"response": "Basle", "ground_truth": "Basle", "score": 1.0}, "entries[7].company_name": {"response": "Beltex Ltd.", "ground_truth": "Beltex Ltd.", "score": 1.0}}, "total_fields": 73}, "raw_response": "{\"id\":\"msg_01BT7bo1Nfn5AxDdeMHfLq8x\",\"container\":null,\"content\":[{\"id\":\"toolu_01JWP5fs63Nr2izdAWiYqAUB\",\"caller\":{\"type\":\"direct\"},\"input\":{\"page_id\":\"74\",\"entries\":[{\"entry_id\":\"74-1\",\"company_name\":\"F. Blumer & Co.\",\"location\":\"Schwanden (Glarus)\"},{\"entry_id\":\"74-2\",\"company_name\":\"Inglo Louis Jablonsky\",\"location\":\"Zurich 2\"},{\"entry_id\":\"74-3\",\"company_name\":\"J. Spiro\",\"location\":\"Zurich 6\"},{\"entry_id\":\"74-4\",\"company_name\":\"Vischer & Co.\",\"location\":\"Basle\"},{\"entry_id\":\"74-5\",\"company_name\":\"Swiss Silk Bolting Cloth Manufacturing Co. Ltd.\",\"location\":\"Thal near Rheineck (St. Gallen)\"},{\"entry_id\":\"74-6\",\"company_name\":\"Swiss Silk Bolting Cloth Manufacturing Co. Ltd.\",\"location\":\"Zurich 2\"},{\"entry_id\":\"74-7\",\"company_name\":\"Tavaro Ltd.\",\"location\":\"Geneva\"},{\"entry_id\":\"74-8\",\"company_name\":\"Beltex Ltd.\",\"location\":\"Arzo (Tessin)\"},{\"entry_id\":\"74-9\",\"company_name\":\"Gust. Metzger Ltd.\",\"location\":\"Basle\"},{\"entry_id\":\"74-10\",\"company_name\":\"Schellenberg, Sax & Co. Ltd.\",\"location\":\"Tr\u00fcbbach (St. Gallen)\"},{\"entry_id\":\"74-11\",\"company_name\":\"Swiss Silk Bolting Cloth Manufacturing Co. Ltd.\",\"location\":\"Thal near Rheineck (St. Gallen)\"},{\"entry_id\":\"74-12\",\"company_name\":\"Swiss Silk Bolting Cloth Manufacturing Co. Ltd.\",\"location\":\"Zurich 2\"},{\"entry_id\":\"74-13\",\"company_name\":\"Abegg & Co. Ltd.\",\"location\":\"Zurich 1\"},{\"entry_id\":\"74-14\",\"company_name\":\"Heer & Co. Ltd.\",\"location\":\"Thalwil\"},{\"entry_id\":\"74-15\",\"company_name\":\"J. Hopf & Co.\",\"location\":\"Basle\"},{\"entry_id\":\"74-16\",\"company_name\":\"P. Marchal & Co.\",\"location\":\"Basle\"},{\"entry_id\":\"74-17\",\"company_name\":\"Saweco Ltd.\",\"location\":\"Basle\"},{\"entry_id\":\"74-18\",\"company_name\":\"Robert Schwarzenbach & Co.\",\"location\":\"Thalwil\"},{\"entry_id\":\"74-19\",\"company_name\":\"Siber, Hegner & Co. Ltd.\",\"location\":\"Zurich 1\"},{\"entry_id\":\"74-20\",\"company_name\":\"Stehli & Co.\",\"location\":\"Zurich 2\"},{\"entry_id\":\"74-21\",\"company_name\":\"Soci\u00e9t\u00e9 Industrielle pour la Schappe\",\"location\":\"Basle\"},{\"entry_id\":\"74-22\",\"company_name\":\"P. Marchal & Co.\",\"location\":\"Basle\"},{\"entry_id\":\"74-23\",\"company_name\":\"Schmid Brothers Ltd.\",\"location\":\"Basle\"},{\"entry_id\":\"74-24\",\"company_name\":\"Siber, Hegner & Co. Ltd.\",\"location\":\"Zurich 1\"}]},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-fable-5\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"inference_geo\":\"global\",\"input_tokens\":4979,\"output_tokens\":1212,\"server_tool_use\":null,\"service_tier\":\"standard\",\"output_tokens_details\":{\"thinking_tokens\":0}},\"stop_details\":null}"}, "scoring": {"f1_micro": 0.5068285280728377, "f1_macro": 0.5146666666666666, "micro_precision": 0.5086294416243655, "micro_recall": 0.5050403225806451, "total_instances": 15, "total_tp": 501, "total_fp": 484, "total_fn": 491, "cost_summary": {"total_input_tokens": 72983, "total_output_tokens": 16883, "total_tokens": 89866, "input_cost_usd": 0.7298300000000001, "output_cost_usd": 0.8441500000000001, "total_cost_usd": 1.57398}}, "normalized_score": 51.46666666666666}