{"_id": {"$oid": "69d4eb12b877594bc57189a9"}, "test_id": "T0234", "benchmark": "bibliographic_data", "date": "2025-10-17", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en"], "layout": ["list"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["pema_frick", "sven_burkhardt", "sorin_marti"], "hidden": false, "config": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a Historian", "legacy_test": false}, "prompt": null, "results": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "duration": 59.35922837257385, "timestamp": "2025-10-17T13:48:48.029128", "parsed": {"metadata": {"title": "HISTORY AND THEORY", "year": "1946", "page_number": 4}, "entries": [{"id": "49", "type": "book", "title": "Historische Zeitschrift CLXXXIV", "container_title": null, "author": [{"family": "Schlenke", "given": "Manfred"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "1957", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "50", "type": "book", "title": "Introduzione allo studio della storia", "container_title": null, "author": [{"family": "Cusin", "given": "Fabio"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "51", "type": "book", "title": "Sophia I", "container_title": null, "author": [{"family": "Cafaro", "given": "F."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "1947", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "52", "type": "book", "title": "L'histoire, science du concret", "container_title": null, "author": [{"family": "Dardel", "given": "Eric"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "53", "type": "book", "title": "Historical Change", "container_title": null, "author": [{"family": "Einstein", "given": "Lewis"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "54", "type": "book", "title": "The Place of War in History", "container_title": null, "author": [{"family": "Falls", "given": "Cyril"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "55", "type": "book", "title": "A Guide to Historical Method", "container_title": null, "author": [{"family": "Garraghan", "given": "Gilbert J."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "56", "type": "book", "title": "Le\u00e7ons sur la philosophie de l'histoire", "container_title": null, "author": [{"family": "Givelin", "given": "J."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "57", "type": "book", "title": "Bilan de l'histoire", "container_title": null, "author": [{"family": "Grousset", "given": "Ren\u00e9"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "58", "type": "book", "title": "Introduction \u00e0 l'histoire", "container_title": null, "author": [{"family": "Halphen", "given": "Louis"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "59", "type": "journal-article", "title": "Sur une forme d'histoire qui n'est pas la n\u00f4tre", "container_title": null, "author": [{"family": "Febvre", "given": "Lucien"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "60", "type": "journal-article", "title": "Avenir de l'histoire", "container_title": null, "author": [{"family": "Lefebvre", "given": "Georges"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "1947", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "61", "type": "book", "title": "Hoe bepaalt de geschiedenis het heden? Een niet gehouden rede", "container_title": null, "author": [{"family": "Huizinga", "given": "J."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "62", "type": "book", "title": "De mensch en de beschaving", "container_title": null, "author": [{"family": "Huizinga", "given": "J."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "63", "type": "book", "title": "Historische Meditationen", "container_title": null, "author": [{"family": "Kaegi", "given": "W."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "64", "type": "book", "title": "Notions d'historiographie moderne", "container_title": null, "author": [{"family": "Lefebvre", "given": "G."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "65", "type": "book", "title": "Essays on Historiography", "container_title": null, "author": [{"family": "Malin", "given": "James C."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "66", "type": "book", "title": "The Old Testament Interpretation of History", "container_title": null, "author": [{"family": "North", "given": "C. R."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "67", "type": "book", "title": "Concord and Liberty", "container_title": null, "author": [{"family": "Ortega y Gasset", "given": "Jos\u00e9"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "68", "type": "book", "title": "Sociolvidenskabelig Metodelaere", "container_title": null, "author": [{"family": "Ranulf", "given": "Svend"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "69", "type": "book", "title": "Geschichte als Bildungsmacht", "container_title": null, "author": [{"family": "Ritter", "given": "Gerhard"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": "", "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}]}, "text": "", "raw_response": {"id": "gen-1760701668-rFD0pUUpXa7sEz6UhRBx", "model": "meta-llama/llama-4-maverick", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"metadata\": {\n    \"title\": \"HISTORY AND THEORY\",\n    \"year\":  \t\"1946\",\n    \"page_number\": 4\n  },\n  \"entries\": [\n    {\n      \"id\":  \t\"49\",\n      \"type\": \t\"book\",\n      \"title\": \"Historische Zeitschrift CLXXXIV\",\n      \"author\": \t[\n        {\n          \"family\": \"Schlenke\",\n          \"given\": \"Manfred\"\n        }\n      ],\n      \"issued\": \t\"1957\",\n      \"pages\": \t\"594-597\",\n      \"relation_type\": \"review\",\n      \"relation_target\": \"German translation\"\n    },\n    {\n      \"id\": \t\"50\",\n      \"type\": \t\"book\",\n      \"title\": \"Introduzione allo studio della storia\",\n      \"author\": \t[\n        {\n          \"family\": \"Cusin\",\n          \"given\": \"Fabio\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Padova\",\n      \"publisher-name\": \t\"Cedam\",\n      \"relation_type\": \"review\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"51\",\n      \"type\": \t\"book\",\n      \"title\": \"Sophia I\",\n      \"author\": \t[\n        {\n          \"family\": \"Cafaro\",\n          \"given\": \"F.\"\n        }\n      ],\n      \"issued\": \t\"1947\",\n      \"pages\": \t\"206-207\",\n      \"publisher-place\": \t\"\",\n      \"publisher-name\": \t\"\",\n      \"relation_type\": \"review\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"52\",\n      \"type\": \t\"book\",\n      \"title\": \"L'histoire, science du concret\",\n      \"author\": \t[\n        {\n          \"family\": \"Dardel\",\n          \"given\": \"Eric\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Paris\",\n      \"publisher-name\": \t\"Presses Universitaires de France\",\n      \"relation_type\": \"review\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"53\",\n      \"type\": \t\"book\",\n      \"title\": \"Historical Change\",\n      \"author\": \t[\n        {\n          \"family\": \"Einstein\",\n          \"given\": \"Lewis\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Cambridge\",\n      \"publisher-name\": \t\"Cambridge University Press\",\n      \"relation_type\": \"review\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"54\",\n      \"type\": \t\"book\",\n      \"title\": \"The Place of War in History\",\n      \"author\": \t[\n        {\n          \"family\": \"Falls\",\n          \"given\": \"Cyril\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Oxford\",\n      \"publisher-name\": \t\"Clarendon\",\n      \"relation_type\": \"review\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"55\",\n      \"type\": \t\"book\",\n      \"title\": \"A Guide to Historical Method\",\n      \"author\": \t[\n        {\n          \"family\": \"Garraghan\",\n          \"given\": \"Gilbert J.\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"New York\",\n      \"publisher-name\": \t\"Fordham University Press\",\n      \"relation_type\": \"review\",\n      \"relation_target\": \"\",\n      \"editor_name\": \t\"Jean Delanglez\"\n    },\n    {\n      \"id\": \t\"56\",\n      \"type\": \t\"book\",\n      \"title\": \"Le\u00e7ons sur la philosophie de l'histoire\",\n      \"author\": \t[\n        {\n          \"family\": \"Givelin\",\n          \"given\": \"J.\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Paris\",\n      \"publisher-name\": \t\"Vrin\",\n      \"relation_type\": \"review\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"57\",\n      \"type\": \t\"book\",\n      \"title\": \"Bilan de l'histoire\",\n      \"author\": \t[\n        {\n          \"family\": \"Grousset\",\n          \"given\": \"Ren\u00e9\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Paris\",\n      \"publisher-name\": \t\"Plon\",\n      \"relation_type\": \"review\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"58\",\n      \"type\": \t\"book\",\n      \"title\": \"Introduction \u00e0 l'histoire\",\n      \"author\": \t[\n        {\n          \"family\": \"Halphen\",\n          \"given\": \"Louis\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Paris\",\n      \"publisher-name\": \t\"Presses Universitaires de France\",\n      \"relation_type\": \"review\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"59\",\n      \"type\": \t\"journal-article\",\n      \"title\": \"Sur une forme d'histoire qui n'est pas la n\u00f4tre\",\n      \"author\": \t[\n        {\n          \"family\": \"Febvre\",\n          \"given\": \"Lucien\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"21-24\",\n      \"container-title\": \t\"Annales: Economies-soci\u00e9t\u00e9s-civilisations III\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"60\",\n      \"type\": \t\"journal-article\",\n      \"title\": \"Avenir de l'histoire\",\n      \"author\": \t[\n        {\n          \"family\": \"Lefebvre\",\n          \"given\": \"Georges\"\n        }\n      ],\n      \"issued\": \t\"1947\",\n      \"pages\": \t\"55-61\",\n      \"container-title\": \t\"Revue historique CXCVII\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"61\",\n      \"type\": \t\"book\",\n      \"title\": \"Hoe bepaalt de geschiedenis het heden? Een niet gehouden rede\",\n      \"author\": \t[\n        {\n          \"family\": \"Huizinga\",\n          \"given\": \"J.\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Haarlem\",\n      \"publisher-name\": \t\"Tjeenk Willink\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"62\",\n      \"type\": \t\"book\",\n      \"title\": \"De mensch en de beschaving\",\n      \"author\": \t[\n        {\n          \"family\": \"Huizinga\",\n          \"given\": \"J.\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Amsterdam, Antwerpen\",\n      \"publisher-name\": \t\"Pantheon\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"63\",\n      \"type\": \t\"book\",\n      \"title\": \"Historische Meditationen\",\n      \"author\": \t[\n        {\n          \"family\": \"Kaegi\",\n          \"given\": \"W.\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"2 vols.\",\n      \"publisher-place\": \t\"Z\u00fcrich\",\n      \"publisher-name\": \t\"Fritz und Wasmuth\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"64\",\n      \"type\": \t\"book\",\n      \"title\": \"Notions d'historiographie moderne\",\n      \"author\": \t[\n        {\n          \"family\": \"Lefebvre\",\n          \"given\": \"G.\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Paris\",\n      \"publisher-name\": \t\"Centre de Documentation Universitaire\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\",\n      \"note:\": \t\"dactylographed for his students\"\n    },\n    {\n      \"id\": \t\"65\",\n      \"type\": \t\"book\",\n      \"title\": \"Essays on Historiography\",\n      \"author\": \t[\n        {\n          \"family\": \"Malin\",\n          \"given\": \"James C.\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Lawrence, Kansas\",\n      \"publisher-name\": \t\"the Author\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"66\",\n      \"type\": \t\"book\",\n      \"title\": \"The Old Testament Interpretation of History\",\n      \"author\": \t[\n        {\n          \"family\": \"North\",\n          \"given\": \"C. R.\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"London\",\n      \"publisher-name\": \t\"The Epworth Press\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"67\",\n      \"type\": \t\"book\",\n      \"title\": \"Concord and Liberty\",\n      \"author\": \t[\n        {\n          \"family\": \"Ortega y Gasset\",\n          \"given\": \"Jos\u00e9\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"New York\",\n      \"publisher-name\": \t\"Norton\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\",\n      \"translator_name\": \t\"Helene Weyl\"\n    },\n    {\n      \"id\": \t\"68\",\n      \"type\": \t\"book\",\n      \"title\": \"Sociolvidenskabelig Metodelaere\",\n      \"author\": \t[\n        {\n          \"family\": \"Ranulf\",\n          \"given\": \"Svend\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"K\u00f8benhavn\",\n      \"publisher-name\": \t\"Einar Munksgaard\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\"\n    },\n    {\n      \"id\": \t\"69\",\n      \"type\": \t\"book\",\n      \"title\": \"Geschichte als Bildungsmacht\",\n      \"author\": \t[\n        {\n          \"family\": \"Ritter\",\n          \"given\": \"Gerhard\"\n        }\n      ],\n      \"issued\": \t\"\",\n      \"pages\": \t\"\",\n      \"publisher-place\": \t\"Stuttgart\",\n      \"publisher-name\": \t\"Deutsche Verlags-Anstalt\",\n      \"relation_type\": \"review-article\",\n      \"relation_target\": \"\",\n      \"incomplete:\": \t\"true\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1983, "completion_tokens": 2516, "total_tokens": 4499}}, "usage": {"input_tokens": 1983, "output_tokens": 2516, "total_tokens": 4499}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.6257180323707402, "cost_summary": {"total_input_tokens": 7725, "total_output_tokens": 8433, "total_tokens": 16158, "input_cost_usd": 0.001159, "output_cost_usd": 0.00506, "total_cost_usd": 0.006219, "pricing_date": "2025-10-17", "input_price_per_million": 0.15, "output_price_per_million": 0.6}}, "normalized_score": 62.57180323707402}