{"_id": {"$oid": "6ac4c7b5164e0f17c0b22e29"}, "test_id": "T0234", "benchmark": "bibliographic_data", "date": "2025-10-17", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en"], "layout": ["list"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["pema_frick", "sven_burkhardt", "sorin_marti"], "hidden": false, "config": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a Historian", "legacy_test": false}, "prompt": null, "results": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "duration": 22.229965686798096, "timestamp": "2025-10-17T13:46:39.111938", "parsed": {"metadata": {"title": "Bibliography", "year": "1947", "page_number": 9}, "entries": [{"id": "140", "type": "book", "title": "Why We Read History", "container_title": null, "author": [{"family": "Smellie", "given": "K. B."}], "note": null, "publisher": "P. Elek", "editor": null, "publisher_place": "London", "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "141", "type": "book", "title": "A Study of History", "container_title": null, "author": [{"family": "Toynbee", "given": "Arnold J."}], "note": null, "publisher": null, "editor": ["D. C. Somervell"], "publisher_place": "New York", "issued": null, "event_date": null, "related": null, "relation": null, "volume": "I-VI", "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "142", "type": "book", "title": "In den spiegel van het verleden: historische opstellen", "container_title": null, "author": [{"family": "Verberne", "given": "L. G. J."}], "note": null, "publisher": "Bruna", "editor": null, "publisher_place": "Utrecht, Antwerpen", "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "143", "type": "book", "title": "Esquemas de metodologia eclesi\u00e1stica", "container_title": null, "author": [{"family": "Vev\u00edr", "given": "Jos\u00e9"}], "note": null, "publisher": "Instituto", "editor": null, "publisher_place": "Barcelona", "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "144", "type": "book", "title": "Sinn und Ziel der Geschichte", "container_title": null, "author": [{"family": "Walz", "given": "Hans H."}], "note": null, "publisher": "Universit\u00e4ts-Verlag", "editor": null, "publisher_place": "Bonn", "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "145", "type": "book", "title": "Atti del congresso internazionale di filosofia 15-20 November, 1946", "container_title": null, "author": null, "note": null, "publisher": "Castellani", "editor": null, "publisher_place": "Milano", "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": null, "fascicle": null, "reprint": null, "edition": null, "incomplete": null}, {"id": "146", "type": "book", "title": "Storia e materialismo storico", "container_title": null, "author": [{"family": "Pietro C.", "given": "Drago"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": "145", "volume": null, "page": "109-117", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "147", "type": "book", "title": "Critica del giudizio storico", "container_title": null, "author": [{"family": "Vito Fazio-Allmayer", "given": ""}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": "145", "volume": null, "page": "281-290", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "148", "type": "book", "title": "Forza e spirito nella storia", "container_title": null, "author": [{"family": "Nyman", "given": "A."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": "145", "volume": null, "page": "191-200", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "149", "type": "book", "title": "Sur le concept d'histoire", "container_title": null, "author": [{"family": "Benjamin", "given": "Walter"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": "145", "volume": null, "page": "623-634", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "150", "type": "book", "title": "Philosophy of History", "container_title": null, "author": [{"family": "Boodin", "given": "John E."}], "note": null, "publisher": null, "editor": ["Dagobert D. Runes"], "publisher_place": "New York", "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "89-106", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "151", "type": "book", "title": "De la perception \u00e0 l'histoire. La philosophie de M. Merleau-Ponty", "container_title": null, "author": [{"family": "Caillois", "given": "R. P."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "57-85", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "152", "type": "book", "title": "Commermorazione di Adolfo Omodeo", "container_title": null, "author": [{"family": "Cantimori", "given": "Delio"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "3-4", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "153", "type": "book", "title": "The Historian and the Philosophy of Science", "container_title": null, "author": [{"family": "Cowan", "given": "Thomas"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "11-18", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "154", "type": "book", "title": "History as a Science", "container_title": null, "author": [{"family": "Crawford", "given": "R. M."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "153-175", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "155", "type": "book", "title": "Christianisme et histoire", "container_title": null, "author": [{"family": "Danielou", "given": "J."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "166-184", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "156", "type": "book", "title": "Wahrheit der Geschichte. Geschichte der Wahrheit", "container_title": null, "author": [{"family": "Dell", "given": "Stanley"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "44-53", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "157", "type": "book", "title": "The Idea of Progress", "container_title": null, "author": [{"family": "Fay", "given": "Sidney B."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "231-246", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "158", "type": "book", "title": "Theologie et histoire", "container_title": null, "author": [{"family": "Fessard", "given": "Gaston"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "39-65", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "159", "type": "book", "title": "Acton, Creighton, and Lea: a Study in History and Ethics", "container_title": null, "author": [{"family": "Fish", "given": "Andrew"}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "59-69", "fascicle": null, "reprint": null, "edition": null, "incomplete": false}, {"id": "160", "type": "book", "title": "Le conscience historique en Italie", "container_title": null, "author": [{"family": "Garosci", "given": "A."}], "note": null, "publisher": null, "editor": null, "publisher_place": null, "issued": null, "event_date": null, "related": null, "relation": null, "volume": null, "page": "23-24", "fascicle": null, "reprint": null, "edition": null, "incomplete": true}]}, "text": "", "raw_response": {"id": "gen-1760701576-iSr1JO5FhSxDahyoCWP3", "model": "meta-llama/llama-4-maverick", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"metadata\": {\"title\": \"Bibliography\", \"year\":  \"1947\", \"page_number\": 9}, \"entries\": [{\"id\":                                                     \"140\", \"type\": \"book\", \"title\": \"Why We Read History\", \"author\":  [  {\"family\": \"Smellie\", \"given\": \"K. B.\"}], \"publisher\": \"P. Elek\", \"publisher_place\": \"London\"}, {\"id\":                                                    \"141\", \"type\": \"book\", \"title\": \"A Study of History\", \"author\":  [  {\"family\": \"Toynbee\", \"given\": \"Arnold J.\"}], \"editor\":  [  \"D. C. Somervell\"], \"publisher_place\": \"New York\", \"volume\": \"I-VI\"}, {\"id\":                                                  \"142\", \"type\": \"book\", \"title\": \"In den spiegel van het verleden: historische opstellen\", \"author\":  [  {\"family\": \"Verberne\", \"given\": \"L. G. J.\"}], \"publisher\": \"Bruna\", \"publisher_place\": \"Utrecht, Antwerpen\"}, {\"id\":                                                  \"143\", \"type\": \"book\", \"title\": \"Esquemas de metodologia eclesi\\u00e1stica\", \"author\":  [  {\"family\": \"Vev\\u00edr\", \"given\": \"Jos\\u00e9\"}], \"publisher\": \"Instituto\", \"publisher_place\": \"Barcelona\"}, {\"id\":                                                  \"144\", \"type\": \"book\", \"title\": \"Sinn und Ziel der Geschichte\", \"author\":  [  {\"family\": \"Walz\", \"given\": \"Hans H.\"}], \"publisher\": \"Universit\\u00e4ts-Verlag\", \"publisher_place\": \"Bonn\"}, {\"id\":                       \"145\", \"type\": \"book\", \"title\": \"Atti del congresso internazionale di filosofia 15-20 November, 1946\", \"publisher\": \"Castellani\", \"publisher_place\": \"Milano\"}, {\"id\":        \"146\", \"type\": \"book\", \"title\": \"Storia e materialismo storico\", \"author\":  [  {\"family\": \"Pietro C.\", \"given\": \"Drago\"}], \"relation\": \"145\", \"page\": \"109-117\", \"incomplete\": false}, {\"id\":        \"147\", \"type\": \"book\", \"title\": \"Critica del giudizio storico\", \"author\":  [  {\"family\": \"Vito Fazio-Allmayer\", \"given\": \"\"}], \"relation\": \"145\", \"page\": \"281-290\", \"incomplete\": false}, {\"id\":                                    \"148\", \"type\": \"book\", \"title\": \"Forza e spirito nella storia\", \"author\":  [  {\"family\": \"Nyman\", \"given\": \"A.\"}], \"relation\": \"145\", \"page\": \"191-200\", \"incomplete\": false}, {\"id\":      \"149\", \"type\": \"book\", \"title\": \"Sur le concept d'histoire\", \"author\":  [  {\"family\": \"Benjamin\", \"given\": \"Walter\"}], \"relation\": \"145\", \"page\": \"623-634\", \"incomplete\": false}, {\"id\":   \"150\", \"type\": \"book\", \"title\": \"Philosophy of History\", \"author\":  [  {\"family\": \"Boodin\", \"given\": \"John E.\"}], \"editor\":  [  \"Dagobert D. Runes\"], \"publisher_place\": \"New York\", \"page\": \"89-106\", \"incomplete\": false}, {\"id\":         \"151\", \"type\": \"book\", \"title\": \"De la perception \\u00e0 l'histoire. La philosophie de M. Merleau-Ponty\", \"author\":  [  {\"family\": \"Caillois\", \"given\": \"R. P.\"}], \"page\": \"57-85\", \"incomplete\": false}, {\"id\":  \"152\", \"type\": \"book\", \"title\": \"Commermorazione di Adolfo Omodeo\", \"author\":  [  {\"family\": \"Cantimori\", \"given\": \"Delio\"}], \"page\": \"3-4\", \"incomplete\": false}, {\"id\":  \"153\", \"type\": \"book\", \"title\": \"The Historian and the Philosophy of Science\", \"author\":  [  {\"family\": \"Cowan\", \"given\": \"Thomas\"}], \"page\": \"11-18\", \"incomplete\": false}, {\"id\":  \"154\", \"type\": \"book\", \"title\": \"History as a Science\", \"author\":  [  {\"family\": \"Crawford\", \"given\": \"R. M.\"}], \"page\": \"153-175\", \"incomplete\": false}, {\"id\":  \"155\", \"type\": \"book\", \"title\": \"Christianisme et histoire\", \"author\":  [  {\"family\": \"Danielou\", \"given\": \"J.\"}], \"page\": \"166-184\", \"incomplete\": false}, {\"id\":  \"156\", \"type\": \"book\", \"title\": \"Wahrheit der Geschichte. Geschichte der Wahrheit\", \"author\":  [  {\"family\": \"Dell\", \"given\": \"Stanley\"}], \"page\": \"44-53\", \"incomplete\": false}, {\"id\":  \"157\", \"type\": \"book\", \"title\": \"The Idea of Progress\", \"author\":  [  {\"family\": \"Fay\", \"given\": \"Sidney B.\"}], \"page\": \"231-246\", \"incomplete\": false}, {\"id\":  \"158\", \"type\": \"book\", \"title\": \"Theologie et histoire\", \"author\":  [  {\"family\": \"Fessard\", \"given\": \"Gaston\"}], \"page\": \"39-65\", \"incomplete\": false}, {\"id\":  \"159\", \"type\": \"book\", \"title\": \"Acton, Creighton, and Lea: a Study in History and Ethics\", \"author\":  [  {\"family\": \"Fish\", \"given\": \"Andrew\"}], \"page\": \"59-69\", \"incomplete\": false}, {\"id\":  \"160\", \"type\": \"book\", \"title\": \"Le conscience historique en Italie\", \"author\":  [  {\"family\": \"Garosci\", \"given\": \"A.\"}], \"page\": \"23-24\", \"incomplete\": true}]}", "role": "assistant"}}], "usage": {"prompt_tokens": 1473, "completion_tokens": 1711, "total_tokens": 3184}}, "usage": {"input_tokens": 1473, "output_tokens": 1711, "total_tokens": 3184, "input_cost_usd": 0.00022094999999999998, "output_cost_usd": 0.0010266, "estimated_cost_usd": 0.0012475499999999998}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.6257180323707402, "cost_summary": {"total_input_tokens": 7725, "total_output_tokens": 8433, "total_tokens": 16158, "input_cost_usd": 0.00115875, "output_cost_usd": 0.0050598, "total_cost_usd": 0.00621855, "pricing_date": "2025-10-17", "input_price_per_million": 0.15, "output_price_per_million": 0.6}}, "pricing": {"bucket_date": "2025-10-17", "age_days": 0, "input_price_per_million": 0.15, "output_price_per_million": 0.6}, "timing": {"total_response_s": 151.02, "mean_response_s": 30.2, "slowest_response_s": 59.36, "inputs_timed": 5, "inputs_stored": 5, "span_s": 128.92}, "normalized_score": 62.57180323707402}