{"_id": {"$oid": "6ac4c7b5164e0f17c0b234ab"}, "test_id": "T1495", "benchmark": "bibliographic_data", "date": "2026-08-14", "tags": {"document-type": ["book-page"], "writing": ["printed"], "century": [20], "language": ["en"], "layout": ["list"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["pema_frick", "sven_burkhardt", "sorin_marti"], "hidden": false, "config": {"provider": "huggingface", "model": "swiss-ai/Apertus-v1.5-8B:publicai", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a Historian", "legacy_test": false}, "prompt": null, "results": {"text": "{\"$defs\": {\"Author\": {\"properties\": {\"family\": {\"title\": \"Family\", \"type\": \"string\"}, \"given\": {\"title\": \"Given\", \"type\": \"string\"}}, \"required\": [\"family\", \"given\"], \"title\": \"Author\", \"type\": \"object\"}, \"Entry\": {\"properties\": {\"id\": {\"title\": \"Id\", \"type\": \"string\"}, \"type\": {\"$ref\": \"#/$defs/EntryType\"}, \"title\": {\"title\": \"Title\", \"type\": \"string\"}, \"container_title\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Container Title\"}, \"author\": {\"anyOf\": [{\"items\": {\"$ref\": \"#/$defs/Author\"}, \"type\": \"array\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Author\"}, \"note\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Note\"}, \"publisher\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Publisher\"}, \"editor\": {\"anyOf\": [{\"items\": {\"type\": \"string\"}, \"type\": \"array\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Editor\"}, \"publisher_place\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Publisher Place\"}, \"issued\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Issued\"}, \"event_date\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Event Date\"}, \"related\": {\"anyOf\": [{\"items\": {\"type\": \"string\"}, \"type\": \"array\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Related\"}, \"relation\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Relation\"}, \"volume\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Volume\"}, \"page\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Page\"}, \"fascicle\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Fascicle\"}, \"reprint\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Reprint\"}, \"edition\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Edition\"}, \"incomplete\": {\"anyOf\": [{\"type\": \"boolean\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Incomplete\"}}, \"EntryType\": {\"enum\": [\"book\", \"journal-article\", \"other\"], \"title\": \"EntryType\", \"type\": \"string\"}, \"Metadata\": {\"properties\": {\"title\": {\"title\": \"Title\", \"type\": \"string\"}, \"year\": {\"title\": \"Year\", \"type\": \"string\"}, \"page_number\": {\"anyOf\": [{\"type\": \"integer\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Page Number\"}}, \"required\": [\"title\", \"year\"], \"title\": \"Metadata\", \"type\": \"object\"}}, \"properties\": {\"metadata\": {\"$ref\": \"#/$defs/Metadata\"}, \"entries\": {\"items\": {\"$ref\": \"#/$defs/Entry\"}, \"title\": \"Entries\", \"type\": \"array\"}}, \"required\": [\"metadata\", \"entries\"], \"title\": \"Document\", \"type\": \"object\"}}", "model": "swiss-ai/apertus-v1.5-8b", "provider": "huggingface", "finish_reason": "stop", "usage": {"input_tokens": 8716, "output_tokens": 872, "total_tokens": 9588, "cached_tokens": 7776, "input_cost_usd": 0.0008716, "output_cost_usd": 0.0001744, "estimated_cost_usd": 0.001046}, "duration": 197.1947054862976, "timestamp": "2026-08-14T16:03:22.161825", "parsed": {"$defs": {"Author": {"properties": {"family": {"title": "Family", "type": "string"}, "given": {"title": "Given", "type": "string"}}, "required": ["family", "given"], "title": "Author", "type": "object"}, "Entry": {"properties": {"id": {"title": "Id", "type": "string"}, "type": {"$ref": "#/$defs/EntryType"}, "title": {"title": "Title", "type": "string"}, "container_title": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Container Title"}, "author": {"anyOf": [{"items": {"$ref": "#/$defs/Author"}, "type": "array"}, {"type": "null"}], "default": null, "title": "Author"}, "note": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Note"}, "publisher": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Publisher"}, "editor": {"anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}], "default": null, "title": "Editor"}, "publisher_place": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Publisher Place"}, "issued": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Issued"}, "event_date": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Event Date"}, "related": {"anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}], "default": null, "title": "Related"}, "relation": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Relation"}, "volume": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Volume"}, "page": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Page"}, "fascicle": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Fascicle"}, "reprint": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Reprint"}, "edition": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Edition"}, "incomplete": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "default": null, "title": "Incomplete"}}, "EntryType": {"enum": ["book", "journal-article", "other"], "title": "EntryType", "type": "string"}, "Metadata": {"properties": {"title": {"title": "Title", "type": "string"}, "year": {"title": "Year", "type": "string"}, "page_number": {"anyOf": [{"type": "integer"}, {"type": "null"}], "default": null, "title": "Page Number"}}, "required": ["title", "year"], "title": "Metadata", "type": "object"}}, "properties": {"metadata": {"$ref": "#/$defs/Metadata"}, "entries": {"items": {"$ref": "#/$defs/Entry"}, "title": "Entries", "type": "array"}}, "required": ["metadata", "entries"], "title": "Document", "type": "object"}}, "conversation_id": "b06c1fa8-5d36-4d3b-baa5-fae9627d453e", "score": {"fuzzy": 0.0}, "raw_response": "{\"id\":\"chatcmpl-4868d083-50b5-4790-8c92-51c678604f1f\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"$defs\\\": {\\\"Author\\\": {\\\"properties\\\": {\\\"family\\\": {\\\"title\\\": \\\"Family\\\", \\\"type\\\": \\\"string\\\"}, \\\"given\\\": {\\\"title\\\": \\\"Given\\\", \\\"type\\\": \\\"string\\\"}}, \\\"required\\\": [\\\"family\\\", \\\"given\\\"], \\\"title\\\": \\\"Author\\\", \\\"type\\\": \\\"object\\\"}, \\\"Entry\\\": {\\\"properties\\\": {\\\"id\\\": {\\\"title\\\": \\\"Id\\\", \\\"type\\\": \\\"string\\\"}, \\\"type\\\": {\\\"$ref\\\": \\\"#/$defs/EntryType\\\"}, \\\"title\\\": {\\\"title\\\": \\\"Title\\\", \\\"type\\\": \\\"string\\\"}, \\\"container_title\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Container Title\\\"}, \\\"author\\\": {\\\"anyOf\\\": [{\\\"items\\\": {\\\"$ref\\\": \\\"#/$defs/Author\\\"}, \\\"type\\\": \\\"array\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Author\\\"}, \\\"note\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Note\\\"}, \\\"publisher\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Publisher\\\"}, \\\"editor\\\": {\\\"anyOf\\\": [{\\\"items\\\": {\\\"type\\\": \\\"string\\\"}, \\\"type\\\": \\\"array\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Editor\\\"}, \\\"publisher_place\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Publisher Place\\\"}, \\\"issued\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Issued\\\"}, \\\"event_date\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Event Date\\\"}, \\\"related\\\": {\\\"anyOf\\\": [{\\\"items\\\": {\\\"type\\\": \\\"string\\\"}, \\\"type\\\": \\\"array\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Related\\\"}, \\\"relation\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Relation\\\"}, \\\"volume\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Volume\\\"}, \\\"page\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Page\\\"}, \\\"fascicle\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Fascicle\\\"}, \\\"reprint\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Reprint\\\"}, \\\"edition\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Edition\\\"}, \\\"incomplete\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"boolean\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Incomplete\\\"}}, \\\"EntryType\\\": {\\\"enum\\\": [\\\"book\\\", \\\"journal-article\\\", \\\"other\\\"], \\\"title\\\": \\\"EntryType\\\", \\\"type\\\": \\\"string\\\"}, \\\"Metadata\\\": {\\\"properties\\\": {\\\"title\\\": {\\\"title\\\": \\\"Title\\\", \\\"type\\\": \\\"string\\\"}, \\\"year\\\": {\\\"title\\\": \\\"Year\\\", \\\"type\\\": \\\"string\\\"}, \\\"page_number\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"integer\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Page Number\\\"}}, \\\"required\\\": [\\\"title\\\", \\\"year\\\"], \\\"title\\\": \\\"Metadata\\\", \\\"type\\\": \\\"object\\\"}}, \\\"properties\\\": {\\\"metadata\\\": {\\\"$ref\\\": \\\"#/$defs/Metadata\\\"}, \\\"entries\\\": {\\\"items\\\": {\\\"$ref\\\": \\\"#/$defs/Entry\\\"}, \\\"title\\\": \\\"Entries\\\", \\\"type\\\": \\\"array\\\"}}, \\\"required\\\": [\\\"metadata\\\", \\\"entries\\\"], \\\"title\\\": \\\"Document\\\", \\\"type\\\": \\\"object\\\"}}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"provider_specific_fields\":{\"refusal\":null,\"reasoning\":null}},\"provider_specific_fields\":{\"routed_experts\":null,\"token_ids\":null,\"stop_reason\":null}}],\"created\":1786716192,\"model\":\"swiss-ai/apertus-v1.5-8b\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":\"vllm-0.23.1rc1.dev1029+ga601a9d99-2d880180\",\"usage\":{\"completion_tokens\":872,\"prompt_tokens\":8716,\"total_tokens\":9588,\"completion_tokens_details\":null,\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":7776,\"multimodal_tokens\":{\"image\":7708}}}}"}, "scoring": {"fuzzy": 0.06605189490645866, "cost_summary": {"total_input_tokens": 42920, "total_output_tokens": 7097, "total_tokens": 50017, "input_cost_usd": 0.004292000000000001, "output_cost_usd": 0.0014194, "total_cost_usd": 0.0057114}}, "pricing": {"bucket_date": "2026-08-10", "age_days": 4, "input_price_per_million": 0.1, "output_price_per_million": 0.2}, "timing": {"total_response_s": 822.9, "mean_response_s": 164.58, "slowest_response_s": 198.02, "inputs_timed": 5, "inputs_stored": 5, "span_s": 162.47}, "normalized_score": 6.6051894906458655}