[{"_id": {"$oid": "69d4eb12b877594bc5718d3c"}, "test_id": "T0884", "benchmark": "library_cards", "date": "2026-03-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-flash-2026-02-23", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "", "model": "qwen3.5-flash-2026-02-23", "provider": "alibaba", "finish_reason": "error", "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, "duration": 23.353549003601074, "timestamp": "2026-03-25T10:24:29.216542", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 8, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": null, "ground_truth": "Rothwell", "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Wilcox", "score": 0.0}, "publication.format": {"response": null, "ground_truth": null, "score": 1.0}, "publication.year": {"response": null, "ground_truth": 1932, "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "Diss.Baltimore 1932", "score": 0.0}, "publication.place": {"response": null, "ground_truth": "Baltimore", "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.title": {"response": null, "ground_truth": "Private secondary education in the association of colleges and secondary schools of the Southern States", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "8", "score": 0.0}}, "total_fields": 12}}, "scoring": {"f1_micro": 0.5335570469798657, "f1_macro": 0.3860836501901141, "micro_precision": 0.8217054263565892, "micro_recall": 0.3950310559006211, "total_instances": 263, "total_tp": 954, "total_fp": 207, "total_fn": 1461, "cost_summary": {"total_input_tokens": 219964, "total_output_tokens": 22189, "total_tokens": 242153, "input_cost_usd": 0.021996400000000013, "output_cost_usd": 0.008875599999999997, "total_cost_usd": 0.030872000000000007}}, "normalized_score": 38.60836501901141}, {"_id": {"$oid": "69d4eb12b877594bc5718d41"}, "test_id": "T0845", "benchmark": "library_cards", "date": "2026-03-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-27b", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"M\u00fcller\",\n    \"first_name\": \"Theodor\"\n  },\n  \"publication\": {\n    \"title\": \"Die K\u00f6rperproportionen und ihre Ver\u00e4nderungen im Kleinkindalter. Anthropologische Untersuchung zwei- bis vierj\u00e4hriger Kinder der Stadt Z\u00fcrich.\",\n    \"year\": 1950,\n    \"place\": \"Z\u00fcrich\",\n    \"pages\": \"375 - 468\",\n    \"publisher\": \"Orell F\u00fcssli\",\n    \"format\": \"8'\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss. Z\u00fcrich Univ. 1950\",\n    \"subjects\": \"Archiv d. Julius Klaus-Stiftung f. Vererbungsforschg, Sozialanthropologie u. Rassenhygiene. Bd.25, H. 3/4.\"\n  }\n}", "model": "qwen3.5-27b", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 1757, "output_tokens": 227, "total_tokens": 1984, "input_cost_usd": 0.0005271, "output_cost_usd": 0.0005447999999999999, "estimated_cost_usd": 0.0010718999999999998}, "duration": 10.789422988891602, "timestamp": "2026-03-25T10:21:37.101869", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "M\u00fcller", "first_name": "Theodor"}, "publication": {"title": "Die K\u00f6rperproportionen und ihre Ver\u00e4nderungen im Kleinkindalter. Anthropologische Untersuchung zwei- bis vierj\u00e4hriger Kinder der Stadt Z\u00fcrich.", "year": 1950, "place": "Z\u00fcrich", "pages": "375 - 468", "publisher": "Orell F\u00fcssli", "format": "8'"}, "library_reference": {"shelfmark": "Diss. Z\u00fcrich Univ. 1950", "subjects": "Archiv d. Julius Klaus-Stiftung f. Vererbungsforschg, Sozialanthropologie u. Rassenhygiene. Bd.25, H. 3/4."}}, "conversation_id": "361c2f6b-6db3-48db-bd20-75b9cd182074", "score": {"f1_score": 0.95, "precision": 0.9090909090909091, "recall": 1.0, "true_positives": 10, "false_positives": 1, "false_negatives": 0, "field_scores": {"publication.title": {"response": "Die K\u00f6rperproportionen und ihre Ver\u00e4nderungen im Kleinkindalter. Anthropologische Untersuchung zwei- bis vierj\u00e4hriger Kinder der Stadt Z\u00fcrich.", "ground_truth": "Die K\u00f6rperproportionen und ihre Ver\u00e4nderungen im Kleinkindalter. Anthropologische Untersuchung zwei-bis vierj\u00e4hriger Kinder der Stadt Z\u00fcrich.", "score": 0.9964664310954063}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.place": {"response": "Z\u00fcrich", "ground_truth": "Z\u00fcrich", "score": 1.0}, "library_reference.shelfmark": {"response": "Diss. Z\u00fcrich Univ. 1950", "ground_truth": "Diss. Z\u00fcrich Univ. 1950", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "library_reference.subjects": {"response": "Archiv d. Julius Klaus-Stiftung f. Vererbungsforschg, Sozialanthropologie u. Rassenhygiene. Bd.25, H. 3/4.", "ground_truth": null, "score": 0.0}, "author.first_name": {"response": "Theodor", "ground_truth": "Theodor", "score": 1.0}, "publication.year": {"response": 1950, "ground_truth": 1950, "score": 1.0}, "publication.publisher": {"response": "Orell F\u00fcssli", "ground_truth": "Orell F\u00fcssli", "score": 1.0}, "publication.pages": {"response": "375 - 468", "ground_truth": "375 - 468", "score": 1.0}, "publication.format": {"response": "8'", "ground_truth": "8'", "score": 1.0}, "author.last_name": {"response": "M\u00fcller", "ground_truth": "M\u00fcller", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-039153f7-5f3d-96c5-9b0c-02a29b94721c\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"M\u00fcller\\\",\\n    \\\"first_name\\\": \\\"Theodor\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Die K\u00f6rperproportionen und ihre Ver\u00e4nderungen im Kleinkindalter. Anthropologische Untersuchung zwei- bis vierj\u00e4hriger Kinder der Stadt Z\u00fcrich.\\\",\\n    \\\"year\\\": 1950,\\n    \\\"place\\\": \\\"Z\u00fcrich\\\",\\n    \\\"pages\\\": \\\"375 - 468\\\",\\n    \\\"publisher\\\": \\\"Orell F\u00fcssli\\\",\\n    \\\"format\\\": \\\"8'\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss. Z\u00fcrich Univ. 1950\\\",\\n    \\\"subjects\\\": \\\"Archiv d. Julius Klaus-Stiftung f. Vererbungsforschg, Sozialanthropologie u. Rassenhygiene. Bd.25, H. 3/4.\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774430497,\"model\":\"qwen3.5-27b\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":227,\"prompt_tokens\":1757,\"total_tokens\":1984,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":227},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":542,\"text_tokens\":1215}}}"}, "scoring": {"f1_micro": 0.875640237656218, "f1_macro": 0.8685171102661596, "micro_precision": 0.8665855636658556, "micro_recall": 0.8848861283643892, "total_instances": 263, "total_tp": 2137, "total_fp": 329, "total_fn": 278, "cost_summary": {"total_input_tokens": 470700, "total_output_tokens": 46874, "total_tokens": 517574, "input_cost_usd": 0.14120999999999992, "output_cost_usd": 0.11249760000000004, "total_cost_usd": 0.25370759999999987}}, "normalized_score": 86.85171102661596}, {"_id": {"$oid": "69d4eb12b877594bc5718d43"}, "test_id": "T0832", "benchmark": "library_cards", "date": "2026-03-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-35b-a3b", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Sydow\",\"first_name\":\"Harald\"},\"publication\":{\"title\":\"Elektronenmikroskopische Untersuchung \u00fcber zytosomale Lamellenk\u00f6rper in den Eizellen des Ovars vom Igel <Erinaceus europaeus L.>\",\"year\":\"1968\",\"place\":\"Kiel\",\"pages\":\"387-407\",\"publisher\":\"\",\"format\":\"8'\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Diss.1969,6496\",\"subjects\":\"2/Elektronen-Mikroskopie/Igel,Igelartige/ Diss\"}}", "model": "qwen3.5-35b-a3b", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 1787, "output_tokens": 209, "total_tokens": 1996, "input_cost_usd": 0.00044675, "output_cost_usd": 0.000418, "estimated_cost_usd": 0.0008647500000000001}, "duration": 3.731600761413574, "timestamp": "2026-03-25T10:15:48.531332", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Sydow", "first_name": "Harald"}, "publication": {"title": "Elektronenmikroskopische Untersuchung \u00fcber zytosomale Lamellenk\u00f6rper in den Eizellen des Ovars vom Igel <Erinaceus europaeus L.>", "year": "1968", "place": "Kiel", "pages": "387-407", "publisher": "", "format": "8'"}, "library_reference": {"shelfmark": "Diss.1969,6496", "subjects": "2/Elektronen-Mikroskopie/Igel,Igelartige/ Diss"}}, "conversation_id": "3ef249b7-acc1-43bc-b06b-658b33b5e4fe", "score": {"f1_score": 1.0, "precision": 1.0, "recall": 1.0, "true_positives": 10, "false_positives": 0, "false_negatives": 0, "field_scores": {"library_reference.subjects": {"response": "2/Elektronen-Mikroskopie/Igel,Igelartige/ Diss", "ground_truth": "2/Elektronen-Mikroskopie/Igel,Igelartige/ Diss", "score": 1.0}, "author.last_name": {"response": "Sydow", "ground_truth": "Sydow", "score": 1.0}, "publication.year": {"response": "1968", "ground_truth": 1968, "score": 1.0}, "publication.place": {"response": "Kiel", "ground_truth": "Kiel", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": "Diss.1969,6496", "ground_truth": "Diss.1969,6496", "score": 1.0}, "author.first_name": {"response": "Harald", "ground_truth": "Harald", "score": 1.0}, "publication.pages": {"response": "387-407", "ground_truth": "387-407", "score": 1.0}, "publication.format": {"response": "8'", "ground_truth": "8'", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.title": {"response": "Elektronenmikroskopische Untersuchung \u00fcber zytosomale Lamellenk\u00f6rper in den Eizellen des Ovars vom Igel <Erinaceus europaeus L.>", "ground_truth": "Elektronenmikroskopische Untersuchung \u00fcber zytosomale Lamellenk\u00f6rper in den Eizellen des Ovars vom Igel (Erinaceus europaeus L.)", "score": 0.984375}}, "total_fields": 11}, "raw_response": "{\"id\":\"chatcmpl-2c9f15b0-4c8b-9c94-958b-5ada8c02bd23\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Sydow\\\",\\n    \\\"first_name\\\": \\\"Harald\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Elektronenmikroskopische Untersuchung \u00fcber zytosomale Lamellenk\u00f6rper in den Eizellen des Ovars vom Igel <Erinaceus europaeus L.>\\\",\\n    \\\"year\\\": \\\"1968\\\",\\n    \\\"place\\\": \\\"Kiel\\\",\\n    \\\"pages\\\": \\\"387-407\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"8'\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss.1969,6496\\\",\\n    \\\"subjects\\\": \\\"2/Elektronen-Mikroskopie/Igel,Igelartige/ Diss\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774430148,\"model\":\"qwen3.5-35b-a3b\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":209,\"prompt_tokens\":1787,\"total_tokens\":1996,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":209},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":572,\"text_tokens\":1215}}}"}, "scoring": {"f1_micro": 0.8444262964475296, "f1_macro": 0.838022813688213, "micro_precision": 0.8328634716069271, "micro_recall": 0.8563146997929607, "total_instances": 263, "total_tp": 2068, "total_fp": 415, "total_fn": 347, "cost_summary": {"total_input_tokens": 470700, "total_output_tokens": 47995, "total_tokens": 518695, "input_cost_usd": 0.11767499999999961, "output_cost_usd": 0.09599000000000003, "total_cost_usd": 0.21366500000000016}}, "normalized_score": 83.8022813688213}, {"_id": {"$oid": "69d4eb12b877594bc5718d66"}, "test_id": "T0858", "benchmark": "library_cards", "date": "2026-03-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-122b-a10b", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Nowak\",\n    \"first_name\": \"Theo\"\n  },\n  \"publication\": {\n    \"title\": \"Winkelabh\u00e4ngigkeit des Polarisationsgrades im Spektrum der Bremsstrahlung relativistischer Elektronen\",\n    \"year\": 1967,\n    \"place\": \"W\u00fcrzburg\",\n    \"pages\": \"32\",\n    \"publisher\": \"\",\n    \"format\": \"8'\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss.1968,5355\",\n    \"subjects\": \"\"\n  }\n}", "model": "qwen3.5-122b-a10b", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 1787, "output_tokens": 170, "total_tokens": 1957, "input_cost_usd": 0.0007148, "output_cost_usd": 0.0005440000000000001, "estimated_cost_usd": 0.0012588}, "duration": 5.5612874031066895, "timestamp": "2026-03-25T10:20:30.445741", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Nowak", "first_name": "Theo"}, "publication": {"title": "Winkelabh\u00e4ngigkeit des Polarisationsgrades im Spektrum der Bremsstrahlung relativistischer Elektronen", "year": 1967, "place": "W\u00fcrzburg", "pages": "32", "publisher": "", "format": "8'"}, "library_reference": {"shelfmark": "Diss.1968,5355", "subjects": ""}}, "conversation_id": "a8720b19-9fca-4e1d-9058-6d1b1ed81257", "score": {"f1_score": 1.0, "precision": 1.0, "recall": 1.0, "true_positives": 9, "false_positives": 0, "false_negatives": 0, "field_scores": {"library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": "Theo", "ground_truth": "Theo", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "32", "ground_truth": "32", "score": 1.0}, "publication.year": {"response": 1967, "ground_truth": 1967, "score": 1.0}, "publication.format": {"response": "8'", "ground_truth": "8'", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "author.last_name": {"response": "Nowak", "ground_truth": "Nowak", "score": 1.0}, "library_reference.shelfmark": {"response": "Diss.1968,5355", "ground_truth": "Diss.1968,5355", "score": 1.0}, "publication.place": {"response": "W\u00fcrzburg", "ground_truth": "W\u00fcrzburg", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.title": {"response": "Winkelabh\u00e4ngigkeit des Polarisationsgrades im Spektrum der Bremsstrahlung relativistischer Elektronen", "ground_truth": "Winkelabh\u00e4ngigkeit des Polarisationsgrades im Spektrum der Bremsstrahlung relativistischer Elektronen", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-1f6110e9-be21-9e04-92f2-e126b439e8e5\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Nowak\\\",\\n    \\\"first_name\\\": \\\"Theo\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Winkelabh\u00e4ngigkeit des Polarisationsgrades im Spektrum der Bremsstrahlung relativistischer Elektronen\\\",\\n    \\\"year\\\": 1967,\\n    \\\"place\\\": \\\"W\u00fcrzburg\\\",\\n    \\\"pages\\\": \\\"32\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"8'\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss.1968,5355\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774430430,\"model\":\"qwen3.5-122b-a10b\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":170,\"prompt_tokens\":1787,\"total_tokens\":1957,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":170},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":572,\"text_tokens\":1215}}}"}, "scoring": {"f1_micro": 0.8603148970528866, "f1_macro": 0.8529277566539923, "micro_precision": 0.8393068137061835, "micro_recall": 0.8824016563146998, "total_instances": 263, "total_tp": 2131, "total_fp": 408, "total_fn": 284, "cost_summary": {"total_input_tokens": 470700, "total_output_tokens": 48029, "total_tokens": 518729, "input_cost_usd": 0.18827999999999973, "output_cost_usd": 0.15369280000000007, "total_cost_usd": 0.3419728000000001}}, "normalized_score": 85.29277566539923}, {"_id": {"$oid": "69d4eb12b877594bc5718d74"}, "test_id": "T0871", "benchmark": "library_cards", "date": "2026-03-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-397b-a17b", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Seisser\",\n    \"first_name\": \"Hans\"\n  },\n  \"publication\": {\n    \"title\": \"Erfolge der Strahlenbehandlung bei Vaginalcarcinomen. <Krankengut d. Strahleninst. d. Univ.Frauenklinik M\u00fcnchen.>\",\n    \"year\": 1938,\n    \"place\": \"D\u00fcsseldorf\",\n    \"pages\": \"13\",\n    \"publisher\": \"Nolte\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"DN: B 1938, 16, 594\",\n    \"subjects\": \"\"\n  }\n}", "model": "qwen3.5-397b-a17b", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 1806, "output_tokens": 195, "total_tokens": 2001, "input_cost_usd": 0.0010836, "output_cost_usd": 0.000702, "estimated_cost_usd": 0.0017856}, "duration": 5.961163759231567, "timestamp": "2026-03-25T10:20:59.313978", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Seisser", "first_name": "Hans"}, "publication": {"title": "Erfolge der Strahlenbehandlung bei Vaginalcarcinomen. <Krankengut d. Strahleninst. d. Univ.Frauenklinik M\u00fcnchen.>", "year": 1938, "place": "D\u00fcsseldorf", "pages": "13", "publisher": "Nolte", "format": "8\u00b0"}, "library_reference": {"shelfmark": "DN: B 1938, 16, 594", "subjects": ""}}, "conversation_id": "c70be667-7811-4de1-8e1e-7a6cbe11bc70", "score": {"f1_score": 0.9, "precision": 0.9, "recall": 0.9, "true_positives": 9, "false_positives": 1, "false_negatives": 1, "field_scores": {"publication.place": {"response": "D\u00fcsseldorf", "ground_truth": "D\u00fcsseldorf", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": "DN: B 1938, 16, 594", "ground_truth": "DN:B 1938, 16, 594", "score": 0.972972972972973}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "author.last_name": {"response": "Seisser", "ground_truth": "Seisser", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "publication.title": {"response": "Erfolge der Strahlenbehandlung bei Vaginalcarcinomen. <Krankengut d. Strahleninst. d. Univ.Frauenklinik M\u00fcnchen.>", "ground_truth": "Erfolge der Strahlenbehandlung bei Vaginalcarcinomen", "score": 0.6303030303030304}, "publication.year": {"response": 1938, "ground_truth": 1938, "score": 1.0}, "publication.pages": {"response": "13", "ground_truth": "13", "score": 1.0}, "publication.publisher": {"response": "Nolte", "ground_truth": "Nolte", "score": 1.0}, "author.first_name": {"response": "Hans", "ground_truth": "Hans", "score": 1.0}}, "total_fields": 11}, "raw_response": "{\"id\":\"chatcmpl-00d26b81-43cb-99b0-b9f1-09f32f09a106\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Seisser\\\",\\n    \\\"first_name\\\": \\\"Hans\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Erfolge der Strahlenbehandlung bei Vaginalcarcinomen. <Krankengut d. Strahleninst. d. Univ.Frauenklinik M\u00fcnchen.>\\\",\\n    \\\"year\\\": 1938,\\n    \\\"place\\\": \\\"D\u00fcsseldorf\\\",\\n    \\\"pages\\\": \\\"13\\\",\\n    \\\"publisher\\\": \\\"Nolte\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"DN: B 1938, 16, 594\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774430459,\"model\":\"qwen3.5-397b-a17b\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":195,\"prompt_tokens\":1806,\"total_tokens\":2001,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":195},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":591,\"text_tokens\":1215}}}"}, "scoring": {"f1_micro": 0.8883096049729297, "f1_macro": 0.8825475285171102, "micro_precision": 0.8611975116640747, "micro_recall": 0.917184265010352, "total_instances": 263, "total_tp": 2215, "total_fp": 357, "total_fn": 200, "cost_summary": {"total_input_tokens": 470700, "total_output_tokens": 47602, "total_tokens": 518302, "input_cost_usd": 0.28241999999999984, "output_cost_usd": 0.17136720000000003, "total_cost_usd": 0.4537872}}, "normalized_score": 88.25475285171102}, {"_id": {"$oid": "69d4eb12b877594bc5718d2d"}, "test_id": "T0819", "benchmark": "library_cards", "date": "2026-03-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-plus-2026-02-15", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"Korsch\",\n    \"first_name\": \"Ernst\"\n  },\n  \"publication\": {\n    \"title\": \"Die verschiedenen Fassungen in dem Werk des Bischofs Otto von Freising \u00fcber die Taten Kaiser Friedrichs I.\",\n    \"year\": 1941,\n    \"place\": \"B.-Steglitz\",\n    \"pages\": \"\",\n    \"publisher\": \"\",\n    \"format\": \"\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss.Univ. Berlin 1941\",\n    \"subjects\": \"Otto Frisingensis ca 1114- 1158 [ biogr.]\"\n  }\n}", "model": "qwen3.5-plus", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 1787, "output_tokens": 187, "total_tokens": 1974, "input_cost_usd": 0.0007148, "output_cost_usd": 0.00044879999999999996, "estimated_cost_usd": 0.0011635999999999999}, "duration": 8.423988342285156, "timestamp": "2026-03-24T18:53:50.338481", "parsed": {"type": {"type": "Reference"}, "author": {"last_name": "Korsch", "first_name": "Ernst"}, "publication": {"title": "Die verschiedenen Fassungen in dem Werk des Bischofs Otto von Freising \u00fcber die Taten Kaiser Friedrichs I.", "year": 1941, "place": "B.-Steglitz", "pages": "", "publisher": "", "format": ""}, "library_reference": {"shelfmark": "Diss.Univ. Berlin 1941", "subjects": "Otto Frisingensis ca 1114- 1158 [ biogr.]"}}, "conversation_id": "94385a23-10d7-429b-b435-23b4a558d9c9", "score": {"f1_score": 0.86, "precision": 0.75, "recall": 1.0, "true_positives": 6, "false_positives": 2, "false_negatives": 0, "field_scores": {"library_reference.subjects": {"response": "Otto Frisingensis ca 1114- 1158 [ biogr.]", "ground_truth": null, "score": 0.0}, "library_reference.shelfmark": {"response": "Diss.Univ. Berlin 1941", "ground_truth": null, "score": 0.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "type.type": {"response": "Reference", "ground_truth": "Reference", "score": 1.0}, "publication.year": {"response": 1941, "ground_truth": 1941, "score": 1.0}, "publication.place": {"response": "B.-Steglitz", "ground_truth": "B.-Steglitz", "score": 1.0}, "author.last_name": {"response": "Korsch", "ground_truth": "Korsch", "score": 1.0}, "publication.pages": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": "Ernst", "ground_truth": "Ernst", "score": 1.0}, "publication.format": {"response": null, "ground_truth": null, "score": 1.0}, "publication.title": {"response": "Die verschiedenen Fassungen in dem Werk des Bischofs Otto von Freising \u00fcber die Taten Kaiser Friedrichs I.", "ground_truth": "Die verschiedenen Fassungen in dem Werk des Bischofs Otto von Freising \u00fcber die Taten Kaiser Friedrichs I.", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-d5fbc97a-d9c1-9dfc-849b-4846f3bba937\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Reference\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Korsch\\\",\\n    \\\"first_name\\\": \\\"Ernst\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Die verschiedenen Fassungen in dem Werk des Bischofs Otto von Freising \u00fcber die Taten Kaiser Friedrichs I.\\\",\\n    \\\"year\\\": 1941,\\n    \\\"place\\\": \\\"B.-Steglitz\\\",\\n    \\\"pages\\\": \\\"\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss.Univ. Berlin 1941\\\",\\n    \\\"subjects\\\": \\\"Otto Frisingensis ca 1114- 1158 [ biogr.]\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774374830,\"model\":\"qwen3.5-plus\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":187,\"prompt_tokens\":1787,\"total_tokens\":1974,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":187},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":572,\"text_tokens\":1215}}}"}, "scoring": {"f1_micro": 0.8901803607214429, "f1_macro": 0.8846007604562738, "micro_precision": 0.8625242718446602, "micro_recall": 0.9196687370600414, "total_instances": 263, "total_tp": 2221, "total_fp": 354, "total_fn": 194, "cost_summary": {"total_input_tokens": 470700, "total_output_tokens": 47670, "total_tokens": 518370, "input_cost_usd": 0.18827999999999973, "output_cost_usd": 0.11440800000000008, "total_cost_usd": 0.3026880000000002}}, "normalized_score": 88.46007604562737}, {"_id": {"$oid": "69d4eb12b877594bc5718d18"}, "test_id": "T0698", "benchmark": "library_cards", "date": "2026-03-23", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.3-codex", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "rules": "{\"api_style\":\"responses\"}", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Tich\u00fd\",\n    \"first_name\": \"Johannes\"\n  },\n  \"publication\": {\n    \"title\": \"Klinischer und experimenteller Beitrag zur Operation der Wanderniere\",\n    \"year\": \"1914\",\n    \"place\": \"Leipzig\",\n    \"pages\": \"40\",\n    \"publisher\": \"Vogel\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"U 14.2443\",\n    \"subjects\": \"Chirurgie; Operation der Wanderniere\"\n  }\n}", "model": "gpt-5.3-codex", "provider": "openai", "finish_reason": "completed", "usage": {"input_tokens": 1885, "output_tokens": 152, "total_tokens": 2037, "input_cost_usd": 0.00329875, "output_cost_usd": 0.002128, "estimated_cost_usd": 0.00542675}, "duration": 2.4848473072052, "timestamp": "2026-03-23T21:14:51.806081", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Tich\u00fd", "first_name": "Johannes"}, "publication": {"title": "Klinischer und experimenteller Beitrag zur Operation der Wanderniere", "year": "1914", "place": "Leipzig", "pages": "40", "publisher": "Vogel", "format": "8\u00b0"}, "library_reference": {"shelfmark": "U 14.2443", "subjects": "Chirurgie; Operation der Wanderniere"}}, "conversation_id": "9ad16e84-d6ef-4db1-8e85-704348d5bb6d", "score": {"f1_score": 0.86, "precision": 0.8181818181818182, "recall": 0.9, "true_positives": 9, "false_positives": 2, "false_negatives": 1, "field_scores": {"publication.year": {"response": "1914", "ground_truth": 1914, "score": 1.0}, "publication.title": {"response": "Klinischer und experimenteller Beitrag zur Operation der Wanderniere", "ground_truth": "Klinischer und experimenteller Beitrag zur Operation der Wanderniere", "score": 1.0}, "publication.place": {"response": "Leipzig", "ground_truth": "Leipzig", "score": 1.0}, "author.first_name": {"response": "Johannes", "ground_truth": "Johannes", "score": 1.0}, "library_reference.subjects": {"response": "Chirurgie; Operation der Wanderniere", "ground_truth": null, "score": 0.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "library_reference.shelfmark": {"response": "U 14.2443", "ground_truth": "U 14. 2443", "score": 0.9473684210526316}, "publication.publisher": {"response": "Vogel", "ground_truth": "Vogel", "score": 1.0}, "publication.pages": {"response": "40", "ground_truth": "40", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "author.last_name": {"response": "Tich\u00fd", "ground_truth": "Tichy", "score": 0.8}}, "total_fields": 11}, "raw_response": "{\"id\":\"resp_00ed0de988d890ef0069c19f3a0444819393ab103d78d84fa8\",\"created_at\":1774296890.0,\"error\":null,\"incomplete_details\":null,\"instructions\":\"You are a historian with keyword knowledge\",\"metadata\":{},\"model\":\"gpt-5.3-codex\",\"object\":\"response\",\"output\":[{\"id\":\"msg_00ed0de988d890ef0069c19f3a741c81939b00a25bf02ad8a2\",\"content\":[{\"annotations\":[],\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Tich\u00fd\\\",\\n    \\\"first_name\\\": \\\"Johannes\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Klinischer und experimenteller Beitrag zur Operation der Wanderniere\\\",\\n    \\\"year\\\": \\\"1914\\\",\\n    \\\"place\\\": \\\"Leipzig\\\",\\n    \\\"pages\\\": \\\"40\\\",\\n    \\\"publisher\\\": \\\"Vogel\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"U 14.2443\\\",\\n    \\\"subjects\\\": \\\"Chirurgie; Operation der Wanderniere\\\"\\n  }\\n}\",\"type\":\"output_text\",\"logprobs\":[]}],\"role\":\"assistant\",\"status\":\"completed\",\"type\":\"message\",\"phase\":\"final_answer\"}],\"parallel_tool_calls\":true,\"temperature\":0.0,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":0.98,\"background\":false,\"completed_at\":1774296892.0,\"conversation\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"previous_response_id\":null,\"prompt\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":null,\"reasoning\":{\"effort\":\"none\",\"generate_summary\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"status\":\"completed\",\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"top_logprobs\":0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1885,\"input_tokens_details\":{\"cached_tokens\":1024},\"output_tokens\":152,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":2037},\"user\":null,\"billing\":{\"payer\":\"developer\"},\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"store\":true}"}, "scoring": {"f1_micro": 0.8554726869582746, "f1_macro": 0.8471482889733841, "micro_precision": 0.8334642576590731, "micro_recall": 0.8786749482401657, "total_instances": 263, "total_tp": 2122, "total_fp": 424, "total_fn": 293, "cost_summary": {"total_input_tokens": 498059, "total_output_tokens": 42068, "total_tokens": 540127, "input_cost_usd": 0.8716032500000012, "output_cost_usd": 0.5889519999999994, "total_cost_usd": 1.4605552500000016}}, "normalized_score": 84.7148288973384}, {"_id": {"$oid": "69d4eb12b877594bc5718d20"}, "test_id": "T0723", "benchmark": "library_cards", "date": "2026-03-23", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "x-ai", "model": "grok-4.20-0309-reasoning", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wettell\",\"first_name\":\"Georg Wilhelm\"},\"publication\":{\"title\":\"Disputatio de qu\u00e6stione, aduersus quem in integrum restitutio imploranda sit.\",\"year\":\"1850\",\"place\":\"Marburgi\",\"pages\":\"46\",\"publisher\":null,\"format\":null,\"editor\":null},\"library_reference\":{\"shelfmark\":\"7\",\"subjects\":\"Rektoratsprogramm\"}}", "model": "grok-4.20-0309-reasoning", "provider": "x-ai", "finish_reason": "stop", "usage": {"input_tokens": 1314, "output_tokens": 151, "total_tokens": 4039, "cached_tokens": 1088, "input_cost_usd": 0.002628, "output_cost_usd": 0.0009060000000000001, "estimated_cost_usd": 0.0035340000000000002}, "duration": 28.54402232170105, "timestamp": "2026-03-23T14:04:00.039635", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wettell", "first_name": "Georg Wilhelm"}, "publication": {"title": "Disputatio de qu\u00e6stione, aduersus quem in integrum restitutio imploranda sit.", "year": "1850", "place": "Marburgi", "pages": "46", "publisher": null, "format": null, "editor": null}, "library_reference": {"shelfmark": "7", "subjects": "Rektoratsprogramm"}}, "conversation_id": "3ca3b527-0c18-4b5f-87fe-2bdf1b312082", "score": {"f1_score": 0.59, "precision": 0.5555555555555556, "recall": 0.625, "true_positives": 5, "false_positives": 4, "false_negatives": 3, "field_scores": {"type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.pages": {"response": "46", "ground_truth": "40", "score": 0.5}, "publication.title": {"response": "Disputatio de qu\u00e6stione, aduersus quem in integrum restitutio imploranda sit.", "ground_truth": "Disputatio de quaestione, adversus quem in integrum restitutio imploranda sit", "score": 0.961038961038961}, "publication.year": {"response": "1850", "ground_truth": 1850, "score": 1.0}, "publication.place": {"response": "Marburgi", "ground_truth": "Marburgi", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": "7", "ground_truth": "AT 7", "score": 0.4}, "author.first_name": {"response": "Georg Wilhelm", "ground_truth": "Georg Wilhelm", "score": 1.0}, "publication.format": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.subjects": {"response": "Rektoratsprogramm", "ground_truth": null, "score": 0.0}, "author.last_name": {"response": "Wettell", "ground_truth": "Wetzell", "score": 0.8571428571428572}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"2e2ef856-cf3b-95b0-8e43-b81fa2df6713\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Wettell\\\",\\n    \\\"first_name\\\": \\\"Georg Wilhelm\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Disputatio de qu\u00e6stione, aduersus quem in integrum restitutio imploranda sit.\\\",\\n    \\\"year\\\": \\\"1850\\\",\\n    \\\"place\\\": \\\"Marburgi\\\",\\n    \\\"pages\\\": \\\"46\\\",\\n    \\\"publisher\\\": null,\\n    \\\"format\\\": null,\\n    \\\"editor\\\": null\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"7\\\",\\n    \\\"subjects\\\": \\\"Rektoratsprogramm\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wettell\",\"first_name\":\"Georg Wilhelm\"},\"publication\":{\"title\":\"Disputatio de qu\u00e6stione, aduersus quem in integrum restitutio imploranda sit.\",\"year\":\"1850\",\"place\":\"Marburgi\",\"pages\":\"46\",\"publisher\":null,\"format\":null,\"editor\":null},\"library_reference\":{\"shelfmark\":\"7\",\"subjects\":\"Rektoratsprogramm\"}}}}],\"created\":1774271039,\"model\":\"grok-4.20-0309-reasoning\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":\"fp_d6e36c82c2\",\"usage\":{\"completion_tokens\":151,\"prompt_tokens\":1314,\"total_tokens\":4039,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":2574,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":1088,\"text_tokens\":1138,\"image_tokens\":176},\"num_sources_used\":0,\"cost_in_usd_ticks\":170196000}}"}, "scoring": {"f1_micro": 0.8645003065603925, "f1_macro": 0.8587072243346008, "micro_precision": 0.8535108958837773, "micro_recall": 0.8757763975155279, "total_instances": 263, "total_tp": 2115, "total_fp": 363, "total_fn": 300, "cost_summary": {"total_input_tokens": 342414, "total_output_tokens": 41178, "total_tokens": 383592, "input_cost_usd": 0.684828000000001, "output_cost_usd": 0.247068, "total_cost_usd": 0.9318959999999993}}, "normalized_score": 85.87072243346007}, {"_id": {"$oid": "69d4eb12b877594bc5718cd0"}, "test_id": "T0698", "benchmark": "library_cards", "date": "2026-03-17", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.3-codex", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "rules": "{\"api_style\":\"responses\"}", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Tichy\",\n    \"first_name\": \"Johannes\"\n  },\n  \"publication\": {\n    \"title\": \"Klinischer und experimenteller Beitrag zur Operation der Wanderniere\",\n    \"year\": \"1914\",\n    \"place\": \"Leipzig\",\n    \"pages\": \"40\",\n    \"publisher\": \"Vogel\",\n    \"format\": \"8\u00b0\",\n    \"editor\": \"\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"U 14.2443\",\n    \"subjects\": \"Chirurgie; Universit\u00e4tsklinik Marburg; Wanderniere\"\n  }\n}", "model": "gpt-5.3-codex", "provider": "openai", "finish_reason": "completed", "usage": {"input_tokens": 1885, "output_tokens": 161, "total_tokens": 2046, "input_cost_usd": 0.00329875, "output_cost_usd": 0.002254, "estimated_cost_usd": 0.00555275}, "duration": 2.628411054611206, "timestamp": "2026-03-17T22:46:30.464914", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Tichy", "first_name": "Johannes"}, "publication": {"title": "Klinischer und experimenteller Beitrag zur Operation der Wanderniere", "year": "1914", "place": "Leipzig", "pages": "40", "publisher": "Vogel", "format": "8\u00b0", "editor": ""}, "library_reference": {"shelfmark": "U 14.2443", "subjects": "Chirurgie; Universit\u00e4tsklinik Marburg; Wanderniere"}}, "conversation_id": "8112610a-f958-4aaf-9a6b-53a0d7f9fc34", "score": {"f1_score": 0.95, "precision": 0.9090909090909091, "recall": 1.0, "true_positives": 10, "false_positives": 1, "false_negatives": 0, "field_scores": {"author.first_name": {"response": "Johannes", "ground_truth": "Johannes", "score": 1.0}, "library_reference.shelfmark": {"response": "U 14.2443", "ground_truth": "U 14. 2443", "score": 0.9473684210526316}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "publication.publisher": {"response": "Vogel", "ground_truth": "Vogel", "score": 1.0}, "library_reference.subjects": {"response": "Chirurgie; Universit\u00e4tsklinik Marburg; Wanderniere", "ground_truth": null, "score": 0.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.pages": {"response": "40", "ground_truth": "40", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": "Tichy", "ground_truth": "Tichy", "score": 1.0}, "publication.year": {"response": "1914", "ground_truth": 1914, "score": 1.0}, "publication.title": {"response": "Klinischer und experimenteller Beitrag zur Operation der Wanderniere", "ground_truth": "Klinischer und experimenteller Beitrag zur Operation der Wanderniere", "score": 1.0}, "publication.place": {"response": "Leipzig", "ground_truth": "Leipzig", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"resp_0af0007fd13ea0100069b9cbb272e88197b2f8b689bd392e23\",\"created_at\":1773783986.0,\"error\":null,\"incomplete_details\":null,\"instructions\":\"You are a historian with keyword knowledge\",\"metadata\":{},\"model\":\"gpt-5.3-codex\",\"object\":\"response\",\"output\":[{\"id\":\"msg_0af0007fd13ea0100069b9cbb2d57881979c01f65bbbe8d994\",\"content\":[{\"annotations\":[],\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Tichy\\\",\\n    \\\"first_name\\\": \\\"Johannes\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Klinischer und experimenteller Beitrag zur Operation der Wanderniere\\\",\\n    \\\"year\\\": \\\"1914\\\",\\n    \\\"place\\\": \\\"Leipzig\\\",\\n    \\\"pages\\\": \\\"40\\\",\\n    \\\"publisher\\\": \\\"Vogel\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\",\\n    \\\"editor\\\": \\\"\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"U 14.2443\\\",\\n    \\\"subjects\\\": \\\"Chirurgie; Universit\u00e4tsklinik Marburg; Wanderniere\\\"\\n  }\\n}\",\"type\":\"output_text\",\"logprobs\":[]}],\"role\":\"assistant\",\"status\":\"completed\",\"type\":\"message\",\"phase\":\"final_answer\"}],\"parallel_tool_calls\":true,\"temperature\":0.0,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":0.98,\"background\":false,\"completed_at\":1773783988.0,\"conversation\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"previous_response_id\":null,\"prompt\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":null,\"reasoning\":{\"effort\":\"none\",\"generate_summary\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"status\":\"completed\",\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"top_logprobs\":0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1885,\"input_tokens_details\":{\"cached_tokens\":1024},\"output_tokens\":161,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":2046},\"user\":null,\"billing\":{\"payer\":\"developer\"},\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"store\":true}"}, "scoring": {"f1_micro": 0.8499899051080154, "f1_macro": 0.8419391634980988, "micro_precision": 0.8293932230102443, "micro_recall": 0.8716356107660456, "total_instances": 263, "total_tp": 2105, "total_fp": 433, "total_fn": 310, "cost_summary": {"total_input_tokens": 498059, "total_output_tokens": 42201, "total_tokens": 540260, "input_cost_usd": 0.8716032500000012, "output_cost_usd": 0.5908140000000002, "total_cost_usd": 1.4624172500000008}}, "normalized_score": 84.19391634980988}, {"_id": {"$oid": "69d4eb12b877594bc5718ca6"}, "test_id": "T0633", "benchmark": "library_cards", "date": "2026-03-16", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-6", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Spiegel\",\"first_name\":\"Kurt\"},\"publication\":{\"title\":\"Das Verl\u00f6bnis im deutschen internationalen Privatrecht\",\"year\":\"1929\",\"place\":\"Walldorf b. Heidelberg\",\"pages\":\"66\",\"publisher\":\"Lamade\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Heidelberg, Jur. Diss. v. 28. Nov. 1929\",\"subjects\":\"U 29. 3005\"}}", "model": "claude-opus-4-6", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2681, "output_tokens": 211, "total_tokens": 2892, "input_cost_usd": 0.013404999999999999, "output_cost_usd": 0.005275, "estimated_cost_usd": 0.01868}, "duration": 5.288168907165527, "timestamp": "2026-03-16T15:53:01.167714", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Spiegel", "first_name": "Kurt"}, "publication": {"title": "Das Verl\u00f6bnis im deutschen internationalen Privatrecht", "year": "1929", "place": "Walldorf b. Heidelberg", "pages": "66", "publisher": "Lamade", "format": "8\u00b0"}, "library_reference": {"shelfmark": "Heidelberg, Jur. Diss. v. 28. Nov. 1929", "subjects": "U 29. 3005"}}, "conversation_id": "02e9b811-040b-4adf-9b43-652b0017cde2", "score": {"f1_score": 0.86, "precision": 0.8181818181818182, "recall": 0.9, "true_positives": 9, "false_positives": 2, "false_negatives": 1, "field_scores": {"publication.pages": {"response": "66", "ground_truth": "66", "score": 1.0}, "publication.title": {"response": "Das Verl\u00f6bnis im deutschen internationalen Privatrecht", "ground_truth": "Das Verl\u00f6bnis im deutschen internationalen Privatrecht", "score": 1.0}, "author.last_name": {"response": "Spiegel", "ground_truth": "Spiegel", "score": 1.0}, "library_reference.subjects": {"response": "U 29. 3005", "ground_truth": null, "score": 0.0}, "author.first_name": {"response": "Kurt", "ground_truth": "Kurt", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "library_reference.shelfmark": {"response": "Heidelberg, Jur. Diss. v. 28. Nov. 1929", "ground_truth": "U 29. 3005", "score": 0.16326530612244894}, "publication.place": {"response": "Walldorf b. Heidelberg", "ground_truth": "Walldorf b. Heidelberg", "score": 1.0}, "publication.publisher": {"response": "Lamade", "ground_truth": "Lamade", "score": 1.0}, "publication.year": {"response": "1929", "ground_truth": 1929, "score": 1.0}}, "total_fields": 11}, "raw_response": "{\"id\":\"msg_014vQiZGPRnwKbWZRwaeLU1i\",\"content\":[{\"id\":\"toolu_01SMiYBNRfkRfxcPzidJDKBc\",\"input\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Spiegel\",\"first_name\":\"Kurt\"},\"publication\":{\"title\":\"Das Verl\u00f6bnis im deutschen internationalen Privatrecht\",\"year\":\"1929\",\"place\":\"Walldorf b. Heidelberg\",\"pages\":\"66\",\"publisher\":\"Lamade\",\"format\":\"8\u00b0\"},\"library_reference\":{\"shelfmark\":\"Heidelberg, Jur. Diss. v. 28. Nov. 1929\",\"subjects\":\"U 29. 3005\"}},\"name\":\"extract_structured_data\",\"type\":\"tool_use\",\"caller\":{\"type\":\"direct\"}}],\"model\":\"claude-opus-4-6\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"inference_geo\":\"global\",\"input_tokens\":2681,\"output_tokens\":211,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"f1_micro": 0.8524525316455696, "f1_macro": 0.8463498098859316, "micro_precision": 0.8159787959106399, "micro_recall": 0.8923395445134575, "total_instances": 263, "total_tp": 2155, "total_fp": 486, "total_fn": 260, "cost_summary": {"total_input_tokens": 706230, "total_output_tokens": 57399, "total_tokens": 763629, "input_cost_usd": 3.5311500000000016, "output_cost_usd": 1.434975000000001, "total_cost_usd": 4.966125000000003}}, "normalized_score": 84.63498098859316}, {"_id": {"$oid": "69d4eb12b877594bc5718cac"}, "test_id": "T0686", "benchmark": "library_cards", "date": "2026-03-16", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3.1-pro-preview", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"M\u00fchlhaus\",\"first_name\":\"Richard\"},\"publication\":{\"title\":\"Untersuchungen des Blutbildes bei akuten u. chronischen Osteomyelitiden.\",\"year\":\"1927\",\"place\":\"M\u00fcnchen\",\"pages\":\"17\",\"publisher\":\"Wolf\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"U 28. 5030\",\"subjects\":null}}", "model": "gemini-3.1-pro-preview", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1529, "output_tokens": 177, "total_tokens": 3011, "input_cost_usd": 0.003058, "output_cost_usd": 0.002124, "estimated_cost_usd": 0.005182}, "duration": 14.158718347549438, "timestamp": "2026-03-16T19:26:35.736558", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "M\u00fchlhaus", "first_name": "Richard"}, "publication": {"title": "Untersuchungen des Blutbildes bei akuten u. chronischen Osteomyelitiden.", "year": "1927", "place": "M\u00fcnchen", "pages": "17", "publisher": "Wolf", "format": "8\u00b0", "editor": null}, "library_reference": {"shelfmark": "U 28. 5030", "subjects": null}}, "conversation_id": "50df3c4e-16a0-4fa7-a56c-8d6efbb7581a", "score": {"f1_score": 1.0, "precision": 1.0, "recall": 1.0, "true_positives": 10, "false_positives": 0, "false_negatives": 0, "field_scores": {"author.last_name": {"response": "M\u00fchlhaus", "ground_truth": "M\u00fchlhaus", "score": 1.0}, "library_reference.shelfmark": {"response": "U 28. 5030", "ground_truth": "U 28. 5030", "score": 1.0}, "publication.pages": {"response": "17", "ground_truth": "17", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "publication.year": {"response": "1927", "ground_truth": 1927, "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.title": {"response": "Untersuchungen des Blutbildes bei akuten u. chronischen Osteomyelitiden.", "ground_truth": "Untersuchungen des Blutbildes bei akuten u. chronischen Osteomyelitiden", "score": 0.993006993006993}, "publication.publisher": {"response": "Wolf", "ground_truth": "Wolf", "score": 1.0}, "author.first_name": {"response": "Richard", "ground_truth": "Richard", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.place": {"response": "M\u00fcnchen", "ground_truth": "M\u00fcnchen", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Mon, 16 Mar 2026 18:26:42 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=14071\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"media_resolution\":null,\"code_execution_result\":null,\"executable_code\":null,\"file_data\":null,\"function_call\":null,\"function_response\":null,\"inline_data\":null,\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"M\u00fchlhaus\\\",\\n    \\\"first_name\\\": \\\"Richard\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Untersuchungen des Blutbildes bei akuten u. chronischen Osteomyelitiden.\\\",\\n    \\\"year\\\": \\\"1927\\\",\\n    \\\"place\\\": \\\"M\u00fcnchen\\\",\\n    \\\"pages\\\": \\\"17\\\",\\n    \\\"publisher\\\": \\\"Wolf\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\",\\n    \\\"editor\\\": null\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"U 28. 5030\\\",\\n    \\\"subjects\\\": null\\n  }\\n}\",\"thought\":null,\"thought_signature\":\"EpQgCpEgAb4-9vsiFy_dEW0tGfWZB04v8aywRXzVnBjpVmHy5YlxRDUQOXaJwNWBbY6QL5Yp7Jm4ugpmQh_8WRMAOFGdlPdlM2pS-X006zhxKatcr4prCIQRGtIUoZrcevXEnpxE5CUw16YmfZ2fcoFrYDyjZBg0Rfz5mlfASXshXmeTVoSOJmQLbmFGkvFgGPutPTq9YNyuqnzptzloJRMtH9YBpnOo3fZwKcOKJ9qJdvxMzvZP67xW_WzvUW9ghjPmA1GZGzTNZOuIrPLgDxmuAgbyfqDaVecXbKRo4DsYpfJ4HFZLcDaskiJsBAKHxhYNnL1RJ5Ta9hBWdkQ4TQkJ4a3bDFSgQeX0yyWGBAGrVzKUuJDyQQehiG2d83p9nFT-9kgxqLLgONo21v3qU67QDkHD3c0bIBUYPofYv3UHKtD3w0XNiivpZuSP_VRxXrlZXpNjtcyV7jq1iAqZWsVXpxSJV-GWiYTIX37lrYGnSaasDblfttEL5YxORcx-VdwrhCjDN1MyyYrNARBS96uFALKBkG3E6eX9r4mmCt_YdjlkkXXHNAMQdqLGW6n_gGcMoijMqMRuDn8Vrirrxf0KmGAG4ueVdbAPAFYXBImkyxi4snMbhd3orC4SUYfRP2GCSw2_3kSWzu05TxsvX_grp3B31XjMXHmT44uQ9AUKyqaq-cu6zNexCV05QE0Le3Q_E5CUsIF-3yLIQGDtvU2fxImCNIqJcPezdZ45mRXgdGxY75AHziOeBLektakqygLScL_1pORfvaKxTZKPG8a6H476KJNQLtaZlFfAQAkDLa3oP_32DUPrvozBOoKwEHo1cvi1Z863w6CHpNssHu2xmvnTwimh_XgAbrCYZXVOXJwq7JxbMyNErQ4-UYAxbeze0iEhdOqaD68-kCpfHXA4BjiO1dhdrvifMV4GOTUiOwpeCuuECQY3HExp_J1Adf99cDcQsWgaxThxHVt8EBjUVvp4tpAjBthHDs6RveXKyF2V8lhma0ApxY8Tz12E9H5-gNHZcOovQ3xDhxa9_D9rey0kVDMIap-MzOc8KRTJQmi2bxIb02n8v2RDdx92J2alr0LeLjKsKZ0KFr9YiWEsSFehIBDOVzV54Luv7QL4a7DVksTmqTodcxgirJeilP9yaf7fJNDW4kpM4hJG_iFsW7QgQjhWXG_1cdUZuM9JZI0ns8nU33gbL47XoF8Y183HUSwByG6j4YORII8SqOGdfHo-9usN73kHZ_kQVzDURR1w2Blh51ovsEZgG9nJ3Vkxx9oYQaKCrn9Pteh5AXefsloHahJsDTKrjX0wsIAwtdpPMBGyELTDFEeJKMTo3eUPkeZ8aMeOUvTRY75fwdFWw3NopKJv3rxQ5QfyyeBskflA861QzBvVSaodZiwseSnZzK1Ryiy0qH11gcvDgNU11zCmvYOI6iwqOQ_lwps-AwWwbeXJIBvu5kV3Li75P5LkSQMOip2u9YwINIE1XxpoNMZfKI6yt4bDHAO1HWyax16wNQeN3prMyQKAj00guFnwM-QCPzLgqszh46v-vgWSJmq2JzYcx2WuxRGvOiJAa7mqQAKZk-Rq5YLngo4qMdL_x9WZIX_fpEdsJUngHbTIjZhkulKIZ32bxUBGUtHt9CNx9uV2Absqk0fIESX59-pI7N_fbGmnsJdly6j_PyDRNEzl9UGomx3RV3eY64r4g0hYKF6dg6dD0E9RZKzccynVXDX4yqbjoJ-5_dv-d7sOVT8JAhd-SqeixmEHrnoBA8qiQIOR_oM819DZPz0OCnbGukmiVWNug4VMPLWnXssf6jNeysGRxd5P1Wr33hUs1LsuG83aOae8EbVVwC5DQfoo6o6iACMiQAoyBE6ZkAcqpq5a71hANip8OCNBRjbNFYvNYLEz3JLLg4Vum3CzZ93vbRpG8FOrrPwvJWy6MyOKZNu-gtc0DSCPANq1FQCBEuhfseqXIhkuMKXrMAXSAFtG7lKexd3dj1zVuISB802lvHCcr2HuEwUN_lnz5hwVF4VUbN2vlse69At27QGLwFdkIB-1i7wvPzAczMIQ5KhkTJGGnX7iRCJewzfKSwZ9tMeCX7Liz2VQoSQNd5A1enIUECTH8clKq5LJMYXYrmdxtrP86u87ZV3KCQJs_H9tp7HMgzhQviG344MfEVjja7rfGFlDsKeAa1VbUaqRDfgnEno5NHc1XGM2410eY7d-1rWyfvEneV9S5YM4UVlqhj8Jd1o0ggaYDlGnk2e2oZYlcFDB2kO9_gkhQr0orq8kqpdpv7TZgDZRpmOSkdLKssfquEVUv5gzyS6RE7HJ-L_gvfGN5-bSdrU0TUMMbapd-L1PGNYRdV3r09m97t8tiY9JbB7Jngu3auFhGn7Y9jXekUZtw-JiUZRR9iEIOf6tjZlEj8BRx3BFdIZ9gjvqqnCyS1CNx-2NEupHTCEe2q7alR22N1wXUenTIbL8ErmzSjLVAh_5bhvi9iIlo99v0F__tW2KX35za4FvNNuPAEui7Wuj7FKm-SeGqWzwShZxw--sqS5JB2nFg_cYhaQu3QngpJ9OuJcQMJDlp38GA2BErD5k5FFobagaIC3BW_3tLwfQXZkMpU-t89JgV_T3psjsoak7H0XyLGBqxDbtssE_v6jKuIu2rM8FYtyuK3Qdjob6mNYqtVf0qwDs93Z3PKCkSsasFL40ca8Cs8dLXEspXy9XBBHSBrrDPpL__cxueRjx59LnD5HQ3HnMGFxf8ktgulCtsiRlHC5fFVC-dXMJgPQoEcQm_GKpkKupMPL_8b-5FrcZ-RphaVRj7lspKrYNQmrO68jOqFGhh0gCiPq_ap5LnGxh5-jKbjbPa-RkOWqyWjHrZ26ARMfV1WHie2IpZDU4o32uBVHBB84dWrk1yeCyvIsA-pTu_fa6sTS6eJac8iJ8_edSLj_u_ldUFZOqlgHfRg3n7NbjQzxQOmxf7ZObF9KrF7HCAwHyme7fX3R-TEnT0SfEZZbi1doHNAMuqd3ta20QOCuCkihpGLLHZ74o90T5aCY16AC_stwKgpRTDMYAEdZPE99NzW_lsC_ZAdTARlR6Nx32nCDiPiIurXOyCTACMG_D2B2SGQKCRaWmq8niKX9_yy9FZjPPpMcF7CYTRkMB_SHaSotz3GDTHx7_u48V71x4j1UtIhBiAgPhmQLtHBcgqiS7qcoX4sN5BgBjtZ4gdrhO-DBg_cGSG_3eqNgepLzCM8278myykHghOFRP-yQGxM0O2L5EqzQ9H6nsNLyVKyqvk6c4WcmtIkFluFMz3BHTInIowYVVdUh74GdqvOKAmCfcLZxRV2Z6uFLLH8MaAYIzpHiNTe0FNgSoUC-7C9aMl_Gd4IvUlWWzwC6iaERVB8Qn0aKZPfkMoOz_Vt_8Q7mxhFuRkxUKVqbMtUqGQi403b4YEaaEZM7AAVHYpmFYrtCHR9KloNQOP0uvxEVNreQ2nrdoyWzf3EdCYmp9p_4UpmDIv4eLOtntsjwD2SHKHvsj2lhuzpK-wPe8LYRVROC5QvC7QqNCRo5D7y5UMYyqNPe4fKldUqmoBkMcSMTB9PZcOsqqpa8p4QKvkSRW2yR9mDvZKeqJnt_-50Eia2EuFxF7nz-14Uq7aYFVMTiL_OW-Wl4rp26leIRN2bpO88R-aOJMssN5cSdgxOwTdLbG77hNYkFZBPGzJYMJ0d8W_c2zK47jWPwyrj4T5HMSBMrUUe5VURa3pkDLv3mRtuBLLi_js8yxVGpvf4rJ8ZZBUpXWujXxX_Mj4r1IVSvT-0xfYDSW5WNdaj1TH1XXyMF1TDghoLyw0RyNMHRB4JCuf_6dg17o4moLupeP05GmHulqYE1ZtXZOQSi8RzaAfWmkN3YGtkR3IlSGvyG_qfRUFrwhObjGvlPExFcufqtcMbEzpz04Ilp7SAd4-zWT_0K6GUepY8FasdmXlM558-DcO-x4WaHaU3Wx6K-HKQXR9j2eTMyrpCOdGGwcrxdU5vleYvZ0IjJNEguBWLsvmaPW9YNzCx2SzLP0RJwgFzebBb2xJJwrHETas9jQnd0not9fuN0Z3wKDIQ-diamaJLljcSKKGX1WuDFo_Ea3VBSRY-mLo0SR3di1HXakD1GjnsbauLXodYEZGpNvrghL7CSm6Qoqn6AzAEL0VX0RFh_WHfMlg6Imc9JYRlAIZgs7qD2q_AGH50mxJzN4kvzht6KdFkfAsNpBFRk-QFIfWKYgz3iY18UqWV2OXK_bVXJBEI4TRmaz3pfCP4XvhXnSYTYNClzctftm7BSxt5clp5kh6dzDxtXkOsuUoPbFI6zWoROOQ2Ud56APZ-ioMNKM7kBegd8cMXLUy623amYypzwXF5ORjhgL8E0o0j_8nik8V74OwroEqv9G3w9F7KSWP7D2-KulvFLNX5vXz8dW4zqzwrrM6AjaesG2S-HtZ0XGkgGalsrZvbrjKipnuo6gkqiZ0zZVgmkdONrdmxPOYXIQbko1trzXF1KcjvYaB2QUgFUvF5WdIgwvuTOWgHB4jHjMCb5dqYYibJ2h0P_ZQnX_pf1uEysWTDNx3MN7C4E4aK5_hvddpTeP1x2vcKAx--pOnK6rnpWN9XtSWZBAh5Qto2kTI22OKSaMf9MRhwnzH9GU8c6uPKa-59gfra7c4Qb68wEHiRo3fAbI1jyV7OU25prf0AHsKQAsPgQUDpsXcmMFWT49jVC0vydMg_2pceKqLW-1yPIlrhnMC-YawOlnsiCbHUHQkNq4LLL5gwXdi_7Pvqft5cR23oWJpfx7UOO0rAelLNNA9UdcY2p5TCalq2-Zb34_xLWbHexEMgZHt0ArZucLT1G0Z8d1GYhyQFRjmaPYwuMqSJ8D4aRI07MxzGVQ3LEV-kmBWAlG_5u60iFWazo-G96bDG7zuFwfPAF3HdgdMM0Ne-N1UrSZos4Qb2zjdROocB4M1Dy5JeIgjIGlQmGETXb46a5Ry_FgJa5y1R7X5RG0SEoJvSjiCY56sw89xJY24MrS0S1Wsx_LIUQ5EYjhq6BhcLfeZxZ7cyVnvUI0zIHrqXvOo_vqm6kZ_Oyaea6aC3jtEDJhksCnv_SRJKYrPiuH5c1cx3RHjFhuED65e1VKydSm2pjybCQ-Yd_Oyq_desiHwactSi6BqPc4qaTOdvdYQr_BnBLwK_ZpPenVC0y29vnMUWoUbE-QHigyBwN-bpSP2h2YPArHh7AvRb_kEKX9pcRkN2DvCVKXM6sGHvi16lhdwAUo8AGvd7q0NP0zzh4_Mmu1rYkJn2zY7d8p9aiP7phKT70XICMXOEsdpcH11GdQvSUTbIpTJN4OWBSgOPDy1k6H5RHEeV3NZi4Gk6jf6JBxiLFGfx1z9NZJrc_F08N9Gu3uqLWIjN9Q7pKXqMS9VtW7T9BePjgB2UbQ7jOu5wbNCXNE9qhwEpicjncxCOw_bAono-Ul-37QV9NclYCANsREv0r4oDYrO-OhTgG-QVbx2Wq6\",\"video_metadata\":null}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null,\"url_context_metadata\":null}],\"create_time\":null,\"model_version\":\"gemini-3.1-pro-preview\",\"prompt_feedback\":null,\"response_id\":\"Yku4abfOIPPMnsEP4Lif4AI\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":177,\"candidates_tokens_details\":null,\"prompt_token_count\":1529,\"prompt_tokens_details\":[{\"modality\":\"IMAGE\",\"token_count\":1075},{\"modality\":\"TEXT\",\"token_count\":454}],\"thoughts_token_count\":1305,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":3011,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"M\u00fchlhaus\",\"first_name\":\"Richard\"},\"publication\":{\"title\":\"Untersuchungen des Blutbildes bei akuten u. chronischen Osteomyelitiden.\",\"year\":\"1927\",\"place\":\"M\u00fcnchen\",\"pages\":\"17\",\"publisher\":\"Wolf\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"U 28. 5030\",\"subjects\":null}}}"}, "scoring": {"f1_micro": 0.89019926799512, "f1_macro": 0.8801140684410647, "micro_precision": 0.8745505393527767, "micro_recall": 0.9064182194616978, "total_instances": 263, "total_tp": 2189, "total_fp": 314, "total_fn": 226, "cost_summary": {"total_input_tokens": 406240, "total_output_tokens": 48633, "total_tokens": 454873, "input_cost_usd": 0.812479999999998, "output_cost_usd": 0.5835960000000002, "total_cost_usd": 1.3960759999999999}}, "normalized_score": 88.01140684410646}, {"_id": {"$oid": "69d4eb12b877594bc5718cb6"}, "test_id": "T0645", "benchmark": "library_cards", "date": "2026-03-16", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-6", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wittenberg\",\"first_name\":\"Joachim\"},\"publication\":{\"title\":\"Unbenutzte Warenzeichen\",\"year\":\"1954\",\"place\":\"o.O.\",\"pages\":\"104\",\"publisher\":\"Diss.d.Rechts-u.Wirtschaftswissenschaftl.Fak.Mainz\",\"format\":\"4'\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Diss.Mainz 1954 (4')\",\"subjects\":\"Marke (Markenrecht). Deutschland\"}}", "model": "claude-sonnet-4-6", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2681, "output_tokens": 222, "total_tokens": 2903, "input_cost_usd": 0.008043, "output_cost_usd": 0.00333, "estimated_cost_usd": 0.011373}, "duration": 4.915768623352051, "timestamp": "2026-03-16T15:53:05.497872", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wittenberg", "first_name": "Joachim"}, "publication": {"title": "Unbenutzte Warenzeichen", "year": "1954", "place": "o.O.", "pages": "104", "publisher": "Diss.d.Rechts-u.Wirtschaftswissenschaftl.Fak.Mainz", "format": "4'"}, "library_reference": {"shelfmark": "Diss.Mainz 1954 (4')", "subjects": "Marke (Markenrecht). Deutschland"}}, "conversation_id": "74e96893-0624-4f32-b1d4-2a9a993fe09f", "score": {"f1_score": 0.8, "precision": 0.7272727272727273, "recall": 0.8888888888888888, "true_positives": 8, "false_positives": 3, "false_negatives": 1, "field_scores": {"publication.pages": {"response": "104", "ground_truth": "104", "score": 1.0}, "author.last_name": {"response": "Wittenberg", "ground_truth": "Wittenberg", "score": 1.0}, "publication.title": {"response": "Unbenutzte Warenzeichen", "ground_truth": "Unbenutzte Warenzeichen", "score": 1.0}, "library_reference.shelfmark": {"response": "Diss.Mainz 1954 (4')", "ground_truth": "Diss.Mainz 1954", "score": 0.8571428571428572}, "publication.publisher": {"response": "Diss.d.Rechts-u.Wirtschaftswissenschaftl.Fak.Mainz", "ground_truth": null, "score": 0.0}, "library_reference.subjects": {"response": "Marke (Markenrecht). Deutschland", "ground_truth": "Marke (Markenrecht). Deutschland", "score": 1.0}, "publication.year": {"response": "1954", "ground_truth": 1954, "score": 1.0}, "publication.place": {"response": "o.O.", "ground_truth": null, "score": 0.0}, "author.first_name": {"response": "Joachim", "ground_truth": "Joachim", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.format": {"response": "4'", "ground_truth": "4'", "score": 1.0}}, "total_fields": 11}, "raw_response": "{\"id\":\"msg_01Dtt8C4tBU6KZduVkTXJwWj\",\"content\":[{\"id\":\"toolu_01N2NEgh5k8bxQLempret1Gk\",\"input\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wittenberg\",\"first_name\":\"Joachim\"},\"publication\":{\"title\":\"Unbenutzte Warenzeichen\",\"year\":\"1954\",\"place\":\"o.O.\",\"pages\":\"104\",\"publisher\":\"Diss.d.Rechts-u.Wirtschaftswissenschaftl.Fak.Mainz\",\"format\":\"4'\"},\"library_reference\":{\"shelfmark\":\"Diss.Mainz 1954 (4')\",\"subjects\":\"Marke (Markenrecht). Deutschland\"}},\"name\":\"extract_structured_data\",\"type\":\"tool_use\",\"caller\":{\"type\":\"direct\"}}],\"model\":\"claude-sonnet-4-6\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"inference_geo\":\"global\",\"input_tokens\":2681,\"output_tokens\":222,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"f1_micro": 0.8714146341463415, "f1_macro": 0.862851711026616, "micro_precision": 0.8239852398523986, "micro_recall": 0.9246376811594202, "total_instances": 263, "total_tp": 2233, "total_fp": 477, "total_fn": 182, "cost_summary": {"total_input_tokens": 706230, "total_output_tokens": 57317, "total_tokens": 763547, "input_cost_usd": 2.11869, "output_cost_usd": 0.8597549999999998, "total_cost_usd": 2.978445000000001}}, "normalized_score": 86.2851711026616}, {"_id": {"$oid": "69d4eb12b877594bc5718cb7"}, "test_id": "T0657", "benchmark": "library_cards", "date": "2026-03-16", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.4-2026-03-05", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Sell\u00e8s\",\"first_name\":\"Manuel\"},\"publication\":{\"title\":\"De la contestation pour inexactitude de la reconnaissance des enfants naturels\",\"year\":\"1945\",\"place\":\"Alger\",\"pages\":\"160\",\"publisher\":\"la Typo-litho et J. Carbonel r\u00e9unies\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Diss.Alger Droit 1945.\",\"subjects\":\"Facult\u00e9 de droit\"}}", "model": "gpt-5.4-2026-03-05", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1682, "output_tokens": 115, "total_tokens": 1797, "input_cost_usd": 0.0042049999999999995, "output_cost_usd": 0.001725, "estimated_cost_usd": 0.0059299999999999995}, "duration": 2.4310460090637207, "timestamp": "2026-03-16T17:07:18.827460", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Sell\u00e8s", "first_name": "Manuel"}, "publication": {"title": "De la contestation pour inexactitude de la reconnaissance des enfants naturels", "year": "1945", "place": "Alger", "pages": "160", "publisher": "la Typo-litho et J. Carbonel r\u00e9unies", "format": "8\u00b0", "editor": null}, "library_reference": {"shelfmark": "Diss.Alger Droit 1945.", "subjects": "Facult\u00e9 de droit"}}, "conversation_id": "cbcda28a-ac8a-4fd1-bff7-bd1178b34e1a", "score": {"f1_score": 0.86, "precision": 0.8181818181818182, "recall": 0.9, "true_positives": 9, "false_positives": 2, "false_negatives": 1, "field_scores": {"publication.year": {"response": "1945", "ground_truth": 1945, "score": 1.0}, "library_reference.shelfmark": {"response": "Diss.Alger Droit 1945.", "ground_truth": "Diss.Alger Droit 1945.", "score": 1.0}, "author.first_name": {"response": "Manuel", "ground_truth": "Manuel", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "author.last_name": {"response": "Sell\u00e8s", "ground_truth": "Sell\u00e8s", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.title": {"response": "De la contestation pour inexactitude de la reconnaissance des enfants naturels", "ground_truth": "De la Contestation pour inexactitude de la reconnaissance des enfants naturels", "score": 0.9871794871794873}, "publication.publisher": {"response": "la Typo-litho et J. Carbonel r\u00e9unies", "ground_truth": "Impr. \u00ab la Typo-litho \u00bb et J. Carbonel r\u00e9unies", "score": 0.878048780487805}, "library_reference.subjects": {"response": "Facult\u00e9 de droit", "ground_truth": null, "score": 0.0}, "publication.place": {"response": "Alger", "ground_truth": "Alger", "score": 1.0}, "publication.pages": {"response": "160", "ground_truth": "160", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-DK4lP66bLpbkyIp5Z3cTP7RH140y3\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"type\\\":{\\\"type\\\":\\\"Dissertation or thesis\\\"},\\\"author\\\":{\\\"last_name\\\":\\\"Sell\u00e8s\\\",\\\"first_name\\\":\\\"Manuel\\\"},\\\"publication\\\":{\\\"title\\\":\\\"De la contestation pour inexactitude de la reconnaissance des enfants naturels\\\",\\\"year\\\":\\\"1945\\\",\\\"place\\\":\\\"Alger\\\",\\\"pages\\\":\\\"160\\\",\\\"publisher\\\":\\\"la Typo-litho et J. Carbonel r\u00e9unies\\\",\\\"format\\\":\\\"8\u00b0\\\",\\\"editor\\\":null},\\\"library_reference\\\":{\\\"shelfmark\\\":\\\"Diss.Alger Droit 1945.\\\",\\\"subjects\\\":\\\"Facult\u00e9 de droit\\\"}}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Sell\u00e8s\",\"first_name\":\"Manuel\"},\"publication\":{\"title\":\"De la contestation pour inexactitude de la reconnaissance des enfants naturels\",\"year\":\"1945\",\"place\":\"Alger\",\"pages\":\"160\",\"publisher\":\"la Typo-litho et J. Carbonel r\u00e9unies\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Diss.Alger Droit 1945.\",\"subjects\":\"Facult\u00e9 de droit\"}}}}],\"created\":1773677243,\"model\":\"gpt-5.4-2026-03-05\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":115,\"prompt_tokens\":1682,\"total_tokens\":1797,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.8310282621767889, "f1_macro": 0.8243346007604563, "micro_precision": 0.8053613053613053, "micro_recall": 0.8583850931677018, "total_instances": 263, "total_tp": 2073, "total_fp": 501, "total_fn": 342, "cost_summary": {"total_input_tokens": 444670, "total_output_tokens": 55735, "total_tokens": 500405, "input_cost_usd": 1.1116750000000013, "output_cost_usd": 0.836025, "total_cost_usd": 1.9476999999999995}}, "normalized_score": 82.43346007604563}, {"_id": {"$oid": "69d4eb12b877594bc5718cbb"}, "test_id": "T0674", "benchmark": "library_cards", "date": "2026-03-16", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3.1-flash-lite-preview", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wiedemann\",\"first_name\":\"August\"},\"publication\":{\"title\":\"Drei bemerkenswerthe F\u00e4lle von Spina bifida\",\"year\":\"1872\",\"place\":\"Breslau\",\"pages\":\"30\",\"publisher\":\"\",\"format\":\"8.\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Breslau. Diss. med. 1872\",\"subjects\":\"\"}}", "model": "gemini-3.1-flash-lite-preview", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1546, "output_tokens": 164, "total_tokens": 1710, "input_cost_usd": 0.0003865, "output_cost_usd": 0.000246, "estimated_cost_usd": 0.0006325}, "duration": 1.6756362915039062, "timestamp": "2026-03-16T19:09:08.261429", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wiedemann", "first_name": "August"}, "publication": {"title": "Drei bemerkenswerthe F\u00e4lle von Spina bifida", "year": "1872", "place": "Breslau", "pages": "30", "publisher": "", "format": "8."}, "library_reference": {"shelfmark": "Breslau. Diss. med. 1872", "subjects": ""}}, "conversation_id": "85b69be4-17b2-459e-99cf-60af776d5615", "score": {"f1_score": 0.88, "precision": 0.7777777777777778, "recall": 1.0, "true_positives": 7, "false_positives": 2, "false_negatives": 0, "field_scores": {"publication.pages": {"response": "30", "ground_truth": "30", "score": 1.0}, "author.last_name": {"response": "Wiedemann", "ground_truth": "Wiedemann", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": "Breslau. Diss. med. 1872", "ground_truth": null, "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.place": {"response": "Breslau", "ground_truth": "Breslau", "score": 1.0}, "publication.format": {"response": "8.", "ground_truth": null, "score": 0.0}, "publication.year": {"response": "1872", "ground_truth": 1872, "score": 1.0}, "author.first_name": {"response": "August", "ground_truth": "August", "score": 1.0}, "publication.title": {"response": "Drei bemerkenswerthe F\u00e4lle von Spina bifida", "ground_truth": "Drei bemerkenswerte F\u00e4lle von Spina bifida", "score": 0.9882352941176471}}, "total_fields": 11}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Mon, 16 Mar 2026 18:09:15 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=1665\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"media_resolution\":null,\"code_execution_result\":null,\"executable_code\":null,\"file_data\":null,\"function_call\":null,\"function_response\":null,\"inline_data\":null,\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Wiedemann\\\",\\n    \\\"first_name\\\": \\\"August\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Drei bemerkenswerthe F\u00e4lle von Spina bifida\\\",\\n    \\\"year\\\": \\\"1872\\\",\\n    \\\"place\\\": \\\"Breslau\\\",\\n    \\\"pages\\\": \\\"30\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"8.\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Breslau. Diss. med. 1872\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\",\"thought\":null,\"thought_signature\":\"EjQKMgG-Pvb7qyvBND9lk_lMwS9U_WNXXyF5HBQntFVOEgqDaLkiPltAmA_ENGY9vilRuu1T\",\"video_metadata\":null}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null,\"url_context_metadata\":null}],\"create_time\":null,\"model_version\":\"gemini-3.1-flash-lite-preview\",\"prompt_feedback\":null,\"response_id\":\"S0e4abyYBJ_pxN8P-Kr0wAw\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":164,\"candidates_tokens_details\":null,\"prompt_token_count\":1546,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":454},{\"modality\":\"IMAGE\",\"token_count\":1092}],\"thoughts_token_count\":null,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":1710,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wiedemann\",\"first_name\":\"August\"},\"publication\":{\"title\":\"Drei bemerkenswerthe F\u00e4lle von Spina bifida\",\"year\":\"1872\",\"place\":\"Breslau\",\"pages\":\"30\",\"publisher\":\"\",\"format\":\"8.\"},\"library_reference\":{\"shelfmark\":\"Breslau. Diss. med. 1872\",\"subjects\":\"\"}}}"}, "scoring": {"f1_micro": 0.8885782747603834, "f1_macro": 0.8809125475285171, "micro_precision": 0.8580794446586965, "micro_recall": 0.9213250517598344, "total_instances": 263, "total_tp": 2225, "total_fp": 368, "total_fn": 190, "cost_summary": {"total_input_tokens": 406240, "total_output_tokens": 47779, "total_tokens": 454019, "input_cost_usd": 0.10155999999999975, "output_cost_usd": 0.07166850000000002, "total_cost_usd": 0.1732285}}, "normalized_score": 88.09125475285171}, {"_id": {"$oid": "69d4eb12b877594bc5718c80"}, "test_id": "T0504", "benchmark": "library_cards", "date": "2026-03-12", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3-flash-preview", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Simpson\",\"first_name\":\"Marcus\"},\"publication\":{\"title\":\"Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale.\",\"year\":\"1898\",\"place\":\"M\u00fcnchen\",\"pages\":\"133\",\"publisher\":\"Kastner & Lossen\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"M\u00fcnchen, Phil. Diss. v. 1898\",\"subjects\":\"\"}}", "model": "gemini-3-flash-preview", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1520, "output_tokens": 113, "total_tokens": 3638, "input_cost_usd": 0.00076, "output_cost_usd": 0.000339, "estimated_cost_usd": 0.0010990000000000002}, "duration": 10.828226327896118, "timestamp": "2026-03-12T22:28:08.405623", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Simpson", "first_name": "Marcus"}, "publication": {"title": "Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale.", "year": "1898", "place": "M\u00fcnchen", "pages": "133", "publisher": "Kastner & Lossen", "format": "8\u00b0", "editor": null}, "library_reference": {"shelfmark": "M\u00fcnchen, Phil. Diss. v. 1898", "subjects": ""}}, "conversation_id": "aba0f400-270e-4697-971f-7cfc7eb6fc78", "score": {"f1_score": 0.95, "precision": 0.9, "recall": 1.0, "true_positives": 9, "false_positives": 1, "false_negatives": 0, "field_scores": {"type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": "Kastner & Lossen", "ground_truth": "Kastner & Lossen", "score": 1.0}, "author.last_name": {"response": "Simpson", "ground_truth": "Simpson", "score": 1.0}, "publication.title": {"response": "Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale.", "ground_truth": "Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale", "score": 0.9935483870967743}, "publication.year": {"response": "1898", "ground_truth": 1898, "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": "Marcus", "ground_truth": "Marcus", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "133", "ground_truth": "133", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "publication.place": {"response": "M\u00fcnchen", "ground_truth": "M\u00fcnchen", "score": 1.0}, "library_reference.shelfmark": {"response": "M\u00fcnchen, Phil. Diss. v. 1898", "ground_truth": null, "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Thu, 12 Mar 2026 21:28:13 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=10814\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"media_resolution\":null,\"code_execution_result\":null,\"executable_code\":null,\"file_data\":null,\"function_call\":null,\"function_response\":null,\"inline_data\":null,\"text\":\"{\\\"type\\\":{\\\"type\\\":\\\"Dissertation or thesis\\\"},\\\"author\\\":{\\\"last_name\\\":\\\"Simpson\\\",\\\"first_name\\\":\\\"Marcus\\\"},\\\"publication\\\":{\\\"title\\\":\\\"Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale.\\\",\\\"year\\\":\\\"1898\\\",\\\"place\\\":\\\"M\u00fcnchen\\\",\\\"pages\\\":\\\"133\\\",\\\"publisher\\\":\\\"Kastner & Lossen\\\",\\\"format\\\":\\\"8\u00b0\\\",\\\"editor\\\":null},\\\"library_reference\\\":{\\\"shelfmark\\\":\\\"M\u00fcnchen, Phil. Diss. v. 1898\\\",\\\"subjects\\\":\\\"\\\"}}\",\"thought\":null,\"thought_signature\":\"EoY1CoM1Ab4-9vvG6mb-fO-1OwgZyrYqMtHDFOoVl2JadWreHuJNK6HiGbcWEf-U6C77UqMISCc6H3z0EXtfEHlC2DxI6eHf3HsdXioSb0wmOUmcg2F6Hq61P72abY-ErCJwqqa_QPDzdijlzDkzojwdfZnoGgOs8JCaQJUJ8n-eYoqJUZcILC2ZMPuvxR-olvtlIfYhR1i6H3RW0VWshdDzssjOchDEskva32DL6UL7t_6EuQ1e0Iiw4vWiy77Z7JD43pzU6mz1OaGYmrED_Rf4loJ6q63EwJOJXrO7TQ6xiSRSu44WWFxJMmftWDmTaWIPZt3-mVEFq2Bb-0AQgSYiaYoWSds_j5BIi9cSBmGXiiX2S_EOISbCuiMjRQpvVenL2Tv9cqYLgvzCyzNXE17kWQy0esAkylYNAhpuWcCgDBADoEbTKxEls3mDYyHa93N-dTTPCV5Cya9MxecLUbJOERuJ3L90RHynyaY3vSPBBZdLbbsVelSz4z_87hBQF0Rnquov7CZkGmSH5_fcm9aTsqDTe28ziwKN8cAk1gtm3X6O0AKpcULod60g4okoxyeZptykbowU8hXinNySpLub2onTaOL0swV_baCTH815I6fkUd_3hwyXzdA6cGZ3NVxISh9mnxlZVkzR69MmQLQJwn0Vkv_cBSmJKDsBD4clfC6cbvl7qBsUO3BVfuY3wgG0o-ATyGRH6xwEgZ6XEqPiCyP6oaWkE8rQqCpvlTbQ2qmgNED-ZROO16L-5CYlv-hoI36rmq2wFNmpG2SL2zIuYZdYtfQjvB-s7NVmItoP8W-6SnfGYOg4Zd9OHyk2xCkGI3ftp8gCZEDAD-RHtfmmuOReg8ND9urYDnFo5ZZ8GwxC-7xn4uaaz575f58Wx01nBDCAFOivOAlvhRVg1r8wmn3KjQrYuePsbAEOl_zKJ15khwGyTwdSvjB8CRMVSL3Ugksp7fy47GrCWaEdSivXmW2zjAdnZPuKhyPhIlxBVsdbLpMIjaPRmV-IDPI1pQUJxT2yEYSdbUS6Vec8ck2e-uoexY6cvsHuEoAvZLxy2XPWq70DwLc8IqbY_2YI9LVYtqSE9trHWnCIGy7ioOxGUAhKC8dhb9KrE3MqPmdyjXF5gzw9PvTht75XThH2qev3zSSWN0zFjxKIMu1MRq_5XmN1h6EynyuIuWAUKUNWlswYDzekwi_ma_usjDiDwzFfx4wic9u230qidANEcM-rwxPe_BwyYiSNzu5wF6BhMJ5OBu7xuC4Dzj1nopH9q3CMFb-oqlURuAadDmgY4Z1F7cYlkM0EhMG4dj4AaAemdE4-uSxzNGQQ4wbcE3-BT0JmBmBgb7PlV-wdjMcCXClTjv_oTBPG1k7EPL-KeqiparicvvqDWDEenUnpe-BuDyZvpK0W9AokwQYWHfiCEUzCR2Ax9KbDHIhZzN0qZkhSlr8QbVRNsayBwG2PhrF6rRfWpgnDJfSMm1p-6ynzITUm8SHLJlLjLOTrQYvQQyZsVSFGdQVQFmIOFowEpZ0u_aFjgcrO12Z-t0motWZe3174EoYHHm_thYtF5wWaEWMXjUdZO9FdtEbh7ccW-neo--eRDJv8ivvSZ2_R9r2g2G6Y9zjICUJRJvLd09iNtRo63LMZ5nKdVuAtb0a96k7a1ySdnt6ZfP-J4Z3A9B5IZ0vPn7UsiksfrDR8JeA-ID4weat6HVOMegvVpMdlIcGrwWry1F1eU02K21BGLXVOc08qBeKhkLU9bT8yvCF19o2jcuFgTlZRwQbxqefvifQi0UMoDeXk1MXE47wGaoSLFqgBe82yKTjvbCVtt5CyX_7ID-t1z97Kd9SKF6ZT7JnmefQKs3-yhRMii_Cgzw-GUSSjjj3mpT0jDX9bm_arQiP0rhN09ugIseusNZB9BliL1EYbhSGrnG_CJAp8cVTUxyO8uRFt3epmv7fF_lEWzAE5FWbIxndgYinVGPVSoiKj-wYQdE8CuevKChX_yG1U5Liwj2ay_wur2Lftfgqt2JPSkpnTyq7qQ94kQ75WBp8clJi5nx_pZ3FegeY2Wc8Ez9oZ2qO6Tsf6NzdkTjLAzP6hhTi6wmV6wiUNXqkAmnXhewzV0UMuSs9TaKcp4Wajzv55LAl_8PoMZa1Kzd0WeLaZPVkhOucZUUwmCAYqJQf5WaY9Y5PIbKrIR5DMIId_S5L8Otn6QJ5VrAgW_uoBe9KXbbd9DgS-ZNjR6XCbbKoDkY5k4sRJVwbyke1QzLcNOXt9pFGpzTm6FCJIsz0XZZGaifb1ZQWEfC_amqtHi_FJw66-2xQ9UqSiN7cWj9qZuUaa5ElXhyrnVlyVvjJjAI33UmdclQT82_5_KWa3s4ewqIl_SkmrdnYhVJeM9V9mysk34eby8UNXGYzbAZ9kCtnmwI4h5cmVS6nTyoVwmwh16MgRtzVroxPkw5vXlP8E03U4IYJoVmdbo6qY-1GYTqMJyFyGuaU7ucmhGqe62cMojDu5XrqHQsAqXjtNo3_rQMbaVUnWUmdLTFqBBEcXAxjuKP9Qf2QfaiNIEf5c6za3syiTqW34O4Yb9XbHO4nkIsNIgMrqu--O69mumWFmy7-obWPVAFIIyeWoEx1vsV12WOpYp42iiR5CHq826Q6gbbMFO2gk5bdoLyJT1j9vtnCibperEMisGkOhbzoa0-8D6-sGAscdK0Jhx4arUHPG8MDBSPpTdAkEts9XgW9aK2p32qLzv2B_sNe9seFGNwpxY5p9lo4TE8ia6NhEv_deYyYpQ7-KS-sAVGooEgwKMF-Ujq0KwUUAT3GcztNU1MvW1IKmMx0FYmjvMm5PKccQgP3iRYC4q6b5w6ujcJ5faFFB2gIvHavz8rVXYPDBlRZBZt44youEfXSErT3HbeM1ZUiZHvhm5mE7th6mcyNwJsXRXLq69UxazqZkUZPTV4LriLNzoo1MRihmA_6YtdVdDfsbBscZCCf1_h9ljJQYP-uhZLsXfnHI4pVj3Gddmp4_ZJn7_nFjikXUu5lBwrglFb-icfUW910sR-c8kOGoGI3SDFhbNnQBZE7VxXkpTiR6w1dLMWXyIKO-tjjTY14ULG04Az_NSA8FsVOM7fkC9GOIA012BpURfJebnb5VqYpghrWPqida3f1c7zCa38cA3abYWsRBtnBI_H26Jwy7ROuecgWU_IEsG5Sv0kBdEUcyYcg_r9T735YrKZA1O6HwIQKw6N2YLHuVtxmycClgKL_R8BqmUtdR_MHivI901sGGxvo5RznIHEnGl8DLzG8QnXZxt-4bUyZabt9HNk3rXuGtvslbZOaP8KhylRdhQcPVR1DFWBauahntENwWRa5YYuZHFtJq8zrwZujh01Sv5smHSYMWSV1O9OuAwWSZTzOvlZIs4POdhZzY41x1xj-aEqlCND92PvO1iSWH0eXZSgnB1UVpGmWvnwGinzanOWJVoy-C85sNIVFO1ImwnXtGzOU_2pG3c8NevvvLZHeFqjYBYmyf5ns6v_E-k1GbzAgZcmfeCjqiPTzyqqwtr-17PqUjczGa3QqpCpzUohbzMWf2c4DjDUUrSGnrxbbTVGQU92injtRflZNAUEKBO69xT00KrZ2NDB7oqiNGysAURjuGKbkxZedA23uTp0nlGMc7qDafFV9oHx2BZFEzTLBmEFoxy9Jycr2XZFarPwpY9TSzmDmCdEAdWLmeuZa3c1kVw5ZbittV2CRO7Rmra_aVYaGOF-Zd0OiTJdJ0vCBrAD3PvxUqMwqjmE8j_-gPQtGZtV886BW7sAmgQ4zPiJ5H6dlN6IVB-UG97GE7AsV1qfK--pqvCiCQEOCutW310cESM-1X-dDTh5lnn0fbRuBpqLqp9lMAybO5GMBqwST-s6JbQZdjPhrV89Xczrk2xw1MxbBEnQ0x7Yp5JDzRXkRo5Fj4KKrZGHEG_2NRgK7kR9rH6TmSI1Tm5mLc7Wb9KkbjiepKigzK1QEROC5PYgq2OUP9TfmgQyWEvjkEktBJbJPKXoA16PTnHdnorCobw3HdT9FeNzxwgVTAEnCFDh0cBCiZ1wRS_HMW6DUM4_9D92w__8qUwnUfN7N4kLrFxW8DEiDK1TFNbVtKzRuRST5YtHLUKXq66qPeAUo4aBAZMO9OWzvmFoJAlaJHi2t2qhghSP_6BZxJooYWQs3arQMcyUZ6xd1sIi7d5sMz7MYw8pm6tySV7r_IUxj1vAGiRSjBE75AGaBhmbMkUEBhpwZMoijx2qG9XM8uwtKcHyO7s49_n0P07XBT9gBOy903uKnrtrCGwitRZILtTh91adjqEwnmrBXxZn2vSR3aKvLcaSE6cusbFT6SJz13jMg2yzHMs7ItOFZREwPFFm3TZN3MPFBs6Yh42BIxwZdyY6lIvMa9N1mfXtjm6VHPvwKa3Fhkl0xQlgGwURU6sLGufJeAeSY9r_IvJgUrSaFTKIs0TVHDt1vGh_-h_wknJU8H5aAwsi-Zt_hwsEiVPM5Cxa6QTIB_oBjjhksf3NSu7fZd1ZKDgwvLQo3BG3uzgqBNcmuRJnIQ1_rJtLo6gcjxGwuQG-PRDXDxcySORfCLhZvzHPnXJW-MFDljwtg5OZk6vAfMHuWNKc0rww_abnMGIT0sar0sQtq1NwFKgIGK7QW8FCDhisx_ip6kEOVrZ_rZMw5cxnoHpsPftqHzyfLdpmVcSGBbBrjDRo5r9MeNfYLXoFJ7QjuCEAjPz42gMWIinF2Cjeh4kPrDwDnqLRZaA2qar7H043SeF2eenZc5x-NHVNfNdoUBJSIwplqihU3PgSjWh5XQ1LMb7KyEp3hSFiulIxru8Hg97MaYZvknwbVWwD5jAWS711_5BfIOIxMyqYsYtwTZMQ4nRPnAwCVBsq2ZT4Ecwoa3OyDrSLbETbdlVqMO3e8E7Zbd6iXIPrxSwnkBySANbNMd2FAr9e1DrvuO7GJP5TEXpezX1wOngxU7KhleKFK6fnPV6z8zX25HBoZ0TxPl9ERzJpXS3E6-e-hKLByOVmTTsUvbv04klnbpDwJ-TuU0fCzsdtZe787dF4ZP3EDanKWOZNPZFCwne_TIJOz33uHoqA8T2jcQ-C1xDVQcKoFjNMWo3NmWacn15B_8SP-zMxKujLkXfyI1-oRAkMDcLbuz_uweHeUolcmCHdurHudrQYYgur_IDd7SpTM_qXrFEnnGe5uQKd0dso8aSWv8Ks3xZy8iDwTR9tiKulLTIse8BX-fZCIU0Njr3aWGvGAau55J114fMf-VQH03tiE1xRVUyyqo_U-nwgv5iQ12nMIizJ-BBjRwOdrimxHeDrf8CsFi19oofYfHleJB6zSgWmli3puBMTJ1XXT-0uFQIEY4w77Pysv3K5sHyrbdnH0Vd6YWlVNFB5qXn9Ppr_y89mfZD6OoQBtaCD4fE1_HVgv91kanSs-XxrOzeYRI5wrTYKLXHa1dl7CjaKpZkgA5SOQV-yO5UJllC6zlW95JUOvYjNfKPwRRs1sCJyqxpp6nKjlqEQ_SrUjxz4nNgO9zd0FMcAJuj3gPe3F7l0BzJzGGkn25k7JwDUpBVQX8aTcOUDFuHdOqavFj8pDsazx3bPqfjnloA2EWdj9FU_zMtFF-a8INCaX2a3Guph5Yx6JyrEW4WPTIdhGyoto4rBoJ-JjLuSjo6fPH1DzMY3sqBcWZj1HWtZf-nSl6r8IHTQZ-AWbYqWI4cAF6342WMl7jFR1ezgQyK0cffJyplEr3pK5zvz_AtZGurDRyJ9UyT3aCmu0zkNkv8vOuHEt-r0fhcH3mMR5Fgd6-qUX-L_UitOB3tcBHvx3hLoUTIvmx8kIQVVueXjaVXDgsBupPp9bRVu6WOF_lp9ly-vtkN0mI0rJRHc24CbEmE0uMmnvLCpiQB9eLgIa4LmeerwAjulz3B0mlBirQFEVqxnX658ZlaptXtRZTMDhBtfgta_3GmYiWUl3jDPs1wkL312RkRBzUwM0Hrc8popDmhITk_2Ev4dESc97MRmRbKN7CdNX8yTtLRp_K5ko-ntqsF3uUqofRaoicopdAXhEpgCSOg1Hrxqxsfp_uN67muF4DCD0XUNF3g2CM5kTf5Qie3W-ogQxfObQI6fpJHakb8tH9G0z5XPY9hJiJhRjHS-8s54fez_29rQMnEFq-Ash2N3Tn61bfW7zHvnpdf84TnGEVpyJolVOLtO0IdDB6lYtaiXjSOX_TLd591_W0bEFgz2N8fuZ0lhpGJj5vy7s_jTR_3fRMQusm2bChGSYnoo5jX7ngSp5dVKENOqAfpvkQ2v6y--y6ScevhMoQMGI_2i_H1i0Xa6k3SPvoVj0Tm763c3JL0ScsRWGPXl_E0GTAH41in43u_GbV68aj-KGRb1q3LESeXInD7_SBQa4s2KbRm2wIQgbK08D6nP98cY21owFFP91V8u_BGwWtC0Da9xhqpholqP2OpWeNA9i8i_EY5wiYTkr8K7RWCthc7QwtC6UeMqbxF3Bvo_zTZP1hMVdrdfVthnGX0wNEHxaq1r5npl3OJPfcyRiTPAww2O-agxNGh6TBmavRfP6jgKUbAkQ7n55Y1kP798xX-ehTnkv0o6UskLGp-q2pXT4QMtTIGTYwlxpzYQM1jKRp9iu7aLhXzjhxPA7laV7z5X8IuoymTvoFH5xZ2VG5B4TSmG90p3vlAmGzKcyaP9P5hqoHLXXXBNwzcVTsJpmBgjvFz5syaW_XgQo56ZvYDrStrpyP66chyGDS2GZJgzCuXqenZr5ySuw1i9KOWTczh468EuyMlh7UgkOFxBt-sFw9b1_C11N9oOQkhJNssQ96RSjzDPRBELTaZUEmX4FlU00dncnDKJJs000VQILv0zSMGXgQ115xE7M6EV0UkXwO7VWOQOST2FybbTsNHQn7GXrxBTMHZSeg8mPerp19DEN_cQB7Z1DStByz19V2Te7Ge-Zg6P-2ed8hKLy03EjcULbYVZWyZeOC7V3-pxVaJyu3Dz5w8glnCQ_LLZWszBePJ_jgBKLP6bim1cjIct4wnMpX3wzVe6PdOjr2v-Sj7CGqX1yd1hWWjXVtqxAVQT8ilUYmTYxcaV_jpwLwFNkLkzUzEY2wftF4QP3AFBjpzzGTO2c75Sk5kSEwXPIgfz-F9LLdinI-vSYcmwXYue8GU7Clxc9vHN39O2lMHvTAV2FFWynL1YcMAUvkTXTymyY1yWp6O9lPuNmewNJI0nyx72ZtnlsZ2aKPnFHITqJMgJUTLTBy8C2N2u0_jMJ5O28tHzKP-623ZfzpqbyWwCOGN0bEjNX1frCN55ftirhdNpwqPs03tVu8pe0_4TR0KnRkp0nYLdYzw4TyTDirNXD1PShQuszMHPARFcrB-MH_n5yDHgedIGabAUZ8Lz28Gunu2oUhKd52_M3Rfz5F3ZkqEqV5Wlny200qqs2EiGXDJazU52GHYKW23HPLR-micGFHmyxUuxQT6iFjJrQZrZ6XcbNsVGa0rhchPrFyZ_57ppi4uyrig6hyOBtPZ1FtibW6od9kMOjFgZdqtLNsFEtQ5bgrBBlLrymkGpZCS4CiEFhn5LlPf67MMJ8M0H1bZ1sUF7XewcM5dWDTPZMOMMOU3JgUPNVDEoI4ztRlpBM-AIS3lOOK-U7edSKO9VhdJWOx4Q9CLEesywXCJsDRuYTxul34QG3x_-GuhHVFMHH0d2bgYZ8sseCNpBffE1yVWMoSdwPR-XOaGqOm_M1UUTEFAl8i8OY-IRHxNJOo4LTAOUZ5g3kjoutlyQaAO7U21TdFTZw3IOvUdHW3rRbBcvSe28k_9Xpcjy2h7Z-zeW6_290DRsD-hxfaIYTDIhj_9d1L1mwZSKmBPDb-WfzdYOdyFq2j5TsWXxWBga4kbiTgLYCKxsCv9bjvXz_SzOeNh_EhsFZu-KmHnAgI3Wz4fRTNTB03SHOLAB1mjFVaLeteUMxInbsszUPPVt45nKkaZsUyVKjg3YQFKBUoMSTDvHdG4eMY1S-0fmAsOogo2XLC5LNzoslG67FxOOwycG757Hd55fpoxxAm2FL0nlhnNxWIj6l-_ZXjgGrkAlZgBGSMLG5FQ2oXlyecqCjbvCPfKYaVI9Ke8KvTz8z623C8Yt9405HKyXnNmirkW_SMFcQwrooUHg_6gBR1aBuHOnZkZjJMHuVs__S6d7-GH4ZUZnWkM-6P04ZJz19nGllM019lSFFRDVdSASUIrHFtMF20iZWaxBaWJ08CXxCiK4P6i1lfHf0UmUzKQN8yRzF5TleDkZiCFwCxv4pTEwAstzhKo6r5ZXTlqrAElPwKhhJbO-4-Dj3eVApCTBe6WYlrU4aGY6jN7FtmC9onybvY03OSdKDd71PeMD-8dXcUcSIH0SXLdp79hsCbgjga9oFmNdU7xavvvFpDOoH9RtTZ1bal1gu9AJVEBf-fhurxwcFB2pf7IpagyU_NscJVQp0Uhlf4xiDUHTnjk6jgofPxf-KrNZ3JRKCKlvbTlS6asW9eLrQGmWIFOq7Zozun-FSCfziZrOFgl8181vHUSBfJQxnomctJW1BzMeJNo015OnWI0vVN4pjDolSFkWf9uCCLYE7kj2ONK6YUGS7OLHzyigktaJPpO25AzJ4ZjdkXgddL-L3EbNIyCVD6j2RqLG_BGve5CeidjD8feAK-RL-UKUCIwrJLU3sSHroK2l3cTVSfEUq2drrwRQQZt7V1qMSde-vkOZWkBw7z3vtgYhz-2J0RYZCDJnt4SSkiZ0ToYz2BTS-Ibod-HzUMzFzUI_7T_MzuzRXZj9n5l54n3vq2oIAWBsTo_Acgl0VBAO9BgKMfmEGtT4URQyCMf5cJPL5y_CCWjKTbCwwgWHPqN2OC1KvYZd9tvODFVKEuNNyg8Sm9etG5VeeB2FxtaaGTHWgdfxQLn1-_1odzSD8KA5PMBUxrFnrEjbEPU7C3PaVP7UtNUfb4xTtcWx0aIjgqSILXNj_yElS2VfH0sRyHBbSVhzJarIoR9vohNuG2HZJTLVerhx4oRrrjcPj1IhoEeSbpNAOvlRyGP7DDrhpxqGbZPvnczv-G-Q==\",\"video_metadata\":null}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null,\"url_context_metadata\":null}],\"create_time\":null,\"model_version\":\"gemini-3-flash-preview\",\"prompt_feedback\":null,\"response_id\":\"7S-zabzOKdukkdUP15jE6Q4\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":113,\"candidates_tokens_details\":null,\"prompt_token_count\":1520,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":454},{\"modality\":\"IMAGE\",\"token_count\":1066}],\"thoughts_token_count\":2005,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":3638,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Simpson\",\"first_name\":\"Marcus\"},\"publication\":{\"title\":\"Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale.\",\"year\":\"1898\",\"place\":\"M\u00fcnchen\",\"pages\":\"133\",\"publisher\":\"Kastner & Lossen\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"M\u00fcnchen, Phil. Diss. v. 1898\",\"subjects\":\"\"}}}"}, "scoring": {"f1_micro": 0.8608644168146831, "f1_macro": 0.8511787072243345, "micro_precision": 0.8223981900452488, "micro_recall": 0.9031055900621118, "total_instances": 263, "total_tp": 2181, "total_fp": 471, "total_fn": 234, "cost_summary": {"total_input_tokens": 406240, "total_output_tokens": 39872, "total_tokens": 446112, "input_cost_usd": 0.2031199999999995, "output_cost_usd": 0.11961600000000001, "total_cost_usd": 0.32273599999999986}}, "normalized_score": 85.11787072243345}, {"_id": {"$oid": "69d4eb12b877594bc5718c77"}, "test_id": "T0166", "benchmark": "library_cards", "date": "2026-02-17", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Schoofs\",\"first_name\":\"F.\"},\"publication\":{\"title\":\"Les impuret\u00e9s des alcools, eaux-de-vie & geni\u00e8vres fabriqu\u00e9s en Belgique\",\"year\":\"place\\\": \\\"Li\u00e8ge\",\"place\":\"Li\u00e8ge\",\"pages\":\"111, 8 Tab.\",\"publisher\":\"\",\"format\":\"\",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Diss. Li\u00e8ge 1912\",\"subjects\":\"\"}}", "model": "gpt-5-mini-2025-08-07", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1719, "output_tokens": 1457, "total_tokens": 3176, "input_cost_usd": 0.00042975, "output_cost_usd": 0.002914, "estimated_cost_usd": 0.00334375}, "duration": 14.821260690689087, "timestamp": "2026-02-17T13:24:43.563371", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Schoofs", "first_name": "F."}, "publication": {"title": "Les impuret\u00e9s des alcools, eaux-de-vie & geni\u00e8vres fabriqu\u00e9s en Belgique", "year": "place\": \"Li\u00e8ge", "place": "Li\u00e8ge", "pages": "111, 8 Tab.", "publisher": "", "format": "", "editor": ""}, "library_reference": {"shelfmark": "Diss. Li\u00e8ge 1912", "subjects": ""}}, "conversation_id": "a63607da-29d0-4e6d-b58d-9469fb6b6acb", "score": {"f1_score": 0.62, "precision": 0.625, "recall": 0.625, "true_positives": 5, "false_positives": 3, "false_negatives": 3, "field_scores": {"publication.title": {"response": "Les impuret\u00e9s des alcools, eaux-de-vie & geni\u00e8vres fabriqu\u00e9s en Belgique", "ground_truth": "Les impuret\u00e9s des alcools, eaux-de-vie & geni\u00e8vres fabriqu\u00e9s en Belgique", "score": 1.0}, "author.first_name": {"response": "F.", "ground_truth": "F.", "score": 1.0}, "publication.place": {"response": "Li\u00e8ge", "ground_truth": "Li\u00e8ge", "score": 1.0}, "library_reference.shelfmark": {"response": "Diss. Li\u00e8ge 1912", "ground_truth": null, "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.year": {"response": "place\": \"Li\u00e8ge", "ground_truth": 1912, "score": 0.0}, "author.last_name": {"response": "Schoofs", "ground_truth": "Schoofs", "score": 1.0}, "publication.format": {"response": null, "ground_truth": "8", "score": 0.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "111, 8 Tab.", "ground_truth": "111", "score": 0.4285714285714286}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-DAEPt7v8vIfckmsBksuToJxJJGByh\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Schoofs\\\",\\n    \\\"first_name\\\": \\\"F.\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Les impuret\u00e9s des alcools, eaux-de-vie & geni\u00e8vres fabriqu\u00e9s en Belgique\\\",\\n    \\\"year\\\": \\n     \\\"place\\\\\\\": \\\\\\\"Li\u00e8ge\\\",\\n    \\\"place\\\": \\\"Li\u00e8ge\\\",\\n    \\\"pages\\\": \\\"111, 8 Tab.\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"\\\"\\n  ,\\n  \\\"editor\\\": \\\"\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss. Li\u00e8ge 1912\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Schoofs\",\"first_name\":\"F.\"},\"publication\":{\"title\":\"Les impuret\u00e9s des alcools, eaux-de-vie & geni\u00e8vres fabriqu\u00e9s en Belgique\",\"year\":\"place\\\": \\\"Li\u00e8ge\",\"place\":\"Li\u00e8ge\",\"pages\":\"111, 8 Tab.\",\"publisher\":\"\",\"format\":\"\",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Diss. Li\u00e8ge 1912\",\"subjects\":\"\"}}}}],\"created\":1771331069,\"model\":\"gpt-5-mini-2025-08-07\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":1457,\"prompt_tokens\":1719,\"total_tokens\":3176,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":1280,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.6287290047630986, "f1_macro": 0.5111406844106464, "micro_precision": 0.7966963151207116, "micro_recall": 0.5192546583850932, "total_instances": 263, "total_tp": 1254, "total_fp": 320, "total_fn": 1161, "cost_summary": {"total_input_tokens": 282332, "total_output_tokens": 467444, "total_tokens": 749776, "input_cost_usd": 0.07058299999999997, "output_cost_usd": 0.9348879999999998, "total_cost_usd": 1.0054710000000002}}, "normalized_score": 51.11406844106464}, {"_id": {"$oid": "69d4eb12b877594bc5718c07"}, "test_id": "T0168", "benchmark": "library_cards", "date": "2026-01-26", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "o3", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Z\u00e4hle\",\"first_name\":\"Werner\"},\"publication\":{\"title\":\"Ergebnisse von Bilirubinbestimmungen im Duodenalsaft von Gesunden u. Leberkranken m.d. Methode v. Jendrassik u. Cleghorn. Auszug a.d. Inaug.-Diss. Med. Fak. M\u00fcnster.\",\"year\":\"1952\",\"place\":\"M\u00fcnster (Westf.)\",\"pages\":\"\",\"publisher\":\"Westf. Vereinsdruckerei\",\"format\":\"8'\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Diss. M\u00fcnster 1952\",\"subjects\":\"\"}}", "model": "o3-2025-04-16", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1649, "output_tokens": 546, "total_tokens": 2195, "input_cost_usd": 0.003298, "output_cost_usd": 0.004368, "estimated_cost_usd": 0.007666000000000001}, "duration": 11.924891948699951, "timestamp": "2026-01-26T21:03:45.184777", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Z\u00e4hle", "first_name": "Werner"}, "publication": {"title": "Ergebnisse von Bilirubinbestimmungen im Duodenalsaft von Gesunden u. Leberkranken m.d. Methode v. Jendrassik u. Cleghorn. Auszug a.d. Inaug.-Diss. Med. Fak. M\u00fcnster.", "year": "1952", "place": "M\u00fcnster (Westf.)", "pages": "", "publisher": "Westf. Vereinsdruckerei", "format": "8'", "editor": null}, "library_reference": {"shelfmark": "Diss. M\u00fcnster 1952", "subjects": ""}}, "conversation_id": "d6c430f5-e83c-4857-b282-db24251df7f3", "score": {"f1_score": 0.89, "precision": 0.8888888888888888, "recall": 0.8888888888888888, "true_positives": 8, "false_positives": 1, "false_negatives": 1, "field_scores": {"publication.publisher": {"response": "Westf. Vereinsdruckerei", "ground_truth": "Westf. Vereinsdruckerei", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.title": {"response": "Ergebnisse von Bilirubinbestimmungen im Duodenalsaft von Gesunden u. Leberkranken m.d. Methode v. Jendrassik u. Cleghorn. Auszug a.d. Inaug.-Diss. Med. Fak. M\u00fcnster.", "ground_truth": "Ergebnisse von Bilirubinbestimmungen im Duodenalsaft von Gesunden u. Leberkranken m.d. Methode v. Jendrassik u. Cleghorn", "score": 0.8421052631578947}, "publication.place": {"response": "M\u00fcnster (Westf.)", "ground_truth": "M\u00fcnster (Westf.)", "score": 1.0}, "author.first_name": {"response": "Werner", "ground_truth": "Werner", "score": 1.0}, "publication.pages": {"response": null, "ground_truth": null, "score": 1.0}, "publication.format": {"response": "8'", "ground_truth": "8'", "score": 1.0}, "author.last_name": {"response": "Z\u00e4hle", "ground_truth": "Z\u00e4hle", "score": 1.0}, "publication.year": {"response": "1952", "ground_truth": 1952, "score": 1.0}, "library_reference.shelfmark": {"response": "Diss. M\u00fcnster 1952", "ground_truth": "Diss.M\u00fcnster 1952", "score": 0.9714285714285714}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-D2N65pDGCC6yY6iNGhDj5b7eCaVqb\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"type\\\":{\\\"type\\\":\\\"Dissertation or thesis\\\"},\\\"author\\\":{\\\"last_name\\\":\\\"Z\u00e4hle\\\",\\\"first_name\\\":\\\"Werner\\\"},\\\"publication\\\":{\\\"title\\\":\\\"Ergebnisse von Bilirubinbestimmungen im Duodenalsaft von Gesunden u. Leberkranken m.d. Methode v. Jendrassik u. Cleghorn. Auszug a.d. Inaug.-Diss. Med. Fak. M\u00fcnster.\\\",\\\"year\\\":\\\"1952\\\",\\\"place\\\":\\\"M\u00fcnster (Westf.)\\\",\\\"pages\\\":\\\"\\\",\\\"publisher\\\":\\\"Westf. Vereinsdruckerei\\\",\\\"format\\\":\\\"8'\\\",\\\"editor\\\":null},\\\"library_reference\\\":{\\\"shelfmark\\\":\\\"Diss. M\u00fcnster 1952\\\",\\\"subjects\\\":\\\"\\\"}}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Z\u00e4hle\",\"first_name\":\"Werner\"},\"publication\":{\"title\":\"Ergebnisse von Bilirubinbestimmungen im Duodenalsaft von Gesunden u. Leberkranken m.d. Methode v. Jendrassik u. Cleghorn. Auszug a.d. Inaug.-Diss. Med. Fak. M\u00fcnster.\",\"year\":\"1952\",\"place\":\"M\u00fcnster (Westf.)\",\"pages\":\"\",\"publisher\":\"Westf. Vereinsdruckerei\",\"format\":\"8'\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Diss. M\u00fcnster 1952\",\"subjects\":\"\"}}}}],\"created\":1769457813,\"model\":\"o3-2025-04-16\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":546,\"prompt_tokens\":1649,\"total_tokens\":2195,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":384,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.8360075015628257, "f1_macro": 0.8299619771863118, "micro_precision": 0.8414429530201343, "micro_recall": 0.8306418219461698, "total_instances": 263, "total_tp": 2006, "total_fp": 378, "total_fn": 409, "cost_summary": {"total_input_tokens": 433687, "total_output_tokens": 190890, "total_tokens": 624577, "input_cost_usd": 0.8673740000000041, "output_cost_usd": 1.5271199999999996, "total_cost_usd": 2.394494}}, "normalized_score": 82.99619771863118}, {"_id": {"$oid": "69d4eb12b877594bc5718c1f"}, "test_id": "T0493", "benchmark": "library_cards", "date": "2026-01-26", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.2-2025-12-11", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "", "model": "gpt-5.2", "provider": "openai", "finish_reason": "error", "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, "duration": 22.381165027618408, "timestamp": "2026-01-27T00:19:36.060951", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 8, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": null, "ground_truth": "Worms", "score": 0.0}, "publication.title": {"response": null, "ground_truth": "De natura et methodo Sociologi\u00e6", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1896, "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "103", "score": 0.0}, "library_reference.shelfmark": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": null, "ground_truth": "Ren\u00e9", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8\u00b0", "score": 0.0}, "publication.place": {"response": null, "ground_truth": "Lutetia Parisiorum", "score": 0.0}}, "total_fields": 12}}, "scoring": {"f1_micro": 0.46877774489493934, "f1_macro": 0.32532319391634984, "micro_precision": 0.8215767634854771, "micro_recall": 0.3279503105590062, "total_instances": 263, "total_tp": 792, "total_fp": 172, "total_fn": 1623, "cost_summary": {"total_input_tokens": 185513, "total_output_tokens": 55962, "total_tokens": 241475, "input_cost_usd": 0.3246477499999993, "output_cost_usd": 0.7834679999999998, "total_cost_usd": 1.1081157500000003}}, "normalized_score": 32.532319391634985}, {"_id": {"$oid": "69d4eb12b877594bc5718c27"}, "test_id": "T0165", "benchmark": "library_cards", "date": "2026-01-26", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Opitz\",\"first_name\":\"Hermann\"},\"publication\":{\"title\":\"Synthesen in der Imidazolreihe\",\"year\":\"1912\",\"place\":\"Freiburg i. B.\",\"pages\":\"34\",\"publisher\":\"Wagner\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"U 12. 1261\",\"subjects\":\"\"}}", "model": "gpt-5-2025-08-07", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1604, "output_tokens": 1887, "total_tokens": 3491, "input_cost_usd": 0.002005, "output_cost_usd": 0.01887, "estimated_cost_usd": 0.020875}, "duration": 47.0479474067688, "timestamp": "2026-01-26T09:05:46.694239", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Opitz", "first_name": "Hermann"}, "publication": {"title": "Synthesen in der Imidazolreihe", "year": "1912", "place": "Freiburg i. B.", "pages": "34", "publisher": "Wagner", "format": "8\u00b0", "editor": null}, "library_reference": {"shelfmark": "U 12. 1261", "subjects": ""}}, "conversation_id": "3a6093cd-d717-4ead-ba6d-b4d0d2dfb679", "score": {"f1_score": 1.0, "precision": 1.0, "recall": 1.0, "true_positives": 10, "false_positives": 0, "false_negatives": 0, "field_scores": {"author.last_name": {"response": "Opitz", "ground_truth": "Opitz", "score": 1.0}, "publication.year": {"response": "1912", "ground_truth": 1912, "score": 1.0}, "author.first_name": {"response": "Hermann", "ground_truth": "Hermann", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": "U 12. 1261", "ground_truth": "U 12. 1261", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "publication.title": {"response": "Synthesen in der Imidazolreihe", "ground_truth": "Synthesen in der Imidazolreihe", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.place": {"response": "Freiburg i. B.", "ground_truth": "Freiburg i. B.", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": "Wagner", "ground_truth": "Wagner", "score": 1.0}, "publication.pages": {"response": "34", "ground_truth": "34", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-D2Bsi7wwYg6M4Wkbx4CJzloIOIx4Z\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Opitz\\\",\\n    \\\"first_name\\\": \\\"Hermann\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Synthesen in der Imidazolreihe\\\",\\n    \\\"year\\\":    \\\"1912\\\",\\n    \\\"place\\\": \\\"Freiburg i. B.\\\",\\n    \\\"pages\\\": \\\"34\\\",\\n    \\\"publisher\\\": \\\"Wagner\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  ,\\n  \\\"editor\\\": null},\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"U 12. 1261\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Opitz\",\"first_name\":\"Hermann\"},\"publication\":{\"title\":\"Synthesen in der Imidazolreihe\",\"year\":\"1912\",\"place\":\"Freiburg i. B.\",\"pages\":\"34\",\"publisher\":\"Wagner\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"U 12. 1261\",\"subjects\":\"\"}}}}],\"created\":1769414700,\"model\":\"gpt-5-2025-08-07\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":1887,\"prompt_tokens\":1604,\"total_tokens\":3491,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":1728,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.7107072691552062, "f1_macro": 0.5955133079847909, "micro_precision": 0.8732649366324683, "micro_recall": 0.5991718426501035, "total_instances": 263, "total_tp": 1447, "total_fp": 210, "total_fn": 968, "cost_summary": {"total_input_tokens": 287116, "total_output_tokens": 414407, "total_tokens": 701523, "input_cost_usd": 0.35889499999999946, "output_cost_usd": 4.144069999999998, "total_cost_usd": 4.502964999999999}}, "normalized_score": 59.551330798479086}, {"_id": {"$oid": "69d4eb12b877594bc5718c2a"}, "test_id": "T0167", "benchmark": "library_cards", "date": "2026-01-26", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wittenberger\",\"first_name\":\"Joachim\"},\"publication\":{\"title\":\"Unbenutzte Warenzeichen.\",\"year\":\"1954\",\"place\":\"Mainz\",\"pages\":\"104\",\"publisher\":\"\",\"format\":\"4'\",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Diss.Mainz 1954 (4')\",\"subjects\":\"Markenrecht\"}}", "model": "gpt-5-nano-2025-08-07", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1905, "output_tokens": 2395, "total_tokens": 4300, "input_cost_usd": 9.525000000000001e-05, "output_cost_usd": 0.0009580000000000001, "estimated_cost_usd": 0.0010532500000000001}, "duration": 23.765233278274536, "timestamp": "2026-01-26T16:55:44.459111", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wittenberger", "first_name": "Joachim"}, "publication": {"title": "Unbenutzte Warenzeichen.", "year": "1954", "place": "Mainz", "pages": "104", "publisher": "", "format": "4'", "editor": ""}, "library_reference": {"shelfmark": "Diss.Mainz 1954 (4')", "subjects": "Markenrecht"}}, "conversation_id": "ce460790-baa9-4752-a969-74ffc41fac43", "score": {"f1_score": 0.63, "precision": 0.6, "recall": 0.6666666666666666, "true_positives": 6, "false_positives": 4, "false_negatives": 3, "field_scores": {"publication.title": {"response": "Unbenutzte Warenzeichen.", "ground_truth": "Unbenutzte Warenzeichen", "score": 0.9787234042553191}, "author.first_name": {"response": "Joachim", "ground_truth": "Joachim", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.format": {"response": "4'", "ground_truth": "4'", "score": 1.0}, "library_reference.subjects": {"response": "Markenrecht", "ground_truth": "Marke (Markenrecht). Deutschland", "score": 0.5116279069767442}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": "Diss.Mainz 1954 (4')", "ground_truth": "Diss.Mainz 1954", "score": 0.8571428571428572}, "publication.pages": {"response": "104", "ground_truth": "104", "score": 1.0}, "publication.year": {"response": "1954", "ground_truth": 1954, "score": 1.0}, "publication.place": {"response": "Mainz", "ground_truth": null, "score": 0.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "author.last_name": {"response": "Wittenberger", "ground_truth": "Wittenberg", "score": 0.9090909090909091}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-D2JDtns4j6aFWjtLbVF48lTc8Xxzr\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Wittenberger\\\",\\n    \\\"first_name\\\": \\\"Joachim\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Unbenutzte Warenzeichen.\\\",\\n    \\\"year\\\": \\t\\\"1954\\\",\\n    \\\"place\\\": \\\"Mainz\\\",\\n    \\\"pages\\\": \\\"104\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"4'\\\"\\n  ,\\n  \\\"editor\\\": \\\"\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss.Mainz 1954 (4')\\\",\\n    \\\"subjects\\\": \\\"Markenrecht\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wittenberger\",\"first_name\":\"Joachim\"},\"publication\":{\"title\":\"Unbenutzte Warenzeichen.\",\"year\":\"1954\",\"place\":\"Mainz\",\"pages\":\"104\",\"publisher\":\"\",\"format\":\"4'\",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Diss.Mainz 1954 (4')\",\"subjects\":\"Markenrecht\"}}}}],\"created\":1769442921,\"model\":\"gpt-5-nano-2025-08-07\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":2395,\"prompt_tokens\":1905,\"total_tokens\":4300,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":2240,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.7375917134225292, "f1_macro": 0.701596958174905, "micro_precision": 0.7701667417755745, "micro_recall": 0.7076604554865424, "total_instances": 263, "total_tp": 1709, "total_fp": 510, "total_fn": 706, "cost_summary": {"total_input_tokens": 446998, "total_output_tokens": 668986, "total_tokens": 1115984, "input_cost_usd": 0.022349900000000065, "output_cost_usd": 0.2675943999999999, "total_cost_usd": 0.28994430000000004}}, "normalized_score": 70.1596958174905}, {"_id": {"$oid": "69d4eb12b877594bc5718bb1"}, "test_id": "T0559", "benchmark": "library_cards", "date": "2026-01-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "ministral-8b-2512", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n  \"type\": \"object\",\n  \"properties\": {\n    \"type\": {},\n    \"author\": {},\n    \"publication\": {},\n    \"library_reference\": {}\n  },\n  \"example\": {\n    \"type\": {\n      \"type\": \"Dissertation or thesis\"\n    },\n    \"author\": {\n      \"last_name\": \"Smith\",\n      \"first_name\": \"John\"\n    },\n    \"publication\": {\n      \"title\": \"The Evolution of Medieval Trade Routes\",\n      \"year\": 2012,\n      \"place\": \"Cambridge\",\n      \"pages\": \"245\",\n      \"publisher\": \"University Press\",\n      \"format\": \"8\u00b0\"\n    },\n    \"library_reference\": {\n      \"shelfmark\": \"Diss. 12345\",\n      \"subjects\": \"Medieval History, Trade, Europe\"\n    }\n  }\n}", "model": "ministral-8b-2512", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 1196, "output_tokens": 208, "total_tokens": 1404, "input_cost_usd": 0.0001794, "output_cost_usd": 3.12e-05, "estimated_cost_usd": 0.0002106}, "duration": 1.1554198265075684, "timestamp": "2026-01-25T00:54:00.519084", "conversation_id": "9a88788f-1d0a-471f-be1e-d492ec2a8995", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 10, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "86", "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": "Selbstverl. d. Roman. Seminars", "score": 0.0}, "author.first_name": {"response": null, "ground_truth": "Fritz", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1921, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "Giessen", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.title": {"response": null, "ground_truth": "Die franz\u00f6sischen Bezeichnungen des Modehelden im 18. und 19. Jahrhundert", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "U 21. 6523", "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Usinger", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8\u00b0", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"8a382586bd40439ebf7de345cdc4c318\",\"object\":\"chat.completion\",\"model\":\"ministral-8b-2512\",\"usage\":{\"prompt_tokens\":1196,\"completion_tokens\":208,\"total_tokens\":1404,\"num_cached_tokens\":1216},\"created\":1769298839,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"$schema\\\": \\\"http://json-schema.org/draft-07/schema#\\\",\\n  \\\"type\\\": \\\"object\\\",\\n  \\\"properties\\\": {\\n    \\\"type\\\": {},\\n    \\\"author\\\": {},\\n    \\\"publication\\\": {},\\n    \\\"library_reference\\\": {}\\n  },\\n  \\\"example\\\": {\\n    \\\"type\\\": {\\n      \\\"type\\\": \\\"Dissertation or thesis\\\"\\n    },\\n    \\\"author\\\": {\\n      \\\"last_name\\\": \\\"Smith\\\",\\n      \\\"first_name\\\": \\\"John\\\"\\n    },\\n    \\\"publication\\\": {\\n      \\\"title\\\": \\\"The Evolution of Medieval Trade Routes\\\",\\n      \\\"year\\\": 2012,\\n      \\\"place\\\": \\\"Cambridge\\\",\\n      \\\"pages\\\": \\\"245\\\",\\n      \\\"publisher\\\": \\\"University Press\\\",\\n      \\\"format\\\": \\\"8\u00b0\\\"\\n    },\\n    \\\"library_reference\\\": {\\n      \\\"shelfmark\\\": \\\"Diss. 12345\\\",\\n      \\\"subjects\\\": \\\"Medieval History, Trade, Europe\\\"\\n    }\\n  }\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.0, "f1_macro": 0.0, "micro_precision": 0.0, "micro_recall": 0.0, "total_instances": 263, "total_tp": 0, "total_fp": 0, "total_fn": 2415, "cost_summary": {"total_input_tokens": 314548, "total_output_tokens": 52935, "total_tokens": 367483, "input_cost_usd": 0.047182200000000236, "output_cost_usd": 0.007940250000000008, "total_cost_usd": 0.05512245}}, "normalized_score": 0}, {"_id": {"$oid": "69d4eb12b877594bc5718bb8"}, "test_id": "T0264", "benchmark": "library_cards", "date": "2026-01-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-8b-instruct", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Vallotton\",\"first_name\":\"Pierre\"},\"publication\":{\"title\":\"Le Christ et la foi. Etude de th\u00e9ologie biblique\",\"year\":\"1960\",\"place\":\"Gen\u00e8ve\",\"pages\":\"171\",\"publisher\":\"Labor et Fides\",\"format\":\"8'\",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Kirch.Bibl.Conv.44 Nr.13\",\"subjects\":\"2/Theologie,biblische/Glaube an Christus/ NK RW Diss\"}}", "model": "qwen/qwen3-vl-8b-instruct", "provider": "openrouter", "finish_reason": "stop", "usage": {"input_tokens": 1912, "output_tokens": 143, "total_tokens": 2055, "input_cost_usd": 0.00015296, "output_cost_usd": 7.15e-05, "estimated_cost_usd": 0.00022446}, "duration": 2.059335470199585, "timestamp": "2026-01-25T09:04:40.616310", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Vallotton", "first_name": "Pierre"}, "publication": {"title": "Le Christ et la foi. Etude de th\u00e9ologie biblique", "year": "1960", "place": "Gen\u00e8ve", "pages": "171", "publisher": "Labor et Fides", "format": "8'", "editor": ""}, "library_reference": {"shelfmark": "Kirch.Bibl.Conv.44 Nr.13", "subjects": "2/Theologie,biblische/Glaube an Christus/ NK RW Diss"}}, "conversation_id": "110720d8-d15e-4323-a4ba-bbab9261abfc", "score": {"f1_score": 0.91, "precision": 0.9090909090909091, "recall": 0.9090909090909091, "true_positives": 10, "false_positives": 1, "false_negatives": 1, "field_scores": {"publication.title": {"response": "Le Christ et la foi. Etude de th\u00e9ologie biblique", "ground_truth": "Le Christ et la foi. Etude de th\u00e9ologie biblique", "score": 1.0}, "library_reference.subjects": {"response": "2/Theologie,biblische/Glaube an Christus/ NK RW Diss", "ground_truth": "2/Theologie, biblische/Glaube an Christus/ NK RW Diss", "score": 0.9904761904761905}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "library_reference.shelfmark": {"response": "Kirch.Bibl.Conv.44 Nr.13", "ground_truth": "f c 3954", "score": 0.1875}, "author.last_name": {"response": "Vallotton", "ground_truth": "Vallotton", "score": 1.0}, "author.first_name": {"response": "Pierre", "ground_truth": "Pierre", "score": 1.0}, "publication.pages": {"response": "171", "ground_truth": "171", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.year": {"response": "1960", "ground_truth": 1960, "score": 1.0}, "publication.publisher": {"response": "Labor et Fides", "ground_truth": "Labor et Fides", "score": 1.0}, "publication.place": {"response": "Gen\u00e8ve", "ground_truth": "Gen\u00e8ve", "score": 1.0}, "publication.format": {"response": "8'", "ground_truth": "8'", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"gen-1769328278-QEIknUn1r4x8fuexyBJM\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"author\\\": {\\n    \\\"first_name\\\": \\\"Pierre\\\",\\n    \\\"last_name\\\": \\\"Vallotton\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Kirch.Bibl.Conv.44 Nr.13\\\",\\n    \\\"subjects\\\": \\\"2/Theologie,biblische/Glaube an Christus/ NK RW Diss\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"editor\\\": \\\"\\\",\\n    \\\"format\\\": \\\"8'\\\",\\n    \\\"pages\\\": \\\"171\\\",\\n    \\\"place\\\": \\\"Gen\u00e8ve\\\",\\n    \\\"publisher\\\": \\\"Labor et Fides\\\",\\n    \\\"title\\\": \\\"Le Christ et la foi. Etude de th\u00e9ologie biblique\\\",\\n    \\\"year\\\":  \\\"1960\\\"\\n  },\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Vallotton\",\"first_name\":\"Pierre\"},\"publication\":{\"title\":\"Le Christ et la foi. Etude de th\u00e9ologie biblique\",\"year\":\"1960\",\"place\":\"Gen\u00e8ve\",\"pages\":\"171\",\"publisher\":\"Labor et Fides\",\"format\":\"8'\",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Kirch.Bibl.Conv.44 Nr.13\",\"subjects\":\"2/Theologie,biblische/Glaube an Christus/ NK RW Diss\"}},\"reasoning\":null},\"native_finish_reason\":\"stop\"}],\"created\":1769328278,\"model\":\"qwen/qwen3-vl-8b-instruct\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":143,\"prompt_tokens\":1912,\"total_tokens\":2055,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0,\"video_tokens\":0},\"cost\":0.00044426,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.00034416,\"upstream_inference_completions_cost\":0.0001001}},\"provider\":\"Alibaba\"}"}, "scoring": {"f1_micro": 0.7013698630136986, "f1_macro": 0.6158555133079848, "micro_precision": 0.7816793893129771, "micro_recall": 0.6360248447204969, "total_instances": 263, "total_tp": 1536, "total_fp": 429, "total_fn": 879, "cost_summary": {"total_input_tokens": 438981, "total_output_tokens": 42635, "total_tokens": 481616, "input_cost_usd": 0.035118480000000014, "output_cost_usd": 0.021317499999999993, "total_cost_usd": 0.05643598}}, "normalized_score": 61.58555133079848}, {"_id": {"$oid": "69d4eb12b877594bc5718bb9"}, "test_id": "T0548", "benchmark": "library_cards", "date": "2026-01-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "ministral-14b-2512", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"\",\"first_name\":\"\"},\"publication\":{\"title\":\"\",\"year\":\"\",\"place\":\"\",\"pages\":\"\",\"publisher\":\"\",\"format\":\"\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"\",\"subjects\":\"\"}}", "model": "ministral-14b-2512", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 1196, "output_tokens": 706, "total_tokens": 1902, "input_cost_usd": 0.00023920000000000001, "output_cost_usd": 0.00014120000000000002, "estimated_cost_usd": 0.00038040000000000003}, "duration": 4.910412549972534, "timestamp": "2026-01-25T00:21:36.642042", "parsed": {"$schema": "http://json-schema.org/draft-07/schema#", "$defs": {"Author": {"description": "Represents an author on a library card.", "properties": {"last_name": {"title": "Last Name", "type": "string"}, "first_name": {"title": "First Name", "type": "string"}}, "required": ["last_name"], "title": "Author", "type": "object"}, "LibraryReference": {"description": "Represents library-specific reference information.", "properties": {"shelfmark": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Shelfmark"}, "subjects": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Subjects"}}, "title": "LibraryReference", "type": "object"}, "Publication": {"description": "Represents publication information from a library card.", "properties": {"title": {"title": "Title", "type": "string"}, "year": {"title": "Year", "type": "string"}, "place": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Place"}, "pages": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Pages"}, "publisher": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Publisher"}, "format": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Format"}}, "required": ["title"], "title": "Publication", "type": "object"}, "WorkType": {"description": "Represents the type of work referenced in a library card.", "properties": {"type": {"enum": ["Dissertation or thesis", "Reference"], "title": "Type", "type": "string"}}, "required": ["type"], "title": "WorkType", "type": "object"}}, "type": {"type": "Dissertation or thesis"}, "author": {"last_name": "", "first_name": ""}, "publication": {"title": "", "year": "", "place": "", "pages": "", "publisher": "", "format": ""}, "library_reference": {"shelfmark": "", "subjects": ""}}, "conversation_id": "33563819-5fe4-4515-a42f-68a63490caf7", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 47, "false_negatives": 6, "field_scores": {"$defs.Publication.properties.format.default": {"response": null, "ground_truth": null, "score": 1.0}, "$defs.Publication.properties.publisher.anyOf[0].type": {"response": "string", "ground_truth": null, "score": 0.0}, "$defs.Author.properties.first_name.type": {"response": "string", "ground_truth": null, "score": 0.0}, "author.first_name": {"response": null, "ground_truth": "Ernst", "score": 0.0}, "$defs.Publication.properties.pages.title": {"response": "Pages", "ground_truth": null, "score": 0.0}, "$defs.LibraryReference.properties.shelfmark.title": {"response": "Shelfmark", "ground_truth": null, "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1941, "score": 0.0}, "$defs.WorkType.properties.type.title": {"response": "Type", "ground_truth": null, "score": 0.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Reference", "score": 0.25806451612903225}, "$defs.Publication.properties.title.title": {"response": "Title", "ground_truth": null, "score": 0.0}, "$defs.Publication.type": {"response": "object", "ground_truth": null, "score": 0.0}, "$defs.WorkType.title": {"response": "WorkType", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.publisher.title": {"response": "Publisher", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.format.anyOf[0].type": {"response": "string", "ground_truth": null, "score": 0.0}, "$defs.LibraryReference.properties.subjects.anyOf[0].type": {"response": "string", "ground_truth": null, "score": 0.0}, "$defs.LibraryReference.properties.shelfmark.anyOf[1].type": {"response": "null", "ground_truth": null, "score": 0.0}, "$schema": {"response": "http://json-schema.org/draft-07/schema#", "ground_truth": null, "score": 0.0}, "$defs.LibraryReference.properties.subjects.anyOf[1].type": {"response": "null", "ground_truth": null, "score": 0.0}, "$defs.LibraryReference.title": {"response": "LibraryReference", "ground_truth": null, "score": 0.0}, "$defs.LibraryReference.type": {"response": "object", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.title.type": {"response": "string", "ground_truth": null, "score": 0.0}, "$defs.Author.description": {"response": "Represents an author on a library card.", "ground_truth": null, "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.format": {"response": null, "ground_truth": null, "score": 1.0}, "$defs.Author.required[0]": {"response": "last_name", "ground_truth": null, "score": 0.0}, "$defs.LibraryReference.properties.subjects.default": {"response": null, "ground_truth": null, "score": 1.0}, "$defs.LibraryReference.properties.shelfmark.default": {"response": null, "ground_truth": null, "score": 1.0}, "publication.title": {"response": null, "ground_truth": "Die verschiedenen Fassungen in dem Werk des Bischofs Otto von Freising \u00fcber die Taten Kaiser Friedrichs I.", "score": 0.0}, "$defs.Publication.properties.year.type": {"response": "string", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.place.title": {"response": "Place", "ground_truth": null, "score": 0.0}, "$defs.Publication.description": {"response": "Represents publication information from a library card.", "ground_truth": null, "score": 0.0}, "$defs.Author.properties.first_name.title": {"response": "First Name", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.pages.default": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": null, "score": 1.0}, "$defs.Publication.properties.year.title": {"response": "Year", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.pages.anyOf[0].type": {"response": "string", "ground_truth": null, "score": 0.0}, "$defs.LibraryReference.properties.shelfmark.anyOf[0].type": {"response": "string", "ground_truth": null, "score": 0.0}, "$defs.Author.title": {"response": "Author", "ground_truth": null, "score": 0.0}, "publication.pages": {"response": null, "ground_truth": null, "score": 1.0}, "$defs.WorkType.properties.type.type": {"response": "string", "ground_truth": null, "score": 0.0}, "$defs.WorkType.properties.type.enum[1]": {"response": "Reference", "ground_truth": null, "score": 0.0}, "$defs.LibraryReference.properties.subjects.title": {"response": "Subjects", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.publisher.default": {"response": null, "ground_truth": null, "score": 1.0}, "$defs.WorkType.required[0]": {"response": "type", "ground_truth": null, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "B.-Steglitz", "score": 0.0}, "$defs.LibraryReference.description": {"response": "Represents library-specific reference information.", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.pages.anyOf[1].type": {"response": "null", "ground_truth": null, "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Korsch", "score": 0.0}, "$defs.WorkType.description": {"response": "Represents the type of work referenced in a library card.", "ground_truth": null, "score": 0.0}, "$defs.Author.properties.last_name.title": {"response": "Last Name", "ground_truth": null, "score": 0.0}, "$defs.WorkType.properties.type.enum[0]": {"response": "Dissertation or thesis", "ground_truth": null, "score": 0.0}, "$defs.Author.properties.last_name.type": {"response": "string", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.place.default": {"response": null, "ground_truth": null, "score": 1.0}, "$defs.Publication.properties.format.anyOf[1].type": {"response": "null", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.publisher.anyOf[1].type": {"response": "null", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.place.anyOf[1].type": {"response": "null", "ground_truth": null, "score": 0.0}, "$defs.Publication.required[0]": {"response": "title", "ground_truth": null, "score": 0.0}, "$defs.Publication.properties.format.title": {"response": "Format", "ground_truth": null, "score": 0.0}, "$defs.WorkType.type": {"response": "object", "ground_truth": null, "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "$defs.Publication.properties.place.anyOf[0].type": {"response": "string", "ground_truth": null, "score": 0.0}, "$defs.Publication.title": {"response": "Publication", "ground_truth": null, "score": 0.0}, "$defs.Author.type": {"response": "object", "ground_truth": null, "score": 0.0}}, "total_fields": 64}, "raw_response": "{\"id\":\"b8b59a5e4f2b48d495a2e8692f02c0c0\",\"object\":\"chat.completion\",\"model\":\"ministral-14b-2512\",\"usage\":{\"prompt_tokens\":1196,\"completion_tokens\":706,\"total_tokens\":1902,\"num_cached_tokens\":1216},\"created\":1769296891,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"$schema\\\": \\\"http://json-schema.org/draft-07/schema#\\\",\\n  \\\"$defs\\\": {\\n    \\\"Author\\\": {\\n      \\\"description\\\": \\\"Represents an author on a library card.\\\",\\n      \\\"properties\\\": {\\n        \\\"last_name\\\": {\\n          \\\"title\\\": \\\"Last Name\\\",\\n          \\\"type\\\": \\\"string\\\"\\n        },\\n        \\\"first_name\\\": {\\n          \\\"title\\\": \\\"First Name\\\",\\n          \\\"type\\\": \\\"string\\\"\\n        }\\n      },\\n      \\\"required\\\": [\\\"last_name\\\"],\\n      \\\"title\\\": \\\"Author\\\",\\n      \\\"type\\\": \\\"object\\\"\\n    },\\n    \\\"LibraryReference\\\": {\\n      \\\"description\\\": \\\"Represents library-specific reference information.\\\",\\n      \\\"properties\\\": {\\n        \\\"shelfmark\\\": {\\n          \\\"anyOf\\\": [\\n            {\\\"type\\\": \\\"string\\\"},\\n            {\\\"type\\\": \\\"null\\\"}\\n          ],\\n          \\\"default\\\": null,\\n          \\\"title\\\": \\\"Shelfmark\\\"\\n        },\\n        \\\"subjects\\\": {\\n          \\\"anyOf\\\": [\\n            {\\\"type\\\": \\\"string\\\"},\\n            {\\\"type\\\": \\\"null\\\"}\\n          ],\\n          \\\"default\\\": null,\\n          \\\"title\\\": \\\"Subjects\\\"\\n        }\\n      },\\n      \\\"title\\\": \\\"LibraryReference\\\",\\n      \\\"type\\\": \\\"object\\\"\\n    },\\n    \\\"Publication\\\": {\\n      \\\"description\\\": \\\"Represents publication information from a library card.\\\",\\n      \\\"properties\\\": {\\n        \\\"title\\\": {\\n          \\\"title\\\": \\\"Title\\\",\\n          \\\"type\\\": \\\"string\\\"\\n        },\\n        \\\"year\\\": {\\n          \\\"title\\\": \\\"Year\\\",\\n          \\\"type\\\": \\\"string\\\"\\n        },\\n        \\\"place\\\": {\\n          \\\"anyOf\\\": [\\n            {\\\"type\\\": \\\"string\\\"},\\n            {\\\"type\\\": \\\"null\\\"}\\n          ],\\n          \\\"default\\\": null,\\n          \\\"title\\\": \\\"Place\\\"\\n        },\\n        \\\"pages\\\": {\\n          \\\"anyOf\\\": [\\n            {\\\"type\\\": \\\"string\\\"},\\n            {\\\"type\\\": \\\"null\\\"}\\n          ],\\n          \\\"default\\\": null,\\n          \\\"title\\\": \\\"Pages\\\"\\n        },\\n        \\\"publisher\\\": {\\n          \\\"anyOf\\\": [\\n            {\\\"type\\\": \\\"string\\\"},\\n            {\\\"type\\\": \\\"null\\\"}\\n          ],\\n          \\\"default\\\": null,\\n          \\\"title\\\": \\\"Publisher\\\"\\n        },\\n        \\\"format\\\": {\\n          \\\"anyOf\\\": [\\n            {\\\"type\\\": \\\"string\\\"},\\n            {\\\"type\\\": \\\"null\\\"}\\n          ],\\n          \\\"default\\\": null,\\n          \\\"title\\\": \\\"Format\\\"\\n        }\\n      },\\n      \\\"required\\\": [\\\"title\\\"],\\n      \\\"title\\\": \\\"Publication\\\",\\n      \\\"type\\\": \\\"object\\\"\\n    },\\n    \\\"WorkType\\\": {\\n      \\\"description\\\": \\\"Represents the type of work referenced in a library card.\\\",\\n      \\\"properties\\\": {\\n        \\\"type\\\": {\\n          \\\"enum\\\": [\\\"Dissertation or thesis\\\", \\\"Reference\\\"],\\n          \\\"title\\\": \\\"Type\\\",\\n          \\\"type\\\": \\\"string\\\"\\n        }\\n      },\\n      \\\"required\\\": [\\\"type\\\"],\\n      \\\"title\\\": \\\"WorkType\\\",\\n      \\\"type\\\": \\\"object\\\"\\n    }\\n  },\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"\\\",\\n    \\\"first_name\\\": \\\"\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"\\\",\\n    \\\"year\\\": \\\"\\\",\\n    \\\"place\\\": \\\"\\\",\\n    \\\"pages\\\": \\\"\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.01897198323406133, "f1_macro": 0.006121673003802282, "micro_precision": 0.020302171860245515, "micro_recall": 0.01780538302277433, "total_instances": 263, "total_tp": 43, "total_fp": 2075, "total_fn": 2372, "cost_summary": {"total_input_tokens": 314548, "total_output_tokens": 161558, "total_tokens": 476106, "input_cost_usd": 0.06290960000000015, "output_cost_usd": 0.03231160000000003, "total_cost_usd": 0.09522120000000023}}, "normalized_score": 0.6121673003802282}, {"_id": {"$oid": "69d4eb12b877594bc5718bbc"}, "test_id": "T0504", "benchmark": "library_cards", "date": "2026-01-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3-flash-preview", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Simpson\",\"first_name\":\"Marcus\"},\"publication\":{\"title\":\"Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale.\",\"year\":\"1898\",\"place\":\"M\u00fcnchen\",\"pages\":\"133\",\"publisher\":\"Kastner & Lossen\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"M\u00fcnchen, Phil. Diss. v. 1898\",\"subjects\":\"\"}}", "model": "gemini-3-flash-preview", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1520, "output_tokens": 184, "total_tokens": 3103, "input_cost_usd": 0.00076, "output_cost_usd": 0.000552, "estimated_cost_usd": 0.001312}, "duration": 12.943895816802979, "timestamp": "2026-01-25T03:22:56.185114", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Simpson", "first_name": "Marcus"}, "publication": {"title": "Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale.", "year": "1898", "place": "M\u00fcnchen", "pages": "133", "publisher": "Kastner & Lossen", "format": "8\u00b0", "editor": null}, "library_reference": {"shelfmark": "M\u00fcnchen, Phil. Diss. v. 1898", "subjects": ""}}, "conversation_id": "f89b419b-877a-486b-8dbc-ebdfe103b735", "score": {"f1_score": 0.95, "precision": 0.9, "recall": 1.0, "true_positives": 9, "false_positives": 1, "false_negatives": 0, "field_scores": {"library_reference.shelfmark": {"response": "M\u00fcnchen, Phil. Diss. v. 1898", "ground_truth": null, "score": 0.0}, "publication.year": {"response": "1898", "ground_truth": 1898, "score": 1.0}, "publication.title": {"response": "Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale.", "ground_truth": "Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale", "score": 0.9935483870967743}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "author.last_name": {"response": "Simpson", "ground_truth": "Simpson", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": "Kastner & Lossen", "ground_truth": "Kastner & Lossen", "score": 1.0}, "author.first_name": {"response": "Marcus", "ground_truth": "Marcus", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "133", "ground_truth": "133", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.place": {"response": "M\u00fcnchen", "ground_truth": "M\u00fcnchen", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sun, 25 Jan 2026 02:22:56 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=12930\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":\"Ep0kCpokAXLI2nzrXr-TExPqSGFNrYiTNj0-zSQuoRKALVrfYDBxKQQW3wVEHD_27cbijiBzu8xBfK76qexZnapbe_hk5uutQ-T092q9kDZwsKD8ZjhoBLH10KIuqJK91vaUj6cRROJYnwl3aoVC4elJKoPzte25aJD3uZBFWuBLdrneW8L5yIuvOi9ekYWZccmvH-yKr-3G5YZDv4UkN-DPhEFNnB3YJQMKKj7tOYjqpUe9na3JRJiA_SCONwt6L1suHpyOzUwTHo-6-bB9IQRTeiWI3akxQHu0akvNyWJ0vhXsSfy_1JQcOS_2IONomh3TfuTk21xtFIH1J2T9QZexWJJIYHanE051e5HQ5JYnjrBLwruP-_kT_uugVu6WlWM94ZLIcHv61Y-EvEX3B3ZnLYPMqb-mUz6ly1Gkg8Zvs33tviYuH134_O76DKZ-5kcQ8dAyNebk3CI1g8H-IOXmm_jadhrakSEdVt7JDipaKlV3mz2DbUb5xyf4Z6hZjeYpr1z835s8baO9wKrWdIIWPvDuImkRNKxAR4n26GIqAdkwUPaRsbMlC1R_KF3ok0PMPQfjS1BttiXqEhMkqJHcAr1qUu9pV1iNLi3AWCgE06ILVp4X9SMcaRSyW2j4KjIFfgfUOQmp0di_H0x_2YZdXRzryV6Io4WS5lPbnbc4EVzFlH3zrcj36GyJG3MfX2XLFMNkHeyF8gQvQLi55kL4174aswPkPiaO6g8HtHhRdvof-nc7uKYNS5gkSdiFJdg7NJdcFKyrd2i6FPlkUR-4dQ7zswyhO0_U0g0scXP9IJiBt_S0EpmzjMldTbyOGOJHtwAzL0LtdHZHn_rTfjoyDjtOiPb2EQT3Zu7GuptkJqlsFIy2EHtb_Qp_NY2DyYBHMIUYsDNVpCWPoZCnV8ktQhSc-iZsTjA7JrV6TeHFfBqLMwK8zu7ObCubw7-igYqtascuUsP927t7mNjvRyBuolcpUVFmwibnLDW_HQD5Svqjetnz3XbcSVX1nQIAlTGp096MrrjgD6OwMdt79ne3NPsSglVXpS3hg9Y00Nhh39EIjcO6FC4Mc5591nysGhvlHyiwKforFArAqOLrJq0oD9RZnuzWdVaUr6l6akIObDjfsd51f_v56Nz763FKLqG2TeU_IOcxcx3zwH7rpir555zWAtzOiTTcagsHuulUS4j6C7qByOCTMGqXx7JdzwWvH7YhB04XjcnLVJWw5fvtzKECu6MbEKqyRY65wvbDpMxRD0XUEbqGSYFK9nA1dfPwl2t17CFyzXVQ74RLYtIKieFv0SckctsVFGtdhCsrZ3Uw91cwVX7iLlAyso3Fa1oQrvX0TcFxCjT1oWGI5FxRrvQHHQuDqFvKGmMzylReenRLrJPlFTTcKwuqQmW2er-hRlSMC3FhFV4ZLY888nXuCRJFjPq_4hyuqRfdqFA-60cO7NaBdI1gtqO1KuhQ933QEko0FHWMDmXLY7I8-NZQ1jOTqO4PYrX_U33yNM-_FAzmZ6ZAixAviENvXXZZH0_9HPCTEcMx5IzoJNPMz1pi9Zr38QG_pKtXCui7OmMgXrHJ0VkJeveWGgapiklkQWnXpnbyk-e_nXuD25c-4zlQtBAPjfR10v9FEBAHVh0XGrvZMD6w6eoVtn9FzY4b_QUikfJqZW6F5qu14hP8gxt9TZVvR0u3bCSQMWF0UNfRvTU1asXuciuL-e3oMdI5V_z8gXJ2Vqheq9hGd6Dq9-y_caiZG6sTZfWcSTwwTfvLRLQwIh9CECmui6HbxyCrN3RFaXwBnZfT6sOg9GXkwC01YvtbeiPEsNIA8ayV643PVxF0igvQFbljU1m9RaPdCvEBcaR4Rknz9f3V10lhHl7vDYoXDmfJ-KnO-42x5tCpEKioRRFP_QIeOAa8Th9GCuQrpwl4yn12LspTbMj2FdfOt5UoredZzQxlK6BO4p12zQP788NzV8D4ZHV51v2X4Le4O-za9gDNRcm12oYJuq5Oqb_7Y59gtjKubFxWEOLRF-phyT4T4CAxCdiv_34Q2pZf_6D80gU8vu240dF9aJu7QTdj_EqCquRXEceZSZO6XpMX8Ck5htXy2GOU0AQLxBTp_wYPKYo_0KVtfl8ic0wHVrQjbrlBPXKJY8jo5ahTj67YoBWX8WB0y29yfuzqD8aorw6vv6Orkgabk6PNA2kSA55nEr_Wkw70rfNAo7BrDQK8wY8Szuvm96oTO0Pd8UOsTVUzJjrM2i8rUqehIvEwIsbuBnhfyD7rCfJJ_wtL70fzHrUYGxi5RYjYxFzmBHkicbGgof7N1NFK9joqGCfcYJGKQwEgyWAfJK7TgjyLQGUV3fgiwilj0nOglDabvGm_Y93oXsbqoahCP5f-qUFFi4mceAA4orH09mO_dr6WlVhQRCfe_VWcHBGOXp9L9y-vvJCtTv4ShqM9VDT_gv-eO3UbtfC168iri4fKUOYz2ZHIYmyENAtBbRPNYrLUHoZ5nsF8Teb1GJ_AOWLiNWTInxGClrQTyWAA0HlEv6TVlLZ3RILnhMIwKQMTbacHTjsWnNLWX0RNlIdV4gN3DJBzBse1EuNYQ4uxWQsmdZ3EuZ3tD7dAQ32QvNYw0g5rrgvPT5wgI9cP8ztCBGksFxdDmbisOmJQysKm9st1uPMHgDZf9Zuw5gRrjtM1JXNkflfeXA8UhWzIFtKhhsWp7BwCeqcz75jAR-p_2CJMmBwS8htBIl1BHeZU_GZTT17XUFQYC3XqxXG83feWo5V2Mxaqprtp7H_LHQQr6yJU_kzpKG5gdL-8PMUYlBir0ghL39RE5tteSCjhjvLDfzNCUjH3osT64alFitMP0eMPJ6BbjQEvAdHAhfo3xdZls1oUT7HY2PGBuyx95uljza3eQd9eTdU5MwtGv596MObIkjCoFlW1roB_jlXDuGXgMlJ1EQGsF3q534ResTrLcS8gu4zlI-vz6Qm_WVA0B-O5fmBMU-cksO0wrNEe-QpvhbrY4px48L4-YUlKmBukpeldNlPnEPAum6gTNGu2IYCmB7bUyxzbtD4NpIQ1mEs_VQxM4flA-gytjCG6t64MvLaCoM7kR7idzTUYbikxHq3Jg5oEXbLTmr8ZNctr-_6nU9MlbUCEcF2hvj1N_9iwkn8nzBL-_IMZRfhZyzuGtvXPHRJOssskJ5lK1IPnLOhD-9k4rk8PJVrXpB-tJAd_vH0BOp8-Xft74MPF_-lDsqVnaZfwVUaCUDsn8p6nyccr5gM43LTpbGOPtF1c6v76G3YDacsHetkQW0eRAMrIu5Sz0RIzkqF9CYylc0BikaI3cZdrU7RxgRD9ANrnXUjvhJBIWgpp80sL_NtdyvWUr7URTmaujOFgNZSpjbQrPhGdc94BfEEdyWv8A4bWeDr4bGaNXcft_U3NM618D0xThq8rmJG_32ZSXc9iJs13B6FY4vnOPsS7iYLxdAh0zq_vlwLne2NNRPAjwJPQyTFrVUSOdR-lrUHcd_OgOb8wqkcqrcBmZ_i9i_twzvIFuRAQdNNaSvbjiZ4s3XBpFbDCIdKsJIKedyFeN9Fp5xzkZJBIXSqyXj7HquKSHW5wO1_s9MSZscEiiFeYLXDjLRSJ1BZ_DXjHnJaWjf4sjDlsNRxVRxqQ_t7ZVuOcL7LhQhsYjwXgvX4lu46K-3jaob-1ztantA8hRwTAAXjMn42yd7TT8h8-o2__KQfnATJjUp9GvmSZqbLPevSFP-_8u2Tnn3ubhTbU89OmhmJllHpz1grUXPbO48oJEXrzdWKRBXc40SwHbEaNrSww0uwbBScLLP5W0noi4F97y6owuPPgJ0z3zxzuXwQTL9PU4RuMQTufIGvxZrJAtXyx916PjylhwfMgg5OP-oo5kZ0-rXVmHoygr10U-6pt2x9MFH4lkjDyQ2j3k_5K4F9oYHBK1GjaQMR8dPTNJBcTxw4KsoFlVRd8zOHHzMaba832y6q4kcRXyDduEKG5OQUEGJ0BNKwE4KAj-z_xmsADXiResC0PK1DLcXZoFKi1UWvamj3UzS8ADKc8mtpasrqjneI8cw4A9PScuOZzgmy5yiao5AcOwXgfbnF9dw8qLSGIUWU4qGvHTSX79f_9uTayDzsQznthegKdJFcxrrGSE8MSFejkSYsla3K1NJUpFF4K51RJAkHojiDGoxGc25orzjNLxxj5dTnnPrJjP_Rtddp2JV16U5eTxXp89tW89Fp087Ce9RZLjgmhGCm21ba0jdDZ3ceQbzfVD4Bq-7iWx3BPTqyFLZ8sf_7mdA9cr7YqvWievD9UYNnIG7MlZv7JvFmmRzDFHJg8dEQ0dPBhfCXGFmGuVd57esCG2LPPpwXxp3JqcaZHCPwhees6gnnvgy3W6i5ZkT99rQbyg-IP3UCrN04xE2ZeUDGWLVkq6hfMX-Md5AJPGyQs6mrUmM9gHH9qI1etW3QDCM1mlVfUyeAaWd4CtAzVDSvMLaAhx-LFlMprTbTWztcsowPShasGUZCMR99OvADS7JPBxzBTa4IpHZESqx-9cSoSgXHdvRV3N7VJ0SlJH2Lxhu9nTo8VFWa1_PR3XTw48OANmDkeAOk3s5WfRqzAvsi0_7NtrNDU9mC8F4mOVvR7B6oqyotzM3EB7mz0iwSy_9k2k7kDTfhHCOrXFFLHaD05f_8xY6ggtHugidKhEPomV8c1hIhhXetY8kFx3O9sLE91hDR2LyLDDWL3g5g-ut3tcl1QsDwWV9HryTav10CNE-Qb1-_2gpTSeEmiC9aDOqCfCVybI2t4haadeiIcd58gAc9K0GZKJ23t2hBh45N5d_8Fd82XujX_X8eHMCoavKTMzQ-8aUgPMxBI2Sd__Oe_8-1aPstT9iAevfhDhNEcwcADFRQz2tvL71PucX154g_zZeusaXXehDvIMZabdHPbwt-RKXPxSfJHonp2WbOFHT14nDMYFeqLX0krOA6gwSgnI3QMXJI_dt4y4b3627GLdN5UG5fsX60xlh-bIt5_lTi__lWYQf6y5_fYZxrfi0twu5o6CcvOaKpkcomxaXPR_xCNaje_UX00Q_-O6-H7FvgRENyUQM2uYmgUN4l0aaEqRFKzXosghuIr8N9mHQ9QXRCd6Zz2WGM0J8qnKNHZQQofgSfA6iM_7_BeIM7wtQan_PxQfLn5xCyfEowBCow3PRXuixkv8DHATFlYjIFlZ_2a_SyLk5c0DzuuH6uqtjewsBzqUEzzWLgAWvxz9SkTVaUhLeGjfQjuKnEoIHIKNEyGgLIjleBq09UVO5OuCKMKF05Oa5A9DvIEL1FiPdWv3dwKQaghJRE8kz8988HCA0tg0bYjAtgQ9oOwBCwch4zlaANa3Hwm6GYaTcB8tkculqUXXpxYZVMQd-7zoEVCg976wFNvGvtVMGIXr5mLzAKd0CxkEdDoVm8SC68SKwiiy1a2or9mhDfJC7XoPr5Fl-uFpyGGZUpYrMkUFoILd_RQyM44AdwSN4byu0UgAbe9ZeQpv1v8-1DQOZT9Z9ci0nh0N-nd9Vdldd1uT7acx75YFfam-Yn-DjV6iwHQwS2E_1Q05ndeaFqbUnn9MwPqqaWdqDk-toLI_j-BeLvKGvft7HJuk46c-hlHXf0FRg9EslTqaSyVXwwTSoG2bOuHPwe3qVZpoiW_5PycZuk4LB85D0PaLZ-D-H6PFcDV-AJsXA4xilL5t3u3Bx9ZZAP20CKSqXnERAPzPdU_DYWrjgMjKWHaJPDPcxBGTUlzq5h1C84QI_Cj_zhIucy4aqwdp3HfKjmmdyra1cQ-QjunHZTymDTbv7-ICPijBUwapTumJnyId2TqxWNFYhTcDToHYWZhN22lhooFhCyGBSScY7MInhU9kY9MTms-Aa7-W4LopMDUGohknK0hRbxs6v4tDnrLTrrsNEuhA6xc0tyUVu_dydravzPKvoOWGKo8jwC_gLCyP_8qHLW8QbWNqi0zUPJ8dT_Asw3heD01vk-XSgCgygTwfUmvPDZD4RikGuW6ZNRxcy_2plxiil-awZhw01mOYIkKtbyIrDEhYtA7JtkUMImN6OEWUzeawlekmv_DCp0C3nKGGPS8GhhqQSWPSeMSALtEOzXF-Wqe-1QzCNoUTlLthOo9F_rnCn-6aBSJWl9BnKsj9jEYdKyNP_xx-MWddPzTdNGm4Fb3gjTdrpE=\",\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Simpson\\\",\\n    \\\"first_name\\\": \\\"Marcus\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale.\\\",\\n    \\\"year\\\": \\\"1898\\\",\\n    \\\"place\\\": \\\"M\u00fcnchen\\\",\\n    \\\"pages\\\": \\\"133\\\",\\n    \\\"publisher\\\": \\\"Kastner & Lossen\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\",\\n    \\\"editor\\\": null\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"M\u00fcnchen, Phil. Diss. v. 1898\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-3-flash-preview\",\"prompt_feedback\":null,\"response_id\":\"gH51adOlDLWLkdUP-dKO-Qo\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":184,\"candidates_tokens_details\":null,\"prompt_token_count\":1520,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":454},{\"modality\":\"IMAGE\",\"token_count\":1066}],\"thoughts_token_count\":1399,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":3103,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Simpson\",\"first_name\":\"Marcus\"},\"publication\":{\"title\":\"Eine Vergleichung der Wieland'schen Shakespeare-\u00dcbersetzung mit dem Originale.\",\"year\":\"1898\",\"place\":\"M\u00fcnchen\",\"pages\":\"133\",\"publisher\":\"Kastner & Lossen\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"M\u00fcnchen, Phil. Diss. v. 1898\",\"subjects\":\"\"}}}"}, "scoring": {"f1_micro": 0.8657957244655582, "f1_macro": 0.8561977186311788, "micro_precision": 0.8293515358361775, "micro_recall": 0.9055900621118013, "total_instances": 263, "total_tp": 2187, "total_fp": 450, "total_fn": 228, "cost_summary": {"total_input_tokens": 406240, "total_output_tokens": 38275, "total_tokens": 444515, "input_cost_usd": 0.2031199999999995, "output_cost_usd": 0.11482500000000012, "total_cost_usd": 0.31794500000000003}}, "normalized_score": 85.61977186311788}, {"_id": {"$oid": "69d4eb12b877594bc5718bc4"}, "test_id": "T0570", "benchmark": "library_cards", "date": "2026-01-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "magistral-small-2509", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"$defs\": {\"Author\": {\"description\": \"Represents an author on a library card.\\n\\nAttributes:\\n    last_name: Author's last name\\n    first_name: Author's first name\", \"properties\": {\"last_name\": {\"title\": \"Last Name\", \"type\": \"string\"}, \"first_name\": {\"title\": \"First Name\", \"type\": \"string\"}}, \"required\": [\"last_name\", \"first_name\"], \"title\": \"Author\", \"type\": \"object\"}, \"LibraryReference\": {\"description\": \"Represents library-specific reference information.\\n\\nAttributes:\\n    shelfmark: Library shelfmark or call number\\n    subjects: Subject classifications or tags\", \"properties\": {\"shelfmark\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Shelfmark\"}, \"subjects\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Subjects\"}}, \"title\": \"LibraryReference\", \"type\": \"object\"}, \"Publication\": {\"description\": \"Represents publication information from a library card.\\n\\nAttributes:\\n    title: Title of the publication\\n    year: Year of publication\\n    place: Place of publication\\n    pages: Number of pages or page range\\n    publisher: Publisher name\\n    format: Format of the publication\\n    editor: Editor name if applicable\", \"properties\": {\"title\": {\"title\": \"Title\", \"type\": \"string\"}, \"year\": {\"title\": \"Year\", \"type\": \"string\"}, \"place\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Place\"}, \"pages\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Pages\"}, \"publisher\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Publisher\"}, \"format\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Format\"}, \"editor\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}], \"default\": null, \"title\": \"Editor\"}}, \"required\": [\"title\", \"year\"], \"title\": \"Publication\", \"type\": \"object\"}, \"WorkType\": {\"description\": \"Represents the type of work referenced in a library card.\\n\\nAttributes:\\n    type: Type of work, either \\\"Dissertation or thesis\\\" or \\\"Reference\\\"\", \"properties\": {\"type\": {\"enum\": [\"Dissertation or thesis\", \"Reference\"], \"title\": \"Type\", \"type\": \"string\"}}, \"required\": [\"type\"], \"title\": \"WorkType\", \"type\": \"object\"}}, \"description\": \"Represents a complete library card document.\\n\\nAttributes:\\n    type: Type of work\\n    author: Author information\\n    publication: Publication details\\n    library_reference: Library reference information\", \"properties\": {\"type\": {\"$ref\": \"#/$defs/WorkType\"}, \"author\": {\"$ref\": \"#/$defs/Author\"}, \"publication\": {\"$ref\": \"#/$defs/Publication\"}, \"library_reference\": {\"$ref\": \"#/$defs/LibraryReference\"}}, \"required\": [\"type\", \"author\", \"publication\", \"library_reference\"], \"title\": \"Document\", \"type\": \"object\"}", "model": "magistral-small-2509", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 1196, "output_tokens": 760, "total_tokens": 1956, "input_cost_usd": 0.000598, "output_cost_usd": 0.00038, "estimated_cost_usd": 0.0009780000000000001}, "duration": 2.9454104900360107, "timestamp": "2026-01-25T01:29:32.304382", "conversation_id": "c7ff0faf-7361-4cc1-86c5-74d3f362c9b8", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 8, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "35", "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": null, "ground_truth": "C W G Eduard", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1869, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "Leipzig", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.title": {"response": null, "ground_truth": "Die Iphigenien-Sage und ihre dramatischen Bearbeitungen", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": null, "ground_truth": "Schwarz", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"64fbdb2a92c44606806ca6dfb7b9dcec\",\"object\":\"chat.completion\",\"model\":\"magistral-small-2509\",\"usage\":{\"prompt_tokens\":1196,\"completion_tokens\":760,\"total_tokens\":1956,\"num_cached_tokens\":1344},\"created\":1769300969,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\\"$defs\\\": {\\\"Author\\\": {\\\"description\\\": \\\"Represents an author on a library card.\\\\n\\\\nAttributes:\\\\n    last_name: Author's last name\\\\n    first_name: Author's first name\\\", \\\"properties\\\": {\\\"last_name\\\": {\\\"title\\\": \\\"Last Name\\\", \\\"type\\\": \\\"string\\\"}, \\\"first_name\\\": {\\\"title\\\": \\\"First Name\\\", \\\"type\\\": \\\"string\\\"}}, \\\"required\\\": [\\\"last_name\\\", \\\"first_name\\\"], \\\"title\\\": \\\"Author\\\", \\\"type\\\": \\\"object\\\"}, \\\"LibraryReference\\\": {\\\"description\\\": \\\"Represents library-specific reference information.\\\\n\\\\nAttributes:\\\\n    shelfmark: Library shelfmark or call number\\\\n    subjects: Subject classifications or tags\\\", \\\"properties\\\": {\\\"shelfmark\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Shelfmark\\\"}, \\\"subjects\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Subjects\\\"}}, \\\"title\\\": \\\"LibraryReference\\\", \\\"type\\\": \\\"object\\\"}, \\\"Publication\\\": {\\\"description\\\": \\\"Represents publication information from a library card.\\\\n\\\\nAttributes:\\\\n    title: Title of the publication\\\\n    year: Year of publication\\\\n    place: Place of publication\\\\n    pages: Number of pages or page range\\\\n    publisher: Publisher name\\\\n    format: Format of the publication\\\\n    editor: Editor name if applicable\\\", \\\"properties\\\": {\\\"title\\\": {\\\"title\\\": \\\"Title\\\", \\\"type\\\": \\\"string\\\"}, \\\"year\\\": {\\\"title\\\": \\\"Year\\\", \\\"type\\\": \\\"string\\\"}, \\\"place\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Place\\\"}, \\\"pages\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Pages\\\"}, \\\"publisher\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Publisher\\\"}, \\\"format\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Format\\\"}, \\\"editor\\\": {\\\"anyOf\\\": [{\\\"type\\\": \\\"string\\\"}, {\\\"type\\\": \\\"null\\\"}], \\\"default\\\": null, \\\"title\\\": \\\"Editor\\\"}}, \\\"required\\\": [\\\"title\\\", \\\"year\\\"], \\\"title\\\": \\\"Publication\\\", \\\"type\\\": \\\"object\\\"}, \\\"WorkType\\\": {\\\"description\\\": \\\"Represents the type of work referenced in a library card.\\\\n\\\\nAttributes:\\\\n    type: Type of work, either \\\\\\\"Dissertation or thesis\\\\\\\" or \\\\\\\"Reference\\\\\\\"\\\", \\\"properties\\\": {\\\"type\\\": {\\\"enum\\\": [\\\"Dissertation or thesis\\\", \\\"Reference\\\"], \\\"title\\\": \\\"Type\\\", \\\"type\\\": \\\"string\\\"}}, \\\"required\\\": [\\\"type\\\"], \\\"title\\\": \\\"WorkType\\\", \\\"type\\\": \\\"object\\\"}}, \\\"description\\\": \\\"Represents a complete library card document.\\\\n\\\\nAttributes:\\\\n    type: Type of work\\\\n    author: Author information\\\\n    publication: Publication details\\\\n    library_reference: Library reference information\\\", \\\"properties\\\": {\\\"type\\\": {\\\"$ref\\\": \\\"#/$defs/WorkType\\\"}, \\\"author\\\": {\\\"$ref\\\": \\\"#/$defs/Author\\\"}, \\\"publication\\\": {\\\"$ref\\\": \\\"#/$defs/Publication\\\"}, \\\"library_reference\\\": {\\\"$ref\\\": \\\"#/$defs/LibraryReference\\\"}}, \\\"required\\\": [\\\"type\\\", \\\"author\\\", \\\"publication\\\", \\\"library_reference\\\"], \\\"title\\\": \\\"Document\\\", \\\"type\\\": \\\"object\\\"}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.0008244023083264633, "f1_macro": 0.0003802281368821293, "micro_precision": 0.09090909090909091, "micro_recall": 0.00041407867494824016, "total_instances": 263, "total_tp": 1, "total_fp": 10, "total_fn": 2414, "cost_summary": {"total_input_tokens": 314548, "total_output_tokens": 158789, "total_tokens": 473337, "input_cost_usd": 0.15727399999999955, "output_cost_usd": 0.07939450000000006, "total_cost_usd": 0.2366685000000007}}, "normalized_score": 0.038022813688212934}, {"_id": {"$oid": "69d4eb12b877594bc5718bc5"}, "test_id": "T0162", "benchmark": "library_cards", "date": "2026-01-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Spiegelberg\",\"first_name\":\"Otto\"},\"publication\":{\"title\":\"Ueber das gegenseitige Verh\u00e4ltnis zwischen dem gerichtlichen Standniss und der Offen-kundigkeit bei dem Gericht nach der deutschen Civilprozessordnung\",\"year\":\"1896\",\"place\":\"Frankfurt a. M.\",\"pages\":\"128\",\"publisher\":\"H. L. Br\u00f6nner's Druckerei\",\"format\":\"8\u00b0\",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"\",\"subjects\":\"\"}}", "model": "gpt-4.1-nano-2025-04-14", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 2425, "output_tokens": 116, "total_tokens": 2541, "input_cost_usd": 0.0002425, "output_cost_usd": 4.64e-05, "estimated_cost_usd": 0.00028890000000000003}, "duration": 1.8077867031097412, "timestamp": "2026-01-25T09:24:26.904504", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Spiegelberg", "first_name": "Otto"}, "publication": {"title": "Ueber das gegenseitige Verh\u00e4ltnis zwischen dem gerichtlichen Standniss und der Offen-kundigkeit bei dem Gericht nach der deutschen Civilprozessordnung", "year": "1896", "place": "Frankfurt a. M.", "pages": "128", "publisher": "H. L. Br\u00f6nner's Druckerei", "format": "8\u00b0", "editor": ""}, "library_reference": {"shelfmark": "", "subjects": ""}}, "conversation_id": "14a580e6-b870-402b-8dfa-ddec74722013", "score": {"f1_score": 0.78, "precision": 0.7777777777777778, "recall": 0.7777777777777778, "true_positives": 7, "false_positives": 2, "false_negatives": 2, "field_scores": {"author.last_name": {"response": "Spiegelberg", "ground_truth": "Spiegelberg", "score": 1.0}, "publication.year": {"response": "1896", "ground_truth": 1896, "score": 1.0}, "publication.title": {"response": "Ueber das gegenseitige Verh\u00e4ltnis zwischen dem gerichtlichen Standniss und der Offen-kundigkeit bei dem Gericht nach der deutschen Civilprozessordnung", "ground_truth": "Ueber das gegenseitige Verh\u00e4ltniss zwischen dem gerichtlichen Gest\u00e4ndniss und der Offenkundigkeit bei dem Gericht nach der deutschen Civilprozessordnung", "score": 0.9735099337748345}, "library_reference.shelfmark": {"response": null, "ground_truth": null, "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.place": {"response": "Frankfurt a. M.", "ground_truth": "Frankfurt a. M.", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "128", "ground_truth": "II, 128", "score": 0.6}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.publisher": {"response": "H. L. Br\u00f6nner's Druckerei", "ground_truth": "H. L. Br\u00f6nner's Druckerei", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8", "score": 0.6666666666666667}, "author.first_name": {"response": "Otto", "ground_truth": "Otto", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-D1phx65BYIgz8muvGlELbf98SVQRh\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"type\\\":{\\\"type\\\":\\\"Dissertation or thesis\\\"},\\\"author\\\":{\\\"last_name\\\":\\\"Spiegelberg\\\",\\\"first_name\\\":\\\"Otto\\\"},\\\"publication\\\":{\\\"title\\\":\\\"Ueber das gegenseitige Verh\u00e4ltnis zwischen dem gerichtlichen Standniss und der Offen-kundigkeit bei dem Gericht nach der deutschen Civilprozessordnung\\\",\\\"year\\\":\\\"1896\\\",\\\"place\\\":\\\"Frankfurt a. M.\\\",\\\"pages\\\":\\\"128\\\",\\\"publisher\\\":\\\"H. L. Br\u00f6nner's Druckerei\\\",\\\"format\\\":\\\"8\u00b0\\\" \\t,\\\"editor\\\":\\\"\\\"},\\\"library_reference\\\":{\\\"shelfmark\\\":\\\"\\\",\\\"subjects\\\":\\\"\\\"}}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Spiegelberg\",\"first_name\":\"Otto\"},\"publication\":{\"title\":\"Ueber das gegenseitige Verh\u00e4ltnis zwischen dem gerichtlichen Standniss und der Offen-kundigkeit bei dem Gericht nach der deutschen Civilprozessordnung\",\"year\":\"1896\",\"place\":\"Frankfurt a. M.\",\"pages\":\"128\",\"publisher\":\"H. L. Br\u00f6nner's Druckerei\",\"format\":\"8\u00b0\",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"\",\"subjects\":\"\"}}}}],\"created\":1769329465,\"model\":\"gpt-4.1-nano-2025-04-14\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_2b9b3207a6\",\"usage\":{\"completion_tokens\":116,\"prompt_tokens\":2425,\"total_tokens\":2541,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.6585197140880794, "f1_macro": 0.645361216730038, "micro_precision": 0.7429760665972945, "micro_recall": 0.591304347826087, "total_instances": 263, "total_tp": 1428, "total_fp": 494, "total_fn": 987, "cost_summary": {"total_input_tokens": 642844, "total_output_tokens": 25695, "total_tokens": 668539, "input_cost_usd": 0.06428439999999995, "output_cost_usd": 0.010278000000000002, "total_cost_usd": 0.0745624}}, "normalized_score": 64.5361216730038}, {"_id": {"$oid": "69d4eb12b877594bc5718bd0"}, "test_id": "T0408", "benchmark": "library_cards", "date": "2026-01-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.1-2025-11-13", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"W\u00f6rz\",\"first_name\":\"Walter\"},\"publication\":{\"title\":\"Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers\",\"year\":\"1934\",\"place\":\"o. O.\",\"pages\":\"36\",\"publisher\":\"T\u00fcbingen, Med. Diss. v. 23. Juni 1934\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"U 34.5324\",\"subjects\":\"\"}}", "model": "gpt-5.1-2025-11-13", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1604, "output_tokens": 121, "total_tokens": 1725, "input_cost_usd": 0.002005, "output_cost_usd": 0.00121, "estimated_cost_usd": 0.003215}, "duration": 2.041754961013794, "timestamp": "2026-01-26T00:28:41.068194", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "W\u00f6rz", "first_name": "Walter"}, "publication": {"title": "Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers", "year": "1934", "place": "o. O.", "pages": "36", "publisher": "T\u00fcbingen, Med. Diss. v. 23. Juni 1934", "format": "8\u00b0", "editor": null}, "library_reference": {"shelfmark": "U 34.5324", "subjects": ""}}, "conversation_id": "6a36707d-b6c5-4742-b605-4888eb501fba", "score": {"f1_score": 0.89, "precision": 0.8, "recall": 1.0, "true_positives": 8, "false_positives": 2, "false_negatives": 0, "field_scores": {"author.first_name": {"response": "Walter", "ground_truth": "Walter", "score": 1.0}, "publication.pages": {"response": "36", "ground_truth": "36", "score": 1.0}, "publication.publisher": {"response": "T\u00fcbingen, Med. Diss. v. 23. Juni 1934", "ground_truth": null, "score": 0.0}, "publication.year": {"response": "1934", "ground_truth": 1934, "score": 1.0}, "author.last_name": {"response": "W\u00f6rz", "ground_truth": "W\u00f6rz", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.title": {"response": "Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers", "ground_truth": "Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "library_reference.shelfmark": {"response": "U 34.5324", "ground_truth": "U 34. 5324", "score": 0.9473684210526316}, "publication.place": {"response": "o. O.", "ground_truth": null, "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-D23p1FjHMOVGxSD2URcRaTSAkjw8i\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"type\\\":{\\\"type\\\":\\\"Dissertation or thesis\\\"},\\\"author\\\":{\\\"last_name\\\":\\\"W\u00f6rz\\\",\\\"first_name\\\":\\\"Walter\\\"},\\\"publication\\\":{\\\"title\\\":\\\"Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers\\\",\\\"year\\\":\\\"1934\\\",\\\"place\\\":\\\"o. O.\\\",\\\"pages\\\":\\\"36\\\",\\\"publisher\\\":\\\"T\u00fcbingen, Med. Diss. v. 23. Juni 1934\\\",\\\"format\\\":\\\"8\u00b0\\\",\\\"editor\\\":null},\\\"library_reference\\\":{\\\"shelfmark\\\":\\\"U 34.5324\\\",\\\"subjects\\\":\\\"\\\"}}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"W\u00f6rz\",\"first_name\":\"Walter\"},\"publication\":{\"title\":\"Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers\",\"year\":\"1934\",\"place\":\"o. O.\",\"pages\":\"36\",\"publisher\":\"T\u00fcbingen, Med. Diss. v. 23. Juni 1934\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"U 34.5324\",\"subjects\":\"\"}}}}],\"created\":1769383719,\"model\":\"gpt-5.1-2025-11-13\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":121,\"prompt_tokens\":1604,\"total_tokens\":1725,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.8469068943970005, "f1_macro": 0.8416349809885931, "micro_precision": 0.8520536462699078, "micro_recall": 0.8418219461697722, "total_instances": 263, "total_tp": 2033, "total_fp": 353, "total_fn": 382, "cost_summary": {"total_input_tokens": 421852, "total_output_tokens": 135132, "total_tokens": 556984, "input_cost_usd": 0.5273149999999984, "output_cost_usd": 1.3513199999999992, "total_cost_usd": 1.8786349999999998}}, "normalized_score": 84.16349809885931}, {"_id": {"$oid": "69d4eb12b877594bc5718bef"}, "test_id": "T0164", "benchmark": "library_cards", "date": "2026-01-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4o-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"May\",\n    \"first_name\": \"Mark\"\n  },\n  \"publication\": {\n    \"title\": \"The institutions that train ministers\",\n    \"year\": 1933,\n    \"place\": \"New York\",\n    \"pages\": \"\",\n    \"publisher\": \"\",\n    \"format\": \"\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"subjects\": \"\"\n  }\n}", "model": "gpt-4o-mini-2024-07-18", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 26682, "output_tokens": 111, "total_tokens": 26793, "input_cost_usd": 0.0040023, "output_cost_usd": 6.659999999999999e-05, "estimated_cost_usd": 0.0040689}, "duration": 299.9439356327057, "timestamp": "2026-01-25T09:45:26.322925", "conversation_id": "1538e9ad-c593-467e-bf8e-a1c38faca817", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 7, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": null, "ground_truth": "May", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1933, "score": 0.0}, "publication.title": {"response": null, "ground_truth": "The institutions that train ministers", "score": 0.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "k 1 XI 11, Vol.3", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.place": {"response": null, "ground_truth": "New York", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": null, "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.format": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": null, "ground_truth": "Mark Arthur", "score": 0.0}}, "total_fields": 13}, "raw_response": "{\"id\":\"chatcmpl-D1q2FOoW3qUKz8AJDZG3Vv54yinyS\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"May\\\",\\n    \\\"first_name\\\": \\\"Mark\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"The institutions that train ministers\\\",\\n    \\\"year\\\": 1933,\\n    \\\"place\\\": \\\"New York\\\",\\n    \\\"pages\\\": \\\"\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null}}],\"created\":1769330723,\"model\":\"gpt-4o-mini-2024-07-18\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bbc38b4db\",\"usage\":{\"completion_tokens\":111,\"prompt_tokens\":26682,\"total_tokens\":26793,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.02190669371196755, "f1_macro": 0.015057034220532318, "micro_precision": 0.54, "micro_recall": 0.011180124223602485, "total_instances": 263, "total_tp": 27, "total_fp": 23, "total_fn": 2388, "cost_summary": {"total_input_tokens": 6535442, "total_output_tokens": 37049, "total_tokens": 6572491, "input_cost_usd": 0.9803163000000028, "output_cost_usd": 0.022229400000000014, "total_cost_usd": 1.0025456999999995}}, "normalized_score": 1.505703422053232}, {"_id": {"$oid": "69d4eb12b877594bc5718bf8"}, "test_id": "T0270", "benchmark": "library_cards", "date": "2026-01-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "x-ai/grok-4", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"de Senarclens\",\"first_name\":\"Victor\"},\"publication\":{\"title\":\"\u00c9pith\u00e9liome malin du Chorion et de ses rapports avec la m\u00f4le hydatique\",\"year\":\"1902\",\"place\":\"Lausanne\",\"pages\":\"52\",\"publisher\":null,\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":null,\"subjects\":null}}", "model": "x-ai/grok-4", "provider": "openrouter", "finish_reason": "stop", "usage": {"input_tokens": 1974, "output_tokens": 4583, "total_tokens": 6557, "cached_tokens": 1713}, "duration": 92.85106372833252, "timestamp": "2026-01-25T17:08:58.450888", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "de Senarclens", "first_name": "Victor"}, "publication": {"title": "\u00c9pith\u00e9liome malin du Chorion et de ses rapports avec la m\u00f4le hydatique", "year": "1902", "place": "Lausanne", "pages": "52", "publisher": null, "format": "8\u00b0", "editor": null}, "library_reference": {"shelfmark": null, "subjects": null}}, "conversation_id": "eb8b80f4-f266-44c2-87e3-2065c26bbd3f", "score": {"f1_score": 0.62, "precision": 0.625, "recall": 0.625, "true_positives": 5, "false_positives": 3, "false_negatives": 3, "field_scores": {"publication.title": {"response": "\u00c9pith\u00e9liome malin du Chorion et de ses rapports avec la m\u00f4le hydatique", "ground_truth": "Contribution \u00e0 l'\u00e9tude de l'\u00e9pith\u00e9liome malin du Chorion et de ses rapports avec la m\u00f4le hydatique", "score": 0.8214285714285714}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": "de Senarclens", "ground_truth": "Senarclens", "score": 0.8695652173913043}, "author.first_name": {"response": "Victor", "ground_truth": "Victor de", "score": 0.8}, "publication.pages": {"response": "52", "ground_truth": "52", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.year": {"response": "1902", "ground_truth": 1902, "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.place": {"response": "Lausanne", "ground_truth": "Lausanne", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"gen-1769357245-q53S0tprx2IxOWsrWnw0\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\" {\\\"type\\\":{\\\"type\\\":\\\"Dissertation or thesis\\\"},\\\"author\\\":{\\\"last_name\\\":\\\"de Senarclens\\\",\\\"first_name\\\":\\\"Victor\\\"},\\\"publication\\\":{\\\"title\\\":\\\"\u00c9pith\u00e9liome malin du Chorion et de ses rapports avec la m\u00f4le hydatique\\\",\\\"year\\\":\\\"1902\\\",\\\"place\\\":\\\"Lausanne\\\",\\\"pages\\\":\\\"52\\\",\\\"publisher\\\":null,\\\"format\\\":\\\"8\u00b0\\\",\\\"editor\\\":null},\\\"library_reference\\\":{\\\"shelfmark\\\":null,\\\"subjects\\\":null}}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"de Senarclens\",\"first_name\":\"Victor\"},\"publication\":{\"title\":\"\u00c9pith\u00e9liome malin du Chorion et de ses rapports avec la m\u00f4le hydatique\",\"year\":\"1902\",\"place\":\"Lausanne\",\"pages\":\"52\",\"publisher\":null,\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":null,\"subjects\":null}},\"reasoning\":null,\"reasoning_details\":[{\"id\":\"rs_49750a7c-b2b6-2bd7-fdcc-b6e8217378b1\",\"format\":\"xai-responses-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"n2Y3VhfsgxuQqkmeau4gaFmJOsf19VPtAk9gaBxiSFVj/InGnQDt0jXtcWU6ksidk8UpapVzg6QOGr5Kz+Mt9UuRgs/aYe1uUpsj4EnchVpDKiFsvGMwxUR+TSP/fXTDcVHjUBsg4GZL5N7IC64Fyh+dIeBsN4FXwzulO31GWelbv7bC4VuGPLdARDAyflHY7Kbcigo4VaztS61sJ4FRJ3VvMSN4EgVn/eJughGDjXyhwz6MoKvfDmgNJgLXQvg746o1AG/HnFaPoPvFd+7WFN2USoLYg9xlCDklcxda4A1WpE2XVq3STxhTMVfBbqBkGwUIGaTmnrRdihP+WMH1lz52Sw7x69a6XNHnaV3sWQoW/jurNc97eBOQJK5ZzFW0AAyrZR96gDrt0ld6E0QtZOXKkI3lL3pzi9Obzk4lK5SbbE2b0cGI1LdyxbQsFLnP7vNKquEX5/pv1CYaZQjpoFaHP/J2dbnXLmio2LeBeRURiPWVWN9QafOYqA0XjXfwvTP4mM+PqnHd8tRbkI3lccMlP85zj6rTo44dEjgvAe5I3juQ1Rl+T+LfV40nh/c3ri/zJMX3Ki+5+07jEOM6psG85kspzMeiUUhQF7coQ0WER47maWhmarZRV6PWNdgNvu+dQy1sKSt8x9flgHZ4OvoErgSIye0bxhFqw0sgfcYoZGFr59nnxMQ8trCiH/yB0Ua7ZHSrm/Mp1wDCITTAsfvCKJPOAeuBOaBkc4Jd8MiskBiYKD4L1KcXo4aO23enysQ7EmD1qCJJwOrciT81IjCvbMD3u6yJZ6HKR/+1Q1l85L+EmT3zW795ELV3ZpG4gTDDxWCGwnRos7Vp0qh7Gz8XM8QYCe0oo7UStMIhhq9ITDTDF2rW5OzNvCRGo213MfZTkqCqpMcmXwqGBy3tbuf+JkTXVxufwxvWqmfLp4l46cRYR1dSno/+PI7GR71L5EepJgcZz1SP1H4u1QyPi9CEN86DlToazK4+QYtOdV/U5EeTJgE83ypU5FMf+Nrvms9zIwgWn5EtglvDGiQNIHOemdS55nOuUGM3XZ/9F2aOQ+42CCntjimDpb0lcyb0OyiyqVFm1L09tciaxVBjw8JUvmeecWi1du3F3ntc7g/rxzIAgk5LHLDheQgq5ExJ8+JTBA1JiVkA5XAWTFt85orjaqTGoo2xLE58ZBq1Tvr2QiTxOcQFmSAJ+gX5s6q4nusQDz21UP2WBc7JjaDI5UGmy8NH76uC1JMUcOsBk+CYTv8YRVHgSgG51KKim7CZHBY9QRNCCWJw3Gmk4jNv3n5s4o6hKZQ57m2odJFQwBtpL1+jJ3iL5VsGYVbZkhs8TwfjEJtQqhy403+5WKFzkAPviXvCl+DIYNXhY/2wzzDij9scCP5hz+WlytnIPtZqtyDy1ciN3AogxxOPRpgXGyVCW2Luh/6IglLvM/QCFbMGbpEsaplt1WoQmn8vVAQiv8iyZvyqZr2ZEaeyKcmI82dWq3caEztAEn80nONLLGshNs2VR28BJ/385eTch7bA+epilMAOCQNEXUR0W05iz3LLtyPsO+a1mKkLaPuaGQhxjlyNwVKh7OgNfSjb0uTc6CiVa70KTkTiSxNXsz6WiUyJrbqBXv0BsYU+z6DMZ8icLA2I9J9om6c7km83AHJHNvUwOrRIYHkpMhcJyG5xjgiZWz3gMp+EBamZjvy+pogot7V5nqLbh1D96dp+WtLDappKQgNuf/9UuoNOKd9ZfSUXJA6MqcmqBlEwGwhXR5DxkidWUdgGp+RIgauIA+vk483ZNFJxD/DwQbiBVVyR4ibAuKjQmeaNeofuQRyaQoVowd69AbaJFPQrUTFMQNHDixOW/Srjcv5F9l7TpDUaEt345Wz7EeBQaRbDXNY12GOHgN5j1YCwyyg29hWtMpQ8YdkYD8utct3h1VnduvMt/9CovjcLRYul8rGAwbhxe7gcuxHBuu0MgS4fVJp2l26md8NUWyIfT9T8C2Yl7IdDOEJ6d3pCDG7j7lDZLxYaU8aDyzPknB1d7gyQzTEB4H4MZOxg/yGVuzVyF8BO8KW6CsCuPZttsc6/bLt+08mURQCf406IaX7WPd2yxAm9z/8K/gg/LHG7A7n3X8V1jIfDcH/LBeLqkcyuUazRJZ0rc4gC9bzCp7EqH/joFGn0YVZ4TBpAzZsx6Mzhc7hvVKmr1cttHBqiYmtvqukKECs3sxFrlWFwMR7t263fHPer+XF0f3QRI8/g14/fyzpoBQjf536+JxUKj2TwwkokcaIFsDJ2pmS1rd82AN1GEneAZQoEFT1vjIW7a47rNP5bYICaN9DbEDM6UEdePqbn4z7YLp4XL+LUNta1azVDYcHDI4fD4ry7adZW6W7a7A65qj6sMsRTFll9lK2NfIbs/9wLMgjGbWU6IzshQJj2shMtBHJTYeX3jQjPD14hwj+eDuQGQHyKUUQX95kTEF7cvQ9I8Ea8iJrySnA9EVWhv1Cb5oH6fTbVMMDhJ+Z75d1FEREdKLPl9kdLSwi+4Dkm91gy1jUwQwkQGgnYP5w5UDeNrIDfeSYUtj1/zSZhbR78uCfleA1wB7g/XLgL7xfQuNC5vu1K8KunyqRM8MZjRxuMVb6w3mcIp9aowRfbAdCk40RyDy/XWK/9ONObdFk/mYkQjEbqngmnydSCLv6SJ19HdjgoSjaJl+qCCVte9Pj+srj83vWQ/5fD2NhhlES/n9KG5WXB1NlniUvS4XYSm+OwYp+O3WZlB4GBdfJIepsp1oFq/3tdWZ6gR8F3ePRKoi1RZTRdzo80T8myp6BSevt8nOcpTBd6gHDzjj8MT00q+Le9OvPRXv257YgkMs7niDpe2mEM4zMiuKBd8Z34hRIRadZCYy8OjNrCHKdODu+/iJ7H+JefweYJvm8+6awws6dGWjkVHzwQFiAVvCeovuaZ9dFfc2VcCEZnOO2t/sP+RCVnauddmVYjCmgCvVSKVtdee8IThiyxVAGqe+PvS56C8X8g4Lr35sffoMCSa5BytW3LUQJ4qnj1VapZcHcoV0T7x7xGLnNODj2izxxn6qB8T3raPtZluu8wUa86wnv4EnRoQh5gySSamihPX4dunMgSnvA6kMlEw3RB7Jy9l5dY4/mY4+SQbpdqgZdFmnkAYSvV0Z3yxR3dhNNM/RI044zjZa1imhOi6PnM8dYWtsqcPo2aJYZ+iOXpnoLWyP+Tp+kscbMaBh1jREnycOgvrOCZPeql2V3kumfutctfOADcX4Gezy6j5V2F9agNBDTSm+Fy21YF2y81pyJ70Ac+UPlvDsf4vZ300EWlmx3Ja5R8+wrVesBhV4VZnsZEXZsXPaJdScMxqwpEz8+wtHPCo0kfgY1bVWKi3rOSwjQN6G4f+AYBozN2TcP8JJX8PyyxaDi2bxrwGqSpCSXekewndsTDKkCbql1TIBlwTn6KhX5TOK/OtXTObrwiTHbU2zLs2WSFHWRSWpk0+fjWWYr5w+wUxGBq7bXyN0IRZEuP0I8miC9rfamssfRbdVJ0WiOAEW1LouVnTX4G+Lkjtm75UPsor3p8JwyuGXFZEvhQK1kh4JndLnGRf9MUGkSUMYfhYqueZAbG2c0SfbAETMufX2rEZmlxo1Ofes2hA6iWOiPdNtYBQJCBTgmB2olwSZyeqaGHrEF7TaFeZFY8d1XFaz1RpwGk1KgGLBB0pRDRMhBQHoASYmGTgowB3GxFla+gKWqObeqRtz6/6I61OqD90AX5ELmY1Uw/my5Ax6yejI6YDO5HJON+kkdsXMhmfRZOfO8vjxWF3YZsnrtKg+D6luKrjET1Kc7LzGw/GiB8Wa4fbNQuflK7QyE9FCZlHfN79dvh9yVylPllLEcD2LJZ89/nd0XHJ5WcNMxM4SCyVuenPW5/xb+gutfx5Ku/rsRb1sJrW0pQUGlDwcc/M0GYTUlzplogmHW7ySUXP7SltyedZ/gxte62vn1qQAHhERT/VdFrCx7G62Y0O8QeUaGaDSoiqfl68Q8mjwh5/SFTxrGU9RzDYGBx7IcMvvQOeHxRmnbLTB1zXq0+hNMC7OzLhlysbtjx5ve+XfAq53TN6fkFyBaMdchHTOoFWmzxwMjJXpXoWMuv601CUXgpIRW0meUMTnu9HyyO5bkRcPSHDRr+R8LBvRVOxNT2u57mhHV/84ln2MuLPdW8ko619hE6dWIKPPeIsn2GdXdcKR7zl1XAPogIxQsoXsS0HZ/44E3c3G18WG/0ue5wtYmud9XPtfBkM4WcbaijKu4b4JsnUxUKPcI7PO86Z2iSvgBYKIbDiC8COQhM2/oDyoWPP/l7VUmMIVAWxt8PD2Hy4d4ylAil6UxskATxAIajS/h8Psqdm5ymwN3WcPorQORK8d5RtCg31EYGo5JS7yg52Tz52WPxlcfJAT7N20SThRrLF7gT2rGFCDF+bnMBbRD5YjkTkDDwTGGbFTdsJrDXa9asA+WOSxzCSOGMymBjpJHrr+/D+A331+F3GZN34shmHOtOg1VBKBEqsO1Y4kN6OJ0AxEL4Ks66d6zbOmXiSuzISsYAnuQ77sIj2ai5XWXFgwmME1pbWtkpYhlU268i+cVLUoyFqYiYAozt246oQpI/NmbZIU1QZEKw1u8kVfbSdZrJJn2PAPy04LlcYqcMLSMM1VP4puowHBIxTVsAu6MT3WmZNUjyA5ULH7uu5g27ruN3jIHKgfikFEluwK3dP3sXvjZ7CxNVZVAF7tJDHon1xjqwXJSaw0HbuWSk+1tUalhRox21PzPfWmdgq1LQKRXQRBNzX4RDV2LuDvxklQhNERKSitx7+Uekzs4J8NnfDMryT0oOZRTU6FpNRix7F0LM01CQajOSr6+broyyTnPq9OUhEbadw6iRMWBt10O+Cxrd/JdhnjAxQO0vXmYN9fmXLSs4mj1R2JlO03hF3JSur6APYv20rtIHW8v3nejoHVNHdgpQXj4urHapkfMY+114G7FMm2mlep+xETANzj9SKY2ztiGR9hGnGQ7Cx6Jm3JXKZAzSK0o9coTiMXxZzCovjxVox5nucoZnXrNrMXC8+KWCKsbLo2Kt9oUjBCv/PZ1O8jivg+AaWtKxbRD4s+vEgfisqubYRl+uawe/6PCyuF9lgpDct3GLKznbLDgV3Z/Z8QSSveHHrc0eW1UgKNiiwx0+xuWVZ+LZflflOFs1Wc+1SmVQ70fQ6vPXgSeE40453xAGDHSt9TDemhNAFGvfaadA63O0kxSoILvT60omd9667xQyAGfwPPoERklpA3J1Dft0CqsPGEoSaBMRSWeU/NcBmdIZMAEoaR967RT4vj9XSk/bMRKozn/irNMh3HXEtpXjyOEt0+3JO3atcSANRiJ5G1jNBFipvi7dUAv2G566CHIRLoOnMPomTPTfKrpqpM/AzsaNmiQtwxNBMwix25HPPfYwM/Qj/hrEhxKd9enLvm1S3pvzjhx4rAMiAjDPHpMapxHelmy7JLJjv/24mvaDVEVVG3V6u/9L3h/lG3VK7Xhb/2DRIMZp6uzNKjm3xpNZs1ZdGvrNvPhlSKyxtVeWpO/VQJxNndYP9DvR7qyLcvEcjb+HAFUmfhQBDkC/+mG9yVd4gh5j2vqf2VfjQbH8xiMCarGuq02vCfiyirMXQ11arKbIKCcgYM1k3y5hNldU+CH+vA4Tj3SKhcYCoaQTmcUccX2eZkllY8fchWtGo/Raouan0vx+sVJeP1ISqiVU6VQ96doTknyrLa7galwrt+aAXcAt+aaJ04yfENTtGsSWFiEwdpEsjNaACMSMlUw7hMVSMgoZPx0eTUdogWahfyRtof2ezmeLhSxVQ99g94EE7t9pAiGS3WeA+mKLcTVtz4+ZARt+AxWfWbfN/Oe8IG5Uh1lcCmJPIze66BXBxpv43Q8pLBHcZeF+qq0ZrZRr14LzuEHzH8RrIAmR17bgyD5QnLoFMVQ85PRKL4KUSqP2appyep9XyIGyHTAT7D1HJw+zb3Mmogr71i89JBHzLvGVC9mWVxB52NndWzrTMF38Dy4/DXytyHCMrmCNhjzUobbHScT6+GlcgZfS45DQ0spPsB6rJS5WKmI7BrRCvG+EEd8aJAn+oEMVatX7LvEu9bD9ePHsYkbBTOF6cdY6i/ybqv2vzKFXDE1ka1rRVpk/LyI3pAhi7pdHNrK2hfPPhgcvjLSSjAfh8a79p4/dbhF93Oa15XRy7TN4hZ7L15uIBga9FkdFExin/fj0n6TaSkgtHjZqRq9ky+WuFTZ2thqgzsNjxZNIH/F8YbIdtxL0501JT0FV6iD4gO7nc7zvEDJGujOp/sdnowZkEM53MMjbDrn7I9H0jMeyhtF4XJF4xWouKCrxaFnG8AJFkjTR2fTPaVAWseCv+TeuxcEri8/Eu/eRd1ZoAV+j0QE+ughaDHa6MHCJgatH11255D3fpklqsxrVL/iDfDnRWTSDM0p//xjK/udjkHAO4pTj4CnYM6nWStshvV8KgMEuKzJ+LZFAvLVCVtZSoPeSz+398AKYf6GA56wKYSIgExgy0eOJTIDzh4mgDl3+H6Bdlp3pwjHVQX88WTwPAFSB0KQ2/5HCR2CeYJD9+ZFoBYLZvoWLzaANr02B0I1gr86h4zmPk9x5ukvQuWE34CF8XiPj6af5CiPaZsLu8FWXj9GCQBgHDsegoexvSyFB0yM/lBHXMpjT/BQA5f0lXi43/p9CDf9ro4LW88aFdeDWn9LtSsVAtrwX/zrgeydigq7OcJbXHkvjs2XYsHQtxcoVPhf81y4jwNCej91gmedGDObxKfdwyHjCIH4S/cILHa5SvqullFfIK/nlJJzMDkq/X1J5G4eFjadFyEUQSbuUzf3/RdGcb0HhsXR/U2SNqSD0pTSjA45u9yNcBqHFxZJRwj5CMfNkqsT3UeN4Bcg5gl69mDjF2fVN61KQC1Chr7SzGtmD9dt8HeqtWToqsc+pQL53bmqhfugBu9uHVXyPnjxKw2dZulphEDAmh60X7nn+X5rNfDmdJW1fOU0WjnGoruKHud+oDj5p63iMHAKPdKirt7kK8mzxdY55KZYFZXzLzVBmszp2umiAuP6RTbxNOSsINvQjRXxw4nQSzX5iwT/YDHVviIbcdgd/G9D5abgU5GhW651ecGJbThdIFcN8+XDIFmaD60WhXOfIayibjXMgeGEfqQjiiLLp5GHKM7rKJLBvsLVliGhrijKhlUqnlscDQyghdtsN1HuX1z32zIxE1EAkEeGh+lQLfz+P4wtVodaSXZQwLVW4zf1NOnjkj1ooPL4ZOgE1JMLKuhqpbxG6crtOtQBuMCG4liM1IUsRdmnmHJrwiXTr4OBZDdLh6Fkm2RwFpbJi0cfAObv99X9D32kvuMZ4vYjzsUY+buDDLFt8NTausfRY7SJh4O8n40qQCtr3gLUwW6uIyFOdh+8Igtzz/sXwvr3EwVOUVuJIUsxEuQaCPzADJX8DThWcGbku8be0r0yfF4Sse3fLxWJYrlpqtUcfkwAHuVDMrf6bfQE07CPbhfAXWJ6oPtF81PJlENeTWaTJvHklDNnRyt8QbTmFEshJk88fEbi+sTEC1kn4+xsRZ22jv7dQCH/7nb71D0CnffIeBv7LUEV+X7V9ZBcDvTuMLK1Tcd6zYyQqGO66RCYxUmB251tr+0ZjjFjHeQHPRhz013X0YagaFLnhMJCszmQuel3BF69ju2gM6R3Cq72rRe7gCQTf83gmE6PNC3114qx+l25qBaZebl0SsIwrqClaEvCyyixBL+gLHOP2uQ0/nCZu1AZMETE3PZkMw0Vl0MNTHTTlpbI2PIY8DcWMbBY0RKAJdtGXHldrxnSUitrJirPSLdtT3ibUOhgwW4/zvoq2/2WNFp8y1HxsBgrput2v4jOnus7VaOgQWOk11TKwix49ycXIsfwas5FUctNAYoQp9+FrBpUP2NcarQ8i54D5/yIidwONxG3uAM7p6xrBtyekFX0GCdWbg1+6BDqpUVHHJKXEvUH+ULfQQL5XEjqbaAdcKhL5HqM5vq0tdfqVFV3fqcYNCsiLMjIwFrt2L8tXzk/1ce0Kqzh+wS9aUGHCI0QsIoQpPDFknKCFcJoc9W5REFhXwsrkJiAj8QvG6AQN0wKFN6dR943BhHewfi4oVXOGKwJiuQVTfr8ao4JXA57TjLr73sbLyu1Fw6KvKkn2muMKU9h3dkeReTRTcK9DYVAURr+fTjN7T1oUafsIUTxP1tn3jXH+WkWfcW5hAyVNp22UTJMPsQwR4oFXcNFiIaBK/WTBFXoCuNdZ7CIkBcQJq4oA3mrcZWum6tAjCO1xFVfOupMhB6F8EnNfKUrsBRtu3FDoxVS1Q1TKHkfwV2+1YPIhp8Yz7oMDYNUl1Yxk5QD0E99fgExc8GAToByG+Jw4uAaPgHDK0u+PO2psRH1QujzKelLUyPNdVMqnawwF3H5kpeIDNwCHWDJ8n8hvtCyGQJ12VC5FOYkVjQwmcjKpbafR2iEaqlTBZcWgBn0Gp5+FRjgEtz++hop7n8EXC7ZWrUW6HT99Job9sR6OTewJpMau5qCIdwbFYJ9CH/a8ed/nwejJCSwwFaJ7BHVKP/2HlL4dRGmByhcS+urv+xOyyhZIUuvF0+qer/cC1ItM3Z7lfx90Sl52/TjsLDzgXifuKUirlEZxNYuW7LJMa4LE83JcoKRMpe1uzZ+U9hfsXdBuq4SIROZp8KphnoyUCuTNQ4+o6nwndGnWC8zuYq5CALeyytSHZ0QKpqphOMUk3bgpr5ajI8TYVB/CcZ8R6Z/368v3p3izYonp9TO9NRQh+ZtqPVTU/BKv5/FEuGlgNKRj59DG8IA5NMMKa0ipjIaDQHT4fkYTYAN3yj+HnaMDhcPSmKOVaEQlrAEwkpvR8AKJo5aoXrhoPIsW+1EuSrk03YBoG7LKCJ/C5u/XJ4JYgISQsKbeHfYRnOZ4q0DUPIzRhDH26ivMUK4tX1l3W7wj51JkHz3CrDD12UCHX3I0O4crOO/aWVJwoSZ1fTizwWWDg+GQ6gmnsKzMma5jTHMZwVgwzB0hrJE+UVwBRK2qPQbznqGzY5ZspL0+Flh2PBziXylbVnWWW2RcPg75QrTcx5tndlMPzO0gz7MCVxTQb0fGRAAtF8rJ90K7uVcyJB+HkVVm7V5tS4DQDXztV1qEEArPUynUC+HjOntoKPfKE10Swc1LA5ypOAUn929ZpxJWPmRz8/rcedcPVjuXpGKAqVHU8RZzDoqZDdOZVMCnInwxwhXqd77Ra1JssbuGLdRwYT6/A0RLH+vEgV/j90gmUzai4CKND65/SqXngmwNVpbmLK9BoVdM/EYGc1Kqui6dL32OaqNsUYSA0HT0kX+HzkkLMrJP8O7VrGEeDx5v3EupzXNGl/PQ3vrA1uhuN6JJgKYGfyl5QkFCVR+cMIT9nwaN/6QPIexRWj1phw3DCUDEtf8zqSFr0JLKWswaeMZzlIxOc3ev5QiRqIOHhmtkDLxwTys8bWatzt1M+Cjss53RMqf0NraNsx1CMqNlZ3ivuo7qFpj/KD2ssmVBqws04xewbLDVj5kUbPK/iP1VTFybVYB75FRZoFiWH73Qy6Tlgw/E1ijeZKBv3OMabSFopRNCV/0+r2GYlSC8oemWnkC8FIOVJkloMb1GSLlH+t4UXqyqb9KmAftg0xURN97BPM0koKLiqMtzvjCpqwsOgWWhuf231BCZr5PEBhtrFNjZ9uluT8/UkrnPuZ0olHksDgNHoo23PzPX4qouYCjAKvSQVz7Q5FtzfBhaJMbvl6pRb86Zvr1J/QBrfC5/AGK76RMgynI4RtlBakKgzqVjAn9knwdJKNezMIDx3eAmayIj028raxbqfHb12PwYNHTWfx4q7EIR6NX2zaH4lIUeOsImMbLrhQHor/4QWmeCfJR2YDyCTe5vBLgPbbigtxF1gPLZbv4ygzCTG2eAA888vmksA/X0G7XAgCtKDEhFqt5JtOaxAPhDth2CWQLUKCNHwiKj7nqDZYwGs6TTMykS2FSM2pYRVyBo2/smcWX8HdcnK9AODRlOZSz0eJLvFlMZosBPqH17LLJJIzj8fWiF7d7IJo/oUz5t9Ys16pql0nDr76DP7sVZsCDa1uKqed0b3m+0mVMZncKCU/Wb+RZ9WQblVccuEr3p1xC2nenpSmZnMDM4niKA9gQSd/rNU2SkBhP07SXQ1yZ1erq8jwb3qbS83TrIzA9wagdmnjDbQ+8B/HXSC92JhnQqL+1GZg3BfF8dyVNWjY6OYRJpGetinFv0rpTljm6zf/hAJYhIRpJgpqi1nir3VdD85VfewoHAbSouo38nwpVZb1TzIAMtzswF5p1FYLINkvu3Ixm8ym2pRwyRJUHVCQx1Y8tcIqbYwc1sCXt/C9ePYSEocCpLBnZovqqQl1VIKHgXjjUoLmsxSFw/T20hIyROnbe4FT/tp2pWq/IRapr6AO53FFg3bBpFF/PxFlsFU9KRogM/xIHbGWCMiQXqvCw9xExSDqAPNRrq+TJF2mE+HgMs/mSXvlRKjyCmIUXUlxd3FRTSpA6JNw2GX8rNpE9B+7O1U68ur8/ygmUds2hAF0Duj/tcFjWlRtwXWDFIuOLFKwIFSq7+DuZBPPgC5cG7Sd1vlS40JCRgdQt1bK1uuNOfE+W8hy65qEj2fzxJiC14Duk4/ltJ2/9Nbed+sSl531IFrGl86qh9ngcFkf3vhGngJZER3hNvnvAmk6epLtqSYDqbE+vCbeBKEygnYuYWtNZW+LPRSwtHiwxBLmn5vZZGJcPtZQBhk0f4ymJRX+91PU1hHBLt1M0CnnZV8/q8sxt4OUR0mhag6ryJ+5Vx6Ej79OytFogGzsvl7xqk5XIc2WHuWqcCHLKyFRfX/4EJsp9F4S94F8wuUQ80G2vbKcTpnb9ljYmeGqJ5Uqxtc3qH4OQWYgp8gOg12DAtD3z7BWbT5HzmfZpYzLQc/a+UwbjD0qymb2J4onuVo9EETj6rLRQxHHOZSJ78cpwvRWtheJL8kKNEPOQiD3WkbUVUi2B0AnN+p+RIh7aJGMC48WRTNgs8AQLYZS4dk+6sbxnRtJs9ncYcv8bY0+7q8+oZDhY+o4sbfSQ3Jm9RfUel0mSU+CZsmCjcClxUePJONKAcY+6G1s7e3vfgRTfibycgjR10amoyHeTXJ9eBY4Z+iKiWvWPiIoT+SlcOpBkqsh2oU7boBw8/jkUKcpn7FS+RZf358tnSznnoxWIQmUogTOaPAv+GBILgYv1RcXhqyr/P18JyO4tS6AcHsQuq7Gj7YmJ+jcPFlh2b1UbrqOTnXBdQ4X6EOyZWNjEKpAcPq/rd3c2Od2gZUTNLt4LRa+TrlwKVm5drC64lmJpcNyZKHYWZ88bFzI9yKm6IvOm02jyL3sEVX/53dV0GHMbi+h0Aarv4RpUGlEYrKmkAfEWqbiMmMBbWpu6tA79E7ZKBHlZPole4cMqvGAaSw5tBDmoNS64z41SEAQJBjiZ6vPkyIZcHpyDQjCuRU9ZIbR+FMCYef14TlZzhPFrUJHcYe2DCkk2MCFcS5ieyG+QcT3GtE6b8oNx7q6ZXoz5nKuxkSkFGXY41UxHy1WOaa4V10MO6SEUoR6dZ/e9HA4HfS5xndaHHL32gjuW4h/PZEQseB7/dp6q7PXc2YB7UKDlDhfpfZgH62y1kU0J+smqjPnDq7/Xt4Ii4HlfQywG6zcpqImcvSkdx5uEX1H/Jjw41QdWSy9bYtsLwmuPFqZPkf4SFNKEatm3WNGV4uINwXqh9WG8MSzLAYlG71HYiIgOmm66YmGdvh+m0HyxDhm5qkXaReWzUl3rKRmDofiauM/pWYQjVcTV83QIRYHJWsm2w/lbdE+R6l14mhmtPwmQiSGlxa+GWL/mGuRjvGT17EtxPK0y89XGxxGhuS1naTAnHcpY85Q/5xG5sClCU5g6MFnqxk/FRRhhG33xyb7E7h6+DiLOEbnCaz8IAVwYT3tuHL/EJu+zFHzBOmvUxtK1hZq1C5u3eeWvdap1eGcBwnjTVVXiARLobJFDzLuhpLBy39mP/nmbQzURKVSyV6HVmkfpfOLYKkGKVbRfA9kg2y3ripkOmcPjWNPWeoPbUzTMoLQ0tEMV4gLLHHBb1gRPSeUYJJel1aZhZSrQGS6/PnCESWhPgTzxHhE5BRWwjDvw86zym/Uira1docISzj0cJ+UsQnqNKMm0FrZ/q8yQp0LO8hffljGfSUBFoWrNHlBUDHxCPmd8O0yToW5d6tcqpDGG83SCVd0MjHgdXZ6iedXsANFNBB3f5P7dL88NlvI9ayREVRw9UTNoTcU3gOSWSYJElswrlTmRNS3TxquMfFhr0VUg1SUSNnOeFhAK7ABk/rxmwaSbBxjrUcLuJ9VcfhlhwGpENp/ttIVzoVAJSj1/+pRIcAl9+T40il8VukrCU/3d6N8/dQ/yN9eJNtkTMetJn6Eebm4b0pLKha0kwwuAMIersLwH/tmKLm1/ixbSsqbX0VA7v7pkp3Vb/VDqgG2vt3swvCESO+9oXUgebjglaB4k3cbBvtWOhe3YRvjnACen0R3PKjasyL2YH4svL5fV1n+/gp0zQZ3qFh/K7y0IyYdOtbQZYbnhYHMFs0lJeeVXqgT3X1KAcKiECzqssd0hVQVRvm4j+qsmr//bXTgNXrgCUBGHg44QMW8OhBjS9IyZl94QoOP0SRIV3hUWY74WYxDcV0KI0oatMgwmN95oFifmeeYt9GIeasf12JGj73C0SysVcEZJ9KUZ2nC19yQKg3yl0S/WYTinU/XXC96hUaJstl7OSf8hviEZr9WHuo9TC93owx/71PibKZ3RLIdoklcsXJBRuGTrYcCyStJKYWDA6eWxV7SLuq2W8NVzpVXfwHTwHfmgZ96OzQJv5nvO8Lq041Ma9GPWW9TMYlGCJXIlIB5WuiXdhZWsaB1ijXZ3Ny7RLIa9vZh98/h0KMuOIGAynBAtChXxFHTdMwSVZHq53UG1YE+Ktf0TWwc2nCxcTH3u5Z5C0CvRal4GGL/NnRim4Ps0KUK7lbfKefY5u5rWgWO4tKYKVsRTJqnvfHgoZNvzVlj4tBOvIm12Ud+/tu3Bw0fqMTonhWpgSnscQUL6XbST1N8ByoHDs0hHTxLS8YX8NQW9LUFOdYLY5eudg8+NGO9r6X1rF1EGFcWLLkhI7JrFXS4krta12/KKqyMDsfkCdLSOVmjIt0+Ii8M25aYRGLXJah1cS6Z88wDQcX7mZQpOWeMxpmSrPWncC6boB7DCKj5CZaOLhNoah59vgp+1NwWHGNaoOQvV2/xwCHUpLXKGuV6jfh0lAKo9dJNnZhH4uHKQvWkw6/zAEE+8yOyLEhRVPL58vmKAR0ZLGWUFeUMHii/7k3uKVt2h/InlQ2cC71Q3/b7AD0BrWrMXXg2BjHVDnCzjH8tcUzRYwQOKng5WLWRmePYjnVZhw9AyubO1iWmlOSxP0v3NgsZ/3QFcvOhypbh6sePQNIjXOe7CiTa87qyvRHXlygkYeDTlk0bSIIQhV8s/7D2z1zIBpwwKnfPbGIV/n39y6Z6eQTgp6RWbKpUSv2Lmarkf5W9+v6furBC9ghXNSYG2IqPm/mamj9m/FLChQMGky7wnKqppVOHkakXFf6HcqAPOHifQmfwuIQ4gK1fXMQ++bMk0gvIi35lF0a6rYyFt01c6lJ5GS5dfZqeKl5PZIjzvh2vdYQ7AnUcaBncGIcCzlYpsnwsWkT67G/UgLN0i3vM2/a6D9gJXd8apmFhsE3WtCpiMtnnv9jl0dqmxMvg15sSq5LrpValTbGTBBPtVmln+uc17CYauIKN9iqXCiyySm3x/eGHXoCFrx8hVrmApkSQPyCs09/Gfw5ryQtIaoecNyLCgH7GCalrdMlECTyM30QJRWNOaiKeGU+mMmanqxmUY+jBoCRFgint7Wx0UZhJJ/6K4D+sxeGIHibwEs2T9nhN6J7fPZgQX1ZYB5F/1bYwon2fDaYS/b+xzgoH0vjBCCbMviOitqyIckDTK/e9J2jfh25iQqI4rE4JjZk7yR49fs/li9ISv9gAtn3tLj7OqM8xneEQzOFnZW95YpMJnBJHuvQERmDMSupn8Dvm4MQCsXTBWu1uMSR373Gz/GJFY4BxpV7LvP13bbm/uwJSEtTEqp1DfEKRsf0vv904HjIFodwkjtU1Vkw0pvRiQatLxbi53wGETQlS1kA1y5hC8ffdoMLmUhRN+UJKVp0IkDafX7KnH9ufzWETimrzUjFfE4SsTGwxpeFk6+ZXoXNAgvZfO0GIMX90FcOjMvLO7cqz3wRZP/uK325axK6LmJlCwS0KYnwaN9F+Ext0oAqFCet3MOA5whHkS9q/y55dgmZb2L+p36S5QpWSkrECq/D3Bn9o69Uux5vlJ+jk2B3WwX/1xseHGcxbvRVkcWJFmbfhUNmmW61RABiLk4WAmk0MBA8zFvdsfNcxUnKz0KOeoBqKM8qxrdajcmuskyqR679Z2s1Syp8/0rPvDUwTlbtAo5Bs1qti948dUhjtvhMMvgzv7Rvnct347jjdEyV95nkaUmjbkDoJinOzII667h5OPZkMTsj3eRcRF+L7htzVBR4FaXLQzcdwUU6QukMYiy6XNCtthTsWp52ccAuPa+xEk87VwPTm0c0bw6Wqs4KO24q3DFOonBKTuYF3Jd907zoLI38AyuadmVP5a6GkdYVzHIW8vjS9aQBeb2xjWl3Ly/+AM49MlrYW1m2VljnBRBE4pelJHtVxpHp6chbtpKS2Kpb1IgZXMIn1uM8QdPDvYe+bq64zYrhwTeAZIPRYKZdmqWjnGJnYCtJ3K4mKm+XnZPkbUxEU4ZUQjvEIOS0xwxnfT3BgEZwCb2azN6ASviWLb0VEVOqCmpoeiJOpbk9j7YJgkmmfPDK6DeWaUuzuOXSUYJeysgpV7Gmt0Cus0h7nSOchyWJpogMt/MrcTSIyxL/xNysLHnUGfpeNDQN8PlhSWU64+yMqs7rrY1+Sq99XxT4y7Ye1zUhzPHh+HUIKLHSxA/5sFY1VYxO+CYGaziQ9TpncVmefJSFkPHbsnN/pSjFrn2WNInPjvoVHWwNPpKQYbj1dZn6yn656K65rFXaqa+y/SAbgdcrDFBTT5t3TAw0ykODgn7Q88T0awNRdqd5HB5vYH2jYmb20Y+Q1BfGVWDDd6pR8yx/fc8gumtvI1AA8a8nQcagL+qS+1wSGM376K8lz2JZZnap14Y/dHj8i8rhazItIk4NCbFUVAjlwbEHvO9/0sw1kQJAG7sKuHQHHp0jKFaXaZ7HZ3az1dkcaRu1c7vaxTPL6ayg9E96IsZ3DQGzgReyPbQfhrRbG6zYja4u3HIkBiTHRdn9jEleT4SYbWwAbOqqz4c536tjfAc5P9pm/Cj9gCQRizdrBH6R7c6K3rKUJLZYS5OvrKXHsauVmslYRyYFbImWUkC6ShOYXi/8OR6u1fzlFgJjl/Ahr4MwgOrBLxlTDE8qaknXec8dQsuzg5qk0flGuu9NDTOSwy9cZRyCApZksrXIIgAZXrwjiAS29dIv8nT9B5p4fRwbypY4sJkQuMF5iklYFC1wKdmbXijI92KLPp2VvMqW3BoRseLR8ZEVNa5pS7hfONUnswHSpHF8hFUd6lbyZ6PlKBOomFnXWnSx/DOS/3r/L3p6Ezetg7GVBXCaUDVV8ZDcIZmm49ROT6Jwj69gMMGjBtsLoVJeoH61u8spg5htaK9woVJ3ehMWxGT5rs3pU3ZqfxZE1D7jJEZccgXLvu3pKWTKlYqfQ/MlgX4IVuyWKnubmetwa/Tzkl5UBjw9HYUNK6Y2einqspi5Qb1MPv7Hpdx/t6JdhJt2ka9iZDcTBngSF8MbJuOQ6c6ZmWAQV3KNos1ZXbdT7fYgYNxdvbH+AngjwuNgOmYQk94DmInmdFtphryuQIFaE4ezOMAaxagXXnhlbS7txFI2dLSIJfSCxkg+qYd7ljtbNYOzN2tcBO9VEkWwHq6FxjNOrSVPgHyewWvFpr4qPcNC00zBZuhTTaZhDuwKn933eBrTWWkzzJkWIPbML44I+H9MQQl/HJDJ9y6y6/uS50cOhf13tm6uPtOy5HSUXJy/iAe/9xCLdsoJkSqmWrBH7TxG+UMfW+fcylDtvocuRRt26dhWGXNE4Artmde9WrnNJLe19TTHT7xYdHCLair2BgSW4Nx44fL+VcelnBs3XL5Lq3Bl3WvNoQ3miviVsc2zogK2Xj+PQK1imW8EhIIOpTDNt1YkNyLlLDlqSgoRkaselX3vWhZRoKlcie1wNFEQYErTW1c4LfBsz31HkINg0iq75xuiWotEQm/UoQUKbo+lyOgRDhKkKzpLCL7iRpP+m0uAZVHeGUMNYtAxaQdy9JJSJb8IGWqgjlYpVtGY86c7gJzZG7O/v+K/KgIo9RYQ/gwm31Qx1PlwuvUFD7sr9/0gqnCbXexS7Os+VlErSvnPpLgAZMQ8AD9ZnQFlg6VAfqSdPFJuotudHKUhm4JBzpjIdjhgte8uiE7lQEdqAVpPwN/PL8buzVE1BrMF9MfbkIO0LpaUaoIhzNYJcShAl8f3iXzvVFpxciKxddMamYEk36LnfH/GyK2oLmGCxeZYMnm1ShA8lvxL8P3CEZc6RCUUyH7DCBl6mHrLFbH/CXRV6rUQZK1g9EVvHs0mdZl4aaXPoiNLVAZ99ao9kPJHsKl5APWX1hE64Ams5+ZEa64MM331YjoU2oPIJoufINm7+bNXf5sSOGwUcGYxDNsBw3tddUncsnMbY5FFYTM7/dp3mlXuG3ZS6kKo46IRD7XPuGTCrhM6YxkugSXmBrXkrOkfZZa6ZVHm+r1lFPV4Ijq5XJzRCKm3jRedl3Ho1ZmIPGaKbMBMkgTdoQbt3RXB0+rCFpmDYBewy3Xefjrjf1wL+O4ObCHHlzd172cUUPi1/A11slw0plE4Wg1s6QKLB/ZuoqUevVWdEdn34276lKa7cMeb63JnKeD5Q+nb5M5OearCHTadQBnXNPnUkhpWnjPk9txagMQ01lplv+2OY1g+ggqPQDZmIPZ9zF2/lV/INZ/DbsIsq7qZHEFM346Ha3uJjjB5WadWpzaGPU/NltuvtZYcIvWvIvtr53ThXGvMYA2vnRQeNaoG+ZwaWlR7dy44wGHCGly9+eTipOTqHAEn36SDPZ0r8PncolhkO8PFsMtyXf3rh5xOp1zyOh32vJZ9mELatIaEGrIeM3PLmwllUCRkFxjAIcbsOw5wFbvlA2F7xJAuBPxXlK+YkjTUNwwODqmo+3+UyZoqMT1KlKnv7oubDhxPteoHhI5srCBxjdHo87fu+RYw7K1MWpeqH6q8fnyhDe2N37X63r040xSiCqJHIxfHa1ExLmEhzPz4sXhF4pDgFYLEFy+uxpT7FZzSY1yabiTr+LmvJK1vHSHg9L8qLRLcVCXMM9P/L2gbN2HR5YP9n0svEkDuBlFUCiQU0RCVYcFUVTzSEoJLk3sS8GGrgKu77R3FjQCHgAsDj1+46akpmZ5c51+D70DKn5ygw91M4dTx9Jh1+1AkYox53+Z+JtsAq0Z8Dpm1LMUkNO0GSDFPZfl5IaJj01egT56K9vtEzSj/pNjlNf0vDdcjiFzu2U0r9jBkZDZ8bMBPBb4VXeNIrdFX9b1EyUkqfur0iNA5tKvwh87d1buQE+RaAuIy0OWA8avohX+Tw+5UR6sBB6eo+EgGIENlE3WgpipMCWI6/P8PbEgo/yV8zJENrOkj+KYQwgjILdXq+JsWLXAKNyc6vomdq/xH2El+6mkiH5haUjat0QKyZvHG1Yhsa1BU8sEvK45n2WRVlIxcAuMmEvfMxRCIZ1uCNns1mPtRh8kJmaY+3ZgVMFauUmLrwf1ru/iWsPdYWz0bjLLhYeoe1g4LqMVeV9Ot9oTLCKYIA7nRAsT01MtbbRHRjVkDLXyE+u+yKwZu+Pq+03sCuZVcJojY/FbSwP1Y81hmkmyf9wyrcJkufRxZE6Ubp+YKkCI28THR/ofG+gNKIu1myWaYROShIhp8Le1bnWYT9E0pzWOdjhd05xclMCM3idCk4C/vMva0k6Of6a3Xh65r44qF/IqeB2mkk+4lyZjMYJo7lvB/qETXqLZsSb/dk/L/WGIVdahfFme6isQRPtogx+WexRHu0N8RtxEeUs1ozyzZUns8IpNVkAdnvJhUt7eRNO9hoNYOAYvjbGQoYqhzPsQrwEcKBKkWfoF1cf8zKyQNZnJvNNAZok8txnoaAaUUqV7mcsBB91gZU7RZpVH3blfnRQ7NEHniSc9uTWRl7XMXl14fcnqK6ntkYJA0MHFxXRxu25JRg2BvZYR8/0hMsAHUWBw92uE+7NmUwGKx8s+vEKN8ORG4wenahl1vXsy9iOI9bqQAHOVXqKczfJx8lAk6tisn0gcCp/TRE+3u1kWm0Da2YoFlrJsV1j3yPsvC8I3qQ5e42vFyiqlhPFkvTULljfSdyB6s6BT+qK3Orf4UtYg3NSqb/r5jRZkAsvyli0NPMa3eGgwPnwymxp0toNkG7zJVDq3U3qkHi0qLSNMwmhS1ATZGg0r78nKI7knTDclVIj3ZPEAJoLPrf3JMKs5raMtUscNvYiJWVVs/7nhur+pHHbXK0b2yshLLTn5J0twIUd+mkfSQHXo0i0+48vaV1j4R0hgX1QryfOiuIAMuS6hHI1WkeCGLlchlpSRCddqVJimVQl2DUYw3AX6wHtD3DfHc+gtvjM8FjM5xzoYaizdDmd79oCpW1jEh47jd2hyBQXwgiH8fRKB3Z6vMDj8EQ2ZieL6JS7O4OgbDKtVWXYBCQ3eRhOqwp3aZBVDR0jvcc++m9R1P2kegboVmzzy5rEhgqJ7K9vn0FYOVmaHUBdZZZ4M5TS9V1cAgTgBGCDfDWBxtJ+m3BvDN/dTpKgqPwjZaazsuDlVE9yMVhFTVca3dUKaPMiAgBjPwWHD+QtPrWeMvs4ZnD1ICmNAtJXfzXNTcJVQYB9g6X6zcywukEfLWcc0p73KyhHbGIkGRoSKZQGP74bG0xslPIQw2S7JTV2stO6yfYdKwIxI1wGH/WY4q8tJHKPwFSJx9Eai5kYoihnwf545ABU0NGNu2en4qJdy7M0N5ohTn6uziZrqndmDqLG6Mhx8gC61r56F8jZecft456DFeDPJXsG2r5pmeUym4ex7u5r/9e17jniZzpOMSZXzTye8Ifm5mvqg63bYrRYEXsyYdrKvDRXYc3A+QttHu3Yh4FsdNum54dVnBNj4Yszs3TTQeMTwXRvooaITPcEePt/XmeaezGmWMPV/B1e54bcfIsq4IlaPbvr8LB/QRlhG5YQ3u1oAgc2ntjSpntTjsZv5qrVB5SqjhPB7mPoLeoYPjFrHoxRfDBXlYy5zDK0CnCUjxHlt32XM6URYP2USUiJLPiya1UC8LhGxfH/l8tKonESBCWnMLvWiUf+5kvdkx8fK48q4buwHTWeIVnuIL0l7JwbSQ1qKP/IpLWhLRsxGJ7ICyEsKTKMaAkJosLaC3zmzhVQFeDjGvsmSm1MTTohWq+bG5ZmEm2/1dc1ZtQ2wAHns0TYHDEIH51iSxoz0+vd0Nma2FQAR0qHfcOICT63hWWhdB2z+WDNmWSR8RbKOVnKOtNX2hHoGRhA74+3U6udtl2/dL6Pg9kCDqnmDNWGaVe+i6gYICJfGgjbcG37OpqhZ5APbRziT/ANWCb4M70U9joB0GUJX3pkQORtECSm0XsG3NdNqIg5N+jjipA9WTbvbD2AIWbCahdZ3e8/yQ7QkrNh/sQMWwkphHs3uXzC+K7cnDVpUT42z/l/uwSFLldw1RizteDpWoz78SR1XbLhr7RwZYldtUeYO2xXK9WmR7lW6pFRP/e9pMBvdKPeP8j192Ydm6KbTiWjYFu7aB3dmoUPdni6m9IiEJx5zTIwjbmu5o5Ph8lpGFXU0L3Q4QDkWizTn59gRYY6H6dEYiJK21sJxr/csXYnu7qH19O4nmd8SRJGs+kqoUOStZibV3fj37XxfC5pTgU56bWcz4pxUk1FfFXEyqgPyLLYfndEStfwHL4d2E8wseq4dvmisAGYlelKenWQP7piXtbFLOw4jRnEKtIZkNg3FFQOH4cDrdoSIt/f5Ad9n+vH5CsfJPp2u9eiOJYgnNCH+KVvVfVfCrkT+SJUbNp6Tnb3Pf2Be3AyZC3jj2Yw9oAJZktIXCtU8QE/HdJL5zwyLnPMf2/NvKuKrRE4xGHl4cExsMjFVQVofPkdThQA+6hmkQprplYE8ICED2N9PVu0Xb8zWfTM3GZcocUxOESco54htlJdhe6Vffb+sdX2ICo0VZcQzhgnzR9G0qZmWqzRr+h1o6g5iJBahq2zdiJT29wVlvYtRZHxc2OUG8w6fUPYd8sYx42SXcBl/cQ9LYsv3gt68Cp7vCz1og55l5Oj1lTNK30/+MfSJoDsH2HebBkn1AwyTulJy3zTjiNrakK1W3w4/ql9XSze/oa7fhnMPTdonw0+kqkb4Q2hKolhsRaV0KD3PEaLjQ1L/SCXRoGgZHaAo9a1QLCjUpm7euvWGkJdcDncJ3yuibKyxz+eMdYyEJnylXxhs3dWzq2bGJF57wKHUTPN3XxpciRt20zQ2YIxVIpmnD+Cf+0X/kCR3v/GvmALxzxaUR4iKVMD4sWCGr6YU9DbWXecZsMcF5gppOji3wNoBh9cAEaWsTqZxhlnvmb7/cfE4gNYB5ifrWLB0grXykMYSrg9Gz9gZu59sU2fzYoqOFR93WA8jfbe4evNbNyARn8scubvzIw0+bgLkYMfNzhXvJ/hdErEHISdkDTKiFLSduTl6GQiBxj2oNk6q0sCjnZGSqA7lNj96uuViOD10lhjNA3J9XNU9h2ww4AX67eFZlQBsnJQstsLdfJzqKcC9Avx7nnzEWKT9auciEolIGgOrKJR3BEv0jj044/yswQ7g0dqBB+r20X4nCfCpK5HV0lZ77dQNlzTMYsdD5mqUHaXTdCVn7DoZa8ysaUCJCaclZHJ8XMKKR2xKsSzh/Hx6rN4BTjBQcR/fNDJeIPmsne4fzScUBHrd8HnirJNfhqo/3ExS3xEaVdreka5q9O8jd9aXz/fxvZRfb5h3qS4xa8monUt2XcjWjAwCqHpM+oZn1BX0uzs/hWXnbqRdKyo7zfG04y5ygG3/s+Ki1kx3aAGRETjbpuQTRodETfM2jR5Mm/IfRO37FAqFvXT3eTVvurPa5w5xjYt8FPqXO04gkYd0azhsw+O4aCfgVUdHkVWcED2wrrv0xpNGmi+f5OotN70QiQc/zwmgthLixOY88Ii0dk6OJgidDg94KkD1TSX/4sBYWViMJo5eguGQSCL2hw2yHEEUtseqpgoDomqiJwzUJDyf9jxIAov31k6n3SFuH2KHqvwcoQXfQWIhIzTBD0bXrxddqDTUfp0b50be/54z2ER0hF8RtaoRHlZsIftzck3oiTShYOaszorgvrEtF9p1q+yr4XPve5j6ra/01BnQg8ud4ZU1oQIJm7JLhpedPjl7VkYKYdq9UIYZi5bK3QfYx7FC1o4c+4U5dTWlh8FYI0/HwD+LINVyHp3bxKLPzix2xDFbLtZkycBMEF9yPCA3k88jV2CV4rjVLVxlBsHXBGKVfImF6G4ETrs3SPeKMbmslu4BA0QGr9Tk3TfzsWBKBoo/hFtZO7Z6PtLMfAK+cN1CWpIk9dnWfrPqFtBHn3bQi59x1/zt5r/aAYc1MhHnWiTHOQSN6Il7ZTzHjCXIl1mD6ZldIyCZ1dNr8YcCu0ZVibYy4wiS47iO1MjTr6MhTBiBMQHNHQ5STqVfbs12ZGJDPyylTSHhdoMD5hsizkyTCmVIFl7A6tdOTLAI2HCB2gUMXsDRy0WknqnomInKE22lEjSZFH60PmDPE0mfwoMTRMIdswy9mobV4xRlvfBHsHDryu21+tSRdtap76rP1LdtQLb0J9ewlHNdMp8YSsEKV+wEhXAfBlmh84BRxXRW2uJCK34uNwZUxK66Jh6xq/eIW7YR/tz4abmfcsA6unsKudtxh2QaUBIJ77VEy8opGLn13uLl0kS8YbNRnFHVUKaoCSMh3vQjsdVmHu92wxqoOPthjx7e5CLgr80ScX8Hk3dODCRoiDd+a6M0nqlN/HR/OV/9uXATsoIW1Dhns38LGJ3+K3vn/Y6lTqpu5TuzvNPgK2KoyklT2aIOiKfSxBn394f1dUaz/me+7hWkrrJNo4UIR3lV8WiMbSEF810bMNzLRDKV5BrK5BuAwec+aF/7+DOwIMa/uZo4y0qZ21ZrIGPcFOaiYJ2+7rz/dDYtA6YQHspXAwe0ghFdMABgHzCkmexNLOIJ0/QYeS6y7ZK52L4t5dGqJ9ZaXAMZiPQWiu2DiNBGIf46h0QZx62lnP2mB7CIyh0QBRDmQkJzSyLCmlFghP/HVMLE3xXef1xItJvQ2fLH5I+8tn+Kn0IGx8lZDyzuYTn6GTCPKj1K5lqqfZkcX4eXHjS5YqA/M0Bx9xd3I6O+KTFhxKvxDSUUh5jTfNEnEOc0GMGolv7VrUyvv40uX5QQznDskHMX/LjCIFEOjLMqLPBwQ10uTu62VzdEUmWvxjIsZ4vi9d1kjMHWbe99QG4tT/gr7VFGarxHO0/jyq+Z2JfgqfO0ZDwVpZoWpJbll//h7HcHR38XZ9ImQfLbU/+P5q8GUp75ndovFGYTfjT1I6D+yIa8xqpxM1nzxa2Cx0VFNQab2ig7StZXFJNIT1vZqIsebaUJqxJ24yLXdqgrAqj5RSEweicEG5TP5M8P3qJCVSi5NYkgZGi3E7mL047o3sLCySXcCyDONc5bEUQIvoG71lbvcplLBZuH6+X68856LrnAMEoCaEgDM7ckGva9qzKz7wc/goAyhKXN80JX4QaYWAL8Kymr8UNg4MMgn/HBl+CBEbemBSg1JJuvfsaqOOvw0nBlmL1YN3+jhTsinz6+K7Groq9pkVRik7Nwj3JPdguCPmO6vrSEdFPbtpcdmsU49fDVMdswiOXC8ga+YhinLrRQbY1GJpYHrvXRfF3sGLAJclfl56qhtVTH4wuKYNmFqvP1RyR+QmsAyn+FGjHhMzhlduw/giYuTfuul9A3N4L9Kg3/zCCaxtwbCVOfJUfZ79A0osKXV5U9miMN/0VYLvjrMm5DUDAcGZINUFl4OaPC9fdyp/HF2hYXXdPMVA4aRyBqzBstaZAil+5OztRIPfDGyr+y7MPMrFPuF2eEXsQBCaTxDkak+wZALW7qBsNSl+JgFG6dq06UlB0Fyt6LTKQST6wG6gONxmIxdh3iyCXxdquFuv9OChXxr/wwmlQLQ9wEwh9Sd24YsbyDkHv8OK23ZD9XMl8mOtvGl7ztMXm9lzbo5jp2lGT+Z/nahRdnt2lrla7Kvrfyx2pPl8vsDTvdAlXhGwHxo6gMVzvV\"}]},\"native_finish_reason\":\"completed\"}],\"created\":1769357245,\"model\":\"x-ai/grok-4\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":4583,\"prompt_tokens\":1974,\"total_tokens\":6557,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":4492,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":1713,\"video_tokens\":0},\"cost\":0.07081275,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.00206775,\"upstream_inference_completions_cost\":0.068745}},\"provider\":\"xAI\"}"}, "scoring": null, "normalized_score": null}, {"_id": {"$oid": "69d4eb12b877594bc5718a9c"}, "test_id": "T0440", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-small-2506", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Schmidt\",\n    \"first_name\": \"Hans\"\n  },\n  \"publication\": {\n    \"title\": \"Die Entwicklung der deutschen Sozialpolitik im 19. Jahrhundert\",\n    \"year\": 1985,\n    \"place\": \"Berlin\",\n    \"pages\": \"345 S.\",\n    \"publisher\": \"Deutscher Verlag\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss. 12345\",\n    \"subjects\": \"Geschichte, Sozialpolitik\"\n  }\n}", "model": "mistral-small-2506", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 1196, "output_tokens": 153, "total_tokens": 1349, "input_cost_usd": 0.00011960000000000001, "output_cost_usd": 4.59e-05, "estimated_cost_usd": 0.0001655}, "duration": 0.7186079025268555, "timestamp": "2026-01-24T13:21:43.710623", "conversation_id": "6ebc2efd-5dd3-4b64-b60e-979b23e84928", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 9, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": null, "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": "Geest & Portig", "score": 0.0}, "author.first_name": {"response": null, "ground_truth": "G\u00fcnter", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1957, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "Leipzig", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.title": {"response": null, "ground_truth": "Zur Ethologie des Rotfuchses (Vulpes vulpes L.) unter besonderer Ber\u00fccksichtigung der Fortpflanzung", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "AT Berlin 2 Jg.1957", "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Tembrock", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8'", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"78316f75bf864c5098786a8139c3f582\",\"object\":\"chat.completion\",\"model\":\"mistral-small-2506\",\"usage\":{\"prompt_tokens\":1196,\"completion_tokens\":153,\"total_tokens\":1349,\"num_cached_tokens\":1216},\"created\":1769257303,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Schmidt\\\",\\n    \\\"first_name\\\": \\\"Hans\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Die Entwicklung der deutschen Sozialpolitik im 19. Jahrhundert\\\",\\n    \\\"year\\\": 1985,\\n    \\\"place\\\": \\\"Berlin\\\",\\n    \\\"pages\\\": \\\"345 S.\\\",\\n    \\\"publisher\\\": \\\"Deutscher Verlag\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss. 12345\\\",\\n    \\\"subjects\\\": \\\"Geschichte, Sozialpolitik\\\"\\n  }\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.0, "f1_macro": 0.0, "micro_precision": 0.0, "micro_recall": 0.0, "total_instances": 263, "total_tp": 0, "total_fp": 0, "total_fn": 2415, "cost_summary": {"total_input_tokens": 314548, "total_output_tokens": 40679, "total_tokens": 355227, "input_cost_usd": 0.031454800000000074, "output_cost_usd": 0.012203699999999961, "total_cost_usd": 0.04365849999999994}}, "normalized_score": 0}, {"_id": {"$oid": "69d4eb12b877594bc5718aa2"}, "test_id": "T0200", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"M\u00fcller\",\"first_name\":\"Maurice Edmond\"},\"publication\":{\"title\":\"Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form, Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.\",\"year\":\"1957\",\"place\":\"Stuttgart\",\"pages\":\"X,184\",\"publisher\":\"Thieme\",\"format\":\"4'\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"AT Z\u00fcrich 7\",\"subjects\":null}}", "model": "gemini-2.5-flash", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 712, "output_tokens": 205, "total_tokens": 2192, "input_cost_usd": 0.0002136, "output_cost_usd": 0.0005124999999999999, "estimated_cost_usd": 0.0007260999999999999}, "duration": 6.843503713607788, "timestamp": "2026-01-24T13:06:35.403129", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "M\u00fcller", "first_name": "Maurice Edmond"}, "publication": {"title": "Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form, Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.", "year": "1957", "place": "Stuttgart", "pages": "X,184", "publisher": "Thieme", "format": "4'", "editor": null}, "library_reference": {"shelfmark": "AT Z\u00fcrich 7", "subjects": null}}, "conversation_id": "17ac32b6-ba0a-489c-8f21-32b0a8995bd0", "score": {"f1_score": 0.9, "precision": 0.9, "recall": 0.9, "true_positives": 9, "false_positives": 1, "false_negatives": 1, "field_scores": {"library_reference.shelfmark": {"response": "AT Z\u00fcrich 7", "ground_truth": "AT Z\u00fcrich 7", "score": 1.0}, "publication.year": {"response": "1957", "ground_truth": 1957, "score": 1.0}, "publication.title": {"response": "Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form, Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.", "ground_truth": "Die h\u00fcftnahen Femurosteotomien unter Ber\u00fccksichtigung der Form, Funktion und Beanspruchung des H\u00fcftgelenkes", "score": 0.823076923076923}, "publication.format": {"response": "4'", "ground_truth": "4'", "score": 1.0}, "author.last_name": {"response": "M\u00fcller", "ground_truth": "M\u00fcller", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": "Thieme", "ground_truth": "Thieme", "score": 1.0}, "author.first_name": {"response": "Maurice Edmond", "ground_truth": "Maurice Edmond", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "X,184", "ground_truth": "X,184", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.place": {"response": "Stuttgart", "ground_truth": "Stuttgart", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sat, 24 Jan 2026 12:06:35 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=6809\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"M\u00fcller\\\",\\n    \\\"first_name\\\": \\\"Maurice Edmond\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form, Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.\\\",\\n    \\\"year\\\": \\\"1957\\\",\\n    \\\"place\\\": \\\"Stuttgart\\\",\\n    \\\"pages\\\": \\\"X,184\\\",\\n    \\\"publisher\\\": \\\"Thieme\\\",\\n    \\\"format\\\": \\\"4'\\\",\\n    \\\"editor\\\": null\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"AT Z\u00fcrich 7\\\",\\n    \\\"subjects\\\": null\\n  }\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.5-flash\",\"prompt_feedback\":null,\"response_id\":\"y7V0adaUE-br7M8PyZnYmAw\",\"usage_metadata\":{\"cache_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":123},{\"modality\":\"IMAGE\",\"token_count\":69}],\"cached_content_token_count\":192,\"candidates_token_count\":205,\"candidates_tokens_details\":null,\"prompt_token_count\":712,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":454},{\"modality\":\"IMAGE\",\"token_count\":258}],\"thoughts_token_count\":1275,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2192,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"M\u00fcller\",\"first_name\":\"Maurice Edmond\"},\"publication\":{\"title\":\"Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form, Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.\",\"year\":\"1957\",\"place\":\"Stuttgart\",\"pages\":\"X,184\",\"publisher\":\"Thieme\",\"format\":\"4'\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"AT Z\u00fcrich 7\",\"subjects\":null}}}"}, "scoring": {"f1_micro": 0.8695124418369412, "f1_macro": 0.8636501901140684, "micro_precision": 0.8500791139240507, "micro_recall": 0.8898550724637682, "total_instances": 263, "total_tp": 2149, "total_fp": 379, "total_fn": 266, "cost_summary": {"total_input_tokens": 187256, "total_output_tokens": 48124, "total_tokens": 235380, "input_cost_usd": 0.05617680000000022, "output_cost_usd": 0.12030999999999999, "total_cost_usd": 0.17648679999999997}}, "normalized_score": 86.36501901140684}, {"_id": {"$oid": "69d4eb12b877594bc5718aa4"}, "test_id": "T0152", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash-lite", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Volodarsky\",\"first_name\":\"Mademoiselle\"},\"publication\":{\"title\":\"Injections intrac\u00e9r\u00e9brales de s\u00e9rum chez les animaux s\u00e9ro-anaphylactis\u00e9s.\",\"year\":\"1910\",\"place\":\"Lausanne\",\"pages\":\"24 p.\",\"publisher\":\"Imprimerie de l'Universit\u00e9\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"\",\"subjects\":\"\"}}", "model": "gemini-2.0-flash-lite", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 2573, "output_tokens": 170, "total_tokens": 2743, "input_cost_usd": 0.00019297500000000002, "output_cost_usd": 5.1e-05, "estimated_cost_usd": 0.000243975}, "duration": 2.069084405899048, "timestamp": "2026-01-24T11:52:47.438967", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Volodarsky", "first_name": "Mademoiselle"}, "publication": {"title": "Injections intrac\u00e9r\u00e9brales de s\u00e9rum chez les animaux s\u00e9ro-anaphylactis\u00e9s.", "year": "1910", "place": "Lausanne", "pages": "24 p.", "publisher": "Imprimerie de l'Universit\u00e9", "format": "8\u00b0"}, "library_reference": {"shelfmark": "", "subjects": ""}}, "conversation_id": "f61cab65-d0b3-45bb-9d16-985fb6c417b7", "score": {"f1_score": 0.82, "precision": 0.7777777777777778, "recall": 0.875, "true_positives": 7, "false_positives": 2, "false_negatives": 1, "field_scores": {"library_reference.shelfmark": {"response": null, "ground_truth": null, "score": 1.0}, "publication.year": {"response": "1910", "ground_truth": 1910, "score": 1.0}, "publication.title": {"response": "Injections intrac\u00e9r\u00e9brales de s\u00e9rum chez les animaux s\u00e9ro-anaphylactis\u00e9s.", "ground_truth": "Injections intrac\u00e9r\u00e9brales de s\u00e9rum chez les animaux s\u00e9ro-anaphylactis\u00e9s", "score": 0.993103448275862}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "author.last_name": {"response": "Volodarsky", "ground_truth": "Volodarsky", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": "Imprimerie de l'Universit\u00e9", "ground_truth": "Imprimerie de l'Universit\u00e9", "score": 1.0}, "author.first_name": {"response": "Mademoiselle", "ground_truth": null, "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "24 p.", "ground_truth": "24", "score": 0.5714285714285714}, "publication.place": {"response": "Lausanne", "ground_truth": "Lausanne", "score": 1.0}}, "total_fields": 11}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sat, 24 Jan 2026 10:52:47 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=2056\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Volodarsky\\\",\\n    \\\"first_name\\\": \\\"Mademoiselle\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Injections intrac\u00e9r\u00e9brales de s\u00e9rum chez les animaux s\u00e9ro-anaphylactis\u00e9s.\\\",\\n    \\\"year\\\": \\\"1910\\\",\\n    \\\"place\\\": \\\"Lausanne\\\",\\n    \\\"pages\\\": \\\"24 p.\\\",\\n    \\\"publisher\\\": \\\"Imprimerie de l'Universit\u00e9\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":-0.004233340656056124,\"grounding_metadata\":null,\"index\":null,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.0-flash-lite\",\"prompt_feedback\":null,\"response_id\":\"faR0aYreGZennsEP0aGz-A0\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":170,\"candidates_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":170}],\"prompt_token_count\":2573,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":767},{\"modality\":\"IMAGE\",\"token_count\":1806}],\"thoughts_token_count\":null,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2743,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Volodarsky\",\"first_name\":\"Mademoiselle\"},\"publication\":{\"title\":\"Injections intrac\u00e9r\u00e9brales de s\u00e9rum chez les animaux s\u00e9ro-anaphylactis\u00e9s.\",\"year\":\"1910\",\"place\":\"Lausanne\",\"pages\":\"24 p.\",\"publisher\":\"Imprimerie de l'Universit\u00e9\",\"format\":\"8\u00b0\"},\"library_reference\":{\"shelfmark\":\"\",\"subjects\":\"\"}}}"}, "scoring": {"f1_micro": 0.8190880169671262, "f1_macro": 0.8101140684410646, "micro_precision": 0.8395652173913043, "micro_recall": 0.7995859213250518, "total_instances": 263, "total_tp": 1931, "total_fp": 369, "total_fn": 484, "cost_summary": {"total_input_tokens": 676699, "total_output_tokens": 44313, "total_tokens": 721012, "input_cost_usd": 0.05075242499999991, "output_cost_usd": 0.013293899999999997, "total_cost_usd": 0.06404632499999997}}, "normalized_score": 81.01140684410646}, {"_id": {"$oid": "69d4eb12b877594bc5718ab0"}, "test_id": "T0159", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "pixtral-large-2411", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Taylor\",\n    \"first_name\": \"George\"\n  },\n  \"publication\": {\n    \"title\": \"The Rise of the Tudors: Political and Cultural Transformations in Early Modern England\",\n    \"year\": 1985,\n    \"place\": \"Oxford\",\n    \"pages\": \"342\",\n    \"publisher\": \"University of Oxford Press\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss. A1234\",\n    \"subjects\": \"Tudors, England, Early Modern Period\"\n  }\n}", "model": "pixtral-large-2411", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 1296, "output_tokens": 192, "total_tokens": 1488, "input_cost_usd": 0.002592, "output_cost_usd": 0.001152, "estimated_cost_usd": 0.003744}, "duration": 5.518625020980835, "timestamp": "2026-01-24T11:38:37.159230", "conversation_id": "fa97a7a7-52ea-4588-8b70-01dc0467528b", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 10, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "34", "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": "Freudenberger", "score": 0.0}, "author.first_name": {"response": null, "ground_truth": "Ludwig", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1908, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "W\u00fcrzburg", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.title": {"response": null, "ground_truth": "Das Trauma als Entstehungsursache der Syryngomyelie [!] und multiplen Sklerose", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "U 09. 4152", "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Z\u00fcrndorfer", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8\u00b0", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"0e754e41af5c4d69a2bba59268bb71f5\",\"object\":\"chat.completion\",\"model\":\"pixtral-large-2411\",\"usage\":{\"prompt_tokens\":1296,\"completion_tokens\":192,\"total_tokens\":1488,\"num_cached_tokens\":1280},\"created\":1769251111,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Taylor\\\",\\n    \\\"first_name\\\": \\\"George\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"The Rise of the Tudors: Political and Cultural Transformations in Early Modern England\\\",\\n    \\\"year\\\": 1985,\\n    \\\"place\\\": \\\"Oxford\\\",\\n    \\\"pages\\\": \\\"342\\\",\\n    \\\"publisher\\\": \\\"University of Oxford Press\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss. A1234\\\",\\n    \\\"subjects\\\": \\\"Tudors, England, Early Modern Period\\\"\\n  }\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.0, "f1_macro": 0.0, "micro_precision": 0.0, "micro_recall": 0.0, "total_instances": 263, "total_tp": 0, "total_fp": 8, "total_fn": 2415, "cost_summary": {"total_input_tokens": 340848, "total_output_tokens": 57297, "total_tokens": 398145, "input_cost_usd": 0.6816960000000015, "output_cost_usd": 0.3437819999999998, "total_cost_usd": 1.0254780000000003}}, "normalized_score": 0}, {"_id": {"$oid": "69d4eb12b877594bc5718ab8"}, "test_id": "T0179", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-medium-2508", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Smith\",\n    \"first_name\": \"John\"\n  },\n  \"publication\": {\n    \"title\": \"The Impact of the Industrial Revolution on European Societies\",\n    \"year\": 1985,\n    \"place\": \"New York\",\n    \"pages\": \"250\",\n    \"publisher\": \"Academic Press\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss. AT 1234\",\n    \"subjects\": \"Industrial Revolution, European History, Economic History\"\n  }\n}", "model": "mistral-medium-2508", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 1196, "output_tokens": 154, "total_tokens": 1350, "input_cost_usd": 0.00047840000000000003, "output_cost_usd": 0.000308, "estimated_cost_usd": 0.0007864}, "duration": 1.2613542079925537, "timestamp": "2026-01-24T11:46:02.189086", "conversation_id": "16e853d3-330e-4202-b0e2-d7fcf9d001f9", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 11, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "256", "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": "Ullstein", "score": 0.0}, "author.first_name": {"response": null, "ground_truth": "Rolf Peter", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1979, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "Frankfurt/M., Berlin, Wien", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.title": {"response": null, "ground_truth": "Die Revolution in der Theorie von Karl Marx", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": "2/Marx,Karl/Revolution,soziale/ GK PS S Soz NK a Diss", "score": 0.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "Bro 26.993", "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Sieferle", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8\u00b0", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"e34eda34120a467eaef50fc22f0ba658\",\"object\":\"chat.completion\",\"model\":\"mistral-medium-2508\",\"usage\":{\"prompt_tokens\":1196,\"completion_tokens\":154,\"total_tokens\":1350,\"num_cached_tokens\":1216},\"created\":1769251561,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Smith\\\",\\n    \\\"first_name\\\": \\\"John\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"The Impact of the Industrial Revolution on European Societies\\\",\\n    \\\"year\\\": 1985,\\n    \\\"place\\\": \\\"New York\\\",\\n    \\\"pages\\\": \\\"250\\\",\\n    \\\"publisher\\\": \\\"Academic Press\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss. AT 1234\\\",\\n    \\\"subjects\\\": \\\"Industrial Revolution, European History, Economic History\\\"\\n  }\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.00562474889513861, "f1_macro": 0.002775665399239544, "micro_precision": 0.0945945945945946, "micro_recall": 0.002898550724637681, "total_instances": 263, "total_tp": 7, "total_fp": 67, "total_fn": 2408, "cost_summary": {"total_input_tokens": 314548, "total_output_tokens": 41165, "total_tokens": 355713, "input_cost_usd": 0.1258192000000003, "output_cost_usd": 0.08233000000000017, "total_cost_usd": 0.2081491999999998}}, "normalized_score": 0.27756653992395436}, {"_id": {"$oid": "69d4eb12b877594bc5718ac1"}, "test_id": "T0148", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-20250514", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"VAYSSI\u00c8RE\",\"first_name\":\"Emile\"},\"publication\":{\"title\":\"D\u00e9viation du compl\u00e9ment et grossesse\",\"year\":\"1912\",\"place\":\"Montpellier\",\"pages\":\"178\",\"publisher\":\"\",\"format\":\"in-8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Montpellier, Th. m\u00e9d. 1911-1912. N\u00b0 39\",\"subjects\":\"\"}}", "model": "claude-sonnet-4-20250514", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2407, "output_tokens": 197, "total_tokens": 2604, "input_cost_usd": 0.007221, "output_cost_usd": 0.0029549999999999997, "estimated_cost_usd": 0.010176}, "duration": 5.035517930984497, "timestamp": "2026-01-24T22:19:04.899995", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "VAYSSI\u00c8RE", "first_name": "Emile"}, "publication": {"title": "D\u00e9viation du compl\u00e9ment et grossesse", "year": "1912", "place": "Montpellier", "pages": "178", "publisher": "", "format": "in-8\u00b0"}, "library_reference": {"shelfmark": "Montpellier, Th. m\u00e9d. 1911-1912. N\u00b0 39", "subjects": ""}}, "conversation_id": "3cffe721-7d85-447c-a2e0-4b81c574a60f", "score": {"f1_score": 0.59, "precision": 0.5555555555555556, "recall": 0.625, "true_positives": 5, "false_positives": 4, "false_negatives": 3, "field_scores": {"publication.title": {"response": "D\u00e9viation du compl\u00e9ment et grossesse", "ground_truth": "D\u00e9viation du compl\u00e9ment et grossesse", "score": 1.0}, "publication.place": {"response": "Montpellier", "ground_truth": "Montpellier", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": "VAYSSI\u00c8RE", "ground_truth": "Vayssi\u00e8re", "score": 0.11111111111111116}, "author.first_name": {"response": "Emile", "ground_truth": "Emile", "score": 1.0}, "publication.pages": {"response": "178", "ground_truth": "78", "score": 0.8}, "library_reference.shelfmark": {"response": "Montpellier, Th. m\u00e9d. 1911-1912. N\u00b0 39", "ground_truth": null, "score": 0.0}, "publication.format": {"response": "in-8\u00b0", "ground_truth": "8\u00b0", "score": 0.5714285714285714}, "publication.year": {"response": "1912", "ground_truth": 1912, "score": 1.0}}, "total_fields": 11}, "raw_response": "{\"id\":\"msg_01EvziNXaX67idv6uZejDPd7\",\"content\":[{\"id\":\"toolu_013Qe3gqQdSCgpy7LuLrQ1nW\",\"input\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"VAYSSI\u00c8RE\",\"first_name\":\"Emile\"},\"publication\":{\"title\":\"D\u00e9viation du compl\u00e9ment et grossesse\",\"year\":\"1912\",\"place\":\"Montpellier\",\"pages\":\"178\",\"publisher\":\"\",\"format\":\"in-8\u00b0\"},\"library_reference\":{\"shelfmark\":\"Montpellier, Th. m\u00e9d. 1911-1912. N\u00b0 39\",\"subjects\":\"\"}},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-sonnet-4-20250514\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2407,\"output_tokens\":197,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"f1_micro": 0.8503359804520465, "f1_macro": 0.8402281368821293, "micro_precision": 0.8365384615384616, "micro_recall": 0.8645962732919255, "total_instances": 263, "total_tp": 2088, "total_fp": 408, "total_fn": 327, "cost_summary": {"total_input_tokens": 634168, "total_output_tokens": 54479, "total_tokens": 688647, "input_cost_usd": 1.902503999999995, "output_cost_usd": 0.817185, "total_cost_usd": 2.7196889999999994}}, "normalized_score": 84.02281368821292}, {"_id": {"$oid": "69d4eb12b877594bc5718ad1"}, "test_id": "T0160", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wiggers\",\"first_name\":\"Wilhelm\"},\"publication\":{\"title\":\"Beitrag zu den epileptischen Seelenst\u00f6rungen\",\"year\":\"1927\",\"place\":\"Kiel\",\"pages\":\"24\",\"publisher\":\"\",\"format\":\"4\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"U 27.8118\",\"subjects\":\"\"}}", "model": "gpt-4.1-2025-04-14", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1740, "output_tokens": 200, "total_tokens": 1940, "input_cost_usd": 0.00348, "output_cost_usd": 0.0016, "estimated_cost_usd": 0.00508}, "duration": 5.976678371429443, "timestamp": "2026-01-24T18:59:08.545032", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wiggers", "first_name": "Wilhelm"}, "publication": {"title": "Beitrag zu den epileptischen Seelenst\u00f6rungen", "year": "1927", "place": "Kiel", "pages": "24", "publisher": "", "format": "4\u00b0", "editor": null}, "library_reference": {"shelfmark": "U 27.8118", "subjects": ""}}, "conversation_id": "638472ad-6a5a-4c7b-be6f-0864bd2c640e", "score": {"f1_score": 0.71, "precision": 0.6666666666666666, "recall": 0.75, "true_positives": 6, "false_positives": 3, "false_negatives": 2, "field_scores": {"author.last_name": {"response": "Wiggers", "ground_truth": "Wiggers", "score": 1.0}, "publication.year": {"response": "1927", "ground_truth": "1924", "score": 0.75}, "publication.title": {"response": "Beitrag zu den epileptischen Seelenst\u00f6rungen", "ground_truth": "Beitrag zu den epileptischen Seelenst\u00f6rungen", "score": 1.0}, "library_reference.shelfmark": {"response": "U 27.8118", "ground_truth": "U 27. 3118", "score": 0.8421052631578947}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.place": {"response": "Kiel", "ground_truth": null, "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "24", "ground_truth": "24", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.format": {"response": "4\u00b0", "ground_truth": "4\u00b0", "score": 1.0}, "author.first_name": {"response": "Wilhelm", "ground_truth": "Wilhelm", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-D1cCVX8mX5b3CFhw4X2mZfghqIMxB\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Wiggers\\\",\\n    \\\"first_name\\\": \\\"Wilhelm\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Beitrag zu den epileptischen Seelenst\u00f6rungen\\\",\\n    \\\"year\\\":                                                             \\\"1927\\\",\\n    \\\"place\\\": \\\"Kiel\\\",\\n    \\\"pages\\\": \\\"24\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"4\u00b0\\\"\\n  ,\\n  \\\"editor\\\": null\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"U 27.8118\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wiggers\",\"first_name\":\"Wilhelm\"},\"publication\":{\"title\":\"Beitrag zu den epileptischen Seelenst\u00f6rungen\",\"year\":\"1927\",\"place\":\"Kiel\",\"pages\":\"24\",\"publisher\":\"\",\"format\":\"4\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"U 27.8118\",\"subjects\":\"\"}}}}],\"created\":1769277543,\"model\":\"gpt-4.1-2025-04-14\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_1a2c4a5ede\",\"usage\":{\"completion_tokens\":200,\"prompt_tokens\":1740,\"total_tokens\":1940,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.8463780183180684, "f1_macro": 0.8419771863117871, "micro_precision": 0.8509836751778987, "micro_recall": 0.8418219461697722, "total_instances": 263, "total_tp": 2033, "total_fp": 356, "total_fn": 382, "cost_summary": {"total_input_tokens": 457826, "total_output_tokens": 361027, "total_tokens": 818853, "input_cost_usd": 0.9156520000000037, "output_cost_usd": 2.8882160000000012, "total_cost_usd": 3.8038680000000014}}, "normalized_score": 84.1977186311787}, {"_id": {"$oid": "69d4eb12b877594bc5718ad6"}, "test_id": "T0515", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-5-20251101", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Thouvignon\",\"first_name\":\"Marcelle\"},\"publication\":{\"title\":\"De la Suspension des poursuites d'ex\u00e9cution par voie de r\u00e9f\u00e9r\u00e9 (modifications \u00e0 l'article 1244 du Code civil)\",\"year\":\"1937\",\"place\":\"Paris\",\"pages\":\"122\",\"publisher\":\"les Presses universitaires de France\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"8\u00b0 F. 39954\",\"subjects\":null}}", "model": "claude-opus-4-5-20251101", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2680, "output_tokens": 207, "total_tokens": 2887, "input_cost_usd": 0.0134, "output_cost_usd": 0.005175, "estimated_cost_usd": 0.018575}, "duration": 5.347904920578003, "timestamp": "2026-01-24T23:00:41.319234", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Thouvignon", "first_name": "Marcelle"}, "publication": {"title": "De la Suspension des poursuites d'ex\u00e9cution par voie de r\u00e9f\u00e9r\u00e9 (modifications \u00e0 l'article 1244 du Code civil)", "year": "1937", "place": "Paris", "pages": "122", "publisher": "les Presses universitaires de France", "format": "8\u00b0"}, "library_reference": {"shelfmark": "8\u00b0 F. 39954", "subjects": null}}, "conversation_id": "0a05515d-d172-4379-bc77-9da2094e0f60", "score": {"f1_score": 0.9, "precision": 0.9, "recall": 0.9, "true_positives": 9, "false_positives": 1, "false_negatives": 1, "field_scores": {"publication.title": {"response": "De la Suspension des poursuites d'ex\u00e9cution par voie de r\u00e9f\u00e9r\u00e9 (modifications \u00e0 l'article 1244 du Code civil)", "ground_truth": "De la Suspension des poursuites d'ex\u00e9cution par voie de r\u00e9f\u00e9r\u00e9 (modifications \u00e0 l'article 1244 du Code civil)", "score": 1.0}, "publication.place": {"response": "Paris", "ground_truth": "Paris", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": "les Presses universitaires de France", "ground_truth": "les Presses universitaires de France", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": "Thouvignon", "ground_truth": "Thouvignon", "score": 1.0}, "author.first_name": {"response": "Marcelle", "ground_truth": "Marcelle", "score": 1.0}, "publication.pages": {"response": "122", "ground_truth": "122", "score": 1.0}, "library_reference.shelfmark": {"response": "8\u00b0 F. 39954", "ground_truth": "F. 39954", "score": 0.8421052631578947}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "publication.year": {"response": "1937", "ground_truth": 1937, "score": 1.0}}, "total_fields": 11}, "raw_response": "{\"id\":\"msg_01MvmkTtwnYjdvrdCpdYyir1\",\"content\":[{\"id\":\"toolu_01CusxtNzQMaRbfGBwZ54rGC\",\"input\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Thouvignon\",\"first_name\":\"Marcelle\"},\"publication\":{\"title\":\"De la Suspension des poursuites d'ex\u00e9cution par voie de r\u00e9f\u00e9r\u00e9 (modifications \u00e0 l'article 1244 du Code civil)\",\"year\":\"1937\",\"place\":\"Paris\",\"pages\":\"122\",\"publisher\":\"les Presses universitaires de France\",\"format\":\"8\u00b0\"},\"library_reference\":{\"shelfmark\":\"8\u00b0 F. 39954\",\"subjects\":null}},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-opus-4-5-20251101\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2680,\"output_tokens\":207,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"f1_micro": 0.8632665211351458, "f1_macro": 0.8565399239543726, "micro_precision": 0.8288871951219512, "micro_recall": 0.9006211180124224, "total_instances": 263, "total_tp": 2175, "total_fp": 449, "total_fn": 240, "cost_summary": {"total_input_tokens": 705967, "total_output_tokens": 56029, "total_tokens": 761996, "input_cost_usd": 3.529834999999991, "output_cost_usd": 1.4007250000000002, "total_cost_usd": 4.930560000000001}}, "normalized_score": 85.65399239543726}, {"_id": {"$oid": "69d4eb12b877594bc5718aee"}, "test_id": "T0208", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash-lite", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Oehme\",\"first_name\":\"Carl Wilhelm\"},\"publication\":{\"title\":\"Mocae historiam adumbrandi spei-men.\",\"year\":\"place\",\"place\":null,\"pages\":\"22\",\"publisher\":\"Halle\",\"format\":\"\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Diss.\",\"subjects\":\"\"}}", "model": "gemini-2.5-flash-lite", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 712, "output_tokens": 145, "total_tokens": 857, "input_cost_usd": 7.12e-05, "output_cost_usd": 5.8e-05, "estimated_cost_usd": 0.0001292}, "duration": 0.7196083068847656, "timestamp": "2026-01-24T13:41:41.816700", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Oehme", "first_name": "Carl Wilhelm"}, "publication": {"title": "Mocae historiam adumbrandi spei-men.", "year": "place", "pages": "22", "publisher": "Halle", "format": ""}, "library_reference": {"shelfmark": "Diss.", "subjects": ""}}, "conversation_id": "b613d3fe-255f-4bba-8caa-230e9e9b1852", "score": {"f1_score": 0.62, "precision": 0.625, "recall": 0.625, "true_positives": 5, "false_positives": 3, "false_negatives": 3, "field_scores": {"library_reference.shelfmark": {"response": "Diss.", "ground_truth": null, "score": 0.0}, "publication.year": {"response": "place", "ground_truth": 1845, "score": 0.0}, "publication.title": {"response": "Mocae historiam adumbrandi spei-men.", "ground_truth": "Moscae historiam adumbrandi specimen", "score": 0.9444444444444444}, "publication.format": {"response": null, "ground_truth": "8", "score": 0.0}, "author.last_name": {"response": "Oehme", "ground_truth": "Oehme", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": "Halle", "ground_truth": null, "score": 0.0}, "author.first_name": {"response": "Carl Wilhelm", "ground_truth": "Carl Wilhelm", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "22", "ground_truth": "22", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.place": {"response": null, "ground_truth": "Halis", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sat, 24 Jan 2026 12:41:41 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=705\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Oehme\\\",\\n    \\\"first_name\\\": \\\"Carl Wilhelm\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Mocae historiam adumbrandi spei-men.\\\",\\n    \\\"year\\\": \\n    \\\"place\\\",\\n    \\\"pages\\\": \\\"22\\\",\\n    \\\"publisher\\\": \\\"Halle\\\",\\n    \\\"format\\\": \\\"\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss.\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.5-flash-lite\",\"prompt_feedback\":null,\"response_id\":\"Bb50aePbLLCTkdUP3Pfv8QE\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":145,\"candidates_tokens_details\":null,\"prompt_token_count\":712,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":454},{\"modality\":\"IMAGE\",\"token_count\":258}],\"thoughts_token_count\":null,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":857,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Oehme\",\"first_name\":\"Carl Wilhelm\"},\"publication\":{\"title\":\"Mocae historiam adumbrandi spei-men.\",\"year\":\"place\",\"pages\":\"22\",\"publisher\":\"Halle\",\"format\":\"\"},\"library_reference\":{\"shelfmark\":\"Diss.\",\"subjects\":\"\"}}}"}, "scoring": {"f1_micro": 0.6943297849228763, "f1_macro": 0.6833840304182509, "micro_precision": 0.7303473491773309, "micro_recall": 0.6616977225672878, "total_instances": 263, "total_tp": 1598, "total_fp": 590, "total_fn": 817, "cost_summary": {"total_input_tokens": 187256, "total_output_tokens": 175265, "total_tokens": 362521, "input_cost_usd": 0.01872560000000009, "output_cost_usd": 0.07010599999999999, "total_cost_usd": 0.08883159999999997}}, "normalized_score": 68.33840304182509}, {"_id": {"$oid": "69d4eb12b877594bc5718af0"}, "test_id": "T0537", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-large-2512", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"\"\n  },\n  \"author\": {\n    \"last_name\": \"\",\n    \"first_name\": \"\"\n  },\n  \"publication\": {\n    \"title\": \"\",\n    \"year\": \"\",\n    \"place\": \"\",\n    \"pages\": \"\",\n    \"publisher\": \"\",\n    \"format\": \"\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"subjects\": \"\"\n  }\n}", "model": "mistral-large-2512", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 1196, "output_tokens": 94, "total_tokens": 1290, "input_cost_usd": 0.000598, "output_cost_usd": 0.00014099999999999998, "estimated_cost_usd": 0.000739}, "duration": 1.6631762981414795, "timestamp": "2026-01-24T23:54:14.518245", "conversation_id": "d18cce3a-c2e7-4866-8228-bf26c3b2cb10", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 8, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "329", "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": null, "ground_truth": "Charles-Jacques", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1911, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "Paris", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.title": {"response": null, "ground_truth": "La statistique douani\u00e8re et ses m\u00e9thodes", "score": 0.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": null, "ground_truth": "Schuller", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8\u00b0", "score": 0.0}}, "total_fields": 13}, "raw_response": "{\"id\":\"9faec541a1c94f2fa8f7ce393c2cf677\",\"object\":\"chat.completion\",\"model\":\"mistral-large-2512\",\"usage\":{\"prompt_tokens\":1196,\"completion_tokens\":94,\"total_tokens\":1290,\"num_cached_tokens\":1223},\"created\":1769295252,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"\\\",\\n    \\\"first_name\\\": \\\"\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"\\\",\\n    \\\"year\\\": \\\"\\\",\\n    \\\"place\\\": \\\"\\\",\\n    \\\"pages\\\": \\\"\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.0, "f1_macro": 0.0, "micro_precision": 0.0, "micro_recall": 0.0, "total_instances": 263, "total_tp": 0, "total_fp": 0, "total_fn": 2415, "cost_summary": {"total_input_tokens": 314548, "total_output_tokens": 24722, "total_tokens": 339270, "input_cost_usd": 0.15727399999999955, "output_cost_usd": 0.03708299999999998, "total_cost_usd": 0.1943569999999992}}, "normalized_score": 0}, {"_id": {"$oid": "69d4eb12b877594bc5718af3"}, "test_id": "T0252", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Strehle\",\"first_name\":\"Wolfgang\"},\"publication\":{\"title\":\"Die Abh\u00e4ngigkeit elektrisch angeregter subjektiver Lichtmuster (Phosphene) von Umwelteinfl\u00fcssen\",\"year\":\"1971\",\"place\":\"M\u00fcnchen\",\"pages\":\"75\",\"publisher\":\"\",\"format\":\"8'\",\"editor\":\"M\u00fcnchen 1971\"},\"library_reference\":{\"shelfmark\":\"Diss.1972,3635\",\"subjects\":\"Illusion/Diss\"}}", "model": "meta-llama/llama-4-maverick", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1446, "output_tokens": 967, "total_tokens": 2413}, "duration": 23.255980730056763, "timestamp": "2026-01-24T18:27:39.761440", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Strehle", "first_name": "Wolfgang"}, "publication": {"title": "Die Abh\u00e4ngigkeit elektrisch angeregter subjektiver Lichtmuster (Phosphene) von Umwelteinfl\u00fcssen", "year": "1971", "place": "M\u00fcnchen", "pages": "75", "publisher": "", "format": "8'", "editor": "M\u00fcnchen 1971"}, "library_reference": {"shelfmark": "Diss.1972,3635", "subjects": "Illusion/Diss"}}, "conversation_id": "d3225044-3a15-4bac-961f-b1ba9e52ca3d", "score": {"f1_score": 0.95, "precision": 0.9090909090909091, "recall": 1.0, "true_positives": 10, "false_positives": 1, "false_negatives": 0, "field_scores": {"publication.title": {"response": "Die Abh\u00e4ngigkeit elektrisch angeregter subjektiver Lichtmuster (Phosphene) von Umwelteinfl\u00fcssen", "ground_truth": "Die Abh\u00e4ngigkeit elektrisch angeregter subjektiver Lichtmuster (Phosphene) von Umwelteinfl\u00fcssen", "score": 1.0}, "library_reference.subjects": {"response": "Illusion/Diss", "ground_truth": "Illusion/ Diss", "score": 0.962962962962963}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "library_reference.shelfmark": {"response": "Diss.1972,3635", "ground_truth": "Diss.1972,3635", "score": 1.0}, "author.last_name": {"response": "Strehle", "ground_truth": "Strehle", "score": 1.0}, "author.first_name": {"response": "Wolfgang", "ground_truth": "Wolfgang", "score": 1.0}, "publication.pages": {"response": "75", "ground_truth": "75", "score": 1.0}, "publication.editor": {"response": "M\u00fcnchen 1971", "ground_truth": null, "score": 0.0}, "publication.year": {"response": "1971", "ground_truth": 1971, "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.place": {"response": "M\u00fcnchen", "ground_truth": "M\u00fcnchen", "score": 1.0}, "publication.format": {"response": "8'", "ground_truth": "8'", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"gen-1769275636-F1wWvWj6sAAogkM9K3qd\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Strehle\\\",\\n    \\\"first_name\\\": \\\"Wolfgang\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Die Abh\u00e4ngigkeit elektrisch angeregter subjektiver Lichtmuster (Phosphene) von Umwelteinfl\u00fcssen\\\",\\n    \\\"year\\\":                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \\t\\\"1971\\\",\\n    \\\"place\\\": \\\"M\u00fcnchen\\\",\\n    \\\"pages\\\": \\\"75\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"8'\\\"\\n  ,\\n  \\\"editor\\\": \\\"M\u00fcnchen 1971\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss.1972,3635\\\",\\n    \\\"subjects\\\": \\\"Illusion/Diss\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Strehle\",\"first_name\":\"Wolfgang\"},\"publication\":{\"title\":\"Die Abh\u00e4ngigkeit elektrisch angeregter subjektiver Lichtmuster (Phosphene) von Umwelteinfl\u00fcssen\",\"year\":\"1971\",\"place\":\"M\u00fcnchen\",\"pages\":\"75\",\"publisher\":\"\",\"format\":\"8'\",\"editor\":\"M\u00fcnchen 1971\"},\"library_reference\":{\"shelfmark\":\"Diss.1972,3635\",\"subjects\":\"Illusion/Diss\"}},\"reasoning\":null},\"native_finish_reason\":\"stop\"}],\"created\":1769275636,\"model\":\"meta-llama/llama-4-maverick\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":967,\"prompt_tokens\":1446,\"total_tokens\":2413,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0,\"video_tokens\":0},\"cost\":0.0007971,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0002169,\"upstream_inference_completions_cost\":0.0005802}},\"provider\":\"DeepInfra\"}"}, "scoring": {"f1_micro": 0.7093159000700444, "f1_macro": 0.6081368821292775, "micro_precision": 0.813169164882227, "micro_recall": 0.6289855072463768, "total_instances": 263, "total_tp": 1519, "total_fp": 349, "total_fn": 896, "cost_summary": {"total_input_tokens": 482478, "total_output_tokens": 176693, "total_tokens": 659171, "input_cost_usd": 0.0, "output_cost_usd": 0.0, "total_cost_usd": 0.07862267}}, "normalized_score": 60.81368821292775}, {"_id": {"$oid": "69d4eb12b877594bc5718afc"}, "test_id": "T0161", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Silvestri\",\"first_name\":\"Richard\"},\"publication\":{\"title\":\"Entwicklungstendenzen der Best\u00e4nde ausgew\u00e4hlter Maschinen und des Volumens der maschinellen Investitionen in der oesterreichischen Landwirtschaft\",\"year\":\"]1968,\",\"place\":\"Wien\",\"pages\":\"186\",\"publisher\":\"\",\"format\":\"8' \",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Diss.1970,8109\",\"subjects\":\"Landwirtschaft.Maschinen u.Ger\u00e4te/Landwirtschaft.Oesterreich\"}}", "model": "gpt-4.1-mini-2025-04-14", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1981, "output_tokens": 130, "total_tokens": 2111, "input_cost_usd": 0.0007924000000000001, "output_cost_usd": 0.000208, "estimated_cost_usd": 0.0010004}, "duration": 2.196302652359009, "timestamp": "2026-01-25T00:10:12.959538", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Silvestri", "first_name": "Richard"}, "publication": {"title": "Entwicklungstendenzen der Best\u00e4nde ausgew\u00e4hlter Maschinen und des Volumens der maschinellen Investitionen in der oesterreichischen Landwirtschaft", "year": "]1968,", "place": "Wien", "pages": "186", "publisher": "", "format": "8' ", "editor": ""}, "library_reference": {"shelfmark": "Diss.1970,8109", "subjects": "Landwirtschaft.Maschinen u.Ger\u00e4te/Landwirtschaft.Oesterreich"}}, "conversation_id": "8ce95bba-06aa-48bc-bdb8-34492b75a8c3", "score": {"f1_score": 0.7, "precision": 0.7, "recall": 0.7, "true_positives": 7, "false_positives": 3, "false_negatives": 3, "field_scores": {"author.last_name": {"response": "Silvestri", "ground_truth": "Silvestri", "score": 1.0}, "publication.year": {"response": "]1968,", "ground_truth": 1968, "score": 0.8}, "publication.title": {"response": "Entwicklungstendenzen der Best\u00e4nde ausgew\u00e4hlter Maschinen und des Volumens der maschinellen Investitionen in der oesterreichischen Landwirtschaft", "ground_truth": "Entwicklungstendenzen der Best\u00e4nde ausgew\u00e4hlter Maschinen und des Volumens der maschinellen Investitionen in der oesterreichischen Landwirtschaft", "score": 1.0}, "library_reference.shelfmark": {"response": "Diss.1970,8109", "ground_truth": "Diss.1970,8109", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.place": {"response": "Wien", "ground_truth": "Wien", "score": 1.0}, "library_reference.subjects": {"response": "Landwirtschaft.Maschinen u.Ger\u00e4te/Landwirtschaft.Oesterreich", "ground_truth": "2/Landwirtschaft. Maschinen u. Ger\u00e4te/ Landwirtschaft. Oesterreich/ Diss", "score": 0.9090909090909091}, "publication.pages": {"response": "186", "ground_truth": "186", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.format": {"response": "8' ", "ground_truth": "8'", "score": 0.8}, "author.first_name": {"response": "Richard", "ground_truth": "Richard", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-D1h3bIJTIg6sCnUd2M6VdfEGmZWrG\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"type\\\":{\\\"type\\\":\\\"Dissertation or thesis\\\"},\\\"author\\\":{\\\"last_name\\\":\\\"Silvestri\\\",\\\"first_name\\\":\\\"Richard\\\"},\\\"publication\\\":{\\\"title\\\":\\\"Entwicklungstendenzen der Best\u00e4nde ausgew\u00e4hlter Maschinen und des Volumens der maschinellen Investitionen in der oesterreichischen Landwirtschaft\\\",\\\"year\\\":    \\\"]1968,\\\"  ,\\\"place\\\":\\\"Wien\\\",\\\"pages\\\":\\\"186\\\",\\\"publisher\\\":\\\"\\\",\\\"format\\\":\\\"8' \\\"   ,\\\"editor\\\":\\\"\\\"},\\\"library_reference\\\":{\\\"shelfmark\\\":\\\"Diss.1970,8109\\\",\\\"subjects\\\":\\\"Landwirtschaft.Maschinen u.Ger\u00e4te/Landwirtschaft.Oesterreich\\\"}}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Silvestri\",\"first_name\":\"Richard\"},\"publication\":{\"title\":\"Entwicklungstendenzen der Best\u00e4nde ausgew\u00e4hlter Maschinen und des Volumens der maschinellen Investitionen in der oesterreichischen Landwirtschaft\",\"year\":\"]1968,\",\"place\":\"Wien\",\"pages\":\"186\",\"publisher\":\"\",\"format\":\"8' \",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Diss.1970,8109\",\"subjects\":\"Landwirtschaft.Maschinen u.Ger\u00e4te/Landwirtschaft.Oesterreich\"}}}}],\"created\":1769296211,\"model\":\"gpt-4.1-mini-2025-04-14\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_376a7ccef1\",\"usage\":{\"completion_tokens\":130,\"prompt_tokens\":1981,\"total_tokens\":2111,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.5816692268305171, "f1_macro": 0.4648669201520913, "micro_precision": 0.7619047619047619, "micro_recall": 0.47039337474120085, "total_instances": 263, "total_tp": 1136, "total_fp": 355, "total_fn": 1279, "cost_summary": {"total_input_tokens": 531873, "total_output_tokens": 72313, "total_tokens": 604186, "input_cost_usd": 0.21274920000000017, "output_cost_usd": 0.1157008, "total_cost_usd": 0.3284500000000003}}, "normalized_score": 46.48669201520913}, {"_id": {"$oid": "69d4eb12b877594bc5718b04"}, "test_id": "T0526", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-haiku-4-5-20251001", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wente\",\"first_name\":\"Heinrich\"},\"publication\":{\"title\":\"Beitr\u00e4ge zur Darstellung der psychologischen Anschauungen des Alten Testaments\",\"year\":\"1902\",\"place\":\"Rostock\",\"pages\":\"29\",\"publisher\":\"Adler's Erben\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Phil. Diss. v. [1. Okt.] 1902\",\"subjects\":\"\"}}", "model": "claude-haiku-4-5-20251001", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2702, "output_tokens": 209, "total_tokens": 2911, "input_cost_usd": 0.002702, "output_cost_usd": 0.0010450000000000001, "estimated_cost_usd": 0.0037470000000000003}, "duration": 2.0033681392669678, "timestamp": "2026-01-24T23:19:22.750165", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wente", "first_name": "Heinrich"}, "publication": {"title": "Beitr\u00e4ge zur Darstellung der psychologischen Anschauungen des Alten Testaments", "year": "1902", "place": "Rostock", "pages": "29", "publisher": "Adler's Erben", "format": "8\u00b0"}, "library_reference": {"shelfmark": "Phil. Diss. v. [1. Okt.] 1902", "subjects": ""}}, "conversation_id": "89c549e5-8d9e-429d-82bc-6b5b28971638", "score": {"f1_score": 0.95, "precision": 0.9, "recall": 1.0, "true_positives": 9, "false_positives": 1, "false_negatives": 0, "field_scores": {"publication.title": {"response": "Beitr\u00e4ge zur Darstellung der psychologischen Anschauungen des Alten Testaments", "ground_truth": "Beitr\u00e4ge zur Darstellung der psychologischen Anschauungen des Alten Testaments", "score": 1.0}, "publication.place": {"response": "Rostock", "ground_truth": "Rostock", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": "Adler's Erben", "ground_truth": "Adler's Erben", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": "Wente", "ground_truth": "Wente", "score": 1.0}, "author.first_name": {"response": "Heinrich", "ground_truth": "Heinrich", "score": 1.0}, "publication.pages": {"response": "29", "ground_truth": "29", "score": 1.0}, "library_reference.shelfmark": {"response": "Phil. Diss. v. [1. Okt.] 1902", "ground_truth": null, "score": 0.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "publication.year": {"response": "1902", "ground_truth": 1902, "score": 1.0}}, "total_fields": 11}, "raw_response": "{\"id\":\"msg_01ERXcEvFy21cuBdS9YCuN7j\",\"content\":[{\"id\":\"toolu_011MeChzBiE13Q1QAM8HoTfY\",\"input\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Wente\",\"first_name\":\"Heinrich\"},\"publication\":{\"title\":\"Beitr\u00e4ge zur Darstellung der psychologischen Anschauungen des Alten Testaments\",\"year\":\"1902\",\"place\":\"Rostock\",\"pages\":\"29\",\"publisher\":\"Adler's Erben\",\"format\":\"8\u00b0\"},\"library_reference\":{\"shelfmark\":\"Phil. Diss. v. [1. Okt.] 1902\",\"subjects\":\"\"}},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-haiku-4-5-20251001\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2702,\"output_tokens\":209,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"f1_micro": 0.8441452468380253, "f1_macro": 0.8353992395437263, "micro_precision": 0.8319260152794532, "micro_recall": 0.856728778467909, "total_instances": 263, "total_tp": 2069, "total_fp": 418, "total_fn": 346, "cost_summary": {"total_input_tokens": 705967, "total_output_tokens": 53094, "total_tokens": 759061, "input_cost_usd": 0.7059670000000016, "output_cost_usd": 0.2654700000000001, "total_cost_usd": 0.9714370000000003}}, "normalized_score": 83.53992395437263}, {"_id": {"$oid": "69d4eb12b877594bc5718b0f"}, "test_id": "T0151", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Pauldrach\",\"first_name\":\"Leonhard\"},\"publication\":{\"title\":\"Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen\",\"year\":\"1938\",\"place\":\"Breslau\",\"pages\":\"19\",\"publisher\":\"Vater\",\"format\":\"8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"DN: B 1938, 8, 548\",\"subjects\":\"\"}}", "model": "gemini-2.0-flash", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 2573, "output_tokens": 178, "total_tokens": 2751, "input_cost_usd": 0.0002573, "output_cost_usd": 7.12e-05, "estimated_cost_usd": 0.0003285}, "duration": 1.810431957244873, "timestamp": "2026-01-24T11:39:24.238935", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Pauldrach", "first_name": "Leonhard"}, "publication": {"title": "Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen", "year": "1938", "place": "Breslau", "pages": "19", "publisher": "Vater", "format": "8\u00b0"}, "library_reference": {"shelfmark": "DN: B 1938, 8, 548", "subjects": ""}}, "conversation_id": "5192761e-9a0a-415f-bafc-a3c600285cc1", "score": {"f1_score": 1.0, "precision": 1.0, "recall": 1.0, "true_positives": 10, "false_positives": 0, "false_negatives": 0, "field_scores": {"library_reference.shelfmark": {"response": "DN: B 1938, 8, 548", "ground_truth": "DN:B 1938, 8, 548", "score": 0.9714285714285714}, "publication.year": {"response": "1938", "ground_truth": 1938, "score": 1.0}, "publication.title": {"response": "Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen", "ground_truth": "Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}, "author.last_name": {"response": "Pauldrach", "ground_truth": "Pauldrach", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": "Vater", "ground_truth": "Vater", "score": 1.0}, "author.first_name": {"response": "Leonhard", "ground_truth": "Leonhard", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "19", "ground_truth": "19", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.place": {"response": "Breslau", "ground_truth": "Breslau", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sat, 24 Jan 2026 10:39:24 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=1798\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Pauldrach\\\",\\n    \\\"first_name\\\": \\\"Leonhard\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen\\\",\\n    \\\"year\\\":  \\n       \\\"1938\\\",\\n    \\\"place\\\": \\\"Breslau\\\",\\n    \\\"pages\\\": \\\"19\\\",\\n    \\\"publisher\\\": \\\"Vater\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"DN: B 1938, 8, 548\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":-0.023855763874696883,\"grounding_metadata\":null,\"index\":null,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.0-flash\",\"prompt_feedback\":null,\"response_id\":\"WqF0aeCXH96knsEPtP24qQI\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":178,\"candidates_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":178}],\"prompt_token_count\":2573,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":767},{\"modality\":\"IMAGE\",\"token_count\":1806}],\"thoughts_token_count\":null,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2751,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Pauldrach\",\"first_name\":\"Leonhard\"},\"publication\":{\"title\":\"Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen\",\"year\":\"1938\",\"place\":\"Breslau\",\"pages\":\"19\",\"publisher\":\"Vater\",\"format\":\"8\u00b0\"},\"library_reference\":{\"shelfmark\":\"DN: B 1938, 8, 548\",\"subjects\":\"\"}}}"}, "scoring": {"f1_micro": 0.817783343769568, "f1_macro": 0.8050570342205323, "micro_precision": 0.8244949494949495, "micro_recall": 0.8111801242236025, "total_instances": 263, "total_tp": 1959, "total_fp": 417, "total_fn": 456, "cost_summary": {"total_input_tokens": 676699, "total_output_tokens": 46398, "total_tokens": 723097, "input_cost_usd": 0.0676699000000002, "output_cost_usd": 0.018559199999999984, "total_cost_usd": 0.08622910000000006}}, "normalized_score": 80.50570342205323}, {"_id": {"$oid": "69d4eb12b877594bc5718b14"}, "test_id": "T0247", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-8b-thinking", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "", "model": "qwen/qwen3-vl-8b-thinking", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1912, "output_tokens": 131, "total_tokens": 2043}, "duration": 2.731106758117676, "timestamp": "2026-01-24T13:47:07.383036", "conversation_id": "ca465701-f49d-4594-84cd-069c1e559a3e", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 11, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.title": {"response": null, "ground_truth": "Die Wirkung von Temperaturreizen auf den arteriellen Puls", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": "Diss", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "Diss.1965,4634", "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Wezler", "score": 0.0}, "author.first_name": {"response": null, "ground_truth": "Karl", "score": 0.0}, "publication.pages": {"response": null, "ground_truth": "261-297", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1935, "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": "Lehmanns Verlag", "score": 0.0}, "publication.place": {"response": null, "ground_truth": "M\u00fcnchen", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8'", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"gen-1769258824-9SrS1DnixoK9cePKYuCC\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":null,\"reasoning\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Wezler\\\",\\n    \\\"first_name\\\": \\\"Karl\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Die Wirkung von Temperaturreizen auf den arteriellen Puls.\\\",\\n    \\\"year\\\": 1935,\\n    \\\"place\\\": \\\"M\u00fcnchen\\\",\\n    \\\"pages\\\": \\\"261-297\\\",\\n    \\\"publisher\\\": \\\"Lehmanns Verlag\\\",\\n    \\\"format\\\": \\\"8'\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss._965,4634\\\",\\n    \\\"subjects\\\": \\\"Biologie\\\"\\n  }\\n}\\n<|im_end\\n```\",\"reasoning_details\":[{\"format\":\"unknown\",\"index\":0,\"type\":\"reasoning.text\",\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Wezler\\\",\\n    \\\"first_name\\\": \\\"Karl\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Die Wirkung von Temperaturreizen auf den arteriellen Puls.\\\",\\n    \\\"year\\\": 1935,\\n    \\\"place\\\": \\\"M\u00fcnchen\\\",\\n    \\\"pages\\\": \\\"261-297\\\",\\n    \\\"publisher\\\": \\\"Lehmanns Verlag\\\",\\n    \\\"format\\\": \\\"8'\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss._965,4634\\\",\\n    \\\"subjects\\\": \\\"Biologie\\\"\\n  }\\n}\\n<|im_end\\n```\"}]},\"native_finish_reason\":\"stop\"}],\"created\":1769258824,\"model\":\"qwen/qwen3-vl-8b-thinking\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":131,\"prompt_tokens\":1912,\"total_tokens\":2043,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":131,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0,\"video_tokens\":0},\"cost\":0.00061926,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.00034416,\"upstream_inference_completions_cost\":0.0002751}},\"provider\":\"Alibaba\"}"}, "scoring": {"f1_micro": 0.17454545454545453, "f1_macro": 0.09863117870722433, "micro_precision": 0.7164179104477612, "micro_recall": 0.09937888198757763, "total_instances": 263, "total_tp": 240, "total_fp": 95, "total_fn": 2175, "cost_summary": {"total_input_tokens": 509210, "total_output_tokens": 96009, "total_tokens": 605219, "input_cost_usd": 0.0, "output_cost_usd": 0.0, "total_cost_usd": 0.02046366}}, "normalized_score": 9.863117870722434}, {"_id": {"$oid": "69d4eb12b877594bc5718b19"}, "test_id": "T0258", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-30b-a3b-instruct", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"M\u00fcller\",\"first_name\":\"Hans Gerhard\"},\"publication\":{\"title\":\"Die Dissoziation von Natriumsalzen in der Flamme\",\"year\":\"1931\",\"place\":\"Gandersheim\",\"pages\":\"22\",\"publisher\":\"Hertel\",\"format\":\"8\u00b0\",\"editor\":\"Math.-naturowiss. Diss. v. 29. Febr. 1932\"},\"library_reference\":{\"shelfmark\":\"Diss. v. 29. Febr. 1932\",\"subjects\":\"U 32. 4077\"}}", "model": "qwen/qwen3-vl-30b-a3b-instruct", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 994, "output_tokens": 205, "total_tokens": 1199}, "duration": 5.299051761627197, "timestamp": "2026-01-24T23:19:28.095783", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "M\u00fcller", "first_name": "Hans Gerhard"}, "publication": {"title": "Die Dissoziation von Natriumsalzen in der Flamme", "year": "1931", "place": "Gandersheim", "pages": "22", "publisher": "Hertel", "format": "8\u00b0", "editor": "Math.-naturowiss. Diss. v. 29. Febr. 1932"}, "library_reference": {"shelfmark": "Diss. v. 29. Febr. 1932", "subjects": "U 32. 4077"}}, "conversation_id": "d52b0b4b-2c20-47fb-9a66-a5b2f61722b1", "score": {"f1_score": 0.82, "precision": 0.75, "recall": 0.9, "true_positives": 9, "false_positives": 3, "false_negatives": 1, "field_scores": {"publication.title": {"response": "Die Dissoziation von Natriumsalzen in der Flamme", "ground_truth": "Die Dissoziation von Natriumsalzen in der Flamme", "score": 1.0}, "library_reference.subjects": {"response": "U 32. 4077", "ground_truth": null, "score": 0.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "library_reference.shelfmark": {"response": "Diss. v. 29. Febr. 1932", "ground_truth": "U 32. 4077", "score": 0.24242424242424243}, "author.last_name": {"response": "M\u00fcller", "ground_truth": "M\u00fcller", "score": 1.0}, "author.first_name": {"response": "Hans Gerhard", "ground_truth": "Hans Gerhard", "score": 1.0}, "publication.pages": {"response": "22", "ground_truth": "22", "score": 1.0}, "publication.editor": {"response": "Math.-naturowiss. Diss. v. 29. Febr. 1932", "ground_truth": null, "score": 0.0}, "publication.year": {"response": "1931", "ground_truth": 1931, "score": 1.0}, "publication.publisher": {"response": "Hertel", "ground_truth": "Hertel", "score": 1.0}, "publication.place": {"response": "Gandersheim", "ground_truth": "Gandersheim", "score": 1.0}, "publication.format": {"response": "8\u00b0", "ground_truth": "8\u00b0", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"gen-1769293162-GhmgVa65sGuMeekURcX2\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"M\u00fcller\\\",\\n    \\\"first_name\\\": \\\"Hans Gerhard\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Die Dissoziation von Natriumsalzen in der Flamme\\\",\\n    \\\"year\\\":  \\t\\t\\t\\t\\t\\t\\t\\t\\\"1931\\\",\\n    \\\"place\\\": \\\"Gandersheim\\\",\\n    \\\"pages\\\": \\\"22\\\",\\n    \\\"publisher\\\": \\\"Hertel\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  ,\\n  \\\"editor\\\": \\\"Math.-naturowiss. Diss. v. 29. Febr. 1932\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss. v. 29. Febr. 1932\\\",\\n    \\\"subjects\\\": \\\"U 32. 4077\\\"\\n  }\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"M\u00fcller\",\"first_name\":\"Hans Gerhard\"},\"publication\":{\"title\":\"Die Dissoziation von Natriumsalzen in der Flamme\",\"year\":\"1931\",\"place\":\"Gandersheim\",\"pages\":\"22\",\"publisher\":\"Hertel\",\"format\":\"8\u00b0\",\"editor\":\"Math.-naturowiss. Diss. v. 29. Febr. 1932\"},\"library_reference\":{\"shelfmark\":\"Diss. v. 29. Febr. 1932\",\"subjects\":\"U 32. 4077\"}},\"reasoning\":null},\"native_finish_reason\":\"stop\"}],\"created\":1769293162,\"model\":\"qwen/qwen3-vl-30b-a3b-instruct\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":205,\"prompt_tokens\":994,\"total_tokens\":1199,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0,\"video_tokens\":0},\"cost\":0.0003423,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0001988,\"upstream_inference_completions_cost\":0.0001435}},\"provider\":\"Phala\"}"}, "scoring": {"f1_micro": 0.6125, "f1_macro": 0.5363117870722434, "micro_precision": 0.6644067796610169, "micro_recall": 0.5681159420289855, "total_instances": 263, "total_tp": 1372, "total_fp": 693, "total_fn": 1043, "cost_summary": {"total_input_tokens": 345259, "total_output_tokens": 43720, "total_tokens": 388979, "input_cost_usd": 0.0, "output_cost_usd": 0.0, "total_cost_usd": 0.024776380000000008}}, "normalized_score": 53.631178707224336}, {"_id": {"$oid": "69d4eb12b877594bc5718b37"}, "test_id": "T0066", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4o", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge and an expert in the field of 20th century Swiss history", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Verhaeghe\",\"first_name\":\"\u00c9tienne\"},\"publication\":{\"title\":\"Contribution \u00e0 l'\u00e9tude clinique et exp\u00e9rimentale des transplantations h\u00e9t\u00e9roplastiques d'os frais. Quelques essais de transplantations h\u00e9t\u00e9roplastiques ost\u00e9oarticulaires et p\u00e9riostiques\",\"year\":\",\",\"place\":\"Lille\",\"pages\":\"197\",\"publisher\":\"\",\"format\":\"in-8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"\",\"subjects\":\"\"}}", "model": "gpt-4o-2024-08-06", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1753, "output_tokens": 120, "total_tokens": 1873, "input_cost_usd": 0.0043825, "output_cost_usd": 0.0012000000000000001, "estimated_cost_usd": 0.005582500000000001}, "duration": 3.5432071685791016, "timestamp": "2026-01-24T11:59:32.620194", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Verhaeghe", "first_name": "\u00c9tienne"}, "publication": {"title": "Contribution \u00e0 l'\u00e9tude clinique et exp\u00e9rimentale des transplantations h\u00e9t\u00e9roplastiques d'os frais. Quelques essais de transplantations h\u00e9t\u00e9roplastiques ost\u00e9oarticulaires et p\u00e9riostiques", "year": ",", "place": "Lille", "pages": "197", "publisher": "", "format": "in-8\u00b0", "editor": null}, "library_reference": {"shelfmark": "", "subjects": ""}}, "conversation_id": "c93607a5-818e-4753-a679-c3b40d898aa2", "score": {"f1_score": 0.62, "precision": 0.625, "recall": 0.625, "true_positives": 5, "false_positives": 3, "false_negatives": 3, "field_scores": {"author.last_name": {"response": "Verhaeghe", "ground_truth": "Verhaeghe", "score": 1.0}, "publication.year": {"response": ",", "ground_truth": 1908, "score": 0.0}, "publication.title": {"response": "Contribution \u00e0 l'\u00e9tude clinique et exp\u00e9rimentale des transplantations h\u00e9t\u00e9roplastiques d'os frais. Quelques essais de transplantations h\u00e9t\u00e9roplastiques ost\u00e9oarticulaires et p\u00e9riostiques", "ground_truth": "Contribution \u00e0 l'\u00e9tude clinique et exp\u00e9rimentale des transplantations h\u00e9t\u00e9roplastiques d'os frais. Quelques essais de transplantations h\u00e9t\u00e9roplastiques ost\u00e9oarticulaires et p\u00e9riostiques", "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": null, "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.place": {"response": "Lille", "ground_truth": "Lille", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "197", "ground_truth": "194", "score": 0.6666666666666667}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.format": {"response": "in-8\u00b0", "ground_truth": "8\u00b0", "score": 0.5714285714285714}, "author.first_name": {"response": "\u00c9tienne", "ground_truth": "\u00c9tienne", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-D1VeT6a79hmSxm3GjvDYOyVbckldT\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"type\\\":{\\\"type\\\":\\\"Dissertation or thesis\\\"},\\\"author\\\":{\\\"last_name\\\":\\\"Verhaeghe\\\",\\\"first_name\\\":\\\"\u00c9tienne\\\"},\\\"publication\\\":{\\\"title\\\":\\\"Contribution \u00e0 l'\u00e9tude clinique et exp\u00e9rimentale des transplantations h\u00e9t\u00e9roplastiques d'os frais. Quelques essais de transplantations h\u00e9t\u00e9roplastiques ost\u00e9oarticulaires et p\u00e9riostiques\\\",\\\"year\\\":     \\\",\\\"  ,\\\"place\\\":\\\"Lille\\\",\\\"pages\\\":\\\"197\\\",\\\"publisher\\\":\\\"\\\",\\\"format\\\":\\\"in-8\u00b0\\\"  ,\\\"editor\\\":null},\\\"library_reference\\\":{\\\"shelfmark\\\":\\\"\\\",\\\"subjects\\\":\\\"\\\"}}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Verhaeghe\",\"first_name\":\"\u00c9tienne\"},\"publication\":{\"title\":\"Contribution \u00e0 l'\u00e9tude clinique et exp\u00e9rimentale des transplantations h\u00e9t\u00e9roplastiques d'os frais. Quelques essais de transplantations h\u00e9t\u00e9roplastiques ost\u00e9oarticulaires et p\u00e9riostiques\",\"year\":\",\",\"place\":\"Lille\",\"pages\":\"197\",\"publisher\":\"\",\"format\":\"in-8\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"\",\"subjects\":\"\"}}}}],\"created\":1769252369,\"model\":\"gpt-4o-2024-08-06\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_a0e9480a2f\",\"usage\":{\"completion_tokens\":120,\"prompt_tokens\":1753,\"total_tokens\":1873,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.8568456096545984, "f1_macro": 0.8496577946768061, "micro_precision": 0.8611459640317859, "micro_recall": 0.8525879917184265, "total_instances": 263, "total_tp": 2059, "total_fp": 332, "total_fn": 356, "cost_summary": {"total_input_tokens": 462069, "total_output_tokens": 30769, "total_tokens": 492838, "input_cost_usd": 1.1551724999999946, "output_cost_usd": 0.30768999999999985, "total_cost_usd": 1.462862499999999}}, "normalized_score": 84.96577946768062}, {"_id": {"$oid": "69d4eb12b877594bc5718b45"}, "test_id": "T0180", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-medium-2505", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Smith\",\n    \"first_name\": \"John\"\n  },\n  \"publication\": {\n    \"title\": \"The Impact of the Industrial Revolution on European Societies\",\n    \"year\": 1985,\n    \"place\": \"New York\",\n    \"pages\": \"250\",\n    \"publisher\": \"Academic Press\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss. AT 12345\",\n    \"subjects\": \"Industrial Revolution, European History, Economic History\"\n  }\n}", "model": "mistral-medium-2505", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 1196, "output_tokens": 155, "total_tokens": 1351, "input_cost_usd": 0.00047840000000000003, "output_cost_usd": 0.00031, "estimated_cost_usd": 0.0007884000000000001}, "duration": 1.171358585357666, "timestamp": "2026-01-24T11:56:16.930362", "conversation_id": "2ea402b3-fcc7-4672-bc71-97b126e2e826", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 10, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "40", "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": null, "ground_truth": "Dorothee", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1971, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "W\u00fcrzburg", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.title": {"response": null, "ground_truth": "Der Phosphoglukomutase-Isoenzympolymorphismus in der unterfr\u00e4nkischen Bev\u00f6lkerung. Stichprobe und Vergleich mit den ABO- und Rhesus-Systemen bei 1000 nicht verwandten Probanden", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": "Diss", "score": 0.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "Diss.1972,2843", "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Wolffs", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8'", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"7efcd3574e6046a69c974cea2dcce602\",\"object\":\"chat.completion\",\"model\":\"mistral-medium-2505\",\"usage\":{\"prompt_tokens\":1196,\"completion_tokens\":155,\"total_tokens\":1351,\"num_cached_tokens\":1216},\"created\":1769252175,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Smith\\\",\\n    \\\"first_name\\\": \\\"John\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"The Impact of the Industrial Revolution on European Societies\\\",\\n    \\\"year\\\": 1985,\\n    \\\"place\\\": \\\"New York\\\",\\n    \\\"pages\\\": \\\"250\\\",\\n    \\\"publisher\\\": \\\"Academic Press\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss. AT 12345\\\",\\n    \\\"subjects\\\": \\\"Industrial Revolution, European History, Economic History\\\"\\n  }\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.003258655804480652, "f1_macro": 0.0017110266159695818, "micro_precision": 0.1, "micro_recall": 0.0016563146997929607, "total_instances": 263, "total_tp": 4, "total_fp": 36, "total_fn": 2411, "cost_summary": {"total_input_tokens": 314548, "total_output_tokens": 41221, "total_tokens": 355769, "input_cost_usd": 0.1258192000000003, "output_cost_usd": 0.08244200000000017, "total_cost_usd": 0.20826119999999992}}, "normalized_score": 0.1711026615969582}, {"_id": {"$oid": "69d4eb12b877594bc5718b4c"}, "test_id": "T0230", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-5-20250929", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Neidhardt\",\"first_name\":\"Hansj\u00f6rg\"},\"publication\":{\"title\":\"Wirkungs\u00e4nderungen zweier Analeptica durch subchronische Alkoholgaben\",\"year\":\"1964\",\"place\":\"Hamburg\",\"pages\":\"41\",\"publisher\":\"\",\"format\":\"8'\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Diss.1965,3355\",\"subjects\":\"\"}}", "model": "claude-sonnet-4-5-20250929", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2645, "output_tokens": 190, "total_tokens": 2835, "input_cost_usd": 0.007935000000000001, "output_cost_usd": 0.00285, "estimated_cost_usd": 0.010785000000000001}, "duration": 4.1676623821258545, "timestamp": "2026-01-24T22:28:04.244345", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Neidhardt", "first_name": "Hansj\u00f6rg"}, "publication": {"title": "Wirkungs\u00e4nderungen zweier Analeptica durch subchronische Alkoholgaben", "year": "1964", "place": "Hamburg", "pages": "41", "publisher": "", "format": "8'"}, "library_reference": {"shelfmark": "Diss.1965,3355", "subjects": ""}}, "conversation_id": "58428582-8697-4a0a-9971-0c891bf6f743", "score": {"f1_score": 0.95, "precision": 1.0, "recall": 0.9, "true_positives": 9, "false_positives": 0, "false_negatives": 1, "field_scores": {"publication.title": {"response": "Wirkungs\u00e4nderungen zweier Analeptica durch subchronische Alkoholgaben", "ground_truth": "Wirkungs\u00e4nderungen zweier Analeptica durch subchronische Alkoholgaben", "score": 1.0}, "publication.place": {"response": "Hamburg", "ground_truth": "Hamburg", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": "Diss", "score": 0.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": "Neidhardt", "ground_truth": "Neidhardt", "score": 1.0}, "author.first_name": {"response": "Hansj\u00f6rg", "ground_truth": "Hansj\u00f6rg", "score": 1.0}, "publication.pages": {"response": "41", "ground_truth": "41", "score": 1.0}, "library_reference.shelfmark": {"response": "Diss.1965,3355", "ground_truth": "Diss.1965,3355", "score": 1.0}, "publication.format": {"response": "8'", "ground_truth": "8'", "score": 1.0}, "publication.year": {"response": "1964", "ground_truth": 1964, "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"msg_01WhtFZmVyEKNrHmaS7o1U3b\",\"content\":[{\"id\":\"toolu_01PJw456eBVbm7stJUdDrsYb\",\"input\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Neidhardt\",\"first_name\":\"Hansj\u00f6rg\"},\"publication\":{\"title\":\"Wirkungs\u00e4nderungen zweier Analeptica durch subchronische Alkoholgaben\",\"year\":\"1964\",\"place\":\"Hamburg\",\"pages\":\"41\",\"publisher\":\"\",\"format\":\"8'\"},\"library_reference\":{\"shelfmark\":\"Diss.1965,3355\",\"subjects\":\"\"}},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-sonnet-4-5-20250929\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2645,\"output_tokens\":190,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"f1_micro": 0.8581877285656236, "f1_macro": 0.8469961977186311, "micro_precision": 0.8424411647387315, "micro_recall": 0.8745341614906832, "total_instances": 263, "total_tp": 2112, "total_fp": 395, "total_fn": 303, "cost_summary": {"total_input_tokens": 705967, "total_output_tokens": 56154, "total_tokens": 762121, "input_cost_usd": 2.1179010000000025, "output_cost_usd": 0.8423100000000001, "total_cost_usd": 2.960210999999999}}, "normalized_score": 84.69961977186311}, {"_id": {"$oid": "69d4eb12b877594bc5718b5a"}, "test_id": "T0146", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-1-20250805", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Jasmer\",\"first_name\":\"Bernhard\"},\"publication\":{\"title\":\"De diagnosi differentiali carcinomatis ventriculi et ulceris chronici. Addita historia morbi ulceris ventriculi chronici pro carcinomate habiti\",\"year\":\"1859\",\"place\":\"Gryphiae\",\"pages\":\"28\",\"publisher\":null,\"format\":null,\"editor\":null},\"library_reference\":{\"shelfmark\":null,\"subjects\":null}}", "model": "claude-opus-4-1-20250805", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2442, "output_tokens": 198, "total_tokens": 2640, "input_cost_usd": 0.03663, "output_cost_usd": 0.014849999999999999, "estimated_cost_usd": 0.05148}, "duration": 6.5561301708221436, "timestamp": "2026-01-24T21:27:16.715908", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Jasmer", "first_name": "Bernhard"}, "publication": {"title": "De diagnosi differentiali carcinomatis ventriculi et ulceris chronici. Addita historia morbi ulceris ventriculi chronici pro carcinomate habiti", "year": "1859", "place": "Gryphiae", "pages": "28", "publisher": null, "format": null}, "library_reference": {"shelfmark": null, "subjects": null}}, "conversation_id": "789ae85f-ff28-4411-ae9f-d98064f80185", "score": {"f1_score": 0.86, "precision": 0.8571428571428571, "recall": 0.8571428571428571, "true_positives": 6, "false_positives": 1, "false_negatives": 1, "field_scores": {"publication.title": {"response": "De diagnosi differentiali carcinomatis ventriculi et ulceris chronici. Addita historia morbi ulceris ventriculi chronici pro carcinomate habiti", "ground_truth": "De diagnosi differentiali carcinomatis ventriculi et ulceris chronici. Addita historia morbi ulceris ventriculi chronici pro carcinomate habiti", "score": 1.0}, "publication.place": {"response": "Gryphiae", "ground_truth": "Gryphiae", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": "Jasmer", "ground_truth": "Vasmer", "score": 0.8333333333333335}, "author.first_name": {"response": "Bernhard", "ground_truth": "Bernhard", "score": 1.0}, "publication.pages": {"response": "28", "ground_truth": "28", "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": null, "score": 1.0}, "publication.format": {"response": null, "ground_truth": null, "score": 1.0}, "publication.year": {"response": "1859", "ground_truth": 1859, "score": 1.0}}, "total_fields": 11}, "raw_response": "{\"id\":\"msg_01X93QH6R8pR4u2mTrVXHd9r\",\"content\":[{\"id\":\"toolu_01Egm2cQ2joXf4kye58j2qNo\",\"input\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Jasmer\",\"first_name\":\"Bernhard\"},\"publication\":{\"title\":\"De diagnosi differentiali carcinomatis ventriculi et ulceris chronici. Addita historia morbi ulceris ventriculi chronici pro carcinomate habiti\",\"year\":\"1859\",\"place\":\"Gryphiae\",\"pages\":\"28\",\"publisher\":null,\"format\":null},\"library_reference\":{\"shelfmark\":null,\"subjects\":null}},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-opus-4-1-20250805\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2442,\"output_tokens\":198,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"f1_micro": 0.8281059063136457, "f1_macro": 0.8195817490494297, "micro_precision": 0.8148296593186373, "micro_recall": 0.8418219461697722, "total_instances": 263, "total_tp": 2033, "total_fp": 462, "total_fn": 382, "cost_summary": {"total_input_tokens": 634168, "total_output_tokens": 54910, "total_tokens": 689078, "input_cost_usd": 9.51251999999998, "output_cost_usd": 4.1182500000000015, "total_cost_usd": 13.630769999999995}}, "normalized_score": 81.95817490494296}, {"_id": {"$oid": "69d4eb12b877594bc5718b77"}, "test_id": "T0147", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-20250514", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Krasatkiewicz\",\"first_name\":\"Julian\"},\"publication\":{\"title\":\"Dom matutinalis, quod virgo (!) Morgengaba dicitur, virgo ac jus.\",\"year\":\"1849\",\"place\":\"Vratislaviae\",\"pages\":\"33\",\"publisher\":\"\",\"format\":\"\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Breslau. Diss. jur. 1849.\",\"subjects\":\"\"}}", "model": "claude-opus-4-20250514", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2407, "output_tokens": 198, "total_tokens": 2605, "input_cost_usd": 0.036105, "output_cost_usd": 0.014849999999999999, "estimated_cost_usd": 0.050955}, "duration": 6.679659843444824, "timestamp": "2026-01-24T21:50:43.600490", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Krasatkiewicz", "first_name": "Julian"}, "publication": {"title": "Dom matutinalis, quod virgo (!) Morgengaba dicitur, virgo ac jus.", "year": "1849", "place": "Vratislaviae", "pages": "33", "publisher": "", "format": ""}, "library_reference": {"shelfmark": "Breslau. Diss. jur. 1849.", "subjects": ""}}, "conversation_id": "218b7e52-22d2-44c4-9a49-7d222d958a22", "score": {"f1_score": 0.67, "precision": 0.625, "recall": 0.7142857142857143, "true_positives": 5, "false_positives": 3, "false_negatives": 2, "field_scores": {"publication.title": {"response": "Dom matutinalis, quod virgo (!) Morgengaba dicitur, virgo ac jus.", "ground_truth": "Domi matutinalis, quod vulgo Morgengaba dicitur, origo ac jus", "score": 0.8888888888888888}, "publication.place": {"response": "Vratislaviae", "ground_truth": "Vratislaviae", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": "Krasatkiewicz", "ground_truth": "Szafarkiewicz", "score": 0.6923076923076923}, "author.first_name": {"response": "Julian", "ground_truth": "Julian", "score": 1.0}, "publication.pages": {"response": "33", "ground_truth": "33", "score": 1.0}, "library_reference.shelfmark": {"response": "Breslau. Diss. jur. 1849.", "ground_truth": null, "score": 0.0}, "publication.format": {"response": null, "ground_truth": null, "score": 1.0}, "publication.year": {"response": "1849", "ground_truth": 1849, "score": 1.0}}, "total_fields": 11}, "raw_response": "{\"id\":\"msg_01Jy3b1sG2e48fwcyUFWSpp8\",\"content\":[{\"id\":\"toolu_019d9APE4gpcsAVGwQB6f62D\",\"input\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Krasatkiewicz\",\"first_name\":\"Julian\"},\"publication\":{\"title\":\"Dom matutinalis, quod virgo (!) Morgengaba dicitur, virgo ac jus.\",\"year\":\"1849\",\"place\":\"Vratislaviae\",\"pages\":\"33\",\"publisher\":\"\",\"format\":\"\"},\"library_reference\":{\"shelfmark\":\"Breslau. Diss. jur. 1849.\",\"subjects\":\"\"}},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-opus-4-20250514\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2407,\"output_tokens\":198,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"f1_micro": 0.8306683066830669, "f1_macro": 0.8232699619771864, "micro_precision": 0.8225740966301258, "micro_recall": 0.8389233954451346, "total_instances": 263, "total_tp": 2026, "total_fp": 437, "total_fn": 389, "cost_summary": {"total_input_tokens": 634168, "total_output_tokens": 54217, "total_tokens": 688385, "input_cost_usd": 9.51251999999998, "output_cost_usd": 4.066274999999998, "total_cost_usd": 13.578795000000014}}, "normalized_score": 82.32699619771864}, {"_id": {"$oid": "69d4eb12b877594bc5718b7a"}, "test_id": "T0155", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-pro", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Hemann\",\"first_name\":\"Christ Ludwig Ernst\"},\"publication\":{\"title\":\"De veterum dotis actionum, rei uxoriae atque ex stipulatu, differentiis.\",\"year\":\"1826\",\"place\":\"Kiliae\",\"pages\":\"40\",\"publisher\":null,\"format\":\"4\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Kiel. Diss. iur. 1826.\",\"subjects\":null}}", "model": "gemini-2.5-pro", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 712, "output_tokens": 181, "total_tokens": 1682, "input_cost_usd": 0.00089, "output_cost_usd": 0.00181, "estimated_cost_usd": 0.0027}, "duration": 8.53559947013855, "timestamp": "2026-01-24T12:21:24.546749", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Hemann", "first_name": "Christ Ludwig Ernst"}, "publication": {"title": "De veterum dotis actionum, rei uxoriae atque ex stipulatu, differentiis.", "year": "1826", "place": "Kiliae", "pages": "40", "publisher": null, "format": "4\u00b0", "editor": null}, "library_reference": {"shelfmark": "Kiel. Diss. iur. 1826.", "subjects": null}}, "conversation_id": "b239837e-27ca-469e-8af2-52fba9541836", "score": {"f1_score": 0.75, "precision": 0.6666666666666666, "recall": 0.8571428571428571, "true_positives": 6, "false_positives": 3, "false_negatives": 1, "field_scores": {"library_reference.shelfmark": {"response": "Kiel. Diss. iur. 1826.", "ground_truth": null, "score": 0.0}, "publication.year": {"response": "1826", "ground_truth": 1826, "score": 1.0}, "publication.title": {"response": "De veterum dotis actionum, rei uxoriae atque ex stipulatu, differentiis.", "ground_truth": "De veterum dotis actionum, rei uxoriae atque ex stipulatu, differentiis", "score": 0.993006993006993}, "publication.format": {"response": "4\u00b0", "ground_truth": null, "score": 0.0}, "author.last_name": {"response": "Hemann", "ground_truth": "Stemann", "score": 0.7692307692307692}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": "Christ Ludwig Ernst", "ground_truth": "Christ Ludwig Ernst", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "40", "ground_truth": "40", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.place": {"response": "Kiliae", "ground_truth": "Kiliae", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sat, 24 Jan 2026 11:21:24 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=8521\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Hemann\\\",\\n    \\\"first_name\\\": \\\"Christ Ludwig Ernst\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"De veterum dotis actionum, rei uxoriae atque ex stipulatu, differentiis.\\\",\\n    \\\"year\\\": \\\"1826\\\",\\n    \\\"place\\\": \\\"Kiliae\\\",\\n    \\\"pages\\\": \\\"40\\\",\\n    \\\"publisher\\\": null,\\n    \\\"format\\\": \\\"4\u00b0\\\",\\n    \\\"editor\\\": null\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Kiel. Diss. iur. 1826.\\\",\\n    \\\"subjects\\\": null\\n  }\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.5-pro\",\"prompt_feedback\":null,\"response_id\":\"NKt0acLGHuDOnsEPj6T7-Ag\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":181,\"candidates_tokens_details\":null,\"prompt_token_count\":712,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":454},{\"modality\":\"IMAGE\",\"token_count\":258}],\"thoughts_token_count\":789,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":1682,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Hemann\",\"first_name\":\"Christ Ludwig Ernst\"},\"publication\":{\"title\":\"De veterum dotis actionum, rei uxoriae atque ex stipulatu, differentiis.\",\"year\":\"1826\",\"place\":\"Kiliae\",\"pages\":\"40\",\"publisher\":null,\"format\":\"4\u00b0\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"Kiel. Diss. iur. 1826.\",\"subjects\":null}}}"}, "scoring": {"f1_micro": 0.868000808570851, "f1_macro": 0.8638783269961977, "micro_precision": 0.8479462875197472, "micro_recall": 0.8890269151138717, "total_instances": 263, "total_tp": 2147, "total_fp": 385, "total_fn": 268, "cost_summary": {"total_input_tokens": 187256, "total_output_tokens": 48901, "total_tokens": 236157, "input_cost_usd": 0.2340700000000004, "output_cost_usd": 0.48900999999999956, "total_cost_usd": 0.72308}}, "normalized_score": 86.38783269961976}, {"_id": {"$oid": "69d4eb12b877594bc5718b82"}, "test_id": "T0216", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash-lite-preview-09-2025", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Oldenburger\",\"first_name\":\"Rufus\"},\"publication\":{\"title\":\"Composition and rank of n-way matrices and multilinear forms.\",\"year\":\"place\",\"place\":null,\"pages\":\"\",\"publisher\":\"University of Chicago\",\"format\":\"\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"AT 36\",\"subjects\":\"\"}}", "model": "gemini-2.5-flash-lite-preview-09-2025", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 712, "output_tokens": 145, "total_tokens": 859, "input_cost_usd": 7.12e-05, "output_cost_usd": 5.8e-05, "estimated_cost_usd": 0.0001292}, "duration": 0.6205329895019531, "timestamp": "2026-01-24T13:56:58.423100", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Oldenburger", "first_name": "Rufus"}, "publication": {"title": "Composition and rank of n-way matrices and multilinear forms.", "year": "place", "pages": "", "publisher": "University of Chicago", "format": ""}, "library_reference": {"shelfmark": "AT 36", "subjects": ""}}, "conversation_id": "8c474075-5a3a-4bc5-987d-f968bfffc498", "score": {"f1_score": 0.57, "precision": 0.5714285714285714, "recall": 0.5714285714285714, "true_positives": 4, "false_positives": 3, "false_negatives": 3, "field_scores": {"library_reference.shelfmark": {"response": "AT 36", "ground_truth": null, "score": 0.0}, "publication.year": {"response": "place", "ground_truth": 1934, "score": 0.0}, "publication.title": {"response": "Composition and rank of n-way matrices and multilinear forms.", "ground_truth": "Composition and rank of n-way matrices and multilinear forms", "score": 0.9917355371900827}, "publication.format": {"response": null, "ground_truth": null, "score": 1.0}, "author.last_name": {"response": "Oldenburger", "ground_truth": "Oldenburger", "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}, "publication.publisher": {"response": "University of Chicago", "ground_truth": null, "score": 0.0}, "author.first_name": {"response": "Rufus", "ground_truth": "Rufus", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "8", "score": 0.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.place": {"response": null, "ground_truth": "Chicago", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sat, 24 Jan 2026 12:56:58 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=608\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Oldenburger\\\",\\n    \\\"first_name\\\": \\\"Rufus\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Composition and rank of n-way matrices and multilinear forms.\\\",\\n    \\\"year\\\": \\n    \\\"place\\\",\\n    \\\"pages\\\": \\\"\\\",\\n    \\\"publisher\\\": \\\"University of Chicago\\\",\\n    \\\"format\\\": \\\"\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"AT 36\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.5-flash-lite-preview-09-2025\",\"prompt_feedback\":null,\"response_id\":\"msF0aa27Fo_OnsEPrqOpgAU\",\"usage_metadata\":{\"cache_tokens_details\":[{\"modality\":\"IMAGE\",\"token_count\":75},{\"modality\":\"TEXT\",\"token_count\":133}],\"cached_content_token_count\":208,\"candidates_token_count\":145,\"candidates_tokens_details\":null,\"prompt_token_count\":712,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":454},{\"modality\":\"IMAGE\",\"token_count\":258}],\"thoughts_token_count\":2,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":859,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Oldenburger\",\"first_name\":\"Rufus\"},\"publication\":{\"title\":\"Composition and rank of n-way matrices and multilinear forms.\",\"year\":\"place\",\"pages\":\"\",\"publisher\":\"University of Chicago\",\"format\":\"\"},\"library_reference\":{\"shelfmark\":\"AT 36\",\"subjects\":\"\"}}}"}, "scoring": {"f1_micro": 0.6629189189189189, "f1_macro": 0.6485171102661597, "micro_precision": 0.6936651583710407, "micro_recall": 0.6347826086956522, "total_instances": 263, "total_tp": 1533, "total_fp": 677, "total_fn": 882, "cost_summary": {"total_input_tokens": 187256, "total_output_tokens": 502964, "total_tokens": 690220, "input_cost_usd": 0.01872560000000009, "output_cost_usd": 0.20118560000000016, "total_cost_usd": 0.2199112}}, "normalized_score": 64.85171102661596}, {"_id": {"$oid": "69d4eb12b877594bc5718b86"}, "test_id": "T0430", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "magistral-medium-2509", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\": {\"type\": \"Dissertation or thesis\"}, \"author\": {\"last_name\": \"string\", \"first_name\": \"string\"}, \"publication\": {\"title\": \"string\", \"year\": 0, \"place\": \"string or empty string\", \"pages\": \"string or empty string\", \"publisher\": \"string or empty string\", \"format\": \"string or empty string\"}, \"library_reference\": {\"shelfmark\": \"string or empty string\", \"subjects\": \"string or empty string\"}}", "model": "magistral-medium-2509", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 1196, "output_tokens": 109, "total_tokens": 1305, "input_cost_usd": 0.002392, "output_cost_usd": 0.000545, "estimated_cost_usd": 0.002937}, "duration": 3.236083984375, "timestamp": "2026-01-24T13:06:50.921572", "conversation_id": "55b95817-9cbf-46f2-8f4a-b565fa88e4a7", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 9, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "10", "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "author.first_name": {"response": null, "ground_truth": "Adrianus Johannes", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1912, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "Utrecht", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.title": {"response": null, "ground_truth": "Stellingen", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "AT Utrecht (Stellingen/1912)", "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Van den Bergh", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"2cddd2eff08c4a138ff75d7aa194f573\",\"object\":\"chat.completion\",\"model\":\"magistral-medium-2509\",\"usage\":{\"prompt_tokens\":1196,\"completion_tokens\":109,\"total_tokens\":1305,\"num_cached_tokens\":1344},\"created\":1769256407,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\\"type\\\": {\\\"type\\\": \\\"Dissertation or thesis\\\"}, \\\"author\\\": {\\\"last_name\\\": \\\"string\\\", \\\"first_name\\\": \\\"string\\\"}, \\\"publication\\\": {\\\"title\\\": \\\"string\\\", \\\"year\\\": 0, \\\"place\\\": \\\"string or empty string\\\", \\\"pages\\\": \\\"string or empty string\\\", \\\"publisher\\\": \\\"string or empty string\\\", \\\"format\\\": \\\"string or empty string\\\"}, \\\"library_reference\\\": {\\\"shelfmark\\\": \\\"string or empty string\\\", \\\"subjects\\\": \\\"string or empty string\\\"}}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.0008254230293025175, "f1_macro": 0.0004182509505703422, "micro_precision": 0.125, "micro_recall": 0.00041407867494824016, "total_instances": 263, "total_tp": 1, "total_fp": 7, "total_fn": 2414, "cost_summary": {"total_input_tokens": 314548, "total_output_tokens": 30251, "total_tokens": 344799, "input_cost_usd": 0.6290959999999982, "output_cost_usd": 0.151255, "total_cost_usd": 0.780351}}, "normalized_score": 0.041825095057034224}, {"_id": {"$oid": "69d4eb12b877594bc5718b8a"}, "test_id": "T0192", "benchmark": "library_cards", "date": "2026-01-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-large-2411", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Caesar\",\n    \"first_name\": \"\"\n  },\n  \"publication\": {\n    \"title\": \"Commentarii de bello Gallico\",\n    \"year\": 58,\n    \"place\": \"Rome\",\n    \"pages\": \"500\",\n    \"publisher\": \"\",\n    \"format\": \"4\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"AT 123456\",\n    \"subjects\": \"Gallic Wars\"\n  }\n}", "model": "mistral-large-2411", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 1296, "output_tokens": 164, "total_tokens": 1460, "input_cost_usd": 0.002592, "output_cost_usd": 0.000984, "estimated_cost_usd": 0.003576}, "duration": 4.140681982040405, "timestamp": "2026-01-24T12:05:00.622313", "conversation_id": "0391e94d-9613-44d3-bb98-8989e31b10a3", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 11, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": null, "ground_truth": "19", "score": 0.0}, "publication.publisher": {"response": null, "ground_truth": "Almqvist & Wiksell", "score": 0.0}, "author.first_name": {"response": null, "ground_truth": "Mato", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1963, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "Uppsala", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "publication.title": {"response": null, "ground_truth": "Enterovirusstammar hos svenska grisar. Isolering, identifikation, patogenicitet, kemiska och fysikaliska egenskaper", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": "Diss NK GK 2Mak", "score": 0.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "Diss.1963,2024", "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Sibal\u00edn", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8'", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"43895f057228489a9476ac8187c11b15\",\"object\":\"chat.completion\",\"model\":\"mistral-large-2411\",\"usage\":{\"prompt_tokens\":1296,\"completion_tokens\":164,\"total_tokens\":1460,\"num_cached_tokens\":1280},\"created\":1769252696,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"Caesar\\\",\\n    \\\"first_name\\\": \\\"\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Commentarii de bello Gallico\\\",\\n    \\\"year\\\": 58,\\n    \\\"place\\\": \\\"Rome\\\",\\n    \\\"pages\\\": \\\"500\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"4\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"AT 123456\\\",\\n    \\\"subjects\\\": \\\"Gallic Wars\\\"\\n  }\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.0, "f1_macro": 0.0, "micro_precision": 0.0, "micro_recall": 0.0, "total_instances": 263, "total_tp": 0, "total_fp": 0, "total_fn": 2415, "cost_summary": {"total_input_tokens": 340848, "total_output_tokens": 41018, "total_tokens": 381866, "input_cost_usd": 0.6816960000000015, "output_cost_usd": 0.24610799999999997, "total_cost_usd": 0.9278039999999999}}, "normalized_score": 0}, {"_id": {"$oid": "69d4eb12b877594bc5718a45"}, "test_id": "T0440", "benchmark": "library_cards", "date": "2025-11-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-small-2506", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"M\u00fcller\",\n    \"first_name\": \"Hans\"\n  },\n  \"publication\": {\n    \"title\": \"Die Entwicklung der deutschen Sprache im Mittelalter\",\n    \"year\": 1985,\n    \"place\": \"Berlin\",\n    \"pages\": \"345\",\n    \"publisher\": \"Universit\u00e4tsverlag\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss. 12345\",\n    \"subjects\": \"German Language, Medieval Studies\"\n  }\n}", "model": "mistral-small-2506", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 968, "output_tokens": 148, "total_tokens": 1116, "input_cost_usd": 9.680000000000001e-05, "output_cost_usd": 4.4399999999999995e-05, "estimated_cost_usd": 0.00014120000000000002}, "duration": 1.5655930042266846, "timestamp": "2025-11-25T10:49:37.278572", "score": {"f1_score": 0.0, "precision": 0.0, "recall": 0.0, "true_positives": 0, "false_positives": 0, "false_negatives": 9, "field_scores": {"": {"response": null, "ground_truth": null, "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": "Geest & Portig", "score": 0.0}, "publication.format": {"response": null, "ground_truth": "8'", "score": 0.0}, "publication.pages": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": null, "ground_truth": "AT Berlin 2 Jg.1957", "score": 0.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.title": {"response": null, "ground_truth": "Zur Ethologie des Rotfuchses (Vulpes vulpes L.) unter besonderer Ber\u00fccksichtigung der Fortpflanzung", "score": 0.0}, "publication.year": {"response": null, "ground_truth": 1957, "score": 0.0}, "publication.place": {"response": null, "ground_truth": "Leipzig", "score": 0.0}, "type.type": {"response": null, "ground_truth": "Dissertation or thesis", "score": 0.0}, "author.last_name": {"response": null, "ground_truth": "Tembrock", "score": 0.0}, "author.first_name": {"response": null, "ground_truth": "G\u00fcnter", "score": 0.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"9e5a656ed3d64c0e8b9afc761cb8ed8f\",\"object\":\"chat.completion\",\"model\":\"mistral-small-2506\",\"usage\":{\"prompt_tokens\":968,\"completion_tokens\":148,\"total_tokens\":1116},\"created\":1764064175,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"M\u00fcller\\\",\\n    \\\"first_name\\\": \\\"Hans\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Die Entwicklung der deutschen Sprache im Mittelalter\\\",\\n    \\\"year\\\": 1985,\\n    \\\"place\\\": \\\"Berlin\\\",\\n    \\\"pages\\\": \\\"345\\\",\\n    \\\"publisher\\\": \\\"Universit\u00e4tsverlag\\\",\\n    \\\"format\\\": \\\"8\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"Diss. 12345\\\",\\n    \\\"subjects\\\": \\\"German Language, Medieval Studies\\\"\\n  }\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"f1_micro": 0.0, "f1_macro": 0.0, "micro_precision": 0.0, "micro_recall": 0.0, "total_instances": 263, "total_tp": 0, "total_fp": 0, "total_fn": 2415, "cost_summary": {"total_input_tokens": 254584, "total_output_tokens": 41283, "total_tokens": 295867, "input_cost_usd": 0.02545840000000006, "output_cost_usd": 0.012384899999999968, "total_cost_usd": 0.03784329999999996}}, "normalized_score": 0}, {"_id": {"$oid": "69d4eb12b877594bc5718a59"}, "test_id": "T0408", "benchmark": "library_cards", "date": "2025-11-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.1-2025-11-13", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"W\u00f6rz\",\"first_name\":\"Walter\"},\"publication\":{\"title\":\"Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers\",\"year\":\"1934\",\"place\":\"o. O.\",\"pages\":\"36\",\"publisher\":\"\",\"format\":\"8\u00b0'},\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"U 34.5324\",\"subjects\":\"\"}}", "model": "gpt-5.1-2025-11-13", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1387, "output_tokens": 107, "total_tokens": 1494}, "duration": 3.463559150695801, "timestamp": "2025-11-25T15:06:13.836687", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "W\u00f6rz", "first_name": "Walter"}, "publication": {"title": "Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers", "year": "1934", "place": "o. O.", "pages": "36", "publisher": "", "format": "8\u00b0'},", "editor": null}, "library_reference": {"shelfmark": "U 34.5324", "subjects": ""}}, "score": {"f1_score": 0.82, "precision": 0.7777777777777778, "recall": 0.875, "true_positives": 7, "false_positives": 2, "false_negatives": 1, "field_scores": {"author.last_name": {"response": "W\u00f6rz", "ground_truth": "W\u00f6rz", "score": 1.0}, "author.first_name": {"response": "Walter", "ground_truth": "Walter", "score": 1.0}, "publication.pages": {"response": "36", "ground_truth": "36", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "library_reference.shelfmark": {"response": "U 34.5324", "ground_truth": "U 34. 5324", "score": 0.9473684210526316}, "publication.place": {"response": "o. O.", "ground_truth": null, "score": 0.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.title": {"response": "Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers", "ground_truth": "Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers", "score": 1.0}, "publication.publisher": {"response": null, "ground_truth": null, "score": 1.0}, "publication.format": {"response": "8\u00b0'},", "ground_truth": "8\u00b0", "score": 0.5714285714285714}, "publication.year": {"response": "1934", "ground_truth": 1934, "score": 1.0}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"id\":\"chatcmpl-CfnyEvJABrzw8UXdTBEaTxnwchwnD\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"type\\\":{\\\"type\\\":\\\"Dissertation or thesis\\\"},\\\"author\\\":{\\\"last_name\\\":\\\"W\u00f6rz\\\",\\\"first_name\\\":\\\"Walter\\\"},\\\"publication\\\":{\\\"title\\\":\\\"Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers\\\",\\\"year\\\":\\\"1934\\\",\\\"place\\\":\\\"o. O.\\\",\\\"pages\\\":\\\"36\\\",\\\"publisher\\\":\\\"\\\",\\\"format\\\":\\\"8\u00b0'},\\\"   ,\\\"editor\\\":null},\\\"library_reference\\\":{\\\"shelfmark\\\":\\\"U 34.5324\\\",\\\"subjects\\\":\\\"\\\"} }\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"W\u00f6rz\",\"first_name\":\"Walter\"},\"publication\":{\"title\":\"Ein Fall von Osteomyelitis acuta traumatica des Oberkiefers\",\"year\":\"1934\",\"place\":\"o. O.\",\"pages\":\"36\",\"publisher\":\"\",\"format\":\"8\u00b0'},\",\"editor\":null},\"library_reference\":{\"shelfmark\":\"U 34.5324\",\"subjects\":\"\"}}}}],\"created\":1764079570,\"model\":\"gpt-5.1-2025-11-13\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":107,\"prompt_tokens\":1387,\"total_tokens\":1494,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"f1_micro": 0.8218438538205981, "f1_macro": 0.818212927756654, "micro_precision": 0.8242399000416493, "micro_recall": 0.8194616977225673, "total_instances": 263, "total_tp": 1979, "total_fp": 422, "total_fn": 436, "cost_summary": {"total_input_tokens": 365175, "total_output_tokens": 33217, "total_tokens": 398392, "input_cost_usd": 0.45646874999999804, "output_cost_usd": 0.33216999999999997, "total_cost_usd": 0.7886387499999997}}, "normalized_score": 81.8212927756654}, {"_id": {"$oid": "69d4eb12b877594bc5718a3d"}, "test_id": "T0200", "benchmark": "library_cards", "date": "2025-11-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"text": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"M\u00fcller\",\"first_name\":\"Maurice Edmond\"},\"publication\":{\"title\":\"Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form,Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.\",\"year\":\"1957\",\"place\":\"Stuttgart\",\"pages\":\"184\",\"publisher\":\"Thieme\",\"format\":\"4'\",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"AT Z\u00fcrich 7\",\"subjects\":\"\"}}", "model": "gemini-2.5-flash", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 712, "output_tokens": 204, "total_tokens": 2076, "input_cost_usd": 0.0002136, "output_cost_usd": 0.00051, "estimated_cost_usd": 0.0007236}, "duration": 7.459839820861816, "timestamp": "2025-11-24T20:48:03.675028", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "M\u00fcller", "first_name": "Maurice Edmond"}, "publication": {"title": "Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form,Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.", "year": "1957", "place": "Stuttgart", "pages": "184", "publisher": "Thieme", "format": "4'", "editor": ""}, "library_reference": {"shelfmark": "AT Z\u00fcrich 7", "subjects": ""}}, "score": {"f1_score": 0.8, "precision": 0.8, "recall": 0.8, "true_positives": 8, "false_positives": 2, "false_negatives": 2, "field_scores": {"publication.publisher": {"response": "Thieme", "ground_truth": "Thieme", "score": 1.0}, "publication.format": {"response": "4'", "ground_truth": "4'", "score": 1.0}, "library_reference.subjects": {"response": null, "ground_truth": null, "score": 1.0}, "publication.title": {"response": "Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form,Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.", "ground_truth": "Die h\u00fcftnahen Femurosteotomien unter Ber\u00fccksichtigung der Form, Funktion und Beanspruchung des H\u00fcftgelenkes", "score": 0.8185328185328186}, "publication.place": {"response": "Stuttgart", "ground_truth": "Stuttgart", "score": 1.0}, "author.first_name": {"response": "Maurice Edmond", "ground_truth": "Maurice Edmond", "score": 1.0}, "publication.year": {"response": "1957", "ground_truth": 1957, "score": 1.0}, "library_reference.shelfmark": {"response": "AT Z\u00fcrich 7", "ground_truth": "AT Z\u00fcrich 7", "score": 1.0}, "author.last_name": {"response": "M\u00fcller", "ground_truth": "M\u00fcller", "score": 1.0}, "publication.editor": {"response": null, "ground_truth": null, "score": 1.0}, "publication.pages": {"response": "184", "ground_truth": "X,184", "score": 0.75}, "type.type": {"response": "Dissertation or thesis", "ground_truth": "Dissertation or thesis", "score": 1.0}}, "total_fields": 12}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Mon, 24 Nov 2025 19:48:04 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=7423\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"type\\\": {\\n    \\\"type\\\": \\\"Dissertation or thesis\\\"\\n  },\\n  \\\"author\\\": {\\n    \\\"last_name\\\": \\\"M\u00fcller\\\",\\n    \\\"first_name\\\": \\\"Maurice Edmond\\\"\\n  },\\n  \\\"publication\\\": {\\n    \\\"title\\\": \\\"Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form,Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.\\\",\\n    \\\"year\\\": \\\"1957\\\",\\n    \\\"place\\\": \\\"Stuttgart\\\",\\n    \\\"pages\\\": \\\"184\\\",\\n    \\\"publisher\\\": \\\"Thieme\\\",\\n    \\\"format\\\": \\\"4'\\\",\\n    \\\"editor\\\": \\\"\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"AT Z\u00fcrich 7\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.5-flash\",\"prompt_feedback\":null,\"response_id\":\"dLYkabmaDLK-vdIP9MuB-A0\",\"usage_metadata\":{\"cache_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":142},{\"modality\":\"IMAGE\",\"token_count\":81}],\"cached_content_token_count\":223,\"candidates_token_count\":204,\"candidates_tokens_details\":null,\"prompt_token_count\":712,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":454},{\"modality\":\"IMAGE\",\"token_count\":258}],\"thoughts_token_count\":1160,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2076,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"M\u00fcller\",\"first_name\":\"Maurice Edmond\"},\"publication\":{\"title\":\"Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form,Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.\",\"year\":\"1957\",\"place\":\"Stuttgart\",\"pages\":\"184\",\"publisher\":\"Thieme\",\"format\":\"4'\",\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"AT Z\u00fcrich 7\",\"subjects\":\"\"}}}"}, "scoring": {"f1_micro": 0.8659251769464105, "f1_macro": 0.8602281368821293, "micro_precision": 0.8462450592885375, "micro_recall": 0.8865424430641822, "total_instances": 263, "total_tp": 2141, "total_fp": 389, "total_fn": 274, "cost_summary": {"total_input_tokens": 187256, "total_output_tokens": 38205, "total_tokens": 225461, "input_cost_usd": 0.05617680000000022, "output_cost_usd": 0.09551250000000004, "total_cost_usd": 0.15168929999999992}}, "normalized_score": 86.02281368821293}, {"_id": {"$oid": "69d4eb12b877594bc57189a6"}, "test_id": "T0252", "benchmark": "library_cards", "date": "2025-10-17", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "duration": 4.685088157653809, "timestamp": "2025-10-18T02:33:09.231798", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Strehle", "first_name": "Wolfgang"}, "publication": {"title": "Die Abh\u00e4ngigkeit elektrisch angeregter subjektiver Lichtmuster (Phosphene) von Umwelteinfl\u00fcssen", "year": "place", "place": null, "pages": "75", "publisher": "", "format": "8'", "editor": null}, "library_reference": {"shelfmark": "Diss.1972,3635", "subjects": "Illusion"}}, "text": "", "raw_response": {"id": "gen-1760747585-r62EmcETyBYrdinJOBD3", "model": "meta-llama/llama-4-maverick", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Strehle\",\n    \"first_name\": \"Wolfgang\"\n  },\n  \"publication\": {\n    \"title\": \"Die Abh\u00e4ngigkeit elektrisch angeregter subjektiver Lichtmuster (Phosphene) von Umwelteinfl\u00fcssen\",\n    \"year\":                                                                                                                                                                                                        \t\"place\" \t,\n    \"pages\": \"75\",\n    \"publisher\": \"\",\n    \"format\": \"8'\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss.1972,3635\",\n    \"subjects\": \"Illusion\"\n  }\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1516, "completion_tokens": 343, "total_tokens": 1859}}, "usage": {"input_tokens": 1516, "output_tokens": 343, "total_tokens": 1859}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.6748292912644219, "f1_macro": 0.6232699619771863, "micro_precision": 0.7822052401746725, "micro_recall": 0.5933747412008281, "total_instances": 263, "total_tp": 1433, "total_fp": 399, "total_fn": 982, "cost_summary": {"total_input_tokens": 390864, "total_output_tokens": 819940, "total_tokens": 1210804, "input_cost_usd": 0.05863, "output_cost_usd": 0.491964, "total_cost_usd": 0.550594, "pricing_date": "2025-10-17", "input_price_per_million": 0.15, "output_price_per_million": 0.6}}, "normalized_score": 62.32699619771863}, {"_id": {"$oid": "69d4eb12b877594bc57189ab"}, "test_id": "T0247", "benchmark": "library_cards", "date": "2025-10-17", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-8b-thinking", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openrouter", "model": "qwen/qwen3-vl-8b-thinking", "duration": 4.78162145614624, "timestamp": "2025-10-17T23:18:53.473682", "text": "", "raw_response": {"id": "gen-1760735929-sKkD3EuVEvj28rcX1XWW", "model": "qwen/qwen3-vl-8b-thinking", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "", "role": "assistant"}}], "usage": {"prompt_tokens": 1883, "completion_tokens": 148, "total_tokens": 2031}}, "usage": {"input_tokens": 1883, "output_tokens": 148, "total_tokens": 2031}, "error": "JSONDecodeError", "error_message": "Expecting value: line 1 column 1 (char 0)", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.10320781032078102, "f1_macro": 0.05897338403041825, "micro_precision": 0.32671081677704195, "micro_recall": 0.06128364389233954, "total_instances": 263, "total_tp": 148, "total_fp": 305, "total_fn": 2267, "cost_summary": {"total_input_tokens": 495229, "total_output_tokens": 44554, "total_tokens": 539783, "input_cost_usd": 0.089141, "output_cost_usd": 0.093563, "total_cost_usd": 0.182705, "pricing_date": "2025-10-17", "input_price_per_million": 0.18, "output_price_per_million": 2.1}}, "normalized_score": 5.897338403041825}, {"_id": {"$oid": "69d4eb12b877594bc57189af"}, "test_id": "T0242", "benchmark": "library_cards", "date": "2025-10-17", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "scicore", "model": "GLM-4.5V-FP8", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "scicore", "model": "GLM-4.5V-FP8", "duration": 307.9977512359619, "timestamp": "2025-10-17T18:00:16.255962", "text": "", "raw_response": {"id": "chatcmpl-3ea5bfdba45842629bb49e06eeddc348", "model": "hosted_vllm/GLM-4.5V-FP8", "choices": [{"finish_reason": "length", "index": 0, "message": {"content": "{\"type\": {\"type\": \"Dissertation or thesis\"}, \"author\": {\"last_name\": \"Papastratigakis\", \"first_name\": \"Constantin\"}, \"publication\": {\"title\": \"Travail du laboratoire et de la clinique des maladies mentales. Spiroch\u00e8te pal\u00e9 et paralysie g\u00e9n\u00e9rale\", \"year\":  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t  \t\t\t\t\t\t\t ", "role": "assistant"}}], "usage": {"prompt_tokens": 1201, "completion_tokens": 23375, "total_tokens": 24576}}, "usage": {"input_tokens": 1201, "output_tokens": 23375, "total_tokens": 24576}, "error": "JSONDecodeError", "error_message": "Expecting value: line 1 column 105106 (char 105105)", "finish_reason": "length"}, "scoring": {"f1_micro": 0.2589058524173028, "f1_macro": 0.19714828897338404, "micro_precision": 0.5582990397805213, "micro_recall": 0.16853002070393375, "total_instances": 263, "total_tp": 407, "total_fp": 322, "total_fn": 2008, "cost_summary": {"total_input_tokens": 263113, "total_output_tokens": 3417252, "total_tokens": 3680365, "input_cost_usd": 0.0, "output_cost_usd": 0.0, "total_cost_usd": 0.0, "pricing_date": "2025-10-17", "input_price_per_million": 0.0, "output_price_per_million": 0.0}}, "normalized_score": 19.714828897338403}, {"_id": {"$oid": "69d4eb12b877594bc571899d"}, "test_id": "T0164", "benchmark": "library_cards", "date": "2025-10-03", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4o-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4o-mini", "duration": 3.8755977153778076, "timestamp": "2025-10-02T18:46:40.189588", "parsed": {"type": {"type": "Reference"}, "author": {"last_name": "May", "first_name": "Mark"}, "publication": {"title": "The institutions that train ministers", "year": "place", "place": "New York", "pages": "", "publisher": "", "format": "", "editor": ""}, "library_reference": {"shelfmark": "", "subjects": ""}}, "text": "", "raw_response": {"id": "chatcmpl-CMGjufgUm3ugch720S5T5EwnlISSh", "model": "gpt-4o-mini-2024-07-18", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"type\": {\n    \"type\": \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"May\",\n    \"first_name\": \"Mark\"\n  },\n  \"publication\": {\n    \"title\": \"The institutions that train ministers\",\n    \"year\":      \t\"place\" \t\t, \n    \"place\": \"New York\",\n    \"pages\": \"\",\n    \"publisher\": \"\",\n    \"format\": \"\"\n \t,\n    \"editor\": \"\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"subjects\": \"\"\n  }\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 26262, "completion_tokens": 127, "total_tokens": 26389}}, "usage": {"input_tokens": 26262, "output_tokens": 127, "total_tokens": 26389}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.014760147601476016, "f1_macro": 0.009505703422053232, "micro_precision": 0.06779661016949153, "micro_recall": 0.008281573498964804, "total_instances": 263, "total_tp": 20, "total_fp": 275, "total_fn": 2395, "cost_summary": {"total_input_tokens": 131310, "total_output_tokens": 727, "total_tokens": 132037, "input_cost_usd": 0.019697, "output_cost_usd": 0.000436, "total_cost_usd": 0.020133, "pricing_date": "2025-10-03", "input_price_per_million": 0.15, "output_price_per_million": 0.6}}, "normalized_score": 0.9505703422053232}, {"_id": {"$oid": "69d4eb12b877594bc57189a2"}, "test_id": "T0162", "benchmark": "library_cards", "date": "2025-10-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4.1-nano", "duration": 2.2592532634735107, "timestamp": "2025-10-02T13:10:02.021396", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Spiegelberg", "first_name": "Otto"}, "publication": {"title": "Ueber das gegenseitige Verh\u00e4ltnis zwischen dem gerichtlichen Standnis und der Offenknadigkeit bei dem Gericht nach der deutschen Civilprozzessordnung", "year": "", "place": "Marburg", "pages": "128 S.", "publisher": "", "format": "8\u00b0", "editor": ""}, "library_reference": {"shelfmark": "Diss. v. 1. Juni 1896", "subjects": ""}}, "text": "", "raw_response": {"id": "chatcmpl-CMBU9Jb086N7f49FnuOvrUKD3E7G3", "model": "gpt-4.1-nano-2025-04-14", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Spiegelberg\",\"first_name\":\"Otto\"},\"publication\":{\"title\":\"Ueber das gegenseitige Verh\u00e4ltnis zwischen dem gerichtlichen Standnis und der Offenknadigkeit bei dem Gericht nach der deutschen Civilprozzessordnung\",\"year\": \"\" ,\"place\":\"Marburg\",\"pages\":\"128 S.\",\"publisher\":\"\",\"format\":\"8\u00b0\"  ,\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Diss. v. 1. Juni 1896\",\"subjects\":\"\"}}", "role": "assistant"}}], "usage": {"prompt_tokens": 2211, "completion_tokens": 114, "total_tokens": 2325}}, "usage": {"input_tokens": 2211, "output_tokens": 114, "total_tokens": 2325}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.6762051507814218, "f1_macro": 0.6669961977186312, "micro_precision": 0.7218045112781954, "micro_recall": 0.6360248447204969, "total_instances": 263, "total_tp": 1536, "total_fp": 592, "total_fn": 879, "cost_summary": null}, "normalized_score": 66.69961977186311}, {"_id": {"$oid": "69d4eb12b877594bc57189a8"}, "test_id": "T0161", "benchmark": "library_cards", "date": "2025-10-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4.1-mini", "duration": 13.869112491607666, "timestamp": "2025-10-01T15:22:40.147442", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Silvestri", "first_name": "Richard"}, "publication": {"title": "Entwicklungstendenzen der Best\u00e4nde ausgew\u00e4hlter Maschinen und des Volumens der maschinellen Investitionen in der oesterreichischen Landwirtschaft", "year": "", "place": "Wien", "pages": "186", "publisher": "", "format": "8' ", "editor": ""}, "library_reference": {"shelfmark": "Diss,1970,8109", "subjects": "Landwirtschaft.Maschinen u.Ger\u00e4te/Landwirtschaft. Oesterreich"}}, "text": "", "raw_response": {"id": "chatcmpl-CLr4vuOS8dbLvDT2Ur1BiWKq1WmxL", "model": "gpt-4.1-mini-2025-04-14", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Silvestri\",\"first_name\":\"Richard\"},\"publication\":{\"title\":\"Entwicklungstendenzen der Best\u00e4nde ausgew\u00e4hlter Maschinen und des Volumens der maschinellen Investitionen in der oesterreichischen Landwirtschaft\",\"year\":     \"\"  ,\"place\":\"Wien\",\"pages\":\"186\",\"publisher\":\"\",\"format\":\"8' \"  ,\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Diss,1970,8109\",\"subjects\":\"Landwirtschaft.Maschinen u.Ger\u00e4te/Landwirtschaft. Oesterreich\"}}", "role": "assistant"}}], "usage": {"prompt_tokens": 1767, "completion_tokens": 130, "total_tokens": 1897}}, "usage": {"input_tokens": 1767, "output_tokens": 130, "total_tokens": 1897}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7589488930271053, "f1_macro": 0.7574144486692015, "micro_precision": 0.7584780810587263, "micro_recall": 0.7594202898550725, "total_instances": 263, "total_tp": 1834, "total_fp": 584, "total_fn": 581, "cost_summary": null}, "normalized_score": 75.74144486692015}, {"_id": {"$oid": "69d4eb12b877594bc571894b"}, "test_id": "T0168", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "o3", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "o3", "duration": 19.141613960266113, "timestamp": "2025-10-01T19:39:55.794116", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Z\u00e4hle", "first_name": "Werner"}, "publication": {"title": "Ergebnisse von Bilirubinbestimmungen im Duodenalsaft von Gesunden u. Leberkranken m.d. Methode v. Jendrassik u. Cleghorn. Auszug a.d. Inaug.-Diss. Med. Fak. M\u00fcnster", "year": "1952", "place": "M\u00fcnster", "pages": "", "publisher": "Westf. Vereinsdruckerei", "format": "8' ", "editor": null}, "library_reference": {"shelfmark": "Diss. M\u00fcnster 1952", "subjects": ""}}, "text": "", "raw_response": {"id": "chatcmpl-CLv5eRHO5EivcFhWZLLB80Ew4xSz9", "model": "o3-2025-04-16", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Z\u00e4hle\",\"first_name\":\"Werner\"},\"publication\":{\"title\":\"Ergebnisse von Bilirubinbestimmungen im Duodenalsaft von Gesunden u. Leberkranken m.d. Methode v. Jendrassik u. Cleghorn. Auszug a.d. Inaug.-Diss. Med. Fak. M\u00fcnster\",\"year\":\"1952\",\"place\":\"M\u00fcnster\",\"pages\":\"\",\"publisher\":\"Westf. Vereinsdruckerei\",\"format\":\"8' \"  ,\"editor\":null},\"library_reference\":{\"shelfmark\":\"Diss. M\u00fcnster 1952\",\"subjects\":\"\"}}", "role": "assistant"}}], "usage": {"prompt_tokens": 1432, "completion_tokens": 608, "total_tokens": 2040}}, "usage": {"input_tokens": 1432, "output_tokens": 608, "total_tokens": 2040}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.833922261484099, "f1_macro": 0.8311026615969582, "micro_precision": 0.837228714524207, "micro_recall": 0.8306418219461698, "total_instances": 263, "total_tp": 2006, "total_fp": 390, "total_fn": 409, "cost_summary": {"total_input_tokens": 376616, "total_output_tokens": 223291, "total_tokens": 599907, "input_cost_usd": 0.753232, "output_cost_usd": 1.786328, "total_cost_usd": 2.53956, "pricing_date": "2025-10-01", "input_price_per_million": 2.0, "output_price_per_million": 8.0}}, "normalized_score": 83.11026615969583}, {"_id": {"$oid": "69d4eb12b877594bc571894e"}, "test_id": "T0152", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash-lite", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "genai", "model": "gemini-2.0-flash-lite", "duration": 1.9496769905090332, "timestamp": "2025-10-01T09:50:10.521859", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Volodarsky", "first_name": "Mademoiselle"}, "publication": {"title": "Injections intrac\u00e9r\u00e9brales de s\u00e9rum chez les animaux s\u00e9ro-anaphylactis\u00e9s.", "year": "place", "pages": "24", "publisher": "Imprimerie de l'Universit\u00e9", "format": "8\n"}, "library_reference": {"shelfmark": "", "subjects": ""}}, "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Volodarsky\",\n    \"first_name\": \"Mademoiselle\"\n  },\n  \"publication\": {\n    \"title\": \"Injections intrac\u00e9r\u00e9brales de s\u00e9rum chez les animaux s\u00e9ro-anaphylactis\u00e9s.\",\n    \"year\": \n    \"place\",\n    \"pages\": \"24\",\n    \"publisher\": \"Imprimerie de l'Universit\u00e9\",\n    \"format\": \"8\\n\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"subjects\": \"\"\n  }\n}", "raw_response": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Volodarsky\",\n    \"first_name\": \"Mademoiselle\"\n  },\n  \"publication\": {\n    \"title\": \"Injections intrac\u00e9r\u00e9brales de s\u00e9rum chez les animaux s\u00e9ro-anaphylactis\u00e9s.\",\n    \"year\": \n    \"place\",\n    \"pages\": \"24\",\n    \"publisher\": \"Imprimerie de l'Universit\u00e9\",\n    \"format\": \"8\\n\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"subjects\": \"\"\n  }\n}", "usage_metadata": {"prompt_token_count": 2329, "candidates_token_count": 159, "total_token_count": 2488}}, "usage": {"input_tokens": 2329, "output_tokens": 159, "total_tokens": 2488}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8121676239098062, "f1_macro": 0.8000760456273763, "micro_precision": 0.8350831146106736, "micro_recall": 0.7904761904761904, "total_instances": 263, "total_tp": 1909, "total_fp": 377, "total_fn": 506, "cost_summary": {"total_input_tokens": 612527, "total_output_tokens": 51712, "total_tokens": 664239, "input_cost_usd": 0.04594, "output_cost_usd": 0.015514, "total_cost_usd": 0.061453, "pricing_date": "2025-10-01", "input_price_per_million": 0.075, "output_price_per_million": 0.3}}, "normalized_score": 80.00760456273764}, {"_id": {"$oid": "69d4eb12b877594bc5718956"}, "test_id": "T0159", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "pixtral-large-2411", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "mistral", "model": "pixtral-large-latest", "duration": 4.241787672042847, "timestamp": "2025-10-01T13:55:09.452487", "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Z\u00fcrndorfer\",\n    \"first_name\": \"Ludwig\"\n  },\n  \"publication\": {\n    \"title\": \"Das Trauma als Entstehungsursache der Syringomyelie und multiplen Sklerose\",\n    \"year\": 1908,\n    \"place\": \"W\u00fcrzburg\",\n    \"pages\": \"34\",\n    \"publisher\": \"Freudenberger\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"U 09: 4152\",\n    \"subjects\": \"\"\n  }\n}", "raw_response": {"id": "2bb0b7cfa0194f879818cbcf0d06ba04", "model": "pixtral-large-latest", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Z\u00fcrndorfer\",\n    \"first_name\": \"Ludwig\"\n  },\n  \"publication\": {\n    \"title\": \"Das Trauma als Entstehungsursache der Syringomyelie und multiplen Sklerose\",\n    \"year\": 1908,\n    \"place\": \"W\u00fcrzburg\",\n    \"pages\": \"34\",\n    \"publisher\": \"Freudenberger\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"U 09: 4152\",\n    \"subjects\": \"\"\n  }\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 3389, "completion_tokens": 192, "total_tokens": 3581}}, "usage": {"input_tokens": 3389, "output_tokens": 192, "total_tokens": 3581}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7782786885245903, "f1_macro": 0.7777566539923955, "micro_precision": 0.7703853955375254, "micro_recall": 0.7863354037267081, "total_instances": 263, "total_tp": 1899, "total_fp": 566, "total_fn": 516, "cost_summary": {"total_input_tokens": 896396, "total_output_tokens": 51940, "total_tokens": 948336, "input_cost_usd": 1.792792, "output_cost_usd": 0.31164, "total_cost_usd": 2.104432, "pricing_date": "2025-10-01", "input_price_per_million": 2.0, "output_price_per_million": 6.0}}, "normalized_score": 77.77566539923954}, {"_id": {"$oid": "69d4eb12b877594bc571895a"}, "test_id": "T0179", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-medium-2508", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "mistral", "model": "mistral-medium-2508", "duration": 5.2062976360321045, "timestamp": "2025-10-01T14:34:20.793411", "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Sieferle\",\n    \"first_name\": \"Rolf Peter\"\n  },\n  \"publication\": {\n    \"title\": \"Die Revolution in der Theorie von Karl Marx\",\n    \"year\": 1979,\n    \"place\": \"Konstanz\",\n    \"pages\": \"256\",\n    \"publisher\": \"Ullstein Buch\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Bro 26.993\",\n    \"subjects\": \"Sozialgeschichtliche Bibliothek\"\n  }\n}", "raw_response": {"id": "47e0d28e1a6e4efeb48ed1cdcc57b6f4", "model": "mistral-medium-2508", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Sieferle\",\n    \"first_name\": \"Rolf Peter\"\n  },\n  \"publication\": {\n    \"title\": \"Die Revolution in der Theorie von Karl Marx\",\n    \"year\": 1979,\n    \"place\": \"Konstanz\",\n    \"pages\": \"256\",\n    \"publisher\": \"Ullstein Buch\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Bro 26.993\",\n    \"subjects\": \"Sozialgeschichtliche Bibliothek\"\n  }\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1753, "completion_tokens": 155, "total_tokens": 1908}}, "usage": {"input_tokens": 1753, "output_tokens": 155, "total_tokens": 1908}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7838858581619808, "f1_macro": 0.7809505703422053, "micro_precision": 0.7945555082943428, "micro_recall": 0.7734989648033126, "total_instances": 263, "total_tp": 1868, "total_fp": 483, "total_fn": 547, "cost_summary": {"total_input_tokens": 462166, "total_output_tokens": 41319, "total_tokens": 503485, "input_cost_usd": 0.184866, "output_cost_usd": 0.082638, "total_cost_usd": 0.267504, "pricing_date": "2025-10-01", "input_price_per_million": 0.4, "output_price_per_million": 2.0}}, "normalized_score": 78.09505703422053}, {"_id": {"$oid": "69d4eb12b877594bc571895d"}, "test_id": "T0148", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-20250514", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "anthropic", "model": "claude-sonnet-4-20250514", "duration": 4.100223779678345, "timestamp": "2025-10-01T08:44:17.100535", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "VAYSSI\u00c8RE", "first_name": "Emile"}, "publication": {"title": "D\u00e9viation du compl\u00e9ment et grossesse", "year": "1912", "place": "Montpellier", "pages": "178", "publisher": "", "format": "in-8\u00b0", "editor": null}, "library_reference": {"shelfmark": "Montpellier, Th. m\u00e9d. 1911-1912, N\u00b0 39", "subjects": ""}}, "text": "", "raw_response": {"id": "msg_019BJTHA9BJw7CUhxCD76hRq", "model": "claude-sonnet-4-20250514", "role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01RJmFWcwH2AiHTrEXb8iGaV", "name": "extract_structured_data", "input": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "VAYSSI\u00c8RE", "first_name": "Emile"}, "publication": {"title": "D\u00e9viation du compl\u00e9ment et grossesse", "year": "1912", "place": "Montpellier", "pages": "178", "publisher": "", "format": "in-8\u00b0"}, "library_reference": {"shelfmark": "Montpellier, Th. m\u00e9d. 1911-1912, N\u00b0 39", "subjects": ""}}}], "stop_reason": "tool_use", "usage": {"input_tokens": 2150, "output_tokens": 197}}, "usage": {"input_tokens": 2150, "output_tokens": 197, "total_tokens": 2347}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8416768416768416, "f1_macro": 0.8314448669201521, "micro_precision": 0.827531012404962, "micro_recall": 0.8563146997929607, "total_instances": 263, "total_tp": 2068, "total_fp": 431, "total_fn": 347, "cost_summary": {"total_input_tokens": 566577, "total_output_tokens": 54291, "total_tokens": 620868, "input_cost_usd": 1.699731, "output_cost_usd": 0.814365, "total_cost_usd": 2.514096, "pricing_date": "2025-10-01", "input_price_per_million": 3.0, "output_price_per_million": 15.0}}, "normalized_score": 83.1444866920152}, {"_id": {"$oid": "69d4eb12b877594bc5718963"}, "test_id": "T0160", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4.1", "duration": 8.13521957397461, "timestamp": "2025-10-01T11:05:41.516811", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wiggers", "first_name": "Wilhelm"}, "publication": {"title": "Beitrag zu den epileptischen Seelenst\u00f6rungen. [Maschinenschrift.]", "year": "", "place": "Kiel", "pages": "24", "publisher": "", "format": "4\u00b0", "editor": null}, "library_reference": {"shelfmark": "U 27.8118", "subjects": ""}}, "text": "", "raw_response": {"id": "chatcmpl-CLn49bR33fzbaQchgoil187ZvZLz8", "model": "gpt-4.1-2025-04-14", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Wiggers\",\n    \"first_name\": \"Wilhelm\"\n  },\n  \"publication\": {\n    \"title\": \"Beitrag zu den epileptischen Seelenst\u00f6rungen. [Maschinenschrift.]\",\n    \"year\":                                                                                                                              \"\",\n    \"place\": \"Kiel\",\n    \"pages\": \"24\",\n    \"publisher\": \"\",\n    \"format\": \"4\u00b0\"\n  ,\n  \"editor\": null\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"U 27.8118\",\n    \"subjects\": \"\"\n  }\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1526, "completion_tokens": 269, "total_tokens": 1795}}, "usage": {"input_tokens": 1526, "output_tokens": 269, "total_tokens": 1795}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8553563984993748, "f1_macro": 0.8483269961977187, "micro_precision": 0.8610994544691565, "micro_recall": 0.8496894409937888, "total_instances": 263, "total_tp": 2052, "total_fp": 331, "total_fn": 363, "cost_summary": {"total_input_tokens": 401338, "total_output_tokens": 236561, "total_tokens": 637899, "input_cost_usd": 0.802676, "output_cost_usd": 1.892488, "total_cost_usd": 2.695164, "pricing_date": "2025-10-01", "input_price_per_million": 2.0, "output_price_per_million": 8.0}}, "normalized_score": 84.83269961977187}, {"_id": {"$oid": "69d4eb12b877594bc5718972"}, "test_id": "T0208", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash-lite", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "genai", "model": "gemini-2.5-flash-lite", "duration": 1.5813243389129639, "timestamp": "2025-10-01T00:25:51.878707", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Oehme", "first_name": "Carl Wilhelm"}, "publication": {"title": "Mocae historiam adumbrandi speicimen", "year": "place", "pages": "22", "publisher": "", "format": ""}, "library_reference": {"shelfmark": "Halle. Diss. med. 1845.", "subjects": ""}}, "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Oehme\",\n    \"first_name\": \"Carl Wilhelm\"\n  },\n  \"publication\": {\n    \"title\": \"Mocae historiam adumbrandi speicimen\",\n    \"year\": \n    \"place\",\n    \"pages\": \"22\",\n    \"publisher\": \"\",\n    \"format\": \"\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Halle. Diss. med. 1845.\",\n    \"subjects\": \"\"\n  }\n}", "raw_response": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Oehme\",\n    \"first_name\": \"Carl Wilhelm\"\n  },\n  \"publication\": {\n    \"title\": \"Mocae historiam adumbrandi speicimen\",\n    \"year\": \n    \"place\",\n    \"pages\": \"22\",\n    \"publisher\": \"\",\n    \"format\": \"\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Halle. Diss. med. 1845.\",\n    \"subjects\": \"\"\n  }\n}", "usage_metadata": {"prompt_token_count": 705, "candidates_token_count": 152, "total_token_count": 857}}, "usage": {"input_tokens": 705, "output_tokens": 152, "total_tokens": 857}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.6974185357596275, "f1_macro": 0.6892395437262357, "micro_precision": 0.7131112072695802, "micro_recall": 0.6824016563146998, "total_instances": 263, "total_tp": 1648, "total_fp": 663, "total_fn": 767, "cost_summary": {"total_input_tokens": 185415, "total_output_tokens": 214729, "total_tokens": 400144, "input_cost_usd": 0.018541, "output_cost_usd": 0.085892, "total_cost_usd": 0.104433, "pricing_date": "2025-10-01", "input_price_per_million": 0.1, "output_price_per_million": 0.4}}, "normalized_score": 68.92395437262357}, {"_id": {"$oid": "69d4eb12b877594bc5718977"}, "test_id": "T0151", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "genai", "model": "gemini-2.0-flash", "duration": 2.3593506813049316, "timestamp": "2025-10-01T08:57:44.592701", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Pauldrach", "first_name": "Leonhard"}, "publication": {"title": "Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen.", "year": ",(1938),", "place": "Breslau", "pages": "19", "publisher": "Vater.", "format": "8\u00b0"}, "library_reference": {"shelfmark": "DN: B 1938, 8, 548", "subjects": ""}}, "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Pauldrach\",\n    \"first_name\": \"Leonhard\"\n  },\n  \"publication\": {\n    \"title\": \"Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen.\",\n    \"year\":  \",(1938),\",\n    \"place\": \"Breslau\",\n    \"pages\": \"19\",\n    \"publisher\": \"Vater.\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"DN: B 1938, 8, 548\",\n    \"subjects\": \"\"\n  }\n}", "raw_response": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Pauldrach\",\n    \"first_name\": \"Leonhard\"\n  },\n  \"publication\": {\n    \"title\": \"Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen.\",\n    \"year\":  \",(1938),\",\n    \"place\": \"Breslau\",\n    \"pages\": \"19\",\n    \"publisher\": \"Vater.\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"DN: B 1938, 8, 548\",\n    \"subjects\": \"\"\n  }\n}", "usage_metadata": {"prompt_token_count": 2329, "candidates_token_count": 174, "total_token_count": 2503}}, "usage": {"input_tokens": 2329, "output_tokens": 174, "total_tokens": 2503}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.841514143094842, "f1_macro": 0.8322813688212928, "micro_precision": 0.8453823652319264, "micro_recall": 0.8376811594202899, "total_instances": 263, "total_tp": 2023, "total_fp": 370, "total_fn": 392, "cost_summary": {"total_input_tokens": 612527, "total_output_tokens": 45652, "total_tokens": 658179, "input_cost_usd": 0.061253, "output_cost_usd": 0.018261, "total_cost_usd": 0.079514, "pricing_date": "2025-10-01", "input_price_per_million": 0.1, "output_price_per_million": 0.4}}, "normalized_score": 83.22813688212928}, {"_id": {"$oid": "69d4eb12b877594bc5718981"}, "test_id": "T0165", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-5", "duration": 43.47502422332764, "timestamp": "2025-10-01T08:38:58.420488", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Opitz", "first_name": "Hermann"}, "publication": {"title": "Synthesen in der Imidazolreihe.", "year": "1912", "place": "Freiburg i. B.", "pages": "34", "publisher": "Wagner", "format": "8\u00b0", "editor": ""}, "library_reference": {"shelfmark": "U 12. 1261", "subjects": ""}}, "text": "", "raw_response": {"id": "chatcmpl-CLklbS3z3GUdGhld1yJ7wk7sWwJmP", "model": "gpt-5-2025-08-07", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Opitz\",\n    \"first_name\": \"Hermann\"\n  },\n  \"publication\": {\n    \"title\": \"Synthesen in der Imidazolreihe.\",\n    \"year\":  \t\"1912\",\n    \"place\": \"Freiburg i. B.\",\n    \"pages\": \"34\",\n    \"publisher\": \"Wagner\",\n    \"format\": \"8\u00b0\"\n  ,\n  \"editor\": \"\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"U 12. 1261\",\n    \"subjects\": \"\"\n  }\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1387, "completion_tokens": 2207, "total_tokens": 3594}}, "usage": {"input_tokens": 1387, "output_tokens": 2207, "total_tokens": 3594}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8701973001038422, "f1_macro": 0.862699619771863, "micro_precision": 0.8729166666666667, "micro_recall": 0.8674948240165632, "total_instances": 263, "total_tp": 2095, "total_fp": 305, "total_fn": 320, "cost_summary": {"total_input_tokens": 364781, "total_output_tokens": 666836, "total_tokens": 1031617, "input_cost_usd": 0.455976, "output_cost_usd": 6.66836, "total_cost_usd": 7.124336, "pricing_date": "2025-10-01", "input_price_per_million": 1.25, "output_price_per_million": 10.0}}, "normalized_score": 86.2699619771863}, {"_id": {"$oid": "69d4eb12b877594bc5718983"}, "test_id": "T0180", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-medium-2505", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "mistral", "model": "mistral-medium-2505", "duration": 3.069485664367676, "timestamp": "2025-10-01T14:57:15.823477", "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Wolffs\",\n    \"first_name\": \"Dorothee\"\n  },\n  \"publication\": {\n    \"title\": \"Der Phosphoglukomutase-Isoenzympolymorphismus in der unterfr\u00e4nkischen Bev\u00f6lkerung. Stichprobe und Vergleich mit den ABO- und Rhesus-Systemen bei 1000 nicht verwandten Probanden\",\n    \"year\": 1971,\n    \"place\": \"W\u00fcrzburg\",\n    \"pages\": \"40\",\n    \"publisher\": \"\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss.1972,2843\",\n    \"subjects\": \"\"\n  }\n}", "raw_response": {"id": "f39ca0f52b004632ae37afc8b79812e9", "model": "mistral-medium-2505", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Wolffs\",\n    \"first_name\": \"Dorothee\"\n  },\n  \"publication\": {\n    \"title\": \"Der Phosphoglukomutase-Isoenzympolymorphismus in der unterfr\u00e4nkischen Bev\u00f6lkerung. Stichprobe und Vergleich mit den ABO- und Rhesus-Systemen bei 1000 nicht verwandten Probanden\",\n    \"year\": 1971,\n    \"place\": \"W\u00fcrzburg\",\n    \"pages\": \"40\",\n    \"publisher\": \"\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss.1972,2843\",\n    \"subjects\": \"\"\n  }\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1753, "completion_tokens": 189, "total_tokens": 1942}}, "usage": {"input_tokens": 1753, "output_tokens": 189, "total_tokens": 1942}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7757244855102897, "f1_macro": 0.7735361216730038, "micro_precision": 0.7869620792501065, "micro_recall": 0.7648033126293996, "total_instances": 263, "total_tp": 1847, "total_fp": 500, "total_fn": 568, "cost_summary": {"total_input_tokens": 462166, "total_output_tokens": 41296, "total_tokens": 503462, "input_cost_usd": 0.184866, "output_cost_usd": 0.082592, "total_cost_usd": 0.267458, "pricing_date": "2025-10-01", "input_price_per_million": 0.4, "output_price_per_million": 2.0}}, "normalized_score": 77.35361216730038}, {"_id": {"$oid": "69d4eb12b877594bc5718988"}, "test_id": "T0146", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-1-20250805", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "anthropic", "model": "claude-opus-4-1-20250805", "duration": 6.205635070800781, "timestamp": "2025-10-01T07:53:02.065576", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Jasmer", "first_name": "Bernhard"}, "publication": {"title": "De diagnosi differentiali carcinomatis ventriculis et ulceris chronici. Adde historia morbi ulceris ventriculi chronici pro carcinomate habiti", "year": "1859", "place": "Gryphiae", "pages": "28", "publisher": "", "format": "", "editor": null}, "library_reference": {"shelfmark": "", "subjects": ""}}, "text": "", "raw_response": {"id": "msg_0177dL6WU8vmEL8zs7grJX84", "model": "claude-opus-4-1-20250805", "role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01TQ5hYFprek931KcZe52JsC", "name": "extract_structured_data", "input": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Jasmer", "first_name": "Bernhard"}, "publication": {"title": "De diagnosi differentiali carcinomatis ventriculis et ulceris chronici. Adde historia morbi ulceris ventriculi chronici pro carcinomate habiti", "year": "1859", "place": "Gryphiae", "pages": "28", "publisher": "", "format": ""}, "library_reference": {"shelfmark": "", "subjects": ""}}}], "stop_reason": "tool_use", "usage": {"input_tokens": 2185, "output_tokens": 197}}, "usage": {"input_tokens": 2185, "output_tokens": 197, "total_tokens": 2382}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8259175722780399, "f1_macro": 0.8153992395437262, "micro_precision": 0.818034118602762, "micro_recall": 0.8339544513457557, "total_instances": 263, "total_tp": 2014, "total_fp": 448, "total_fn": 401, "cost_summary": {"total_input_tokens": 566577, "total_output_tokens": 54253, "total_tokens": 620830, "input_cost_usd": 8.498655, "output_cost_usd": 4.068975, "total_cost_usd": 12.56763, "pricing_date": "2025-10-01", "input_price_per_million": 15.0, "output_price_per_million": 75.0}}, "normalized_score": 81.53992395437261}, {"_id": {"$oid": "69d4eb12b877594bc571898a"}, "test_id": "T0167", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-5-nano", "duration": 10.945187091827393, "timestamp": "2025-10-01T21:02:59.889999", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wittenberg", "first_name": "Joachim"}, "publication": {"title": "Unbenutzte Warenzeichen.", "year": "place\": \"\",", "place": "", "pages": "104", "publisher": "", "format": "4\u00b0", "editor": ""}, "library_reference": {"shelfmark": "Diss.d.Rechts-u.Wirtschaftswissenschaftl.Fak.Mainz.", "subjects": ""}}, "text": "", "raw_response": {"id": "chatcmpl-CLwO9w1CEmFvFD8lplpxQQrwGdYtT", "model": "gpt-5-nano-2025-08-07", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Wittenberg\",\n    \"first_name\": \"Joachim\"\n  },\n  \"publication\": {\n    \"title\": \"Unbenutzte Warenzeichen.\",\n    \"year\":      \t\"place\\\": \\\"\\\",\" \t,\n    \"place\": \"\",\n    \"pages\": \"104\",\n    \"publisher\": \"\",\n    \"format\": \"4\u00b0\"\n  ,\n    \"editor\": \"\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss.d.Rechts-u.Wirtschaftswissenschaftl.Fak.Mainz.\",\n    \"subjects\": \"\"\n  }\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1688, "completion_tokens": 1698, "total_tokens": 3386}}, "usage": {"input_tokens": 1688, "output_tokens": 1698, "total_tokens": 3386}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7705185790118416, "f1_macro": 0.7659315589353612, "micro_precision": 0.7599677809101892, "micro_recall": 0.7813664596273292, "total_instances": 263, "total_tp": 1887, "total_fp": 596, "total_fn": 528, "cost_summary": {"total_input_tokens": 434613, "total_output_tokens": 807745, "total_tokens": 1242358, "input_cost_usd": 0.021731, "output_cost_usd": 0.323098, "total_cost_usd": 0.344829, "pricing_date": "2025-10-01", "input_price_per_million": 0.05, "output_price_per_million": 0.4}}, "normalized_score": 76.59315589353612}, {"_id": {"$oid": "69d4eb12b877594bc5718990"}, "test_id": "T0166", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-5-mini", "duration": 19.616859197616577, "timestamp": "2025-10-01T23:49:01.520962", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Schoofs", "first_name": "F."}, "publication": {"title": "Les impuret\u00e9s des alcools, eaux-de-vie & geni\u00e8vres fabriqu\u00e9s en Belgique", "year": ": 1912", "place": "Li\u00e8ge", "pages": "111", "publisher": "", "format": "", "editor": ""}, "library_reference": {"shelfmark": "Diss. Li\u00e8ge", "subjects": "alcools; eaux-de-vie; geni\u00e8vres; Belgique"}}, "text": "", "raw_response": {"id": "chatcmpl-CLyygVoQ4q8r9MDs43FD7Nq5hLlOm", "model": "gpt-5-mini-2025-08-07", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Schoofs\",\"first_name\":\"F.\"},\"publication\":{\"title\":\"Les impuret\u00e9s des alcools, eaux-de-vie & geni\u00e8vres fabriqu\u00e9s en Belgique\",\"year\":\": 1912\",\"place\":\"Li\u00e8ge\",\"pages\":\"111\",\"publisher\":\"\",\"format\":\"\"    ,\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"Diss. Li\u00e8ge\",\"subjects\":\"alcools; eaux-de-vie; geni\u00e8vres; Belgique\"}}", "role": "assistant"}}], "usage": {"prompt_tokens": 1502, "completion_tokens": 1336, "total_tokens": 2838}}, "usage": {"input_tokens": 1502, "output_tokens": 1336, "total_tokens": 2838}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7910357359176255, "f1_macro": 0.7893536121673004, "micro_precision": 0.7718676122931442, "micro_recall": 0.8111801242236025, "total_instances": 263, "total_tp": 1959, "total_fp": 579, "total_fn": 456, "cost_summary": {"total_input_tokens": 387529, "total_output_tokens": 609480, "total_tokens": 997009, "input_cost_usd": 0.096882, "output_cost_usd": 1.21896, "total_cost_usd": 1.315842, "pricing_date": "2025-10-01", "input_price_per_million": 0.25, "output_price_per_million": 2.0}}, "normalized_score": 78.93536121673004}, {"_id": {"$oid": "69d4eb12b877594bc5718992"}, "test_id": "T0147", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-20250514", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "anthropic", "model": "claude-opus-4-20250514", "duration": 7.011705160140991, "timestamp": "2025-10-01T08:17:14.735921", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Skalatkiewicz", "first_name": "Julian"}, "publication": {"title": "Dom matutinalis, quod virglo (!) Morgengaba dicitur, virgo ac jus.", "year": "1849", "place": "Vratislaviae", "pages": "33", "publisher": "", "format": "", "editor": null}, "library_reference": {"shelfmark": "Breslau. Diss. jur. 1849.", "subjects": ""}}, "text": "", "raw_response": {"id": "msg_01HpG6X371avjmbPZU5KJ24Y", "model": "claude-opus-4-20250514", "role": "assistant", "content": [{"type": "tool_use", "id": "toolu_014K9GTSEouCkK7pJE8BxBQD", "name": "extract_structured_data", "input": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Skalatkiewicz", "first_name": "Julian"}, "publication": {"title": "Dom matutinalis, quod virglo (!) Morgengaba dicitur, virgo ac jus.", "year": "1849", "place": "Vratislaviae", "pages": "33", "publisher": "", "format": ""}, "library_reference": {"shelfmark": "Breslau. Diss. jur. 1849.", "subjects": ""}}}], "stop_reason": "tool_use", "usage": {"input_tokens": 2150, "output_tokens": 198}}, "usage": {"input_tokens": 2150, "output_tokens": 198, "total_tokens": 2348}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7851458885941645, "f1_macro": 0.7787832699619771, "micro_precision": 0.7739340305711987, "micro_recall": 0.7966873706004141, "total_instances": 263, "total_tp": 1924, "total_fp": 562, "total_fn": 491, "cost_summary": {"total_input_tokens": 566577, "total_output_tokens": 56103, "total_tokens": 622680, "input_cost_usd": 8.498655, "output_cost_usd": 4.207725, "total_cost_usd": 12.70638, "pricing_date": "2025-10-01", "input_price_per_million": 15.0, "output_price_per_million": 75.0}}, "normalized_score": 77.87832699619771}, {"_id": {"$oid": "69d4eb12b877594bc5718996"}, "test_id": "T0155", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-pro", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "genai", "model": "gemini-2.5-pro", "duration": 12.97180461883545, "timestamp": "2025-10-01T10:21:17.432634", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Hemann", "first_name": "Christ Ludwig Ernst"}, "publication": {"title": "De veterum dotis actionum, rei uxoriae atque ex stipulatu, differentiis.", "year": "1826", "place": "Kiliae", "pages": "40", "publisher": "", "format": "4\u00b0"}, "library_reference": {"shelfmark": "", "subjects": "Kiel. Diss. iur. 1826."}}, "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Hemann\",\n    \"first_name\": \"Christ Ludwig Ernst\"\n  },\n  \"publication\": {\n    \"title\": \"De veterum dotis actionum, rei uxoriae atque ex stipulatu, differentiis.\",\n    \"year\":       \"1826\",\n    \"place\": \"Kiliae\",\n    \"pages\": \"40\",\n    \"publisher\": \"\",\n    \"format\": \"4\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"subjects\": \"Kiel. Diss. iur. 1826.\"\n  }\n}", "raw_response": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Hemann\",\n    \"first_name\": \"Christ Ludwig Ernst\"\n  },\n  \"publication\": {\n    \"title\": \"De veterum dotis actionum, rei uxoriae atque ex stipulatu, differentiis.\",\n    \"year\":       \"1826\",\n    \"place\": \"Kiliae\",\n    \"pages\": \"40\",\n    \"publisher\": \"\",\n    \"format\": \"4\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"subjects\": \"Kiel. Diss. iur. 1826.\"\n  }\n}", "usage_metadata": {"prompt_token_count": 705, "candidates_token_count": 175, "total_token_count": 1818}}, "usage": {"input_tokens": 705, "output_tokens": 175, "total_tokens": 1818}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8697948405443835, "f1_macro": 0.8647908745247148, "micro_precision": 0.8536682615629984, "micro_recall": 0.8865424430641822, "total_instances": 263, "total_tp": 2141, "total_fp": 367, "total_fn": 274, "cost_summary": {"total_input_tokens": 185415, "total_output_tokens": 47772, "total_tokens": 233187, "input_cost_usd": 0.231769, "output_cost_usd": 0.47772, "total_cost_usd": 0.709489, "pricing_date": "2025-10-01", "input_price_per_million": 1.25, "output_price_per_million": 10.0}}, "normalized_score": 86.47908745247148}, {"_id": {"$oid": "69d4eb12b877594bc5718998"}, "test_id": "T0216", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash-lite-preview-09-2025", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "genai", "model": "gemini-2.5-flash-lite-preview-09-2025", "duration": 0.6779627799987793, "timestamp": "2025-10-01T00:39:40.163323", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Oldenburger", "first_name": "Rufus"}, "publication": {"title": "Composition and rank of n-way matrices and multilinear forms.", "year": 1934, "place": "Chicago", "pages": "", "publisher": "University of Chicago", "format": "8."}, "library_reference": {"shelfmark": "Diss.Chicago 1934", "subjects": ""}}, "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Oldenburger\",\n    \"first_name\": \"Rufus\"\n  },\n  \"publication\": {\n    \"title\": \"Composition and rank of n-way matrices and multilinear forms.\",\n    \"year\": 1934,\n    \"place\": \"Chicago\",\n    \"pages\": \"\",\n    \"publisher\": \"University of Chicago\",\n    \"format\": \"8.\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss.Chicago 1934\",\n    \"subjects\": \"\"\n  }\n}", "raw_response": {"text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Oldenburger\",\n    \"first_name\": \"Rufus\"\n  },\n  \"publication\": {\n    \"title\": \"Composition and rank of n-way matrices and multilinear forms.\",\n    \"year\": 1934,\n    \"place\": \"Chicago\",\n    \"pages\": \"\",\n    \"publisher\": \"University of Chicago\",\n    \"format\": \"8.\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss.Chicago 1934\",\n    \"subjects\": \"\"\n  }\n}", "usage_metadata": {"prompt_token_count": 705, "candidates_token_count": 159, "total_token_count": 864}}, "usage": {"input_tokens": 705, "output_tokens": 159, "total_tokens": 864}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.684657883552911, "f1_macro": 0.6662737642585551, "micro_precision": 0.7031863814927979, "micro_recall": 0.6670807453416149, "total_instances": 263, "total_tp": 1611, "total_fp": 680, "total_fn": 804, "cost_summary": {"total_input_tokens": 185415, "total_output_tokens": 766627, "total_tokens": 952042, "input_cost_usd": 0.018541, "output_cost_usd": 0.306651, "total_cost_usd": 0.325192, "pricing_date": "2025-10-01", "input_price_per_million": 0.1, "output_price_per_million": 0.4}}, "normalized_score": 66.62737642585552}, {"_id": {"$oid": "69d4eb12b877594bc5718999"}, "test_id": "T0192", "benchmark": "library_cards", "date": "2025-10-01", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-large-2411", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "mistral", "model": "mistral-large-latest", "duration": 3.682055711746216, "timestamp": "2025-10-01T15:52:48.639711", "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Sibalin\",\n    \"first_name\": \"Mato\"\n  },\n  \"publication\": {\n    \"title\": \"Enterovirusstammar hos svenska grisar. Isolering, identifikation, patogenicitet, kemiska och fysikaliska egenskaper\",\n    \"year\": 1963,\n    \"place\": \"Uppsala\",\n    \"pages\": \"19\",\n    \"publisher\": \"Almqvist & Wiksell\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss. NK GK 2Mak\",\n    \"subjects\": \"\"\n  }\n}", "raw_response": {"id": "2d3d90ecf8124a4c9f229a0e9517b141", "model": "mistral-large-latest", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Sibalin\",\n    \"first_name\": \"Mato\"\n  },\n  \"publication\": {\n    \"title\": \"Enterovirusstammar hos svenska grisar. Isolering, identifikation, patogenicitet, kemiska och fysikaliska egenskaper\",\n    \"year\": 1963,\n    \"place\": \"Uppsala\",\n    \"pages\": \"19\",\n    \"publisher\": \"Almqvist & Wiksell\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss. NK GK 2Mak\",\n    \"subjects\": \"\"\n  }\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1753, "completion_tokens": 170, "total_tokens": 1923}}, "usage": {"input_tokens": 1753, "output_tokens": 170, "total_tokens": 1923}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7804061126229852, "f1_macro": 0.7769201520912548, "micro_precision": 0.7891617273497037, "micro_recall": 0.7718426501035197, "total_instances": 263, "total_tp": 1864, "total_fp": 498, "total_fn": 551, "cost_summary": {"total_input_tokens": 462166, "total_output_tokens": 41452, "total_tokens": 503618, "input_cost_usd": 0.924332, "output_cost_usd": 0.248712, "total_cost_usd": 1.173044, "pricing_date": "2025-10-01", "input_price_per_million": 2.0, "output_price_per_million": 6.0}}, "normalized_score": 77.69201520912547}, {"_id": {"$oid": "69d4eb12b877594bc57188e7"}, "test_id": "T0200", "benchmark": "library_cards", "date": "2025-09-30", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "genai", "model": "gemini-2.5-flash", "duration": 7.568887948989868, "timestamp": "2025-09-30T23:33:23.140893", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "M\u00fcller", "first_name": "Maurice Edmond"}, "publication": {"title": "Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form,Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.", "year": "1957", "place": "Stuttgart", "pages": "184", "publisher": "Thieme", "format": "4'"}, "library_reference": {"shelfmark": "AT Z\u00fcrich 7", "subjects": ""}}, "text": "{\"type\": {\"type\": \"Dissertation or thesis\"},\"author\": {\"last_name\": \"M\u00fcller\",\"first_name\": \"Maurice Edmond\"},\"publication\": {\"title\": \"Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form,Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.\",\"year\": \n                      \"1957\",\n                     \"place\": \"Stuttgart\",\"pages\": \"184\",\"publisher\": \"Thieme\",\"format\": \"4'\"},\"library_reference\": {\"shelfmark\": \"AT Z\u00fcrich 7\",\"subjects\": \"\"}}", "raw_response": {"text": "{\"type\": {\"type\": \"Dissertation or thesis\"},\"author\": {\"last_name\": \"M\u00fcller\",\"first_name\": \"Maurice Edmond\"},\"publication\": {\"title\": \"Die h\u00fcftnahen Femur-osteotomien unter Ber\u00fccksichtigung der Form,Funktion und Beanspruchung des H\u00fcftgelenkes. Hab.Schr. Med.Fak.Univ.Z\u00fcrich. Mit 282 Abb.\",\"year\": \n                      \"1957\",\n                     \"place\": \"Stuttgart\",\"pages\": \"184\",\"publisher\": \"Thieme\",\"format\": \"4'\"},\"library_reference\": {\"shelfmark\": \"AT Z\u00fcrich 7\",\"subjects\": \"\"}}", "usage_metadata": {"prompt_token_count": 705, "candidates_token_count": 151, "total_token_count": 1878}}, "usage": {"input_tokens": 705, "output_tokens": 151, "total_tokens": 1878}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8572004028197381, "f1_macro": 0.8498098859315589, "micro_precision": 0.8345098039215686, "micro_recall": 0.881159420289855, "total_instances": 263, "total_tp": 2128, "total_fp": 422, "total_fn": 287, "cost_summary": {"total_input_tokens": 185415, "total_output_tokens": 45117, "total_tokens": 230532, "input_cost_usd": 0.055625, "output_cost_usd": 0.112792, "total_cost_usd": 0.168417, "pricing_date": "2025-09-30", "input_price_per_million": 0.3, "output_price_per_million": 2.5}}, "normalized_score": 84.9809885931559}, {"_id": {"$oid": "69d4eb12b877594bc57188fd"}, "test_id": "T0160", "benchmark": "library_cards", "date": "2025-09-30", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4.1", "duration": 160.2987585067749, "timestamp": "2025-10-01T02:21:16.553703", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wiggers", "first_name": "Wilhelm"}, "publication": {"title": "Beitrag zu den epileptischen Seelenst\u00f6rungen", "year": "place\": \"Kiel\",\n    \"pages\": \"24\",\n    \"publisher\": \"\",\n    \"format\": \"4\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"U 27.8118\",\n    \"subjects\": \"\"\n  }\n}", "place": "Kiel", "pages": "24", "publisher": "", "format": "4\u00b0", "editor": null}, "library_reference": {"shelfmark": "U 27.8118", "subjects": ""}}, "text": "", "raw_response": {"id": "chatcmpl-CLeqDbrzHwp29JSJY11vakfUDBzHo", "model": "gpt-4.1-2025-04-14", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Wiggers\",\n    \"first_name\": \"Wilhelm\"\n  },\n  \"publication\": {\n    \"title\": \"Beitrag zu den epileptischen Seelenst\u00f6rungen\",\n    \"year\":                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        \"place\\\": \\\"Kiel\\\",\\n    \\\"pages\\\": \\\"24\\\",\\n    \\\"publisher\\\": \\\"\\\",\\n    \\\"format\\\": \\\"4\u00b0\\\"\\n  },\\n  \\\"library_reference\\\": {\\n    \\\"shelfmark\\\": \\\"U 27.8118\\\",\\n    \\\"subjects\\\": \\\"\\\"\\n  }\\n}\"\n  ,\n  \"place\": \"Kiel\",\n  \"pages\": \"24\",\n  \"publisher\": \"\",\n  \"format\": \"4\u00b0\"\n  ,\n  \"editor\": null\n  }\n  ,\n  \"library_reference\": {\n    \"shelfmark\": \"U 27.8118\",\n    \"subjects\": \"\"\n  }\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1526, "completion_tokens": 6407, "total_tokens": 7933}}, "usage": {"input_tokens": 1526, "output_tokens": 6407, "total_tokens": 7933}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8592314118629908, "f1_macro": 0.8538403041825096, "micro_precision": 0.866835229667088, "micro_recall": 0.85175983436853, "total_instances": 263, "total_tp": 2057, "total_fp": 316, "total_fn": 358, "cost_summary": {"total_input_tokens": 401338, "total_output_tokens": 245201, "total_tokens": 646539, "input_cost_usd": 0.802676, "output_cost_usd": 1.961608, "total_cost_usd": 2.764284, "pricing_date": "2025-09-30", "input_price_per_million": 2.0, "output_price_per_million": 8.0}}, "normalized_score": 85.38403041825096}, {"_id": {"$oid": "69d4eb12b877594bc5718929"}, "test_id": "T0066", "benchmark": "library_cards", "date": "2025-09-30", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4o", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge and an expert in the field of 20th century Swiss history", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4o", "duration": 9.363641023635864, "timestamp": "2025-09-30T16:40:58.234669", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Verhaeghe", "first_name": "\u00c9tienne"}, "publication": {"title": "Contribution \u00e0 l'\u00e9tude clinique et exp\u00e9rimentale des transplantations h\u00e9t\u00e9roplastiques d'os frais. Quelques essais de transplantations h\u00e9t\u00e9roplastiques ost\u00e9oarticulaires et p\u00e9riostiques", "year": ",", "place": "Lille", "pages": "197", "publisher": "", "format": "in-8\u00b0", "editor": ""}, "library_reference": {"shelfmark": "", "subjects": ""}}, "text": "", "raw_response": {"id": "chatcmpl-CLVoxgmM0fDo6z4oIHxveqEwfIjPp", "model": "gpt-4o-2024-08-06", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"type\":{\"type\":\"Dissertation or thesis\"},\"author\":{\"last_name\":\"Verhaeghe\",\"first_name\":\"\u00c9tienne\"},\"publication\":{\"title\":\"Contribution \u00e0 l'\u00e9tude clinique et exp\u00e9rimentale des transplantations h\u00e9t\u00e9roplastiques d'os frais. Quelques essais de transplantations h\u00e9t\u00e9roplastiques ost\u00e9oarticulaires et p\u00e9riostiques\",\"year\":     \",\"  ,\"place\":\"Lille\",\"pages\":\"197\",\"publisher\":\"\",\"format\":\"in-8\u00b0\"  ,\"editor\":\"\"},\"library_reference\":{\"shelfmark\":\"\",\"subjects\":\"\"}}", "role": "assistant"}}], "usage": {"prompt_tokens": 1539, "completion_tokens": 120, "total_tokens": 1659}}, "usage": {"input_tokens": 1539, "output_tokens": 120, "total_tokens": 1659}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8248447204968944, "f1_macro": 0.8211787072243346, "micro_precision": 0.8248447204968944, "micro_recall": 0.8248447204968944, "total_instances": 263, "total_tp": 1992, "total_fp": 423, "total_fn": 423, "cost_summary": {"total_input_tokens": 404757, "total_output_tokens": 29124, "total_tokens": 433881, "input_cost_usd": 1.011892, "output_cost_usd": 0.29124, "total_cost_usd": 1.303132, "pricing_date": "2025-09-30", "input_price_per_million": 2.5, "output_price_per_million": 10.0}}, "normalized_score": 82.11787072243347}, {"_id": {"$oid": "69d4eb12b877594bc5718937"}, "test_id": "T0230", "benchmark": "library_cards", "date": "2025-09-30", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-5-20250929", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "anthropic", "model": "claude-sonnet-4-5-20250929", "duration": 4.475728273391724, "timestamp": "2025-09-30T23:03:32.436908", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Neidhardt", "first_name": "Hansj\u00f6rg"}, "publication": {"title": "Wirkungs\u00e4nderungen zweier Analeptica durch subchronische Alkoholgaben", "year": "1964", "place": "Hamburg", "pages": "41", "publisher": "", "format": "8'", "editor": null}, "library_reference": {"shelfmark": "Diss.1965,3355", "subjects": ""}}, "text": "", "raw_response": {"id": "msg_01Xaov67p5pqQjU6C3QvH7tV", "model": "claude-sonnet-4-5-20250929", "role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01J8PRrfCfFhArjPnckNirtV", "name": "extract_structured_data", "input": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Neidhardt", "first_name": "Hansj\u00f6rg"}, "publication": {"title": "Wirkungs\u00e4nderungen zweier Analeptica durch subchronische Alkoholgaben", "year": "1964", "place": "Hamburg", "pages": "41", "publisher": "", "format": "8'"}, "library_reference": {"shelfmark": "Diss.1965,3355", "subjects": ""}}}], "stop_reason": "tool_use", "usage": {"input_tokens": 2388, "output_tokens": 190}}, "usage": {"input_tokens": 2388, "output_tokens": 190, "total_tokens": 2578}, "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8062269561655059, "f1_macro": 0.792851711026616, "micro_precision": 0.7977300364815565, "micro_recall": 0.8149068322981367, "total_instances": 263, "total_tp": 1968, "total_fp": 499, "total_fn": 447, "cost_summary": {"total_input_tokens": 638376, "total_output_tokens": 57192, "total_tokens": 695568, "input_cost_usd": 1.915128, "output_cost_usd": 0.85788, "total_cost_usd": 2.773008, "pricing_date": "2025-09-30", "input_price_per_million": 3.0, "output_price_per_million": 15.0}}, "normalized_score": 79.2851711026616}, {"_id": {"$oid": "69d4eb12b877594bc57188d4"}, "test_id": "T0179", "benchmark": "library_cards", "date": "2025-09-26", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-medium-2508", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "mistral", "model": "mistral-medium-2508", "duration": 2.8012421131134033, "timestamp": "2025-09-26T09:18:03.642834", "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Sieferle\",\n    \"first_name\": \"Rolf Peter\"\n  },\n  \"publication\": {\n    \"title\": \"Die Revolution in der Theorie von Karl Marx\",\n    \"year\": 1979,\n    \"place\": \"Konstanz\",\n    \"pages\": \"256\",\n    \"publisher\": \"Ullstein Buch\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Bro 26.993\",\n    \"subjects\": \"Sozial-geschichtliche Bibliothek\"\n  }\n}", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7804366078925273, "f1_macro": 0.7777566539923955, "micro_precision": 0.7914005959982972, "micro_recall": 0.7697722567287785, "total_instances": 263, "total_tp": 1859, "total_fp": 490, "total_fn": 556}, "normalized_score": 77.77566539923954}, {"_id": {"$oid": "69d4eb12b877594bc57188dd"}, "test_id": "T0180", "benchmark": "library_cards", "date": "2025-09-26", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-medium-2505", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "mistral", "model": "mistral-medium-2505", "duration": 3.934082269668579, "timestamp": "2025-09-26T09:42:28.181306", "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Wolff\",\n    \"first_name\": \"Dorothee\"\n  },\n  \"publication\": {\n    \"title\": \"Der Phosphoglukomutase-Isoenzympolymorphismus in der unterfr\u00e4nkischen Bev\u00f6lkerung. Stichprobe und Vergleich mit den ABO- und Rhesus-Systemen bei 1000 nicht verwandten Probanden\",\n    \"year\": 1971,\n    \"place\": \"W\u00fcrzburg\",\n    \"pages\": \"40\",\n    \"publisher\": \"\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Diss.1972,2843\",\n    \"subjects\": \"\"\n  }\n}", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7782672540381792, "f1_macro": 0.7755133079847909, "micro_precision": 0.7886904761904762, "micro_recall": 0.7681159420289855, "total_instances": 263, "total_tp": 1855, "total_fp": 497, "total_fn": 560}, "normalized_score": 77.55133079847909}, {"_id": {"$oid": "69d4eb12b877594bc57188d1"}, "test_id": "T0159", "benchmark": "library_cards", "date": "2025-09-25", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "pixtral-large-2411", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "mistral", "model": "pixtral-large-latest", "duration": 6.079905033111572, "timestamp": "2025-09-25T20:53:28.841569", "text": "{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Z\u00fcrndorfer\",\n    \"first_name\": \"Ludwig\"\n  },\n  \"publication\": {\n    \"title\": \"Das Trauma als Entstehungsursache der Syringomyelie und multiplen Sklerose\",\n    \"year\": 1908,\n    \"place\": \"W\u00fcrzburg\",\n    \"pages\": \"34\",\n    \"publisher\": \"Freudenberger\",\n    \"format\": \"8\u00b0\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Med. Diss. v. 12. Dez. 1908, Ref. v. Leube\",\n    \"subjects\": \"[Geb. 13. Juli 81 Rexingen; Wohnort: Rexingen; Staatsangeh.: W\u00fcrttemberg; Vorbildung: Gymn. W\u00fcrzburg Reife Juli 02; Studium: M\u00fcnchen 1, W\u00fcrzburg 4, Berlin 1, Stra\u00dfburg 1, W\u00fcrzburg 4 S.; Coll. 15. Jan. 08; Approb. 12. Dez. 07.]\"\n  }\n}", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7775731532637611, "f1_macro": 0.7760456273764258, "micro_precision": 0.7686084142394822, "micro_recall": 0.7867494824016563, "total_instances": 263, "total_tp": 1900, "total_fp": 572, "total_fn": 515}, "normalized_score": 77.60456273764258}, {"_id": {"$oid": "69d4eb12b877594bc57188c6"}, "test_id": "T0162", "benchmark": "library_cards", "date": "2025-09-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4.1-nano", "duration": 2.2541215419769287, "timestamp": "2025-09-24T23:59:16.332445", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Spiegelberg", "first_name": "Otto"}, "publication": {"title": "Ueber das gegenseitige Verh\u00e4ltnis zwischen dem gerichtlichen Gest\u00e4ndnis und der Offen-kundigkeit bei dem Gericht nach der deutschen Civilprozessordnung", "year": "1896", "place": "Marburg", "pages": "128 S. 8", "publisher": "H. L. Br\u00f6nner's Druckerei", "format": "8\u00b0", "editor": ""}, "library_reference": {"shelfmark": "Diss. Marburg, Jur. Fak., Inaug.-Diss. v. 1. Juni 1896", "subjects": ""}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.611930442438917, "f1_macro": 0.604106463878327, "micro_precision": 0.6531954887218046, "micro_recall": 0.5755693581780539, "total_instances": 263, "total_tp": 1390, "total_fp": 738, "total_fn": 1025}, "normalized_score": 60.4106463878327}, {"_id": {"$oid": "69d4eb12b877594bc57188c8"}, "test_id": "T0151", "benchmark": "library_cards", "date": "2025-09-24", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "genai", "model": "gemini-2.0-flash", "duration": 2.128584384918213, "timestamp": "2025-09-24T23:36:02.574631", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Pauldrach", "first_name": "Leonhard"}, "publication": {"title": "Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen", "year": 1938, "place": "Breslau", "pages": "19", "publisher": "Vater", "format": "8\u00b0", "editor": null}, "library_reference": {"shelfmark": "DN: B 1938, 8, 548", "subjects": ""}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8579055441478439, "f1_macro": 0.849277566539924, "micro_precision": 0.8509164969450101, "micro_recall": 0.8650103519668737, "total_instances": 263, "total_tp": 2089, "total_fp": 366, "total_fn": 326}, "normalized_score": 84.9277566539924}, {"_id": {"$oid": "69d4eb12b877594bc57188b1"}, "test_id": "T0168", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "o3", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "o3", "duration": 11.284327268600464, "timestamp": "2025-09-10T02:05:14.162538", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Z\u00e4hle", "first_name": "Werner"}, "publication": {"title": "Ergebnisse von Bilirubinbestimmungen im Duodenalsaft von Gesunden u. Leberkranken m.d. Methode v. Jendrassik u. Cleghorn. Auszug a.d. Inaug.-Diss. Med. Fak. M\u00fcnster.", "year": 1952, "place": "M\u00fcnster (Westf.)", "pages": "", "publisher": "Westf. Vereinsdruckerei", "format": "8'\"},", "editor": null}, "library_reference": {"shelfmark": "Diss. M\u00fcnster 1952", "subjects": ""}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.871041192299731, "f1_macro": 0.8666539923954373, "micro_precision": 0.8708609271523179, "micro_recall": 0.8712215320910973, "total_instances": 263, "total_tp": 2104, "total_fp": 312, "total_fn": 311}, "normalized_score": 86.66539923954373}, {"_id": {"$oid": "69d4eb12b877594bc57188b3"}, "test_id": "T0152", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash-lite", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "genai", "model": "gemini-2.0-flash-lite", "duration": 5.286254405975342, "timestamp": "2025-09-05T11:23:09.355123", "text": "```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Volodarsky\",\n    \"first_name\": \"Mademoiselle\"\n  },\n  \"publication\": {\n    \"title\": \"Injections intrac\u00e9r\u00e9brales de s\u00e9rum chez les animaux s\u00e9ro-anaphylactis\u00e9s.\",\n    \"year\": 1910,\n    \"place\": \"Lausanne\",\n    \"pages\": \"24\",\n    \"publisher\": \"Imprimerie de l'Universit\u00e9\",\n    \"format\": \"8\u00b0\",\n    \"reprint_note\": \"\"\n  },\n  \"examination\": {\n    \"place\": \"Lausanne\",\n    \"year\": 1910\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"publication_number\": \"\",\n    \"subjects\": \"\"\n  }\n}\n```", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8706914560267389, "f1_macro": 0.8652471482889734, "micro_precision": 0.8785834738617201, "micro_recall": 0.8629399585921325, "total_instances": 263, "total_tp": 2084, "total_fp": 288, "total_fn": 331}, "normalized_score": 86.52471482889734}, {"_id": {"$oid": "69d4eb12b877594bc57188b4"}, "test_id": "T0159", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "pixtral-large-2411", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "mistral", "model": "pixtral-large-latest", "duration": 6.170723915100098, "timestamp": "2025-09-03T21:29:51.351394", "text": "```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Z\u00fcrndorfer\",\n    \"first_name\": \"Ludwig\"\n  },\n  \"publication\": {\n    \"title\": \"Das Trauma als Entstehungsursache der Syringomyelie [] und multiplen Sklerose\",\n    \"year\": 1908,\n    \"place\": \"W\u00fcrzburg\",\n    \"pages\": \"34\",\n    \"publisher\": \"Freudenberger\",\n    \"format\": \"8\u00b0\",\n    \"reprint_note\": \"\"\n  },\n  \"examination\": {\n    \"place\": \"W\u00fcrzburg\",\n    \"year\": 1908\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"Med. Diss. v. 12. Dez. 1908, Ref. v. Leube\",\n    \"publication_number\": \"U 09: 4152\",\n    \"subjects\": \"\"\n  }\n}\n```", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7890251506963208, "f1_macro": 0.7788973384030418, "micro_precision": 0.7921535893155259, "micro_recall": 0.7859213250517598, "total_instances": 263, "total_tp": 1898, "total_fp": 498, "total_fn": 517}, "normalized_score": 77.88973384030417}, {"_id": {"$oid": "69d4eb12b877594bc57188b5"}, "test_id": "T0160", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4.1", "duration": 3.931912660598755, "timestamp": "2025-09-03T21:57:32.855213", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wiggers", "first_name": "Wilhelm"}, "publication": {"title": "Beitrag zu den epileptischen Seelenst\u00f6rungen", "year": 1927, "place": "Kiel", "pages": "24", "publisher": "", "format": "4\u00b0", "reprint_note": ""}, "examination": {"place": "Kiel", "year": 1927}, "library_reference": {"shelfmark": "U 27.3118", "publication_number": "", "subjects": ""}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8970743001473375, "f1_macro": 0.8939163498098859, "micro_precision": 0.9122431506849316, "micro_recall": 0.8824016563146998, "total_instances": 263, "total_tp": 2131, "total_fp": 205, "total_fn": 284}, "normalized_score": 89.39163498098858}, {"_id": {"$oid": "69d4eb12b877594bc57188b7"}, "test_id": "T0161", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4.1-mini", "duration": 6.351454734802246, "timestamp": "2025-09-03T22:06:03.309755", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Silvestri", "first_name": "Richard"}, "publication": {"title": "Entwicklungstendenzen der Best\u00e4nde ausgew\u00e4hlter Maschinen und des Volumens der maschinellen Investitionen in der oesterreichischen Landwirtschaft", "year": 1968, "place": "Wien", "pages": "186", "publisher": "", "format": "8'", "reprint_note": ""}, "examination": {"place": "Wien", "year": 1968}, "library_reference": {"shelfmark": "Diss.", "publication_number": "1970,8109", "subjects": "Landwirtschaft. Maschinen u. Ger\u00e4te/Landwirtschaft. Oesterreich"}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.87017099430019, "f1_macro": 0.8673764258555133, "micro_precision": 0.8875968992248062, "micro_recall": 0.853416149068323, "total_instances": 263, "total_tp": 2061, "total_fp": 261, "total_fn": 354}, "normalized_score": 86.73764258555133}, {"_id": {"$oid": "69d4eb12b877594bc57188ba"}, "test_id": "T0151", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "genai", "model": "gemini-2.0-flash", "duration": 4.942450523376465, "timestamp": "2025-09-05T10:44:51.679467", "text": "```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Pauldrach\",\n    \"first_name\": \"Leonhard\"\n  },\n  \"publication\": {\n    \"title\": \"Gefrierhistologische Untersuchungen an einigen inneren Organen menschlicher Leichen.\",\n    \"year\": 1938,\n    \"place\": \"Breslau\",\n    \"pages\": \"19\",\n    \"publisher\": \"Vater\",\n    \"format\": \"8\u00b0\",\n    \"reprint_note\": \"\"\n  },\n  \"examination\": {\n    \"place\": \"Breslau\",\n    \"year\": 1938\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"publication_number\": \"DN: B 1938, 8, 548\",\n    \"subjects\": \"\"\n  }\n}\n```", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8374052232518955, "f1_macro": 0.829657794676806, "micro_precision": 0.8521217316759537, "micro_recall": 0.8231884057971014, "total_instances": 263, "total_tp": 1988, "total_fp": 345, "total_fn": 427}, "normalized_score": 82.9657794676806}, {"_id": {"$oid": "69d4eb12b877594bc57188bb"}, "test_id": "T0148", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-20250514", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "anthropic", "model": "claude-sonnet-4-20250514", "duration": 5.478160381317139, "timestamp": "2025-09-03T19:12:57.697265", "text": "```json\n{\n  \"type\": \"Dissertation or thesis\",\n  \"author\": {\n    \"last_name\": \"VAYSSI\u00c8RE\",\n    \"first_name\": \"Emile\"\n  },\n  \"publication\": {\n    \"title\": \"D\u00e9viation du compl\u00e9ment et grossesse\",\n    \"year\": 1912,\n    \"place\": \"Montpellier\",\n    \"pages\": \"178\",\n    \"publisher\": \"\",\n    \"format\": \"8\u00b0\",\n    \"reprint_note\": \"\"\n  },\n  \"examination\": {\n    \"place\": \"Montpellier\",\n    \"year\": 1912\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"publication_number\": \"N\u00b0 39\",\n    \"subjects\": \"Th. m\u00e9d.\"\n  }\n}\n```", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7933058030862856, "f1_macro": 0.7815209125475285, "micro_precision": 0.8348581884720951, "micro_recall": 0.7556935817805382, "total_instances": 263, "total_tp": 1825, "total_fp": 361, "total_fn": 590}, "normalized_score": 78.15209125475285}, {"_id": {"$oid": "69d4eb12b877594bc57188be"}, "test_id": "T0164", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4o-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4o-mini", "duration": 3.071946620941162, "timestamp": "2025-09-03T22:51:53.065955", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "May", "first_name": "Mark A"}, "publication": {"title": "The institutions that train ministers", "year": 1933, "place": "New York", "pages": "", "publisher": "", "format": "", "reprint_note": ""}, "examination": {"place": "", "year": 0}, "library_reference": {"shelfmark": "", "publication_number": "", "subjects": ""}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.807258064516129, "f1_macro": 0.8032319391634981, "micro_precision": 0.7866404715127702, "micro_recall": 0.8289855072463768, "total_instances": 263, "total_tp": 2002, "total_fp": 543, "total_fn": 413}, "normalized_score": 80.32319391634981}, {"_id": {"$oid": "69d4eb12b877594bc57188bf"}, "test_id": "T0162", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4.1-nano", "duration": 2.2131683826446533, "timestamp": "2025-09-03T22:40:27.615189", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Spiegelberg", "first_name": "Otto"}, "publication": {"title": "Ueber das gegenseitige Verh\u00e4ltnis zwischen dem gerichtlichen Standnis und der Offen-kundigkeit bei dem Gericht nach der deutschen Civilprozessordnung", "year": 1896, "place": "Frankfurt a. M.", "pages": "128", "publisher": "H. L. Br\u00f6nner's Druckerei", "format": "8\u00b0", "reprint_note": ""}, "examination": {"place": "Marburg, Jur. Fak., Inaug.-Diss.", "year": 1896}, "library_reference": {"shelfmark": "Diss.", "publication_number": "", "subjects": ""}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.7296943231441049, "f1_macro": 0.7179087452471483, "micro_precision": 0.771824480369515, "micro_recall": 0.6919254658385093, "total_instances": 263, "total_tp": 1671, "total_fp": 494, "total_fn": 744}, "normalized_score": 71.79087452471484}, {"_id": {"$oid": "69d4eb12b877594bc57188c0"}, "test_id": "T0146", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-1-20250805", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "anthropic", "model": "claude-opus-4-1-20250805", "duration": 6.943610429763794, "timestamp": "2025-09-03T17:59:40.678385", "text": "```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Jasmer\",\n    \"first_name\": \"Bernhard\"\n  },\n  \"publication\": {\n    \"title\": \"De diagnosi differentiali carcinomatis ventriculi et ulceris chronici. Addita historia morbi ulceris ventriculi chronici pro carcinomate habiti\",\n    \"year\": 1859,\n    \"place\": \"Gryphiae\",\n    \"pages\": \"28\",\n    \"publisher\": \"\",\n    \"format\": \"\",\n    \"reprint_note\": \"\"\n  },\n  \"examination\": {\n    \"place\": \"Greifswald\",\n    \"year\": 1859\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"publication_number\": \"\",\n    \"subjects\": \"\"\n  }\n}\n```", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.844688954718578, "f1_macro": 0.8333079847908745, "micro_precision": 0.8636953699697101, "micro_recall": 0.8265010351966874, "total_instances": 263, "total_tp": 1996, "total_fp": 315, "total_fn": 419}, "normalized_score": 83.33079847908745}, {"_id": {"$oid": "69d4eb12b877594bc57188c1"}, "test_id": "T0167", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-5-nano", "duration": 14.242206335067749, "timestamp": "2025-09-10T00:54:22.163991", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Wittenberg", "first_name": "Joachim"}, "publication": {"title": "Unbenutzte Warenzeichen.", "year": 1954, "place": "Mainz", "pages": "104", "publisher": "", "format": "4\u00b0", "editor": null}, "library_reference": {"shelfmark": "Diss.Mainz 1954 (4')", "subjects": "Marke (Markenrecht)"}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8020366598778004, "f1_macro": 0.7969961977186312, "micro_precision": 0.7891783567134268, "micro_recall": 0.8153209109730849, "total_instances": 263, "total_tp": 1969, "total_fp": 526, "total_fn": 446}, "normalized_score": 79.69961977186311}, {"_id": {"$oid": "69d4eb12b877594bc57188c2"}, "test_id": "T0166", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-5-mini", "duration": 23.818702220916748, "timestamp": "2025-09-03T10:26:43.097682", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Schoofs", "first_name": "F."}, "publication": {"title": "Les impuret\u00e9s des alcools, eaux-de-vie & geni\u00e8vres fabriqu\u00e9s en Belgique", "year": 1912, "place": "Li\u00e8ge", "pages": "111", "publisher": null, "format": null, "reprint_note": "8 Tab."}, "examination": {"place": "Li\u00e8ge", "year": 1912}, "library_reference": {"shelfmark": null, "publication_number": null, "subjects": null}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8440528634361233, "f1_macro": 0.840532319391635, "micro_precision": 0.9016470588235294, "micro_recall": 0.7933747412008282, "total_instances": 263, "total_tp": 1916, "total_fp": 209, "total_fn": 499}, "normalized_score": 84.0532319391635}, {"_id": {"$oid": "69d4eb12b877594bc57188c3"}, "test_id": "T0147", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-20250514", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "anthropic", "model": "claude-opus-4-20250514", "duration": 5.4295289516448975, "timestamp": "2025-09-03T18:45:45.631090", "text": "```json\n{\n  \"type\": {\n    \"type\": \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"Kra\u015batkiewicz\",\n    \"first_name\": \"Julian\"\n  },\n  \"publication\": {\n    \"title\": \"Dom matutinalis, quod vuglo (!) Morgengabe dicitur, virgo ac jus\",\n    \"year\": 1849,\n    \"place\": \"Vratislaviae\",\n    \"pages\": \"33\",\n    \"publisher\": \"\",\n    \"format\": \"\",\n    \"reprint_note\": \"\"\n  },\n  \"examination\": {\n    \"place\": \"Breslau\",\n    \"year\": 1849\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"publication_number\": \"\",\n    \"subjects\": \"\"\n  }\n}\n```", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8390243902439023, "f1_macro": 0.8247148288973384, "micro_precision": 0.86, "micro_recall": 0.819047619047619, "total_instances": 263, "total_tp": 1978, "total_fp": 322, "total_fn": 437}, "normalized_score": 82.47148288973384}, {"_id": {"$oid": "69d4eb12b877594bc57188c4"}, "test_id": "T0155", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-pro", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "genai", "model": "gemini-2.5-pro", "duration": 14.106214761734009, "timestamp": "2025-09-05T12:15:54.617240", "text": "```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\"\n  },\n  \"author\": {\n    \"last_name\": \"Hemann\",\n    \"first_name\": \"Christ Ludwig Ernst\"\n  },\n  \"publication\": {\n    \"title\": \"De veterum dotis actionum, rei uxoriae atque ex stipulatu, differentiis.\",\n    \"year\": 1826,\n    \"place\": \"Kiliae\",\n    \"pages\": \"40\",\n    \"publisher\": \"\",\n    \"format\": \"4\u00b0\",\n    \"reprint_note\": \"\"\n  },\n  \"examination\": {\n    \"place\": \"Kiel\",\n    \"year\": 1826\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"\",\n    \"publication_number\": \"\",\n    \"subjects\": \"Diss. iur.\"\n  }\n}\n```", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8858638743455498, "f1_macro": 0.879657794676806, "micro_precision": 0.8961864406779662, "micro_recall": 0.8757763975155279, "total_instances": 263, "total_tp": 2115, "total_fp": 245, "total_fn": 300}, "normalized_score": 87.9657794676806}, {"_id": {"$oid": "69d4eb12b877594bc57188ca"}, "test_id": "T0066", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4o", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge and an expert in the field of 20th century Swiss history", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-4o", "duration": 3.6560862064361572, "timestamp": "2025-09-03T22:32:15.928185", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Verhaeghe", "first_name": "\u00c9tienne"}, "publication": {"title": "Contribution \u00e0 l'\u00e9tude clinique et exp\u00e9rimentale des transplantations h\u00e9t\u00e9roplastiques d'os frais. Quelques essais de transplantations h\u00e9t\u00e9roplastiques ost\u00e9oarticulaires et p\u00e9riostiques", "year": 1908, "place": "Lille", "pages": "196", "publisher": "", "format": "in-8\u00b0", "reprint_note": ""}, "examination": {"place": "", "year": 1908}, "library_reference": {"shelfmark": "", "publication_number": "", "subjects": ""}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8960573476702509, "f1_macro": 0.8936121673003803, "micro_precision": 0.9128006872852233, "micro_recall": 0.8799171842650103, "total_instances": 263, "total_tp": 2125, "total_fp": 203, "total_fn": 290}, "normalized_score": 89.36121673003802}, {"_id": {"$oid": "69d4eb12b877594bc57188d0"}, "test_id": "T0165", "benchmark": "library_cards", "date": "2025-09-02", "tags": {"document-type": ["index-card"], "writing": ["typed", "printed", "handwritten"], "century": [20, 19], "language": ["de", "fr", "en", "la", "el", "fi", "sv", "pl"], "layout": ["index"], "entry-type": ["bibliographic"], "task": ["information-extraction"]}, "contributors": ["gabriel_muller", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with keyword knowledge", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "Extract the bibliographic information about a historical dissertation from this index card and return it as a structured JSON object with the following exact format:\n\n```json\n{\n  \"type\": {\n    \"type\": \"Dissertation or thesis\" OR \"Reference\"\n  },\n  \"author\": {\n    \"last_name\": \"string\",\n    \"first_name\": \"string\"\n  },\n  \"publication\": {\n    \"title\": \"string\",\n    \"year\": integer,\n    \"place\": \"string or empty string\",\n    \"pages\": \"string or empty string\",\n    \"publisher\": \"string or empty string\",\n    \"format\": \"string or empty string\"\n  },\n  \"library_reference\": {\n    \"shelfmark\": \"string or empty string\",\n    \"subjects\": \"string or empty string\"\n  }\n}\n```\n\nEXTRACTION RULES:\n1. **Card Type**: If a card contains the note \"s.\" on a separate line, it is a \"Reference\". Otherwise, it is a \"Dissertation or thesis\".\n\n2. **Author**: Extract last_name and first_name. If only one name is given, put it in last_name and leave first_name empty.\n\n3. **Publication**:\n   - title: The main title of the work\n   - year: Publication year as integer\n   - place: Publication place\n   - pages: Page count (remove \" S.\" suffix if present)\n   - publisher: Publishing house/institution\n   - format: Usually \"8\u00b0\", \"8'\", or \"4\u00b0\" - single value only\n\n4. **Library Reference**:\n   - shelfmark: Often begins with \"Diss.\" or \"AT\", may be marked with \"Standort:\"\n   - subjects: Subject classifications or keywords\n\n5. **Missing Information**: Use empty string \"\" for missing text fields, omit year fields entirely if not present.\n\n6. **Ignore**: Disregard any information that doesn't fit into these categories.\n\nReturn ONLY the JSON object, no additional text or explanation.", "results": {"provider": "openai", "model": "gpt-5", "duration": 24.99132990837097, "timestamp": "2025-09-03T23:45:21.237550", "parsed": {"type": {"type": "Dissertation or thesis"}, "author": {"last_name": "Opitz", "first_name": "Hermann"}, "publication": {"title": "Synthesen in der Imidazolreihe.", "year": 1912, "place": "Freiburg i. B.", "pages": "34", "publisher": "Wagner", "format": "8\u00b0", "reprint_note": ""}, "examination": {"place": "Freiburg i. B.", "year": 1912}, "library_reference": {"shelfmark": "", "publication_number": "U 12. 1261", "subjects": ""}}, "text": "", "finish_reason": "stop"}, "scoring": {"f1_micro": 0.8972515856236787, "f1_macro": 0.8950570342205323, "micro_precision": 0.9166306695464362, "micro_recall": 0.8786749482401657, "total_instances": 263, "total_tp": 2122, "total_fp": 193, "total_fn": 293}, "normalized_score": 89.50570342205323}]