[{"_id": {"$oid": "69d4eb12b877594bc5718d78"}, "test_id": "T0549", "benchmark": "medieval_manuscripts", "date": "2026-03-31", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "ministral-14b-2512", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[]}", "model": "ministral-14b-2512", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 753, "output_tokens": 9, "total_tokens": 762, "input_cost_usd": 0.0001506, "output_cost_usd": 1.8000000000000001e-06, "estimated_cost_usd": 0.0001524}, "duration": 0.6302366256713867, "timestamp": "2026-03-31T11:25:16.793965", "parsed": {"folios": []}, "conversation_id": "e572882b-186d-4d85-b9b6-c7399ed1d994", "score": {"fuzzy": 0.0, "cer": 1.0}, "raw_response": "{\"id\":\"241e60e84ac948eca4d18010f9084679\",\"object\":\"chat.completion\",\"model\":\"ministral-14b-2512\",\"usage\":{\"prompt_tokens\":753,\"completion_tokens\":9,\"total_tokens\":762,\"prompt_tokens_details\":{\"cached_tokens\":0}},\"created\":1774949116,\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\n  \\\"folios\\\": []\\n}\",\"tool_calls\":null,\"prefix\":false},\"finish_reason\":\"stop\"}]}"}, "scoring": {"fuzzy": 0.024, "cer": 1.0, "cost_summary": {"total_input_tokens": 9036, "total_output_tokens": 1153, "total_tokens": 10189, "input_cost_usd": 0.0018072000000000003, "output_cost_usd": 0.00023060000000000002, "total_cost_usd": 0.0020377999999999998}}, "normalized_score": 0}, {"_id": {"$oid": "69d4eb12b877594bc5718d79"}, "test_id": "T0286", "benchmark": "medieval_manuscripts", "date": "2026-03-31", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash-lite-preview-09-2025", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Jof Dem partnay in groffen\\nfiir fiir Zirund\\nJof ze emao ein groffen\\nJof ze Jhenspalen i groff z\\ncurtifiq von Sin eplen\\nJof Dem conful z grob\\nJof in Dem tempel ij grob\\nJof ze Betlagen ein grob\\nJof ze fant Jofans ein grob\\nJof wider ze Jhenspalen ein\\ngrob ze curtifiq aber i grob\\nJof wider in Den tempel in\\ngrob\\nJof ze Dem Jorda\u00ff z\u00ff grob\\nJof Dey eplen ze lor\\nJof aber ein grob ze curtifiet\\nJof aber ze Jhenspale i grob.\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gemini-2.5-flash-lite-preview-09-2025", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 772, "output_tokens": 229, "total_tokens": 1003, "input_cost_usd": 7.72e-05, "output_cost_usd": 9.16e-05, "estimated_cost_usd": 0.0001688}, "duration": 14.069963216781616, "timestamp": "2026-03-31T11:04:49.554508", "parsed": {"folios": [{"folio": "", "text": "Jof Dem partnay in groffen\nfiir fiir Zirund\nJof ze emao ein groffen\nJof ze Jhenspalen i groff z\ncurtifiq von Sin eplen\nJof Dem conful z grob\nJof in Dem tempel ij grob\nJof ze Betlagen ein grob\nJof ze fant Jofans ein grob\nJof wider ze Jhenspalen ein\ngrob ze curtifiq aber i grob\nJof wider in Den tempel in\ngrob\nJof ze Dem Jorda\u00ff z\u00ff grob\nJof Dey eplen ze lor\nJof aber ein grob ze curtifiet\nJof aber ze Jhenspale i grob.", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "6e5ef62e-89e1-414d-94c2-65cecb41801b", "score": {"fuzzy": 0.719, "cer": 0.272}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"x-gemini-service-tier\":\"standard\",\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Tue, 31 Mar 2026 09:04:49 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=13712\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"media_resolution\":null,\"code_execution_result\":null,\"executable_code\":null,\"file_data\":null,\"function_call\":null,\"function_response\":null,\"inline_data\":null,\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Jof Dem partnay in groffen\\\\nfiir fiir Zirund\\\\nJof ze emao ein groffen\\\\nJof ze Jhenspalen i groff z\\\\ncurtifiq von Sin eplen\\\\nJof Dem conful z grob\\\\nJof in Dem tempel ij grob\\\\nJof ze Betlagen ein grob\\\\nJof ze fant Jofans ein grob\\\\nJof wider ze Jhenspalen ein\\\\ngrob ze curtifiq aber i grob\\\\nJof wider in Den tempel in\\\\ngrob\\\\nJof ze Dem Jorda\u00ff z\u00ff grob\\\\nJof Dey eplen ze lor\\\\nJof aber ein grob ze curtifiet\\\\nJof aber ze Jhenspale i grob.\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"thought\":null,\"thought_signature\":null,\"video_metadata\":null}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"grounding_metadata\":null,\"avg_logprobs\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null,\"url_context_metadata\":null}],\"create_time\":null,\"model_version\":\"gemini-2.5-flash-lite-preview-09-2025\",\"prompt_feedback\":null,\"response_id\":\"MY7LabiXILWZkdUP5Om3iQQ\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":229,\"candidates_tokens_details\":null,\"prompt_token_count\":772,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":514},{\"modality\":\"IMAGE\",\"token_count\":258}],\"thoughts_token_count\":2,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":1003,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"Jof Dem partnay in groffen\\nfiir fiir Zirund\\nJof ze emao ein groffen\\nJof ze Jhenspalen i groff z\\ncurtifiq von Sin eplen\\nJof Dem conful z grob\\nJof in Dem tempel ij grob\\nJof ze Betlagen ein grob\\nJof ze fant Jofans ein grob\\nJof wider ze Jhenspalen ein\\ngrob ze curtifiq aber i grob\\nJof wider in Den tempel in\\ngrob\\nJof ze Dem Jorda\u00ff z\u00ff grob\\nJof Dey eplen ze lor\\nJof aber ein grob ze curtifiet\\nJof aber ze Jhenspale i grob.\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}}"}, "scoring": {"fuzzy": 0.686, "cer": 0.31, "cost_summary": {"total_input_tokens": 8492, "total_output_tokens": 2580, "total_tokens": 11072, "input_cost_usd": 0.0008492000000000003, "output_cost_usd": 0.001032, "total_cost_usd": 0.0018812}}, "normalized_score": 69.0}, {"_id": {"$oid": "69d4eb12b877594bc5718d7a"}, "test_id": "T0285", "benchmark": "medieval_manuscripts", "date": "2026-03-31", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash-lite", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"18\",\"text\":\"Das yrof von Ruarzburg\\nund fupf all ander ritter\\nund Encof wolten mit\\nDar Sey in was enteilagt\\nRicg Ngenomen Ges. Jakob\\nvon Gategunt und gant\\nvon hertgen\u00dfbeig ritter\\nDie zwen werend gery\\nmit mir gefis aber des\\npatroy wollt vnser mit\\nBeiten So sy meltcil mit\\nDas walt und also mu\u00dfen\\nwir mit groffen Vnwilley\\nSamid Regerden Sey wir\\npnit dem patroy Und\\noct sey bri\u00dfdervn dar\\nvmb Garent ze zypren\\nwaa\u00dfge Sy zuber\",\"addition1\":\"Sarchani\\nin Cypro\\nnaskir\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gemini-2.5-flash-lite", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 772, "output_tokens": 225, "total_tokens": 997, "input_cost_usd": 7.72e-05, "output_cost_usd": 9e-05, "estimated_cost_usd": 0.0001672}, "duration": 3.430695056915283, "timestamp": "2026-03-31T10:53:15.729719", "parsed": {"folios": [{"folio": "18", "text": "Das yrof von Ruarzburg\nund fupf all ander ritter\nund Encof wolten mit\nDar Sey in was enteilagt\nRicg Ngenomen Ges. Jakob\nvon Gategunt und gant\nvon hertgen\u00dfbeig ritter\nDie zwen werend gery\nmit mir gefis aber des\npatroy wollt vnser mit\nBeiten So sy meltcil mit\nDas walt und also mu\u00dfen\nwir mit groffen Vnwilley\nSamid Regerden Sey wir\npnit dem patroy Und\noct sey bri\u00dfdervn dar\nvmb Garent ze zypren\nwaa\u00dfge Sy zuber", "addition1": "Sarchani\nin Cypro\nnaskir", "addition2": "", "addition3": ""}]}, "conversation_id": "05a09889-6c6b-423d-a06b-b752c8adf49c", "score": {"fuzzy": 0.833, "cer": 0.152}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"x-gemini-service-tier\":\"standard\",\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Tue, 31 Mar 2026 08:53:15 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=3080\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"media_resolution\":null,\"code_execution_result\":null,\"executable_code\":null,\"file_data\":null,\"function_call\":null,\"function_response\":null,\"inline_data\":null,\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"18\\\",\\n      \\\"text\\\": \\\"Das yrof von Ruarzburg\\\\nund fupf all ander ritter\\\\nund Encof wolten mit\\\\nDar Sey in was enteilagt\\\\nRicg Ngenomen Ges. Jakob\\\\nvon Gategunt und gant\\\\nvon hertgen\u00dfbeig ritter\\\\nDie zwen werend gery\\\\nmit mir gefis aber des\\\\npatroy wollt vnser mit\\\\nBeiten So sy meltcil mit\\\\nDas walt und also mu\u00dfen\\\\nwir mit groffen Vnwilley\\\\nSamid Regerden Sey wir\\\\npnit dem patroy Und\\\\noct sey bri\u00dfdervn dar\\\\nvmb Garent ze zypren\\\\nwaa\u00dfge Sy zuber\\\",\\n      \\\"addition1\\\": \\\"Sarchani\\\\nin Cypro\\\\nnaskir\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"thought\":null,\"thought_signature\":null,\"video_metadata\":null}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"grounding_metadata\":null,\"avg_logprobs\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null,\"url_context_metadata\":null}],\"create_time\":null,\"model_version\":\"gemini-2.5-flash-lite\",\"prompt_feedback\":null,\"response_id\":\"e4vLaaChKe3s7M8Pp-G2GQ\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":225,\"candidates_tokens_details\":null,\"prompt_token_count\":772,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":514},{\"modality\":\"IMAGE\",\"token_count\":258}],\"thoughts_token_count\":null,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":997,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"folios\":[{\"folio\":\"18\",\"text\":\"Das yrof von Ruarzburg\\nund fupf all ander ritter\\nund Encof wolten mit\\nDar Sey in was enteilagt\\nRicg Ngenomen Ges. Jakob\\nvon Gategunt und gant\\nvon hertgen\u00dfbeig ritter\\nDie zwen werend gery\\nmit mir gefis aber des\\npatroy wollt vnser mit\\nBeiten So sy meltcil mit\\nDas walt und also mu\u00dfen\\nwir mit groffen Vnwilley\\nSamid Regerden Sey wir\\npnit dem patroy Und\\noct sey bri\u00dfdervn dar\\nvmb Garent ze zypren\\nwaa\u00dfge Sy zuber\",\"addition1\":\"Sarchani\\nin Cypro\\nnaskir\",\"addition2\":\"\",\"addition3\":\"\"}]}}"}, "scoring": {"fuzzy": 0.601, "cer": 0.413, "cost_summary": {"total_input_tokens": 8492, "total_output_tokens": 2753, "total_tokens": 11245, "input_cost_usd": 0.0008492000000000003, "output_cost_usd": 0.0011012, "total_cost_usd": 0.0019504000000000001}}, "normalized_score": 58.7}, {"_id": {"$oid": "69d4eb12b877594bc5718d39"}, "test_id": "T0885", "benchmark": "medieval_manuscripts", "date": "2026-03-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-flash-2026-02-23", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"vff den erzbischof do selbs\\nvff zungtmay vnd dz heilthum\\nin dem thurn do selbs ist\\nsmit hundert haupt der do se\\nhynniger emer wgtt stat\\nBeware eme kunig vnd zwen\\ntarster haupt ist smit blei\u00dfen\\nvnd zungtmay men ist smit andernemen ist\\nvmb sich ze bad\\nNud smit stoffenb men ist vn\\ndie ein weg\\nrob darau\u00df solz zwey zen die do smit dippels\\nwie in f\u00fcnfft dien Ngeflagge wurden ist\\nteuteil blaugt\\nvff montag nach smit Johans\\nwirdet man\\ntrinket die blutkarg ze Ouregottkey f\u00fcrrey wir\\ngrey und halb\\nund gantz zug Nach Kandia die f\u00fcrend an d\\nwas sie nit mitwiel f\u00fcr die fest gezi\\nund aus L die zey do kam zu Inne Pattela\\nzwegen Neb land hier fast do versprach sich die\\nnig k\u00fcnft wenig\\nnun ist die vol pfifflut all gemeind vnd\\ngesegzt Spital nemend all burgerin ire\\nin den kleine\\nbaufusse Kloster\",\"addition1\":\"vnd zungtmay\\nvmb sich ze bad\\ndie ein weg\\nrob darau\u00df solz\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "qwen3.5-flash-2026-02-23", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 3284, "output_tokens": 365, "total_tokens": 3649, "input_cost_usd": 0.0003284, "output_cost_usd": 0.000146, "estimated_cost_usd": 0.00047440000000000004}, "duration": 12.498748302459717, "timestamp": "2026-03-25T10:25:21.555601", "parsed": {"folios": [{"folio": "", "text": "vff den erzbischof do selbs\nvff zungtmay vnd dz heilthum\nin dem thurn do selbs ist\nsmit hundert haupt der do se\nhynniger emer wgtt stat\nBeware eme kunig vnd zwen\ntarster haupt ist smit blei\u00dfen\nvnd zungtmay men ist smit andernemen ist\nvmb sich ze bad\nNud smit stoffenb men ist vn\ndie ein weg\nrob darau\u00df solz zwey zen die do smit dippels\nwie in f\u00fcnfft dien Ngeflagge wurden ist\nteuteil blaugt\nvff montag nach smit Johans\nwirdet man\ntrinket die blutkarg ze Ouregottkey f\u00fcrrey wir\ngrey und halb\nund gantz zug Nach Kandia die f\u00fcrend an d\nwas sie nit mitwiel f\u00fcr die fest gezi\nund aus L die zey do kam zu Inne Pattela\nzwegen Neb land hier fast do versprach sich die\nnig k\u00fcnft wenig\nnun ist die vol pfifflut all gemeind vnd\ngesegzt Spital nemend all burgerin ire\nin den kleine\nbaufusse Kloster", "addition1": "vnd zungtmay\nvmb sich ze bad\ndie ein weg\nrob darau\u00df solz", "addition2": "", "addition3": ""}]}, "conversation_id": "34c8d5f6-f9dc-4106-abc9-4d6f5024190c", "score": {"fuzzy": 0.422, "cer": 0.782}, "raw_response": "{\"id\":\"chatcmpl-98333a22-68ec-9362-a8ee-82ed69a36276\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"vff den erzbischof do selbs\\\\nvff zungtmay vnd dz heilthum\\\\nin dem thurn do selbs ist\\\\nsmit hundert haupt der do se\\\\nhynniger emer wgtt stat\\\\nBeware eme kunig vnd zwen\\\\ntarster haupt ist smit blei\u00dfen\\\\nvnd zungtmay men ist smit andernemen ist\\\\nvmb sich ze bad\\\\nNud smit stoffenb men ist vn\\\\ndie ein weg\\\\nrob darau\u00df solz zwey zen die do smit dippels\\\\nwie in f\u00fcnfft dien Ngeflagge wurden ist\\\\nteuteil blaugt\\\\nvff montag nach smit Johans\\\\nwirdet man\\\\ntrinket die blutkarg ze Ouregottkey f\u00fcrrey wir\\\\ngrey und halb\\\\nund gantz zug Nach Kandia die f\u00fcrend an d\\\\nwas sie nit mitwiel f\u00fcr die fest gezi\\\\nund aus L die zey do kam zu Inne Pattela\\\\nzwegen Neb land hier fast do versprach sich die\\\\nnig k\u00fcnft wenig\\\\nnun ist die vol pfifflut all gemeind vnd\\\\ngesegzt Spital nemend all burgerin ire\\\\nin den kleine\\\\nbaufusse Kloster\\\",\\n      \\\"addition1\\\": \\\"vnd zungtmay\\\\nvmb sich ze bad\\\\ndie ein weg\\\\nrob darau\u00df solz\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774430721,\"model\":\"qwen3.5-flash-2026-02-23\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":365,\"prompt_tokens\":3284,\"total_tokens\":3649,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":365},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":2503,\"text_tokens\":781}}}"}, "scoring": {"fuzzy": 0.705, "cer": 0.32, "cost_summary": {"total_input_tokens": 39408, "total_output_tokens": 2795, "total_tokens": 42203, "input_cost_usd": 0.0039408, "output_cost_usd": 0.001118, "total_cost_usd": 0.0050588}}, "normalized_score": 68.0}, {"_id": {"$oid": "69d4eb12b877594bc5718d42"}, "test_id": "T0872", "benchmark": "medieval_manuscripts", "date": "2026-03-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-397b-a17b", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Selbs im fronaltar die folgend\\nwir gruntz vnd em stuck voy\\ndem heiligen fron cr\u00fctz dz\\nlit ir vff ir brust vnd ein\\nbein von sant marien magdalene\\nringg vrot ist off samstag nach\\ndem ostertag der do was der\\nander tag des abretten do\\nf\u00fcrend wir goy messay do\\nligend im sant steffans\\nkilchen bi hundert der kindlin\\ndie herodes erschl\u00fcg in einem\\naltar die folgend wir vnd\\nsicht man noch bestendenclich\\nin hend vnd f\u00fc\u00df vnd die\\nnagel dar an vnd des ay\\nDem sunnentag fr\u00fcg do\",\"addition1\":\"Muray\\nglab\\ns\u00fcch\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "qwen3.5-397b-a17b", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 3284, "output_tokens": 223, "total_tokens": 3507, "input_cost_usd": 0.0019703999999999998, "output_cost_usd": 0.0008028, "estimated_cost_usd": 0.0027732}, "duration": 11.737691402435303, "timestamp": "2026-03-25T10:22:07.100536", "parsed": {"folios": [{"folio": "", "text": "Selbs im fronaltar die folgend\nwir gruntz vnd em stuck voy\ndem heiligen fron cr\u00fctz dz\nlit ir vff ir brust vnd ein\nbein von sant marien magdalene\nringg vrot ist off samstag nach\ndem ostertag der do was der\nander tag des abretten do\nf\u00fcrend wir goy messay do\nligend im sant steffans\nkilchen bi hundert der kindlin\ndie herodes erschl\u00fcg in einem\naltar die folgend wir vnd\nsicht man noch bestendenclich\nin hend vnd f\u00fc\u00df vnd die\nnagel dar an vnd des ay\nDem sunnentag fr\u00fcg do", "addition1": "Muray\nglab\ns\u00fcch", "addition2": "", "addition3": ""}]}, "conversation_id": "a35bdeaa-3fa7-42a4-8ac9-dfddbc102773", "score": {"fuzzy": 0.742, "cer": 0.258}, "raw_response": "{\"id\":\"chatcmpl-1d2ccf84-8224-98a7-a3ee-198b70e3961f\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Selbs im fronaltar die folgend\\\\nwir gruntz vnd em stuck voy\\\\ndem heiligen fron cr\u00fctz dz\\\\nlit ir vff ir brust vnd ein\\\\nbein von sant marien magdalene\\\\nringg vrot ist off samstag nach\\\\ndem ostertag der do was der\\\\nander tag des abretten do\\\\nf\u00fcrend wir goy messay do\\\\nligend im sant steffans\\\\nkilchen bi hundert der kindlin\\\\ndie herodes erschl\u00fcg in einem\\\\naltar die folgend wir vnd\\\\nsicht man noch bestendenclich\\\\nin hend vnd f\u00fc\u00df vnd die\\\\nnagel dar an vnd des ay\\\\nDem sunnentag fr\u00fcg do\\\",\\n      \\\"addition1\\\": \\\"Muray\\\\nglab\\\\ns\u00fcch\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774430527,\"model\":\"qwen3.5-397b-a17b\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":223,\"prompt_tokens\":3284,\"total_tokens\":3507,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":223},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":2503,\"text_tokens\":781}}}"}, "scoring": {"fuzzy": 0.707, "cer": 0.326, "cost_summary": {"total_input_tokens": 39408, "total_output_tokens": 3137, "total_tokens": 42545, "input_cost_usd": 0.023644800000000004, "output_cost_usd": 0.0112932, "total_cost_usd": 0.034938}}, "normalized_score": 67.4}, {"_id": {"$oid": "69d4eb12b877594bc5718d49"}, "test_id": "T0859", "benchmark": "medieval_manuscripts", "date": "2026-03-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-122b-a10b", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"73\",\"text\":\"Ist zu dem ersten Min herr graff albrecht vo\\nlmhald\\nIst her ludwig lanugraff von huchtenberg\\nIst greff ott von hemmenberg\\nIst greff Bernhart von Regenstem\\nIst greff g\u00f6tfrid von hahenloch\\nIst greff gebhard von Mansfeld\\nIst greff ludwig von Arenburg\\nIst her Gotifrid von Blaff \u2022 bespanchind\\nIst her hans von turg\u00f6w\\nIst J\u00f6rg Bappenheim\\nIst her Suppchin vom stein wz vorhin Ritter\\nIst luz vom Rotenhan\\nIst J\u00f6rg von waldenfels\\nIst hilprand von Hungen\\nIst J\u00f6rg von Elicben\\nIst henijn quast\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "qwen3.5-122b-a10b", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 3284, "output_tokens": 232, "total_tokens": 3516, "input_cost_usd": 0.0013136, "output_cost_usd": 0.0007424, "estimated_cost_usd": 0.002056}, "duration": 9.762769222259521, "timestamp": "2026-03-25T10:21:44.718076", "parsed": {"folios": [{"folio": "73", "text": "Ist zu dem ersten Min herr graff albrecht vo\nlmhald\nIst her ludwig lanugraff von huchtenberg\nIst greff ott von hemmenberg\nIst greff Bernhart von Regenstem\nIst greff g\u00f6tfrid von hahenloch\nIst greff gebhard von Mansfeld\nIst greff ludwig von Arenburg\nIst her Gotifrid von Blaff \u2022 bespanchind\nIst her hans von turg\u00f6w\nIst J\u00f6rg Bappenheim\nIst her Suppchin vom stein wz vorhin Ritter\nIst luz vom Rotenhan\nIst J\u00f6rg von waldenfels\nIst hilprand von Hungen\nIst J\u00f6rg von Elicben\nIst henijn quast", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "00435595-4e0d-47f7-bdff-ab6fafdf5762", "score": {"fuzzy": 0.926, "cer": 0.09}, "raw_response": "{\"id\":\"chatcmpl-7f0578bb-f6f0-95fa-90be-985c568b8772\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"73\\\",\\n      \\\"text\\\": \\\"Ist zu dem ersten Min herr graff albrecht vo\\\\nlmhald\\\\nIst her ludwig lanugraff von huchtenberg\\\\nIst greff ott von hemmenberg\\\\nIst greff Bernhart von Regenstem\\\\nIst greff g\u00f6tfrid von hahenloch\\\\nIst greff gebhard von Mansfeld\\\\nIst greff ludwig von Arenburg\\\\nIst her Gotifrid von Blaff \u2022 bespanchind\\\\nIst her hans von turg\u00f6w\\\\nIst J\u00f6rg Bappenheim\\\\nIst her Suppchin vom stein wz vorhin Ritter\\\\nIst luz vom Rotenhan\\\\nIst J\u00f6rg von waldenfels\\\\nIst hilprand von Hungen\\\\nIst J\u00f6rg von Elicben\\\\nIst henijn quast\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774430505,\"model\":\"qwen3.5-122b-a10b\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":232,\"prompt_tokens\":3284,\"total_tokens\":3516,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":232},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":2503,\"text_tokens\":781}}}"}, "scoring": {"fuzzy": 0.719, "cer": 0.294, "cost_summary": {"total_input_tokens": 39408, "total_output_tokens": 2741, "total_tokens": 42149, "input_cost_usd": 0.0157632, "output_cost_usd": 0.0087712, "total_cost_usd": 0.0245344}}, "normalized_score": 70.6}, {"_id": {"$oid": "69d4eb12b877594bc5718d63"}, "test_id": "T0846", "benchmark": "medieval_manuscripts", "date": "2026-03-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-27b", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Gelbs im fronaltar die folgend\\n wir gantz vnd em stuck von\\ndem heiligen fron cr\u00fctz dz\\n lie in uff ir brust vnd ein\\n bein von sant marien madalene\\n ringgrot pf off samstag nach\\ndem ostertag der do was der\\n ander tag des abrelten do\\n f\u00fcrend wir gon merway do\\n ligend im sant # steffan\u00df\\n bildtzen bi hundert der kindlin\\ndie derselb erplug in einem\\n altar die folgend wir und\\n sieht man noch bestgerdenlich\\n in hand und fu\u00df und die\\n nagel dar an st\u00fcnden an\\ndem funentag fr\u00fcg do\",\"addition1\":\"Murgau\\n glas\\n s\u00fcttin\",\"addition2\":null,\"addition3\":null}]}", "model": "qwen3.5-27b", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 3284, "output_tokens": 218, "total_tokens": 3502, "input_cost_usd": 0.0009851999999999999, "output_cost_usd": 0.0005232, "estimated_cost_usd": 0.0015084}, "duration": 17.148881196975708, "timestamp": "2026-03-25T10:23:38.733878", "parsed": {"folios": [{"folio": "", "text": "Gelbs im fronaltar die folgend\n wir gantz vnd em stuck von\ndem heiligen fron cr\u00fctz dz\n lie in uff ir brust vnd ein\n bein von sant marien madalene\n ringgrot pf off samstag nach\ndem ostertag der do was der\n ander tag des abrelten do\n f\u00fcrend wir gon merway do\n ligend im sant # steffan\u00df\n bildtzen bi hundert der kindlin\ndie derselb erplug in einem\n altar die folgend wir und\n sieht man noch bestgerdenlich\n in hand und fu\u00df und die\n nagel dar an st\u00fcnden an\ndem funentag fr\u00fcg do", "addition1": "Murgau\n glas\n s\u00fcttin", "addition2": null, "addition3": null}]}, "conversation_id": "68c924b5-79fb-4366-b147-0b8501522031", "score": {"fuzzy": 0.854, "cer": 0.18}, "raw_response": "{\"id\":\"chatcmpl-53214993-d44f-9ff4-8a77-a5fe1f6a55bc\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Gelbs im fronaltar die folgend\\\\n wir gantz vnd em stuck von\\\\ndem heiligen fron cr\u00fctz dz\\\\n lie in uff ir brust vnd ein\\\\n bein von sant marien madalene\\\\n ringgrot pf off samstag nach\\\\ndem ostertag der do was der\\\\n ander tag des abrelten do\\\\n f\u00fcrend wir gon merway do\\\\n ligend im sant # steffan\u00df\\\\n bildtzen bi hundert der kindlin\\\\ndie derselb erplug in einem\\\\n altar die folgend wir und\\\\n sieht man noch bestgerdenlich\\\\n in hand und fu\u00df und die\\\\n nagel dar an st\u00fcnden an\\\\ndem funentag fr\u00fcg do\\\",\\n      \\\"addition1\\\": \\\"Murgau\\\\n glas\\\\n s\u00fcttin\\\",\\n      \\\"addition2\\\": null,\\n      \\\"addition3\\\": null\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774430619,\"model\":\"qwen3.5-27b\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":218,\"prompt_tokens\":3284,\"total_tokens\":3502,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":218},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":2503,\"text_tokens\":781}}}"}, "scoring": {"fuzzy": 0.727, "cer": 0.314, "cost_summary": {"total_input_tokens": 39408, "total_output_tokens": 3014, "total_tokens": 42422, "input_cost_usd": 0.011822400000000002, "output_cost_usd": 0.0072336, "total_cost_usd": 0.019056}}, "normalized_score": 68.6}, {"_id": {"$oid": "69d4eb12b877594bc5718d75"}, "test_id": "T0833", "benchmark": "medieval_manuscripts", "date": "2026-03-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-35b-a3b", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"18\",\"text\":\"Der graf von Wurzburg\\nund fuft all ander ritter\\nvnd knecht wolten mit\\ndaz sie in wab enteilwert\\nreich vgenomen her Jacob\\nvon Hagenawt und Hans\\nvon Herchawberig Ritter\\nSie zwen weren gern\\nmit mir gesin aber der\\npatron woll unser mit\\nreiten So der meitel mit\\ndar woll vnd also musten\\nwir mit grossem vnwillen\\ndamit fcheiden der wir\\nmit dem patron vnd\\noch den brideren dar\\nvnd patrand ze cipren\\nwagte dz zuer\",\"addition1\":\"Sarcasini\\nin Cypro\\nnascitur\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "qwen3.5-35b-a3b", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 3284, "output_tokens": 205, "total_tokens": 3489, "input_cost_usd": 0.000821, "output_cost_usd": 0.00041, "estimated_cost_usd": 0.0012309999999999999}, "duration": 8.312419652938843, "timestamp": "2026-03-25T10:16:25.128079", "parsed": {"folios": [{"folio": "18", "text": "Der graf von Wurzburg\nund fuft all ander ritter\nvnd knecht wolten mit\ndaz sie in wab enteilwert\nreich vgenomen her Jacob\nvon Hagenawt und Hans\nvon Herchawberig Ritter\nSie zwen weren gern\nmit mir gesin aber der\npatron woll unser mit\nreiten So der meitel mit\ndar woll vnd also musten\nwir mit grossem vnwillen\ndamit fcheiden der wir\nmit dem patron vnd\noch den brideren dar\nvnd patrand ze cipren\nwagte dz zuer", "addition1": "Sarcasini\nin Cypro\nnascitur", "addition2": "", "addition3": ""}]}, "conversation_id": "e75a5e47-d823-4471-b517-8dace161606c", "score": {"fuzzy": 0.861, "cer": 0.166}, "raw_response": "{\"id\":\"chatcmpl-6012bcd5-ec6f-9458-82c7-9b4464bc3aef\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"18\\\",\\n      \\\"text\\\": \\\"Der graf von Wurzburg\\\\nund fuft all ander ritter\\\\nvnd knecht wolten mit\\\\ndaz sie in wab enteilwert\\\\nreich vgenomen her Jacob\\\\nvon Hagenawt und Hans\\\\nvon Herchawberig Ritter\\\\nSie zwen weren gern\\\\nmit mir gesin aber der\\\\npatron woll unser mit\\\\nreiten So der meitel mit\\\\ndar woll vnd also musten\\\\nwir mit grossem vnwillen\\\\ndamit fcheiden der wir\\\\nmit dem patron vnd\\\\noch den brideren dar\\\\nvnd patrand ze cipren\\\\nwagte dz zuer\\\",\\n      \\\"addition1\\\": \\\"Sarcasini\\\\nin Cypro\\\\nnascitur\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774430185,\"model\":\"qwen3.5-35b-a3b\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":205,\"prompt_tokens\":3284,\"total_tokens\":3489,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":205},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":2503,\"text_tokens\":781}}}"}, "scoring": {"fuzzy": 0.663, "cer": 0.359, "cost_summary": {"total_input_tokens": 39408, "total_output_tokens": 2763, "total_tokens": 42171, "input_cost_usd": 0.009852, "output_cost_usd": 0.005526, "total_cost_usd": 0.015378000000000001}}, "normalized_score": 64.1}, {"_id": {"$oid": "69d4eb12b877594bc5718d27"}, "test_id": "T0820", "benchmark": "medieval_manuscripts", "date": "2026-03-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "alibaba", "model": "qwen3.5-plus-2026-02-15", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "rules": "{\"base_url\": \"https://ws-wkjpbv6ov7sqlgqe.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1\"}", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Vnd die in\u017fel i\u017ft bi ccc mil\\nwit vnd i\u017ft der venediger\\nI\u017ft von kurch\u017fung vntz gen\\nmodon \u017find cc mil vnd\\ni\u017ft och ein h\u00fcb\u017fch \u017ftat vnd\\nein ertzbistum vnd i\u017ft och\\nder venediger ze modon\\nwach\u017fet der rumanier vnd\\ni\u017ft och do bi ein heilig ge\u017ftet\\ndas bi ij milen\",\"addition1\":\"Candia I\u017ft von modon bis gen candia\\nCreta. \u017find cc mil vnd candia\\nVinum i\u017ft ein grob in\u017fel vj mil\\nMalua wit vnd wach\u017fet der\\nriem malua\u017fier do vnd i\u017ft och\\nder venediger\",\"addition2\":\"Rhody. I\u017ft von candia vntz gen rodis\\n\u017find cc mil wer aber\",\"addition3\":\"\"}]}", "model": "qwen3.5-plus", "provider": "alibaba", "finish_reason": "stop", "usage": {"input_tokens": 3284, "output_tokens": 281, "total_tokens": 3565, "input_cost_usd": 0.0013136, "output_cost_usd": 0.0006744, "estimated_cost_usd": 0.001988}, "duration": 12.496056318283081, "timestamp": "2026-03-24T18:57:10.205560", "parsed": {"folios": [{"folio": "", "text": "Vnd die in\u017fel i\u017ft bi ccc mil\nwit vnd i\u017ft der venediger\nI\u017ft von kurch\u017fung vntz gen\nmodon \u017find cc mil vnd\ni\u017ft och ein h\u00fcb\u017fch \u017ftat vnd\nein ertzbistum vnd i\u017ft och\nder venediger ze modon\nwach\u017fet der rumanier vnd\ni\u017ft och do bi ein heilig ge\u017ftet\ndas bi ij milen", "addition1": "Candia I\u017ft von modon bis gen candia\nCreta. \u017find cc mil vnd candia\nVinum i\u017ft ein grob in\u017fel vj mil\nMalua wit vnd wach\u017fet der\nriem malua\u017fier do vnd i\u017ft och\nder venediger", "addition2": "Rhody. I\u017ft von candia vntz gen rodis\n\u017find cc mil wer aber", "addition3": ""}]}, "conversation_id": "a86bb086-b667-4606-b8c7-3a56634e4b24", "score": {"fuzzy": 0.252, "cer": 0.873}, "raw_response": "{\"id\":\"chatcmpl-a9dfaf4b-65aa-9348-a465-fa5fcb02ea6b\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Vnd die in\u017fel i\u017ft bi ccc mil\\\\nwit vnd i\u017ft der venediger\\\\nI\u017ft von kurch\u017fung vntz gen\\\\nmodon \u017find cc mil vnd\\\\ni\u017ft och ein h\u00fcb\u017fch \u017ftat vnd\\\\nein ertzbistum vnd i\u017ft och\\\\nder venediger ze modon\\\\nwach\u017fet der rumanier vnd\\\\ni\u017ft och do bi ein heilig ge\u017ftet\\\\ndas bi ij milen\\\",\\n      \\\"addition1\\\": \\\"Candia I\u017ft von modon bis gen candia\\\\nCreta. \u017find cc mil vnd candia\\\\nVinum i\u017ft ein grob in\u017fel vj mil\\\\nMalua wit vnd wach\u017fet der\\\\nriem malua\u017fier do vnd i\u017ft och\\\\nder venediger\\\",\\n      \\\"addition2\\\": \\\"Rhody. I\u017ft von candia vntz gen rodis\\\\n\u017find cc mil wer aber\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"reasoning_content\":\"\"}}],\"created\":1774375030,\"model\":\"qwen3.5-plus\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":281,\"prompt_tokens\":3284,\"total_tokens\":3565,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":null,\"rejected_prediction_tokens\":null,\"text_tokens\":281},\"prompt_tokens_details\":{\"audio_tokens\":null,\"cached_tokens\":null,\"image_tokens\":2503,\"text_tokens\":781}}}"}, "scoring": {"fuzzy": 0.72, "cer": 0.304, "cost_summary": {"total_input_tokens": 39408, "total_output_tokens": 3023, "total_tokens": 42431, "input_cost_usd": 0.0157632, "output_cost_usd": 0.0072552, "total_cost_usd": 0.0230184}}, "normalized_score": 69.6}, {"_id": {"$oid": "69d4eb12b877594bc5718cf0"}, "test_id": "T0724", "benchmark": "medieval_manuscripts", "date": "2026-03-23", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "x-ai", "model": "grok-4.20-0309-reasoning", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"\u2020 vsser magt got mary in die\\nh\u00ffmel des heiligen grabes\\nAl ist ein closter vff dem mer\\ngenant vsser lieben frowen z\u016f gen\u016fsz\\nvnd wirt genant in t\u00fctschen\\nvsser lieben frowen z\u016f gen\u016fsz\\nvnd wirt genant in t\u00fctschen\\nvsser frowen z\u016f gen\u016fsz\",\"addition1\":\"vnd wirt genant in t\u00fctschen\\nvnd wirt genant in t\u00fctschen\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "grok-4.20-0309-reasoning", "provider": "x-ai", "finish_reason": "stop", "usage": {"input_tokens": 970, "output_tokens": 153, "total_tokens": 7545, "cached_tokens": 64, "input_cost_usd": 0.00194, "output_cost_usd": 0.000918, "estimated_cost_usd": 0.0028580000000000003}, "duration": 1264.5698318481445, "timestamp": "2026-03-23T14:26:04.733154", "parsed": {"folios": [{"folio": "", "text": "\u2020 vsser magt got mary in die\nh\u00ffmel des heiligen grabes\nAl ist ein closter vff dem mer\ngenant vsser lieben frowen z\u016f gen\u016fsz\nvnd wirt genant in t\u00fctschen\nvsser lieben frowen z\u016f gen\u016fsz\nvnd wirt genant in t\u00fctschen\nvsser frowen z\u016f gen\u016fsz", "addition1": "vnd wirt genant in t\u00fctschen\nvnd wirt genant in t\u00fctschen", "addition2": "", "addition3": ""}]}, "conversation_id": "1f3bb3b6-9e5f-4d03-b60a-cdee4c0eaadf", "score": {"fuzzy": 0.211, "cer": 0.842}, "raw_response": "{\"id\":\"13dcf05c-10eb-93ec-989b-a4cdcf2fb09b\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"\u2020 vsser magt got mary in die\\\\nh\u00ffmel des heiligen grabes\\\\nAl ist ein closter vff dem mer\\\\ngenant vsser lieben frowen z\u016f gen\u016fsz\\\\nvnd wirt genant in t\u00fctschen\\\\nvsser lieben frowen z\u016f gen\u016fsz\\\\nvnd wirt genant in t\u00fctschen\\\\nvsser frowen z\u016f gen\u016fsz\\\",\\n      \\\"addition1\\\": \\\"vnd wirt genant in t\u00fctschen\\\\nvnd wirt genant in t\u00fctschen\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"\u2020 vsser magt got mary in die\\nh\u00ffmel des heiligen grabes\\nAl ist ein closter vff dem mer\\ngenant vsser lieben frowen z\u016f gen\u016fsz\\nvnd wirt genant in t\u00fctschen\\nvsser lieben frowen z\u016f gen\u016fsz\\nvnd wirt genant in t\u00fctschen\\nvsser frowen z\u016f gen\u016fsz\",\"addition1\":\"vnd wirt genant in t\u00fctschen\\nvnd wirt genant in t\u00fctschen\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1774272363,\"model\":\"grok-4.20-0309-reasoning\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":\"fp_d6e36c82c2\",\"usage\":{\"completion_tokens\":153,\"prompt_tokens\":970,\"total_tokens\":7545,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":6422,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":64,\"text_tokens\":794,\"image_tokens\":176},\"num_sources_used\":0,\"cost_in_usd_ticks\":412748000}}"}, "scoring": {"fuzzy": 0.533, "cer": 0.512, "cost_summary": {"total_input_tokens": 11640, "total_output_tokens": 2174, "total_tokens": 13814, "input_cost_usd": 0.023280000000000006, "output_cost_usd": 0.013044000000000002, "total_cost_usd": 0.036323999999999995}}, "normalized_score": 48.8}, {"_id": {"$oid": "69d4eb12b877594bc5718d05"}, "test_id": "T0699", "benchmark": "medieval_manuscripts", "date": "2026-03-23", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.3-codex", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "rules": "{\"api_style\":\"responses\"}", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\n  \"folios\": [\n    {\n      \"folio\": \"69\",\n      \"text\": \"Dar nach kame ir pfiffer vnd\\n trumeter zu immen her vnd dorff\\n die gallen vnd hofierten do\\n f\u00fcren grubden des glich komd\\n hie nundertzig pfiffer vnd\\n trumeter vnd besigffen plaher\\n vnd hofierten do ach nach iren\\n sitten also ward mins gnedige\\n her\ua75b vnd allen den sinen vil\\n erey\ua75b erbotten Jst niderand\ua76d ge-\\ngen tag fr\u00fcent wie da denne\\n vnd kamen off hettag nach sunt\\n velome tag det do voz det erst\\n tag im krieghonert gon kurchon\\n Jst ein pfaner pal knut zulech\\n hurliffen Wellffl\ua770fey ob der\\n stat von off sinnetag darnach\\n f\u00fcren wir da denne vnd off\\n h\u00fcrslag var sant vits tag Jst\\n kerten zu vns in gallen by got\",\n      \"addition1\": \"Monages\\nMarehio\\nin erbi\\nbit.\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "model": "gpt-5.3-codex", "provider": "openai", "finish_reason": "completed", "usage": {"input_tokens": 2482, "output_tokens": 277, "total_tokens": 2759, "input_cost_usd": 0.0043435, "output_cost_usd": 0.0038780000000000004, "estimated_cost_usd": 0.0082215}, "duration": 8.738424062728882, "timestamp": "2026-03-23T21:15:16.636457", "parsed": {"folios": [{"folio": "69", "text": "Dar nach kame ir pfiffer vnd\n trumeter zu immen her vnd dorff\n die gallen vnd hofierten do\n f\u00fcren grubden des glich komd\n hie nundertzig pfiffer vnd\n trumeter vnd besigffen plaher\n vnd hofierten do ach nach iren\n sitten also ward mins gnedige\n her\ua75b vnd allen den sinen vil\n erey\ua75b erbotten Jst niderand\ua76d ge-\ngen tag fr\u00fcent wie da denne\n vnd kamen off hettag nach sunt\n velome tag det do voz det erst\n tag im krieghonert gon kurchon\n Jst ein pfaner pal knut zulech\n hurliffen Wellffl\ua770fey ob der\n stat von off sinnetag darnach\n f\u00fcren wir da denne vnd off\n h\u00fcrslag var sant vits tag Jst\n kerten zu vns in gallen by got", "addition1": "Monages\nMarehio\nin erbi\nbit.", "addition2": "", "addition3": ""}]}, "conversation_id": "c4340237-d37e-428f-8448-c60d29bf6c5e", "score": {"fuzzy": 0.488, "cer": 0.524}, "raw_response": "{\"id\":\"resp_08b0f778fac519f40069c19f4d38788190880c656993d63092\",\"created_at\":1774296909.0,\"error\":null,\"incomplete_details\":null,\"instructions\":\"You are a historian with expertise in medieval manuscripts and paleography\",\"metadata\":{},\"model\":\"gpt-5.3-codex\",\"object\":\"response\",\"output\":[{\"id\":\"msg_08b0f778fac519f40069c19f4dd6ec81909ebcb20b6b527b2a\",\"content\":[{\"annotations\":[],\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"69\\\",\\n      \\\"text\\\": \\\"Dar nach kame ir pfiffer vnd\\\\n trumeter zu immen her vnd dorff\\\\n die gallen vnd hofierten do\\\\n f\u00fcren grubden des glich komd\\\\n hie nundertzig pfiffer vnd\\\\n trumeter vnd besigffen plaher\\\\n vnd hofierten do ach nach iren\\\\n sitten also ward mins gnedige\\\\n her\ua75b vnd allen den sinen vil\\\\n erey\ua75b erbotten Jst niderand\ua76d ge-\\\\ngen tag fr\u00fcent wie da denne\\\\n vnd kamen off hettag nach sunt\\\\n velome tag det do voz det erst\\\\n tag im krieghonert gon kurchon\\\\n Jst ein pfaner pal knut zulech\\\\n hurliffen Wellffl\ua770fey ob der\\\\n stat von off sinnetag darnach\\\\n f\u00fcren wir da denne vnd off\\\\n h\u00fcrslag var sant vits tag Jst\\\\n kerten zu vns in gallen by got\\\",\\n      \\\"addition1\\\": \\\"Monages\\\\nMarehio\\\\nin erbi\\\\nbit.\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"type\":\"output_text\",\"logprobs\":[]}],\"role\":\"assistant\",\"status\":\"completed\",\"type\":\"message\",\"phase\":\"final_answer\"}],\"parallel_tool_calls\":true,\"temperature\":0.0,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":0.98,\"background\":false,\"completed_at\":1774296917.0,\"conversation\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"previous_response_id\":null,\"prompt\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":null,\"reasoning\":{\"effort\":\"none\",\"generate_summary\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"status\":\"completed\",\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"top_logprobs\":0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":2482,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":277,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":2759},\"user\":null,\"billing\":{\"payer\":\"developer\"},\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"store\":true}"}, "scoring": {"fuzzy": 0.782, "cer": 0.244, "cost_summary": {"total_input_tokens": 29784, "total_output_tokens": 2864, "total_tokens": 32648, "input_cost_usd": 0.052122, "output_cost_usd": 0.040096, "total_cost_usd": 0.092218}}, "normalized_score": 75.6}, {"_id": {"$oid": "69d4eb12b877594bc5718d22"}, "test_id": "T0494", "benchmark": "medieval_manuscripts", "date": "2026-03-23", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.2-2025-12-11", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Des pilley tags gieng Sebastian Jn der wue 2\\ngey \u00dft an dem balmitag herdo wz\\nDer sulen2feude tag Witten wue\\ngey \u00dft an dem tag achzehende\\ntag gieng ter\u00df Nid billy zwey\\ntag do vnd firend an dem \u00dfo\u00dflen\\ndunrag der do wz der achtr\u0169\\nzwenzigft tag des marzen, Vff\\ndem Waffer wo ter\u00df gieng Vnd \u00dfig\\nJft an dem osteroben der do wz der\\nder tag im merzen, do \u00dfo\u00dflen wir\\netwe manigen ftikungen Vnd groft\\npfeff als mechtige h\u016ffer do \u00dfo\u00dfe\\nvnd \u00dfant marco m\u00fcnfter vnd der\\nhertzog palafl der \u00ff z mal \u00dft\u016fg\\nVnd vnd hau\u00df vnd \u00dflow Vatter\\nvnd knechten wappen, die dem\\nze Venedig gewefen \u00dfind do \u00fft\\nlin cappell an dem m\u00fcnfter do \u00fft\\ndie pe\u00ff in ein altar gewaffet do\\nvifer be\u00dfalter Jfch Vff \u00dfat\",\"addition1\":\"Veneti\\nDucis\\npalati\\nmarchi\\ncum.\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-5.2-2025-12-11", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 2410, "output_tokens": 319, "total_tokens": 2729, "input_cost_usd": 0.0042175, "output_cost_usd": 0.004466, "estimated_cost_usd": 0.0086835}, "duration": 10.943171977996826, "timestamp": "2026-03-23T14:50:19.156489", "parsed": {"folios": [{"folio": "", "text": "Des pilley tags gieng Sebastian Jn der wue 2\ngey \u00dft an dem balmitag herdo wz\nDer sulen2feude tag Witten wue\ngey \u00dft an dem tag achzehende\ntag gieng ter\u00df Nid billy zwey\ntag do vnd firend an dem \u00dfo\u00dflen\ndunrag der do wz der achtr\u0169\nzwenzigft tag des marzen, Vff\ndem Waffer wo ter\u00df gieng Vnd \u00dfig\nJft an dem osteroben der do wz der\nder tag im merzen, do \u00dfo\u00dflen wir\netwe manigen ftikungen Vnd groft\npfeff als mechtige h\u016ffer do \u00dfo\u00dfe\nvnd \u00dfant marco m\u00fcnfter vnd der\nhertzog palafl der \u00ff z mal \u00dft\u016fg\nVnd vnd hau\u00df vnd \u00dflow Vatter\nvnd knechten wappen, die dem\nze Venedig gewefen \u00dfind do \u00fft\nlin cappell an dem m\u00fcnfter do \u00fft\ndie pe\u00ff in ein altar gewaffet do\nvifer be\u00dfalter Jfch Vff \u00dfat", "addition1": "Veneti\nDucis\npalati\nmarchi\ncum.", "addition2": "", "addition3": ""}]}, "conversation_id": "95228213-c1f2-4581-96f9-89304b6ca3ea", "score": {"fuzzy": 0.779, "cer": 0.231}, "raw_response": "{\"id\":\"chatcmpl-DMZxSTyGHvG8TTZ1yvgkSmrvdndz3\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"folios\\\":[{\\\"folio\\\":\\\"\\\",\\\"text\\\":\\\"Des pilley tags gieng Sebastian Jn der wue 2\\\\ngey \u00dft an dem balmitag herdo wz\\\\nDer sulen2feude tag Witten wue\\\\ngey \u00dft an dem tag achzehende\\\\ntag gieng ter\u00df Nid billy zwey\\\\ntag do vnd firend an dem \u00dfo\u00dflen\\\\ndunrag der do wz der achtr\u0169\\\\nzwenzigft tag des marzen, Vff\\\\ndem Waffer wo ter\u00df gieng Vnd \u00dfig\\\\nJft an dem osteroben der do wz der\\\\nder tag im merzen, do \u00dfo\u00dflen wir\\\\netwe manigen ftikungen Vnd groft\\\\npfeff als mechtige h\u016ffer do \u00dfo\u00dfe\\\\nvnd \u00dfant marco m\u00fcnfter vnd der\\\\nhertzog palafl der \u00ff z mal \u00dft\u016fg\\\\nVnd vnd hau\u00df vnd \u00dflow Vatter\\\\nvnd knechten wappen, die dem\\\\nze Venedig gewefen \u00dfind do \u00fft\\\\nlin cappell an dem m\u00fcnfter do \u00fft\\\\ndie pe\u00ff in ein altar gewaffet do\\\\nvifer be\u00dfalter Jfch Vff \u00dfat\\\",\\\"addition1\\\":\\\"Veneti\\\\nDucis\\\\npalati\\\\nmarchi\\\\ncum.\\\",\\\"addition2\\\":\\\"\\\",\\\"addition3\\\":\\\"\\\"}]}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"Des pilley tags gieng Sebastian Jn der wue 2\\ngey \u00dft an dem balmitag herdo wz\\nDer sulen2feude tag Witten wue\\ngey \u00dft an dem tag achzehende\\ntag gieng ter\u00df Nid billy zwey\\ntag do vnd firend an dem \u00dfo\u00dflen\\ndunrag der do wz der achtr\u0169\\nzwenzigft tag des marzen, Vff\\ndem Waffer wo ter\u00df gieng Vnd \u00dfig\\nJft an dem osteroben der do wz der\\nder tag im merzen, do \u00dfo\u00dflen wir\\netwe manigen ftikungen Vnd groft\\npfeff als mechtige h\u016ffer do \u00dfo\u00dfe\\nvnd \u00dfant marco m\u00fcnfter vnd der\\nhertzog palafl der \u00ff z mal \u00dft\u016fg\\nVnd vnd hau\u00df vnd \u00dflow Vatter\\nvnd knechten wappen, die dem\\nze Venedig gewefen \u00dfind do \u00fft\\nlin cappell an dem m\u00fcnfter do \u00fft\\ndie pe\u00ff in ein altar gewaffet do\\nvifer be\u00dfalter Jfch Vff \u00dfat\",\"addition1\":\"Veneti\\nDucis\\npalati\\nmarchi\\ncum.\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1774273810,\"model\":\"gpt-5.2-2025-12-11\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":319,\"prompt_tokens\":2410,\"total_tokens\":2729,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.701, "cer": 0.319, "cost_summary": {"total_input_tokens": 28920, "total_output_tokens": 2923, "total_tokens": 31843, "input_cost_usd": 0.050609999999999995, "output_cost_usd": 0.040922, "total_cost_usd": 0.09153200000000002}}, "normalized_score": 68.1}, {"_id": {"$oid": "69d4eb12b877594bc5718ccb"}, "test_id": "T0699", "benchmark": "medieval_manuscripts", "date": "2026-03-17", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.3-codex", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "rules": "{\"api_style\":\"responses\"}", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\n  \"folios\": [\n    {\n      \"folio\": \"69\",\n      \"text\": \"Dar nach kame ir pfiffer vnd\\n trumeter zu immen h\ua75ben dorff\\n die gallen vnd hofierten do\\n f\u00fcren grubden des glich komd\\n hie nundertfichen pfiffer vnd\\n trumeter vnd besoffen flaffer\\n die hofierten do ach nach irs\\n sitten also ward mins gnedige\\n h\ua75b\ua75b vnd allen den sinen vil\\n erey\ua75b erbotten Jst niderndes ge-\\n gen tag fr\u00fcent wir do danns\\n vnd kamen uff freitag nach sunt\\n velone tag des do waz der erst\\n tag im heidmonet gon kurfen\\n Jst ein schone pal vnd z\u00fclich\\n gehalten Wellflerfey ob der\\n stat von uff suntag darnach\\n f\u00fcren wir da danns vnd uff\\n z\u00fcnstag vor sant vits tag Jst\\n komen zu vns in gallen by ge\ua75b\",\n      \"addition1\": \"Monachus\\nMarelio\\nin exi\\nbit.\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "model": "gpt-5.3-codex", "provider": "openai", "finish_reason": "completed", "usage": {"input_tokens": 2482, "output_tokens": 275, "total_tokens": 2757, "input_cost_usd": 0.0043435, "output_cost_usd": 0.00385, "estimated_cost_usd": 0.0081935}, "duration": 10.928551435470581, "timestamp": "2026-03-17T22:46:57.484059", "parsed": {"folios": [{"folio": "69", "text": "Dar nach kame ir pfiffer vnd\n trumeter zu immen h\ua75ben dorff\n die gallen vnd hofierten do\n f\u00fcren grubden des glich komd\n hie nundertfichen pfiffer vnd\n trumeter vnd besoffen flaffer\n die hofierten do ach nach irs\n sitten also ward mins gnedige\n h\ua75b\ua75b vnd allen den sinen vil\n erey\ua75b erbotten Jst niderndes ge-\n gen tag fr\u00fcent wir do danns\n vnd kamen uff freitag nach sunt\n velone tag des do waz der erst\n tag im heidmonet gon kurfen\n Jst ein schone pal vnd z\u00fclich\n gehalten Wellflerfey ob der\n stat von uff suntag darnach\n f\u00fcren wir da danns vnd uff\n z\u00fcnstag vor sant vits tag Jst\n komen zu vns in gallen by ge\ua75b", "addition1": "Monachus\nMarelio\nin exi\nbit.", "addition2": "", "addition3": ""}]}, "conversation_id": "e112b9ba-556c-4844-900c-c252de7d92a5", "score": {"fuzzy": 0.486, "cer": 0.537}, "raw_response": "{\"id\":\"resp_00a9841efc54b51c0069b9cbc63a188197a97d38e117d25db0\",\"created_at\":1773784008.0,\"error\":null,\"incomplete_details\":null,\"instructions\":\"You are a historian with expertise in medieval manuscripts and paleography\",\"metadata\":{},\"model\":\"gpt-5.3-codex\",\"object\":\"response\",\"output\":[{\"id\":\"msg_00a9841efc54b51c0069b9cbc8a6f081978a9adc85c61a66b2\",\"content\":[{\"annotations\":[],\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"69\\\",\\n      \\\"text\\\": \\\"Dar nach kame ir pfiffer vnd\\\\n trumeter zu immen h\ua75ben dorff\\\\n die gallen vnd hofierten do\\\\n f\u00fcren grubden des glich komd\\\\n hie nundertfichen pfiffer vnd\\\\n trumeter vnd besoffen flaffer\\\\n die hofierten do ach nach irs\\\\n sitten also ward mins gnedige\\\\n h\ua75b\ua75b vnd allen den sinen vil\\\\n erey\ua75b erbotten Jst niderndes ge-\\\\n gen tag fr\u00fcent wir do danns\\\\n vnd kamen uff freitag nach sunt\\\\n velone tag des do waz der erst\\\\n tag im heidmonet gon kurfen\\\\n Jst ein schone pal vnd z\u00fclich\\\\n gehalten Wellflerfey ob der\\\\n stat von uff suntag darnach\\\\n f\u00fcren wir da danns vnd uff\\\\n z\u00fcnstag vor sant vits tag Jst\\\\n komen zu vns in gallen by ge\ua75b\\\",\\n      \\\"addition1\\\": \\\"Monachus\\\\nMarelio\\\\nin exi\\\\nbit.\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"type\":\"output_text\",\"logprobs\":[]}],\"role\":\"assistant\",\"status\":\"completed\",\"type\":\"message\",\"phase\":\"final_answer\"}],\"parallel_tool_calls\":true,\"temperature\":0.0,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":0.98,\"background\":false,\"completed_at\":1773784015.0,\"conversation\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"previous_response_id\":null,\"prompt\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":null,\"reasoning\":{\"effort\":\"none\",\"generate_summary\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"status\":\"completed\",\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"top_logprobs\":0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":2482,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":275,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":2757},\"user\":null,\"billing\":{\"payer\":\"developer\"},\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"store\":true}"}, "scoring": {"fuzzy": 0.746, "cer": 0.285, "cost_summary": {"total_input_tokens": 29784, "total_output_tokens": 2867, "total_tokens": 32651, "input_cost_usd": 0.052122, "output_cost_usd": 0.040138, "total_cost_usd": 0.09226}}, "normalized_score": 71.5}, {"_id": {"$oid": "69d4eb12b877594bc5718c84"}, "test_id": "T0634", "benchmark": "medieval_manuscripts", "date": "2026-03-16", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-6", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"vnd die jnsel ist bi cc mil\\nwit vnd ist der venediger\\nIt\ufffd765 von kurfun vntz gon\\nmodun sind cc mil vnd\\nist och ein hu\u0364bsch stat vnd\\nein ertzbistum vnd ist och\\nder venediger ze modun\\nwachset der rumanier vnd\\nlit och do bi ein heilig heisset\\ndo bi ij milen\\nIt\ufffd765 von modun bis gon candia\\nsind cc mil vnd candia\\nist ein gro\u00df jnsel vnf mil\\nwit vnd wachset der\\nmaluasier do vnd ist och\\nder venedier\\nIt\ufffd765 von candia vntz gon rodis\\nsind cc mil wer aber\",\"addition1\":\"Candia vel\\nCreta.\",\"addition2\":\"Vinum\\nMalua\\nticum.\",\"addition3\":\"Rhod.\"}]}", "model": "claude-opus-4-6", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2992, "output_tokens": 283, "total_tokens": 3275, "input_cost_usd": 0.01496, "output_cost_usd": 0.007075, "estimated_cost_usd": 0.022035}, "duration": 14.846066951751709, "timestamp": "2026-03-16T15:54:15.718799", "parsed": {"folios": [{"folio": "", "text": "vnd die jnsel ist bi cc mil\nwit vnd ist der venediger\nIt\ufffd765 von kurfun vntz gon\nmodun sind cc mil vnd\nist och ein hu\u0364bsch stat vnd\nein ertzbistum vnd ist och\nder venediger ze modun\nwachset der rumanier vnd\nlit och do bi ein heilig heisset\ndo bi ij milen\nIt\ufffd765 von modun bis gon candia\nsind cc mil vnd candia\nist ein gro\u00df jnsel vnf mil\nwit vnd wachset der\nmaluasier do vnd ist och\nder venedier\nIt\ufffd765 von candia vntz gon rodis\nsind cc mil wer aber", "addition1": "Candia vel\nCreta.", "addition2": "Vinum\nMalua\nticum.", "addition3": "Rhod."}]}, "conversation_id": "e4320bcc-4c2c-424d-a97a-4235b9c36746", "score": {"fuzzy": 0.857, "cer": 0.163}, "raw_response": "{\"id\":\"msg_01HvubdJS4KfBin5WyokWTfp\",\"content\":[{\"id\":\"toolu_015MF68QTuSwQSLumNcMi343\",\"input\":{\"folios\":[{\"folio\":\"\",\"text\":\"vnd die jnsel ist bi cc mil\\nwit vnd ist der venediger\\nIt\ufffd765 von kurfun vntz gon\\nmodun sind cc mil vnd\\nist och ein hu\u0364bsch stat vnd\\nein ertzbistum vnd ist och\\nder venediger ze modun\\nwachset der rumanier vnd\\nlit och do bi ein heilig heisset\\ndo bi ij milen\\nIt\ufffd765 von modun bis gon candia\\nsind cc mil vnd candia\\nist ein gro\u00df jnsel vnf mil\\nwit vnd wachset der\\nmaluasier do vnd ist och\\nder venedier\\nIt\ufffd765 von candia vntz gon rodis\\nsind cc mil wer aber\",\"addition1\":\"Candia vel\\nCreta.\",\"addition2\":\"Vinum\\nMalua\\nticum.\",\"addition3\":\"Rhod.\"}]},\"name\":\"extract_structured_data\",\"type\":\"tool_use\",\"caller\":{\"type\":\"direct\"}}],\"model\":\"claude-opus-4-6\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"inference_geo\":\"global\",\"input_tokens\":2992,\"output_tokens\":283,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"fuzzy": 0.807, "cer": 0.202, "cost_summary": {"total_input_tokens": 35904, "total_output_tokens": 3320, "total_tokens": 39224, "input_cost_usd": 0.17951999999999999, "output_cost_usd": 0.08299999999999999, "total_cost_usd": 0.26252000000000003}}, "normalized_score": 79.8}, {"_id": {"$oid": "69d4eb12b877594bc5718c91"}, "test_id": "T0687", "benchmark": "medieval_manuscripts", "date": "2026-03-16", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3.1-pro-preview", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Item dar noch got man in die\\nkappel des heiligen grabes\\nDo ist ein altar von marmel\\nstein gemacht vber das heilig\\ngrab do vnser herre gewarer\\ngot vnd mensche in gelegen\\nist vnd do von vferstanden\\nan dem dritten tag von\\ndem tod\\nItem in dem cor des tempels\\nda ist ein loch vnd stat dar\\nbi ein klein stein do sprach\\nvnser herre dz were das mittel\\nder welt\\nItem hie nochgeschriben stand die\\ndie bilgerstet vswendig des\\ntempels in der stat\\nItem das hus des Richen mans\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gemini-3.1-pro-preview", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1594, "output_tokens": 199, "total_tokens": 1793}, "duration": 4.858447790145874, "timestamp": "2026-03-16T19:37:11.071810", "parsed": {"folios": [{"folio": "", "text": "Item dar noch got man in die\nkappel des heiligen grabes\nDo ist ein altar von marmel\nstein gemacht vber das heilig\ngrab do vnser herre gewarer\ngot vnd mensche in gelegen\nist vnd do von vferstanden\nan dem dritten tag von\ndem tod\nItem in dem cor des tempels\nda ist ein loch vnd stat dar\nbi ein klein stein do sprach\nvnser herre dz were das mittel\nder welt\nItem hie nochgeschriben stand die\ndie bilgerstet vswendig des\ntempels in der stat\nItem das hus des Richen mans", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "1b2a20fa-a79a-4ccb-a02e-f20a3be66acf", "score": {"fuzzy": 0.936, "cer": 0.055}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Mon, 16 Mar 2026 18:37:17 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=4553\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"media_resolution\":null,\"code_execution_result\":null,\"executable_code\":null,\"file_data\":null,\"function_call\":null,\"function_response\":null,\"inline_data\":null,\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Item dar noch got man in die\\\\nkappel des heiligen grabes\\\\nDo ist ein altar von marmel\\\\nstein gemacht vber das heilig\\\\ngrab do vnser herre gewarer\\\\ngot vnd mensche in gelegen\\\\nist vnd do von vferstanden\\\\nan dem dritten tag von\\\\ndem tod\\\\nItem in dem cor des tempels\\\\nda ist ein loch vnd stat dar\\\\nbi ein klein stein do sprach\\\\nvnser herre dz were das mittel\\\\nder welt\\\\nItem hie nochgeschriben stand die\\\\ndie bilgerstet vswendig des\\\\ntempels in der stat\\\\nItem das hus des Richen mans\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"thought\":null,\"thought_signature\":\"EjQKMgG-Pvb7hvNmc0oemEgd5f1qbrRYkRBkVEFWpzfRgFdWAPrPwikEUXPKS7YX6Qjq3tSv\",\"video_metadata\":null}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null,\"url_context_metadata\":null}],\"create_time\":null,\"model_version\":\"gemini-3.1-pro-preview\",\"prompt_feedback\":null,\"response_id\":\"3U24aaKOOLaWkdUP9YDD0AE\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":199,\"candidates_tokens_details\":null,\"prompt_token_count\":1594,\"prompt_tokens_details\":[{\"modality\":\"IMAGE\",\"token_count\":1080},{\"modality\":\"TEXT\",\"token_count\":514}],\"thoughts_token_count\":null,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":1793,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"Item dar noch got man in die\\nkappel des heiligen grabes\\nDo ist ein altar von marmel\\nstein gemacht vber das heilig\\ngrab do vnser herre gewarer\\ngot vnd mensche in gelegen\\nist vnd do von vferstanden\\nan dem dritten tag von\\ndem tod\\nItem in dem cor des tempels\\nda ist ein loch vnd stat dar\\nbi ein klein stein do sprach\\nvnser herre dz were das mittel\\nder welt\\nItem hie nochgeschriben stand die\\ndie bilgerstet vswendig des\\ntempels in der stat\\nItem das hus des Richen mans\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}}"}, "scoring": null, "normalized_score": null}, {"_id": {"$oid": "69d4eb12b877594bc5718c92"}, "test_id": "T0675", "benchmark": "medieval_manuscripts", "date": "2026-03-16", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3.1-flash-lite-preview", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"vn dem erzbischoff do selbs\\n vn zongt man vns dz heilig\\n in dem thurm do selbs ist\\n sant stephans houpt der do de\\n bexiij hunger emer wz ist sit\\n barbara eins kunigs do zipern\\n tochter houpt ist sant blasius\\n vn sant andres emer ist\\n vnd sant stephans emer ist von\\n zwen zehen die do sant hippol\\n geschlage wurdent ist\\n off mentag vor sant johans\\n tag ze dungestey f\u00fcrch wir\\n laudia von f\u00fcrend an d\\n mitwoch f\u00fcr die pfort ze zi\\n do kam zu der battela\\n fast do wappnete sich die\\n geflut all gemeinlich vnd\\n komend all bilgerin hie\",\"addition1\":\"vn zongt man\\nvns ach ze pa\\ndie ein von\\nrot daran\\nwir in f\u00fcrst mien\\nenteil blangt\\nnoch ent von\\ntriebt die bil\\ngrin vnd hall\\nund gang zu\\nware vn sit\\nmd vns diepeck\\nvonze des anker\\nny fenst twing\\nund ist die rat\\ngegen de spital\\nny dem deme\\nBarf\u00fcsse kloster\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gemini-3.1-flash-lite-preview", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1594, "output_tokens": 341, "total_tokens": 1935, "input_cost_usd": 0.0003985, "output_cost_usd": 0.0005115, "estimated_cost_usd": 0.00091}, "duration": 9.240050315856934, "timestamp": "2026-03-16T19:09:27.550040", "parsed": {"folios": [{"folio": "", "text": "vn dem erzbischoff do selbs\n vn zongt man vns dz heilig\n in dem thurm do selbs ist\n sant stephans houpt der do de\n bexiij hunger emer wz ist sit\n barbara eins kunigs do zipern\n tochter houpt ist sant blasius\n vn sant andres emer ist\n vnd sant stephans emer ist von\n zwen zehen die do sant hippol\n geschlage wurdent ist\n off mentag vor sant johans\n tag ze dungestey f\u00fcrch wir\n laudia von f\u00fcrend an d\n mitwoch f\u00fcr die pfort ze zi\n do kam zu der battela\n fast do wappnete sich die\n geflut all gemeinlich vnd\n komend all bilgerin hie", "addition1": "vn zongt man\nvns ach ze pa\ndie ein von\nrot daran\nwir in f\u00fcrst mien\nenteil blangt\nnoch ent von\ntriebt die bil\ngrin vnd hall\nund gang zu\nware vn sit\nmd vns diepeck\nvonze des anker\nny fenst twing\nund ist die rat\ngegen de spital\nny dem deme\nBarf\u00fcsse kloster", "addition2": "", "addition3": ""}]}, "conversation_id": "1846925b-12ac-4797-b61d-0572cb2efc9e", "score": {"fuzzy": 0.754, "cer": 0.322}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Mon, 16 Mar 2026 18:09:34 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=8955\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"media_resolution\":null,\"code_execution_result\":null,\"executable_code\":null,\"file_data\":null,\"function_call\":null,\"function_response\":null,\"inline_data\":null,\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"vn dem erzbischoff do selbs\\\\n vn zongt man vns dz heilig\\\\n in dem thurm do selbs ist\\\\n sant stephans houpt der do de\\\\n bexiij hunger emer wz ist sit\\\\n barbara eins kunigs do zipern\\\\n tochter houpt ist sant blasius\\\\n vn sant andres emer ist\\\\n vnd sant stephans emer ist von\\\\n zwen zehen die do sant hippol\\\\n geschlage wurdent ist\\\\n off mentag vor sant johans\\\\n tag ze dungestey f\u00fcrch wir\\\\n laudia von f\u00fcrend an d\\\\n mitwoch f\u00fcr die pfort ze zi\\\\n do kam zu der battela\\\\n fast do wappnete sich die\\\\n geflut all gemeinlich vnd\\\\n komend all bilgerin hie\\\",\\n      \\\"addition1\\\": \\\"vn zongt man\\\\nvns ach ze pa\\\\ndie ein von\\\\nrot daran\\\\nwir in f\u00fcrst mien\\\\nenteil blangt\\\\nnoch ent von\\\\ntriebt die bil\\\\ngrin vnd hall\\\\nund gang zu\\\\nware vn sit\\\\nmd vns diepeck\\\\nvonze des anker\\\\nny fenst twing\\\\nund ist die rat\\\\ngegen de spital\\\\nny dem deme\\\\nBarf\u00fcsse kloster\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"thought\":null,\"thought_signature\":\"EjQKMgG-Pvb7jKbKWU8apuF6g0lVuxVQigMwK983vSDn1kZxNSOcmy2qdYKJe1NfTLsTBum5\",\"video_metadata\":null}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null,\"url_context_metadata\":null}],\"create_time\":null,\"model_version\":\"gemini-3.1-flash-lite-preview\",\"prompt_feedback\":null,\"response_id\":\"Xke4aeCEFuG7vdIP1eqw-Aw\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":341,\"candidates_tokens_details\":null,\"prompt_token_count\":1594,\"prompt_tokens_details\":[{\"modality\":\"IMAGE\",\"token_count\":1080},{\"modality\":\"TEXT\",\"token_count\":514}],\"thoughts_token_count\":null,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":1935,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"vn dem erzbischoff do selbs\\n vn zongt man vns dz heilig\\n in dem thurm do selbs ist\\n sant stephans houpt der do de\\n bexiij hunger emer wz ist sit\\n barbara eins kunigs do zipern\\n tochter houpt ist sant blasius\\n vn sant andres emer ist\\n vnd sant stephans emer ist von\\n zwen zehen die do sant hippol\\n geschlage wurdent ist\\n off mentag vor sant johans\\n tag ze dungestey f\u00fcrch wir\\n laudia von f\u00fcrend an d\\n mitwoch f\u00fcr die pfort ze zi\\n do kam zu der battela\\n fast do wappnete sich die\\n geflut all gemeinlich vnd\\n komend all bilgerin hie\",\"addition1\":\"vn zongt man\\nvns ach ze pa\\ndie ein von\\nrot daran\\nwir in f\u00fcrst mien\\nenteil blangt\\nnoch ent von\\ntriebt die bil\\ngrin vnd hall\\nund gang zu\\nware vn sit\\nmd vns diepeck\\nvonze des anker\\nny fenst twing\\nund ist die rat\\ngegen de spital\\nny dem deme\\nBarf\u00fcsse kloster\",\"addition2\":\"\",\"addition3\":\"\"}]}}"}, "scoring": {"fuzzy": 0.783, "cer": 0.221, "cost_summary": {"total_input_tokens": 19128, "total_output_tokens": 2721, "total_tokens": 21849, "input_cost_usd": 0.004782, "output_cost_usd": 0.004081499999999999, "total_cost_usd": 0.008863499999999998}}, "normalized_score": 77.9}, {"_id": {"$oid": "69d4eb12b877594bc5718ca5"}, "test_id": "T0658", "benchmark": "medieval_manuscripts", "date": "2026-03-16", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.4-2026-03-05", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Item sant patton in groffen\\nf\u00fcr sin beruung\\nItem ze enius ein groffen\\nItem ze Iherusalem j groff ze\\nkurtzsch von den eplon\\nItem dem conful in grob\\nItem in den tempel iij grob\\nItem ze bethlahem ein grob\\nItem ze sant Iohans ein grob\\nItem wider ze Iherusalem ein\\ngrob ze cruntzsch \u00fcber j grob\\nItem wider in den tempel iiij\\ngrob\\nItem ze dem Jordan gein grob\\nIn den eplon ze lon\\nItem aber ein grob ze f\u00fcnft\\nItem aber ze Iherusalem j grob\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-5.4-2026-03-05", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 3353, "output_tokens": 173, "total_tokens": 3526, "input_cost_usd": 0.008382500000000001, "output_cost_usd": 0.002595, "estimated_cost_usd": 0.010977500000000001}, "duration": 5.822008371353149, "timestamp": "2026-03-16T17:18:16.632799", "parsed": {"folios": [{"folio": "", "text": "Item sant patton in groffen\nf\u00fcr sin beruung\nItem ze enius ein groffen\nItem ze Iherusalem j groff ze\nkurtzsch von den eplon\nItem dem conful in grob\nItem in den tempel iij grob\nItem ze bethlahem ein grob\nItem ze sant Iohans ein grob\nItem wider ze Iherusalem ein\ngrob ze cruntzsch \u00fcber j grob\nItem wider in den tempel iiij\ngrob\nItem ze dem Jordan gein grob\nIn den eplon ze lon\nItem aber ein grob ze f\u00fcnft\nItem aber ze Iherusalem j grob", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "32e1022b-0c86-44c4-9e2d-2470b94c35fb", "score": {"fuzzy": 0.798, "cer": 0.229}, "raw_response": "{\"id\":\"chatcmpl-DK4vztdjfs1MGx0DmC6MJZOvPRs1e\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"folios\\\":[{\\\"folio\\\":\\\"\\\",\\\"text\\\":\\\"Item sant patton in groffen\\\\nf\u00fcr sin beruung\\\\nItem ze enius ein groffen\\\\nItem ze Iherusalem j groff ze\\\\nkurtzsch von den eplon\\\\nItem dem conful in grob\\\\nItem in den tempel iij grob\\\\nItem ze bethlahem ein grob\\\\nItem ze sant Iohans ein grob\\\\nItem wider ze Iherusalem ein\\\\ngrob ze cruntzsch \u00fcber j grob\\\\nItem wider in den tempel iiij\\\\ngrob\\\\nItem ze dem Jordan gein grob\\\\nIn den eplon ze lon\\\\nItem aber ein grob ze f\u00fcnft\\\\nItem aber ze Iherusalem j grob\\\",\\\"addition1\\\":\\\"\\\",\\\"addition2\\\":\\\"\\\",\\\"addition3\\\":\\\"\\\"}]}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"Item sant patton in groffen\\nf\u00fcr sin beruung\\nItem ze enius ein groffen\\nItem ze Iherusalem j groff ze\\nkurtzsch von den eplon\\nItem dem conful in grob\\nItem in den tempel iij grob\\nItem ze bethlahem ein grob\\nItem ze sant Iohans ein grob\\nItem wider ze Iherusalem ein\\ngrob ze cruntzsch \u00fcber j grob\\nItem wider in den tempel iiij\\ngrob\\nItem ze dem Jordan gein grob\\nIn den eplon ze lon\\nItem aber ein grob ze f\u00fcnft\\nItem aber ze Iherusalem j grob\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1773677899,\"model\":\"gpt-5.4-2026-03-05\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":173,\"prompt_tokens\":3353,\"total_tokens\":3526,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.765, "cer": 0.264, "cost_summary": {"total_input_tokens": 40236, "total_output_tokens": 2650, "total_tokens": 42886, "input_cost_usd": 0.10059000000000001, "output_cost_usd": 0.03975, "total_cost_usd": 0.14034000000000002}}, "normalized_score": 73.6}, {"_id": {"$oid": "69d4eb12b877594bc5718cba"}, "test_id": "T0646", "benchmark": "medieval_manuscripts", "date": "2026-03-16", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-6", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"73\",\"text\":\"Jt z\u00fc dem ersten Min hern graff albrecht vo\\nanhalt\\nJt her ludwig lantgraff von kurhtenberg\\nJt graff ott von hemienberg\\nJt groff Bernhart von Regenstein\\nJt groff g\u00f6ttfrid von hohenloch\\nJt groff gebhart von Mansfelt\\nJt groff ludwig von Isenburg\\nJt her Gotsthalk von bless\\nJt her hans von turg\u00f6w\\nJt Jerg Bappemhein\\nJt her Buppelin von stein wz vorhm Ritter\\nJt luz vam Rotenhan\\nJt Jerg von waldenfels\\nJt hylprand von thungen\\nJt Jerg von Sleben\\nJt henin quast\",\"addition1\":\"bedpanclud\",\"addition2\":null,\"addition3\":null}]}", "model": "claude-sonnet-4-6", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2992, "output_tokens": 275, "total_tokens": 3267, "input_cost_usd": 0.008976, "output_cost_usd": 0.004125, "estimated_cost_usd": 0.013101}, "duration": 8.979059934616089, "timestamp": "2026-03-16T15:53:21.019044", "parsed": {"folios": [{"folio": "73", "text": "Jt z\u00fc dem ersten Min hern graff albrecht vo\nanhalt\nJt her ludwig lantgraff von kurhtenberg\nJt graff ott von hemienberg\nJt groff Bernhart von Regenstein\nJt groff g\u00f6ttfrid von hohenloch\nJt groff gebhart von Mansfelt\nJt groff ludwig von Isenburg\nJt her Gotsthalk von bless\nJt her hans von turg\u00f6w\nJt Jerg Bappemhein\nJt her Buppelin von stein wz vorhm Ritter\nJt luz vam Rotenhan\nJt Jerg von waldenfels\nJt hylprand von thungen\nJt Jerg von Sleben\nJt henin quast", "addition1": "bedpanclud"}]}, "conversation_id": "0aed2d5b-0ded-498c-a622-e4a5191c021f", "score": {"fuzzy": 0.626, "cer": 0.379}, "raw_response": "{\"id\":\"msg_01V71Aht56b22d7k6e6DxvSU\",\"content\":[{\"id\":\"toolu_013BL9obY8CQjiMqsvpLZgWt\",\"input\":{\"folios\":[{\"folio\":\"73\",\"text\":\"Jt z\u00fc dem ersten Min hern graff albrecht vo\\nanhalt\\nJt her ludwig lantgraff von kurhtenberg\\nJt graff ott von hemienberg\\nJt groff Bernhart von Regenstein\\nJt groff g\u00f6ttfrid von hohenloch\\nJt groff gebhart von Mansfelt\\nJt groff ludwig von Isenburg\\nJt her Gotsthalk von bless\\nJt her hans von turg\u00f6w\\nJt Jerg Bappemhein\\nJt her Buppelin von stein wz vorhm Ritter\\nJt luz vam Rotenhan\\nJt Jerg von waldenfels\\nJt hylprand von thungen\\nJt Jerg von Sleben\\nJt henin quast\",\"addition1\":\"bedpanclud\"}]},\"name\":\"extract_structured_data\",\"type\":\"tool_use\",\"caller\":{\"type\":\"direct\"}}],\"model\":\"claude-sonnet-4-6\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"inference_geo\":\"global\",\"input_tokens\":2992,\"output_tokens\":275,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"fuzzy": 0.759, "cer": 0.239, "cost_summary": {"total_input_tokens": 35904, "total_output_tokens": 3347, "total_tokens": 39251, "input_cost_usd": 0.10771199999999999, "output_cost_usd": 0.05020500000000001, "total_cost_usd": 0.157917}}, "normalized_score": 76.1}, {"_id": {"$oid": "69d4eb12b877594bc5718c7a"}, "test_id": "T0538", "benchmark": "medieval_manuscripts", "date": "2026-02-18", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-large-2512", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "mistral-large-2512", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 753, "output_tokens": 46, "total_tokens": 799, "input_cost_usd": 0.0003765, "output_cost_usd": 6.9e-05, "estimated_cost_usd": 0.0004455}, "duration": 1.1263446807861328, "timestamp": "2026-02-18T11:46:23.629851", "parsed": {"folios": [{"folio": "", "text": "", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "402c3277-06e0-48a4-9ea9-ec8bd4c0ec71", "score": {"fuzzy": 0.0, "cer": 1.0}, "raw_response": "{\"id\":\"929a19a72dab4c3389400d77f1ad3645\",\"object\":\"chat.completion\",\"model\":\"mistral-large-2512\",\"usage\":{\"prompt_tokens\":753,\"completion_tokens\":46,\"total_tokens\":799,\"prompt_tokens_details\":{\"cached_tokens\":16}},\"created\":1771411582,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"fuzzy": 0.0, "cer": 1.0, "cost_summary": {"total_input_tokens": 9036, "total_output_tokens": 552, "total_tokens": 9588, "input_cost_usd": 0.004518, "output_cost_usd": 0.0008280000000000002, "total_cost_usd": 0.005345999999999999}}, "normalized_score": 0}, {"_id": {"$oid": "69d4eb12b877594bc5718c61"}, "test_id": "T0431", "benchmark": "medieval_manuscripts", "date": "2026-02-17", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "magistral-medium-2509", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"15\",\"text\":\"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\"addition1\":\"Anno domini 1444\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "magistral-medium-2509", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 753, "output_tokens": 108, "total_tokens": 861, "input_cost_usd": 0.001506, "output_cost_usd": 0.00054, "estimated_cost_usd": 0.002046}, "duration": 6.106011390686035, "timestamp": "2026-02-17T15:43:14.438020", "parsed": {"folios": [{"folio": "15", "text": "In disem jare kam der kunig\n mit grossem here in daz lant\n vnd belagerte die stat Basel\n do wertend sich die burger\n mit grosser kraft vnd tugent\n vnd triben den kunig dannen\n mit schanden vnd verlust", "addition1": "Anno domini 1444", "addition2": "", "addition3": ""}]}, "conversation_id": "c1c87aa3-deb2-427c-9f46-d638e516dd3b", "score": {"fuzzy": 0.119, "cer": 0.919}, "raw_response": "{\"id\":\"7934e9ee8d7b46dc9c35a63c61b88517\",\"object\":\"chat.completion\",\"model\":\"magistral-medium-2509\",\"usage\":{\"prompt_tokens\":753,\"completion_tokens\":108,\"total_tokens\":861,\"prompt_tokens_details\":{\"cached_tokens\":128}},\"created\":1771339388,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\\"folios\\\": [{\\\"folio\\\": \\\"15\\\", \\\"text\\\": \\\"In disem jare kam der kunig\\\\n mit grossem here in daz lant\\\\n vnd belagerte die stat Basel\\\\n do wertend sich die burger\\\\n mit grosser kraft vnd tugent\\\\n vnd triben den kunig dannen\\\\n mit schanden vnd verlust\\\", \\\"addition1\\\": \\\"Anno domini 1444\\\", \\\"addition2\\\": \\\"\\\", \\\"addition3\\\": \\\"\\\"}]}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"fuzzy": 0.168, "cer": 0.883, "cost_summary": {"total_input_tokens": 9036, "total_output_tokens": 1296, "total_tokens": 10332, "input_cost_usd": 0.018072, "output_cost_usd": 0.006479999999999999, "total_cost_usd": 0.024551999999999994}}, "normalized_score": 11.700000000000003}, {"_id": {"$oid": "69d4eb12b877594bc5718c67"}, "test_id": "T0276", "benchmark": "medieval_manuscripts", "date": "2026-02-17", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4o-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"18\",\"text\":\"Das grof von Swarnzberg\\n und pest all under viter\\n vnd ancht woltey mit\\n dasey in was entlafft\\n rictig astronomi\ua75bia. Jedab\\n von hertzunt und gans\\n das heidenspeys vnd\\n die zwey nerend gien\\n mit mir ge\u00dfis oder der\\n patron velt myr mit\\n betung. So der mitteil mit\\n das walt und al\u00df m\u00fcgen\\n wir mit gro\u00dfen unwilley\\n sampt begiden, dey wir\\n mit dem patron vnd\\n o\u00df der br\u00fcderin jar\\n und ratend ze gipen\\n vnd g\u00fctel\",\"addition1\":\"Sachsen in Cypro\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-4o-mini-2024-07-18", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 37496, "output_tokens": 198, "total_tokens": 37694, "input_cost_usd": 0.0056244, "output_cost_usd": 0.00011879999999999999, "estimated_cost_usd": 0.0057431999999999995}, "duration": 8.481578826904297, "timestamp": "2026-02-17T11:51:32.957775", "parsed": {"folios": [{"folio": "18", "text": "Das grof von Swarnzberg\n und pest all under viter\n vnd ancht woltey mit\n dasey in was entlafft\n rictig astronomi\ua75bia. Jedab\n von hertzunt und gans\n das heidenspeys vnd\n die zwey nerend gien\n mit mir ge\u00dfis oder der\n patron velt myr mit\n betung. So der mitteil mit\n das walt und al\u00df m\u00fcgen\n wir mit gro\u00dfen unwilley\n sampt begiden, dey wir\n mit dem patron vnd\n o\u00df der br\u00fcderin jar\n und ratend ze gipen\n vnd g\u00fctel", "addition1": "Sachsen in Cypro", "addition2": "", "addition3": ""}]}, "conversation_id": "6a4425aa-0f53-464b-9249-ca6bf2ce7ad8", "score": {"fuzzy": 0.776, "cer": 0.267}, "raw_response": "{\"id\":\"chatcmpl-DACxpnojdUPD9V1CBgbUicDo4zGX0\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"18\\\",\\n      \\\"text\\\": \\\"Das grof von Swarnzberg\\\\n und pest all under viter\\\\n vnd ancht woltey mit\\\\n dasey in was entlafft\\\\n rictig astronomi\ua75bia. Jedab\\\\n von hertzunt und gans\\\\n das heidenspeys vnd\\\\n die zwey nerend gien\\\\n mit mir ge\u00dfis oder der\\\\n patron velt myr mit\\\\n betung. So der mitteil mit\\\\n das walt und al\u00df m\u00fcgen\\\\n wir mit gro\u00dfen unwilley\\\\n sampt begiden, dey wir\\\\n mit dem patron vnd\\\\n o\u00df der br\u00fcderin jar\\\\n und ratend ze gipen\\\\n vnd g\u00fctel\\\",\\n      \\\"addition1\\\": \\\"Sachsen in Cypro\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"18\",\"text\":\"Das grof von Swarnzberg\\n und pest all under viter\\n vnd ancht woltey mit\\n dasey in was entlafft\\n rictig astronomi\ua75bia. Jedab\\n von hertzunt und gans\\n das heidenspeys vnd\\n die zwey nerend gien\\n mit mir ge\u00dfis oder der\\n patron velt myr mit\\n betung. So der mitteil mit\\n das walt und al\u00df m\u00fcgen\\n wir mit gro\u00dfen unwilley\\n sampt begiden, dey wir\\n mit dem patron vnd\\n o\u00df der br\u00fcderin jar\\n und ratend ze gipen\\n vnd g\u00fctel\",\"addition1\":\"Sachsen in Cypro\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1771325485,\"model\":\"gpt-4o-mini-2024-07-18\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_373a14eb6f\",\"usage\":{\"completion_tokens\":198,\"prompt_tokens\":37496,\"total_tokens\":37694,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.562, "cer": 0.483, "cost_summary": {"total_input_tokens": 449952, "total_output_tokens": 2434, "total_tokens": 452386, "input_cost_usd": 0.0674928, "output_cost_usd": 0.0014604000000000002, "total_cost_usd": 0.06895319999999999}}, "normalized_score": 51.7}, {"_id": {"$oid": "69d4eb12b877594bc5718c6e"}, "test_id": "T0278", "benchmark": "medieval_manuscripts", "date": "2026-02-17", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Gelobt mu fronlaster die bo\u017fend\\n mir gantz vnd ein s\u00ffde vnd\\n dant heiligen fron v\u00fft ez\\n lie in uff e brucht vnd ein\\n beym v\u00ffn pant marien madalene\\n v\u00ffgkhot \ua75bp\ua75bf samptag noch\\n dem oster tag der so was der\\n ander t\u00fff die abwelten do\\n\\nmenschn fuend v\u00ffn v\u00ffn me\u00ffn v\u00ffn\\n guet in pont \ua75bf\u017ftumb\\n \\n\\nkatek\u00ffn zu funden der end\u00ffn\\n die k\u00ffn\u017ftig ar\u017f\u017flig in emen\\n al\u017far die bo\u017fend v\u00ffn v\u00ffn\\n s\u00fft man noch ab\u017f\u017ferentlich\\n in fund v\u00ffn f\u00fcr v\u00ffn die\\n\\nnagel don ay v\u00ffn\u017ft an\\n dem funentag pr\u00ffg e\",\"addition1\":\"Anno domini 1444\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-4.1-nano-2025-04-14", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 4247, "output_tokens": 245, "total_tokens": 4492, "input_cost_usd": 0.0004247, "output_cost_usd": 9.8e-05, "estimated_cost_usd": 0.0005227000000000001}, "duration": 4.212749719619751, "timestamp": "2026-02-17T11:51:04.985875", "parsed": {"folios": [{"folio": "", "text": "Gelobt mu fronlaster die bo\u017fend\n mir gantz vnd ein s\u00ffde vnd\n dant heiligen fron v\u00fft ez\n lie in uff e brucht vnd ein\n beym v\u00ffn pant marien madalene\n v\u00ffgkhot \ua75bp\ua75bf samptag noch\n dem oster tag der so was der\n ander t\u00fff die abwelten do\n\nmenschn fuend v\u00ffn v\u00ffn me\u00ffn v\u00ffn\n guet in pont \ua75bf\u017ftumb\n \n\nkatek\u00ffn zu funden der end\u00ffn\n die k\u00ffn\u017ftig ar\u017f\u017flig in emen\n al\u017far die bo\u017fend v\u00ffn v\u00ffn\n s\u00fft man noch ab\u017f\u017ferentlich\n in fund v\u00ffn f\u00fcr v\u00ffn die\n\nnagel don ay v\u00ffn\u017ft an\n dem funentag pr\u00ffg e", "addition1": "Anno domini 1444", "addition2": "", "addition3": ""}]}, "conversation_id": "30cfc7b6-f72e-4e6f-bf67-8b6ca8094d4b", "score": {"fuzzy": 0.443, "cer": 0.646}, "raw_response": "{\"id\":\"chatcmpl-DACxRYuSKBylSOqiPV5nJBEqkoqIG\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"folios\\\": [{\\\"folio\\\": \\\"\\\", \\\"text\\\": \\\"Gelobt mu fronlaster die bo\u017fend\\\\n mir gantz vnd ein s\u00ffde vnd\\\\n dant heiligen fron v\u00fft ez\\\\n lie in uff e brucht vnd ein\\\\n beym v\u00ffn pant marien madalene\\\\n v\u00ffgkhot \ua75bp\ua75bf samptag noch\\\\n dem oster tag der so was der\\\\n ander t\u00fff die abwelten do\\\\n\\\\nmenschn fuend v\u00ffn v\u00ffn me\u00ffn v\u00ffn\\\\n guet in pont \ua75bf\u017ftumb\\\\n \\\\n\\\\nkatek\u00ffn zu funden der end\u00ffn\\\\n die k\u00ffn\u017ftig ar\u017f\u017flig in emen\\\\n al\u017far die bo\u017fend v\u00ffn v\u00ffn\\\\n s\u00fft man noch ab\u017f\u017ferentlich\\\\n in fund v\u00ffn f\u00fcr v\u00ffn die\\\\n\\\\nnagel don ay v\u00ffn\u017ft an\\\\n dem funentag pr\u00ffg e\\\", \\\"addition1\\\": \\\"Anno domini 1444\\\", \\\"addition2\\\": \\\"\\\", \\\"addition3\\\": \\\"\\\"}]}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"Gelobt mu fronlaster die bo\u017fend\\n mir gantz vnd ein s\u00ffde vnd\\n dant heiligen fron v\u00fft ez\\n lie in uff e brucht vnd ein\\n beym v\u00ffn pant marien madalene\\n v\u00ffgkhot \ua75bp\ua75bf samptag noch\\n dem oster tag der so was der\\n ander t\u00fff die abwelten do\\n\\nmenschn fuend v\u00ffn v\u00ffn me\u00ffn v\u00ffn\\n guet in pont \ua75bf\u017ftumb\\n \\n\\nkatek\u00ffn zu funden der end\u00ffn\\n die k\u00ffn\u017ftig ar\u017f\u017flig in emen\\n al\u017far die bo\u017fend v\u00ffn v\u00ffn\\n s\u00fft man noch ab\u017f\u017ferentlich\\n in fund v\u00ffn f\u00fcr v\u00ffn die\\n\\nnagel don ay v\u00ffn\u017ft an\\n dem funentag pr\u00ffg e\",\"addition1\":\"Anno domini 1444\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1771325461,\"model\":\"gpt-4.1-nano-2025-04-14\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b0b94f3709\",\"usage\":{\"completion_tokens\":245,\"prompt_tokens\":4247,\"total_tokens\":4492,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.538, "cer": 0.516, "cost_summary": {"total_input_tokens": 51114, "total_output_tokens": 2160, "total_tokens": 53274, "input_cost_usd": 0.0051113999999999995, "output_cost_usd": 0.0008640000000000001, "total_cost_usd": 0.0059754000000000005}}, "normalized_score": 48.4}, {"_id": {"$oid": "69d4eb12b877594bc5718c0d"}, "test_id": "T0279", "benchmark": "medieval_manuscripts", "date": "2026-01-26", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-5-2025-08-07", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1570, "output_tokens": 4707, "total_tokens": 6277, "input_cost_usd": 0.0019625, "output_cost_usd": 0.04707, "estimated_cost_usd": 0.0490325}, "duration": 114.36210894584656, "timestamp": "2026-01-26T15:59:46.498625", "parsed": {"folios": [{"folio": "", "text": "", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "b233e3fc-9c24-44a3-af71-97c1c392a45d", "score": {"fuzzy": 0.0, "cer": 1.0}, "raw_response": "{\"id\":\"chatcmpl-D2IKIF13eYZ1mPIRQlNzWIDRZvn61\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"folios\\\":[{\\\"folio\\\":\\\"\\\",\\\"text\\\":\\\"\\\",\\\"addition1\\\":\\\"\\\",\\\"addition2\\\":\\\"\\\",\\\"addition3\\\":\\\"\\\"}]}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1769439474,\"model\":\"gpt-5-2025-08-07\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":4707,\"prompt_tokens\":1570,\"total_tokens\":6277,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":4672,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.462, "cer": 0.566, "cost_summary": {"total_input_tokens": 18840, "total_output_tokens": 62926, "total_tokens": 81766, "input_cost_usd": 0.02354999999999999, "output_cost_usd": 0.6292599999999999, "total_cost_usd": 0.65281}}, "normalized_score": 43.400000000000006}, {"_id": {"$oid": "69d4eb12b877594bc5718c0e"}, "test_id": "T0275", "benchmark": "medieval_manuscripts", "date": "2026-01-26", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4o", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"73\",\"text\":\"Jch zu dem ersten Mij heid graff allwart vo\\ninhalt\\nJch her ludwig lantgraff von hursenberg\\nJch graff out von hemmenberg\\nJch graff Beu\u0364nha\u0364rt von segenstein\\nJch graff giffried von hassenbach\\nJch graff Eberhart von Mansfelt\\nJch her ludwig von Altenburg\\nJch her Gottschalk von Wese\u00df\\nJch her kuntz von turgen\\nJch her Sigemund von\\nJch her Ruprecht von stein waz woch her Ruter\\nJch hus vom Rotenhan\\nJch her von waldorff\\nJch hisprand von thungen\\nJch her von Elcben\\nJch henin quap\",\"addition1\":\"bespancket\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-4o-2024-08-06", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1766, "output_tokens": 202, "total_tokens": 1968, "input_cost_usd": 0.004415, "output_cost_usd": 0.00202, "estimated_cost_usd": 0.006435}, "duration": 10.777731657028198, "timestamp": "2026-01-26T17:26:00.630036", "parsed": {"folios": [{"folio": "73", "text": "Jch zu dem ersten Mij heid graff allwart vo\ninhalt\nJch her ludwig lantgraff von hursenberg\nJch graff out von hemmenberg\nJch graff Beu\u0364nha\u0364rt von segenstein\nJch graff giffried von hassenbach\nJch graff Eberhart von Mansfelt\nJch her ludwig von Altenburg\nJch her Gottschalk von Wese\u00df\nJch her kuntz von turgen\nJch her Sigemund von\nJch her Ruprecht von stein waz woch her Ruter\nJch hus vom Rotenhan\nJch her von waldorff\nJch hisprand von thungen\nJch her von Elcben\nJch henin quap", "addition1": "bespancket", "addition2": "", "addition3": ""}]}, "conversation_id": "df2b580b-2b58-48fc-b8db-358fa4ef740a", "score": {"fuzzy": 0.566, "cer": 0.444}, "raw_response": "{\"id\":\"chatcmpl-D2JhPspwcl1XZnVBgmJVHHQyQdwpl\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"folios\\\":[{\\\"folio\\\":\\\"73\\\",\\\"text\\\":\\\"Jch zu dem ersten Mij heid graff allwart vo\\\\ninhalt\\\\nJch her ludwig lantgraff von hursenberg\\\\nJch graff out von hemmenberg\\\\nJch graff Beu\u0364nha\u0364rt von segenstein\\\\nJch graff giffried von hassenbach\\\\nJch graff Eberhart von Mansfelt\\\\nJch her ludwig von Altenburg\\\\nJch her Gottschalk von Wese\u00df\\\\nJch her kuntz von turgen\\\\nJch her Sigemund von\\\\nJch her Ruprecht von stein waz woch her Ruter\\\\nJch hus vom Rotenhan\\\\nJch her von waldorff\\\\nJch hisprand von thungen\\\\nJch her von Elcben\\\\nJch henin quap\\\",\\\"addition1\\\":\\\"bespancket\\\",\\\"addition2\\\":\\\"\\\",\\\"addition3\\\":\\\"\\\"}]}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"73\",\"text\":\"Jch zu dem ersten Mij heid graff allwart vo\\ninhalt\\nJch her ludwig lantgraff von hursenberg\\nJch graff out von hemmenberg\\nJch graff Beu\u0364nha\u0364rt von segenstein\\nJch graff giffried von hassenbach\\nJch graff Eberhart von Mansfelt\\nJch her ludwig von Altenburg\\nJch her Gottschalk von Wese\u00df\\nJch her kuntz von turgen\\nJch her Sigemund von\\nJch her Ruprecht von stein waz woch her Ruter\\nJch hus vom Rotenhan\\nJch her von waldorff\\nJch hisprand von thungen\\nJch her von Elcben\\nJch henin quap\",\"addition1\":\"bespancket\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1769444751,\"model\":\"gpt-4o-2024-08-06\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_deacdd5f6f\",\"usage\":{\"completion_tokens\":202,\"prompt_tokens\":1766,\"total_tokens\":1968,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.65, "cer": 0.383, "cost_summary": {"total_input_tokens": 21192, "total_output_tokens": 2274, "total_tokens": 23466, "input_cost_usd": 0.052980000000000006, "output_cost_usd": 0.022740000000000003, "total_cost_usd": 0.07571999999999998}}, "normalized_score": 61.7}, {"_id": {"$oid": "69d4eb12b877594bc5718c16"}, "test_id": "T0409", "benchmark": "medieval_manuscripts", "date": "2026-01-26", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.1-2025-11-13", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"3\",\"text\":\"vnd ein pferit die mir vnd\\nminen knechten vber hulffend\\ndas So was miners t\u00e4nt weg\\nsame das wir machtend\\nund vilclach die knecht dut\\nund vil in vntz an den ardt\\nund die pferit vntz an die\\nstatl und was ze mal ein groszer\\nnebel das wir kun ge\u017fehend\\nund als mit groszer arbait kamen\\nwir ze mitten tag z\u016f sant\\nfrantzstol vff den berg So\\nSo sach ich die bu\u0364rger So gar\\nvil herren wap\u1e9den zu pferd\\nvnd in aller scheyn gestenck gantz\\nv\u00f1 sinnd miners vatters flagen\\nwap\u00f1 auch in dem schein\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-5.1-2025-11-13", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1570, "output_tokens": 200, "total_tokens": 1770, "input_cost_usd": 0.0019625, "output_cost_usd": 0.002, "estimated_cost_usd": 0.0039625}, "duration": 10.364962577819824, "timestamp": "2026-01-26T00:30:43.511695", "parsed": {"folios": [{"folio": "3", "text": "vnd ein pferit die mir vnd\nminen knechten vber hulffend\ndas So was miners t\u00e4nt weg\nsame das wir machtend\nund vilclach die knecht dut\nund vil in vntz an den ardt\nund die pferit vntz an die\nstatl und was ze mal ein groszer\nnebel das wir kun ge\u017fehend\nund als mit groszer arbait kamen\nwir ze mitten tag z\u016f sant\nfrantzstol vff den berg So\nSo sach ich die bu\u0364rger So gar\nvil herren wap\u1e9den zu pferd\nvnd in aller scheyn gestenck gantz\nv\u00f1 sinnd miners vatters flagen\nwap\u00f1 auch in dem schein", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "a0a236b9-ddd8-43c8-b7bd-25283fbd22e6", "score": {"fuzzy": 0.895, "cer": 0.118}, "raw_response": "{\"id\":\"chatcmpl-D23qszuKj0T0A3ta9S5RmYdnFXdmF\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"folios\\\":[{\\\"folio\\\":\\\"3\\\",\\\"text\\\":\\\"vnd ein pferit die mir vnd\\\\nminen knechten vber hulffend\\\\ndas So was miners t\u00e4nt weg\\\\nsame das wir machtend\\\\nund vilclach die knecht dut\\\\nund vil in vntz an den ardt\\\\nund die pferit vntz an die\\\\nstatl und was ze mal ein groszer\\\\nnebel das wir kun ge\u017fehend\\\\nund als mit groszer arbait kamen\\\\nwir ze mitten tag z\u016f sant\\\\nfrantzstol vff den berg So\\\\nSo sach ich die bu\u0364rger So gar\\\\nvil herren wap\u1e9den zu pferd\\\\nvnd in aller scheyn gestenck gantz\\\\nv\u00f1 sinnd miners vatters flagen\\\\nwap\u00f1 auch in dem schein\\\",\\\"addition1\\\":\\\"\\\",\\\"addition2\\\":\\\"\\\",\\\"addition3\\\":\\\"\\\"}]}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"3\",\"text\":\"vnd ein pferit die mir vnd\\nminen knechten vber hulffend\\ndas So was miners t\u00e4nt weg\\nsame das wir machtend\\nund vilclach die knecht dut\\nund vil in vntz an den ardt\\nund die pferit vntz an die\\nstatl und was ze mal ein groszer\\nnebel das wir kun ge\u017fehend\\nund als mit groszer arbait kamen\\nwir ze mitten tag z\u016f sant\\nfrantzstol vff den berg So\\nSo sach ich die bu\u0364rger So gar\\nvil herren wap\u1e9den zu pferd\\nvnd in aller scheyn gestenck gantz\\nv\u00f1 sinnd miners vatters flagen\\nwap\u00f1 auch in dem schein\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1769383834,\"model\":\"gpt-5.1-2025-11-13\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":200,\"prompt_tokens\":1570,\"total_tokens\":1770,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.655, "cer": 0.371, "cost_summary": {"total_input_tokens": 18840, "total_output_tokens": 2626, "total_tokens": 21466, "input_cost_usd": 0.02354999999999999, "output_cost_usd": 0.026260000000000002, "total_cost_usd": 0.04981000000000001}}, "normalized_score": 62.9}, {"_id": {"$oid": "69d4eb12b877594bc5718c17"}, "test_id": "T0282", "benchmark": "medieval_manuscripts", "date": "2026-01-26", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "o3", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"93\",\"text\":\"u\u00df dem land ze\\nmeagklingung\\nEinrat Tobe Gut\\nDietrich N\u00fctzow\\npeter sin bos priester\\n\\nu\u00df Entringen\\nJakob von Garzhant stat\\nSebastian de m\u00fcnrat priester\\nCota von Langer sin knecht\\n\\nu\u00df to\u00dfens lant\\nJakob Neslang\\nHans Mundt\\nHeinrich Lang\\n\\nvon Frosburg\\nHans der Kenselt ein pfir\\nfrut Jacob Orsin\u00df\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "o3-2025-04-16", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1635, "output_tokens": 3827, "total_tokens": 5462, "input_cost_usd": 0.00327, "output_cost_usd": 0.030616, "estimated_cost_usd": 0.033886}, "duration": 98.50842022895813, "timestamp": "2026-01-26T15:12:47.171010", "parsed": {"folios": [{"folio": "93", "text": "u\u00df dem land ze\nmeagklingung\nEinrat Tobe Gut\nDietrich N\u00fctzow\npeter sin bos priester\n\nu\u00df Entringen\nJakob von Garzhant stat\nSebastian de m\u00fcnrat priester\nCota von Langer sin knecht\n\nu\u00df to\u00dfens lant\nJakob Neslang\nHans Mundt\nHeinrich Lang\n\nvon Frosburg\nHans der Kenselt ein pfir\nfrut Jacob Orsin\u00df", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "a799a36f-59d5-4d01-a25e-c400734b8831", "score": {"fuzzy": 0.841, "cer": 0.172}, "raw_response": "{\"id\":\"chatcmpl-D2Hb473n2HLgUCfewomdVyH50pbN7\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"93\\\",\\n      \\\"text\\\": \\\"u\u00df dem land ze\\\\nmeagklingung\\\\nEinrat Tobe Gut\\\\nDietrich N\u00fctzow\\\\npeter sin bos priester\\\\n\\\\nu\u00df Entringen\\\\nJakob von Garzhant stat\\\\nSebastian de m\u00fcnrat priester\\\\nCota von Langer sin knecht\\\\n\\\\nu\u00df to\u00dfens lant\\\\nJakob Neslang\\\\nHans Mundt\\\\nHeinrich Lang\\\\n\\\\nvon Frosburg\\\\nHans der Kenselt ein pfir\\\\nfrut Jacob Orsin\u00df\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"93\",\"text\":\"u\u00df dem land ze\\nmeagklingung\\nEinrat Tobe Gut\\nDietrich N\u00fctzow\\npeter sin bos priester\\n\\nu\u00df Entringen\\nJakob von Garzhant stat\\nSebastian de m\u00fcnrat priester\\nCota von Langer sin knecht\\n\\nu\u00df to\u00dfens lant\\nJakob Neslang\\nHans Mundt\\nHeinrich Lang\\n\\nvon Frosburg\\nHans der Kenselt ein pfir\\nfrut Jacob Orsin\u00df\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1769436670,\"model\":\"o3-2025-04-16\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":3827,\"prompt_tokens\":1635,\"total_tokens\":5462,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":3648,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.63, "cer": 0.403, "cost_summary": {"total_input_tokens": 19620, "total_output_tokens": 39534, "total_tokens": 59154, "input_cost_usd": 0.03924, "output_cost_usd": 0.316272, "total_cost_usd": 0.355512}}, "normalized_score": 59.699999999999996}, {"_id": {"$oid": "69d4eb12b877594bc5718c25"}, "test_id": "T0281", "benchmark": "medieval_manuscripts", "date": "2026-01-26", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"18\",\"text\":\"\",\"addition1\":\"Sarcharin in Cypro nasfin\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-5-nano-2025-08-07", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 2847, "output_tokens": 2502, "total_tokens": 5349, "input_cost_usd": 0.00014235000000000002, "output_cost_usd": 0.0010008, "estimated_cost_usd": 0.0011431500000000001}, "duration": 27.426353931427002, "timestamp": "2026-01-26T15:04:25.627737", "parsed": {"folios": [{"folio": "18", "text": "", "addition1": "Sarcharin in Cypro nasfin", "addition2": "", "addition3": ""}]}, "conversation_id": "3ef9cb3c-5b24-46a1-b484-2d04cebcaa8f", "score": {"fuzzy": 0.595, "cer": 0.403}, "raw_response": "{\"id\":\"chatcmpl-D2HU8lSWRRz3pieSWFr94VNioZmge\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"18\\\",\\n      \\\"text\\\": \\\"\\\",\\n      \\\"addition1\\\": \\\"Sarcharin in Cypro nasfin\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"18\",\"text\":\"\",\"addition1\":\"Sarcharin in Cypro nasfin\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1769436240,\"model\":\"gpt-5-nano-2025-08-07\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":2502,\"prompt_tokens\":2847,\"total_tokens\":5349,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":2432,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.216, "cer": 0.784, "cost_summary": {"total_input_tokens": 34164, "total_output_tokens": 22119, "total_tokens": 56283, "input_cost_usd": 0.0017082000000000006, "output_cost_usd": 0.0088476, "total_cost_usd": 0.010555800000000002}}, "normalized_score": 21.599999999999994}, {"_id": {"$oid": "69d4eb12b877594bc5718c28"}, "test_id": "T0560", "benchmark": "medieval_manuscripts", "date": "2026-01-26", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "ministral-8b-2512", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"In dem namen des herren amen. \\n\\nGelobet sey got der herre. \\n\\nVnd nu wil ich euch sagen von \\nvnderm wunderlichen geschicht \\nvon dem heiligen martin. \\n\\nEs was ein man in dem lande \\nGallien, der hete namen \\nGundobad, der was ein kunig \\nvnd ein edler man. Der hatte \\nzwei sun, die heten namen \\nChlodovig vnd Childebert. \\n\\nDer Chlodovig was ein junger \\nman, der was ein guter \\nvnd ein fromer man. Der \\nwolt got dienen vnd \\nseine sunnen erziehen. \\n\\nEs was aber ein man in dem \\nlande, der hete namen \\nGundobad, der was ein heiden \\nvnd ein boser man. Der \\nwolt den Chlodovig vnd \\nseine sunnen umbringen. \\n\\nDoch got hat den Chlodovig \\nvnd seine sunnen beschirmet. \\n\\nDer Chlodovig hat sich getauft \\nvnd ist ein christen worden. \\n\\nDer heilige martin hat den \\nChlodovig getauft vnd \\nihm den glauben gebracht. \\n\\nDer Chlodovig hat dann \\nseine sunnen getauft vnd \\nihm den glauben gebracht. \\n\\nDer Chlodovig hat dann \\nsein land in drey teil geteilt \\nvnd hat seinen sunnen \\nChildebert vnd Chlotar \\ndas land gegeben. \\n\\nDer Chlodovig ist dann \\nin got gegangen vnd hat \\nsein seel in got gegeben. \\n\\nVnd nu wil ich euch sagen \\nvon dem wunderlichen geschicht \\nvon dem heiligen martin. \\n\\nEs was ein man in dem lande \\nGallien, der hete namen \\nGundobad, der was ein kunig \\nvnd ein edler man. Der hatte \\nzwei sun, die heten namen \\nChlodovig vnd Childebert. \\n\\nDer Chlodovig was ein junger \\nman, der was ein guter \\nvnd ein fromer man. Der \\nwolt got dienen vnd \\nseine sunnen erziehen. \\n\\nEs was aber ein man in dem \\nlande, der hete namen \\nGundobad, der was ein heiden \\nvnd ein boser man. Der \\nwolt den Chlodovig vnd \\nseine sunnen umbringen. \\n\\nDoch got hat den Chlodovig \\nvnd seine sunnen beschirmet. \\n\\nDer Chlodovig hat sich getauft \\nvnd ist ein christen worden. \\n\\nDer heilige martin hat den \\nChlodovig getauft vnd \\nihm den glauben gebracht. \\n\\nDer Chlodovig hat dann \\nseine sunnen getauft vnd \\nihm den glauben gebracht. \\n\\nDer Chlodovig hat dann \\nsein land in drey teil geteilt \\nvnd hat seinen sunnen \\nChildebert vnd Chlotar \\ndas land gegeben. \\n\\nDer Chlodovig ist dann \\nin got gegangen vnd hat \\nsein seel in got gegeben. \\n\\nAnno domini MCCCCLXXIIII\",\"addition1\":\"Hic incipit liber de sancto Martino. \\n\\nNotandum: Hic liber fuit scriptus \\nper manus magistri Henrici \\nBasileae anno domini MCCCCLXXIIII.\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "ministral-8b-2512", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 753, "output_tokens": 897, "total_tokens": 1650, "input_cost_usd": 0.00011295, "output_cost_usd": 0.00013455, "estimated_cost_usd": 0.0002475}, "duration": 5.179831504821777, "timestamp": "2026-01-26T23:00:26.409910", "parsed": {"$schema": "http://json-schema.org/draft-07/schema#", "folios": [{"folio": "", "text": "In dem namen des herren amen. \n\nGelobet sey got der herre. \n\nVnd nu wil ich euch sagen von \nvnderm wunderlichen geschicht \nvon dem heiligen martin. \n\nEs was ein man in dem lande \nGallien, der hete namen \nGundobad, der was ein kunig \nvnd ein edler man. Der hatte \nzwei sun, die heten namen \nChlodovig vnd Childebert. \n\nDer Chlodovig was ein junger \nman, der was ein guter \nvnd ein fromer man. Der \nwolt got dienen vnd \nseine sunnen erziehen. \n\nEs was aber ein man in dem \nlande, der hete namen \nGundobad, der was ein heiden \nvnd ein boser man. Der \nwolt den Chlodovig vnd \nseine sunnen umbringen. \n\nDoch got hat den Chlodovig \nvnd seine sunnen beschirmet. \n\nDer Chlodovig hat sich getauft \nvnd ist ein christen worden. \n\nDer heilige martin hat den \nChlodovig getauft vnd \nihm den glauben gebracht. \n\nDer Chlodovig hat dann \nseine sunnen getauft vnd \nihm den glauben gebracht. \n\nDer Chlodovig hat dann \nsein land in drey teil geteilt \nvnd hat seinen sunnen \nChildebert vnd Chlotar \ndas land gegeben. \n\nDer Chlodovig ist dann \nin got gegangen vnd hat \nsein seel in got gegeben. \n\nVnd nu wil ich euch sagen \nvon dem wunderlichen geschicht \nvon dem heiligen martin. \n\nEs was ein man in dem lande \nGallien, der hete namen \nGundobad, der was ein kunig \nvnd ein edler man. Der hatte \nzwei sun, die heten namen \nChlodovig vnd Childebert. \n\nDer Chlodovig was ein junger \nman, der was ein guter \nvnd ein fromer man. Der \nwolt got dienen vnd \nseine sunnen erziehen. \n\nEs was aber ein man in dem \nlande, der hete namen \nGundobad, der was ein heiden \nvnd ein boser man. Der \nwolt den Chlodovig vnd \nseine sunnen umbringen. \n\nDoch got hat den Chlodovig \nvnd seine sunnen beschirmet. \n\nDer Chlodovig hat sich getauft \nvnd ist ein christen worden. \n\nDer heilige martin hat den \nChlodovig getauft vnd \nihm den glauben gebracht. \n\nDer Chlodovig hat dann \nseine sunnen getauft vnd \nihm den glauben gebracht. \n\nDer Chlodovig hat dann \nsein land in drey teil geteilt \nvnd hat seinen sunnen \nChildebert vnd Chlotar \ndas land gegeben. \n\nDer Chlodovig ist dann \nin got gegangen vnd hat \nsein seel in got gegeben. \n\nAnno domini MCCCCLXXIIII", "addition1": "Hic incipit liber de sancto Martino. \n\nNotandum: Hic liber fuit scriptus \nper manus magistri Henrici \nBasileae anno domini MCCCCLXXIIII.", "addition2": "", "addition3": ""}]}, "conversation_id": "66a3b7a0-53cc-4672-960f-562b3d8758b2", "score": {"fuzzy": 0.065, "cer": 1.0}, "raw_response": "{\"id\":\"12e558ce269345388aa77e96a7a7d58b\",\"object\":\"chat.completion\",\"model\":\"ministral-8b-2512\",\"usage\":{\"prompt_tokens\":753,\"completion_tokens\":897,\"total_tokens\":1650,\"num_cached_tokens\":0},\"created\":1769464821,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"$schema\\\": \\\"http://json-schema.org/draft-07/schema#\\\",\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"In dem namen des herren amen. \\\\n\\\\nGelobet sey got der herre. \\\\n\\\\nVnd nu wil ich euch sagen von \\\\nvnderm wunderlichen geschicht \\\\nvon dem heiligen martin. \\\\n\\\\nEs was ein man in dem lande \\\\nGallien, der hete namen \\\\nGundobad, der was ein kunig \\\\nvnd ein edler man. Der hatte \\\\nzwei sun, die heten namen \\\\nChlodovig vnd Childebert. \\\\n\\\\nDer Chlodovig was ein junger \\\\nman, der was ein guter \\\\nvnd ein fromer man. Der \\\\nwolt got dienen vnd \\\\nseine sunnen erziehen. \\\\n\\\\nEs was aber ein man in dem \\\\nlande, der hete namen \\\\nGundobad, der was ein heiden \\\\nvnd ein boser man. Der \\\\nwolt den Chlodovig vnd \\\\nseine sunnen umbringen. \\\\n\\\\nDoch got hat den Chlodovig \\\\nvnd seine sunnen beschirmet. \\\\n\\\\nDer Chlodovig hat sich getauft \\\\nvnd ist ein christen worden. \\\\n\\\\nDer heilige martin hat den \\\\nChlodovig getauft vnd \\\\nihm den glauben gebracht. \\\\n\\\\nDer Chlodovig hat dann \\\\nseine sunnen getauft vnd \\\\nihm den glauben gebracht. \\\\n\\\\nDer Chlodovig hat dann \\\\nsein land in drey teil geteilt \\\\nvnd hat seinen sunnen \\\\nChildebert vnd Chlotar \\\\ndas land gegeben. \\\\n\\\\nDer Chlodovig ist dann \\\\nin got gegangen vnd hat \\\\nsein seel in got gegeben. \\\\n\\\\nVnd nu wil ich euch sagen \\\\nvon dem wunderlichen geschicht \\\\nvon dem heiligen martin. \\\\n\\\\nEs was ein man in dem lande \\\\nGallien, der hete namen \\\\nGundobad, der was ein kunig \\\\nvnd ein edler man. Der hatte \\\\nzwei sun, die heten namen \\\\nChlodovig vnd Childebert. \\\\n\\\\nDer Chlodovig was ein junger \\\\nman, der was ein guter \\\\nvnd ein fromer man. Der \\\\nwolt got dienen vnd \\\\nseine sunnen erziehen. \\\\n\\\\nEs was aber ein man in dem \\\\nlande, der hete namen \\\\nGundobad, der was ein heiden \\\\nvnd ein boser man. Der \\\\nwolt den Chlodovig vnd \\\\nseine sunnen umbringen. \\\\n\\\\nDoch got hat den Chlodovig \\\\nvnd seine sunnen beschirmet. \\\\n\\\\nDer Chlodovig hat sich getauft \\\\nvnd ist ein christen worden. \\\\n\\\\nDer heilige martin hat den \\\\nChlodovig getauft vnd \\\\nihm den glauben gebracht. \\\\n\\\\nDer Chlodovig hat dann \\\\nseine sunnen getauft vnd \\\\nihm den glauben gebracht. \\\\n\\\\nDer Chlodovig hat dann \\\\nsein land in drey teil geteilt \\\\nvnd hat seinen sunnen \\\\nChildebert vnd Chlotar \\\\ndas land gegeben. \\\\n\\\\nDer Chlodovig ist dann \\\\nin got gegangen vnd hat \\\\nsein seel in got gegeben. \\\\n\\\\nAnno domini MCCCCLXXIIII\\\",\\n      \\\"addition1\\\": \\\"Hic incipit liber de sancto Martino. \\\\n\\\\nNotandum: Hic liber fuit scriptus \\\\nper manus magistri Henrici \\\\nBasileae anno domini MCCCCLXXIIII.\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": null, "normalized_score": null}, {"_id": {"$oid": "69d4eb12b877594bc5718c2b"}, "test_id": "T0280", "benchmark": "medieval_manuscripts", "date": "2026-01-26", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Selbs im fronaltar die pschand\\nnim gantz vnd ein st\u00fcck von\\ndem heiligin frey cruz dz\\nsit in uff ne brust vnd ein\\nbarn von sant marien madalene\\nvingstrot yff off samstag noch\\nden osterstag daz do was der\\nander tag dis abrechtin do\\nmunan f\u00fcrend wir gen mensch do\\nlegend in sant steffans\\nkelsin bi hundirt der kundlin\\ndie herodes caplich in ernen\\naltar die pschend wir vnd\\nricht may noch bestenderlich\\nin sand vnd fur vnd die\\nnagil dar abr xpinstag on\\nden fundamentag frig do\",\"addition1\":\"Muran glau zi\u00fctten\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-5-mini-2025-08-07", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 2410, "output_tokens": 2074, "total_tokens": 4484, "input_cost_usd": 0.0006025, "output_cost_usd": 0.004148, "estimated_cost_usd": 0.0047504999999999995}, "duration": 37.227211236953735, "timestamp": "2026-01-26T14:57:29.512415", "parsed": {"folios": [{"folio": "", "text": "Selbs im fronaltar die pschand\nnim gantz vnd ein st\u00fcck von\ndem heiligin frey cruz dz\nsit in uff ne brust vnd ein\nbarn von sant marien madalene\nvingstrot yff off samstag noch\nden osterstag daz do was der\nander tag dis abrechtin do\nmunan f\u00fcrend wir gen mensch do\nlegend in sant steffans\nkelsin bi hundirt der kundlin\ndie herodes caplich in ernen\naltar die pschend wir vnd\nricht may noch bestenderlich\nin sand vnd fur vnd die\nnagil dar abr xpinstag on\nden fundamentag frig do", "addition1": "Muran glau zi\u00fctten", "addition2": "", "addition3": ""}]}, "conversation_id": "50bcccb5-66fe-4161-9d63-21bb2ecaad20", "score": {"fuzzy": 0.814, "cer": 0.19}, "raw_response": "{\"id\":\"chatcmpl-D2HNFWXEvom6xofn5TBqJqwJX1vyH\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Selbs im fronaltar die pschand\\\\nnim gantz vnd ein st\u00fcck von\\\\ndem heiligin frey cruz dz\\\\nsit in uff ne brust vnd ein\\\\nbarn von sant marien madalene\\\\nvingstrot yff off samstag noch\\\\nden osterstag daz do was der\\\\nander tag dis abrechtin do\\\\nmunan f\u00fcrend wir gen mensch do\\\\nlegend in sant steffans\\\\nkelsin bi hundirt der kundlin\\\\ndie herodes caplich in ernen\\\\naltar die pschend wir vnd\\\\nricht may noch bestenderlich\\\\nin sand vnd fur vnd die\\\\nnagil dar abr xpinstag on\\\\nden fundamentag frig do\\\",\\n      \\\"addition1\\\": \\\"Muran glau zi\u00fctten\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"Selbs im fronaltar die pschand\\nnim gantz vnd ein st\u00fcck von\\ndem heiligin frey cruz dz\\nsit in uff ne brust vnd ein\\nbarn von sant marien madalene\\nvingstrot yff off samstag noch\\nden osterstag daz do was der\\nander tag dis abrechtin do\\nmunan f\u00fcrend wir gen mensch do\\nlegend in sant steffans\\nkelsin bi hundirt der kundlin\\ndie herodes caplich in ernen\\naltar die pschend wir vnd\\nricht may noch bestenderlich\\nin sand vnd fur vnd die\\nnagil dar abr xpinstag on\\nden fundamentag frig do\",\"addition1\":\"Muran glau zi\u00fctten\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1769435813,\"model\":\"gpt-5-mini-2025-08-07\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":2074,\"prompt_tokens\":2410,\"total_tokens\":4484,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":1856,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.749, "cer": 0.29, "cost_summary": {"total_input_tokens": 28995, "total_output_tokens": 24398, "total_tokens": 53393, "input_cost_usd": 0.007248749999999998, "output_cost_usd": 0.048796, "total_cost_usd": 0.05604475}}, "normalized_score": 71.0}, {"_id": {"$oid": "69d4eb12b877594bc5718c2c"}, "test_id": "T0273", "benchmark": "medieval_manuscripts", "date": "2026-01-26", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"93\",\"text\":\"V\u00df dem land ze\\n magt\u00dflinting\\n Eimrat Goli pritt\\n Dietrint liebzow\\n peter ins das fu\u00df pri\u00dft\\n\\nV\u00df luttringen\\n Jakob von Hartzunt pritt\\n Sebastian de mintrant pri\u00dft\\n Cola von Gayoy pri Ewalt\\n\\nV\u00df golssen land\\n Jakob Melang\\n Hans Ruddech\\n Heinriech Lang\\n\\nVon grosburg\\n Hans von kanselt ein pri\u00dft\\n Jant Jakons dresin\u00df\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-4.1-2025-04-14", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1766, "output_tokens": 152, "total_tokens": 1918, "input_cost_usd": 0.003532, "output_cost_usd": 0.001216, "estimated_cost_usd": 0.0047480000000000005}, "duration": 7.363040447235107, "timestamp": "2026-01-26T17:26:12.940490", "parsed": {"folios": [{"folio": "93", "text": "V\u00df dem land ze\n magt\u00dflinting\n Eimrat Goli pritt\n Dietrint liebzow\n peter ins das fu\u00df pri\u00dft\n\nV\u00df luttringen\n Jakob von Hartzunt pritt\n Sebastian de mintrant pri\u00dft\n Cola von Gayoy pri Ewalt\n\nV\u00df golssen land\n Jakob Melang\n Hans Ruddech\n Heinriech Lang\n\nVon grosburg\n Hans von kanselt ein pri\u00dft\n Jant Jakons dresin\u00df", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "d03a0f22-da00-459b-84e0-c22778b2850b", "score": {"fuzzy": 0.865, "cer": 0.147}, "raw_response": "{\"id\":\"chatcmpl-D2JhfIoqTA4I9WUw4xST25596qgMr\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"93\\\",\\n      \\\"text\\\": \\\"V\u00df dem land ze\\\\n magt\u00dflinting\\\\n Eimrat Goli pritt\\\\n Dietrint liebzow\\\\n peter ins das fu\u00df pri\u00dft\\\\n\\\\nV\u00df luttringen\\\\n Jakob von Hartzunt pritt\\\\n Sebastian de mintrant pri\u00dft\\\\n Cola von Gayoy pri Ewalt\\\\n\\\\nV\u00df golssen land\\\\n Jakob Melang\\\\n Hans Ruddech\\\\n Heinriech Lang\\\\n\\\\nVon grosburg\\\\n Hans von kanselt ein pri\u00dft\\\\n Jant Jakons dresin\u00df\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"93\",\"text\":\"V\u00df dem land ze\\n magt\u00dflinting\\n Eimrat Goli pritt\\n Dietrint liebzow\\n peter ins das fu\u00df pri\u00dft\\n\\nV\u00df luttringen\\n Jakob von Hartzunt pritt\\n Sebastian de mintrant pri\u00dft\\n Cola von Gayoy pri Ewalt\\n\\nV\u00df golssen land\\n Jakob Melang\\n Hans Ruddech\\n Heinriech Lang\\n\\nVon grosburg\\n Hans von kanselt ein pri\u00dft\\n Jant Jakons dresin\u00df\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1769444767,\"model\":\"gpt-4.1-2025-04-14\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_1a2c4a5ede\",\"usage\":{\"completion_tokens\":152,\"prompt_tokens\":1766,\"total_tokens\":1918,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.678, "cer": 0.36, "cost_summary": {"total_input_tokens": 21192, "total_output_tokens": 2437, "total_tokens": 23629, "input_cost_usd": 0.042384, "output_cost_usd": 0.019495999999999996, "total_cost_usd": 0.06187999999999999}}, "normalized_score": 64.0}, {"_id": {"$oid": "69d4eb12b877594bc5718c30"}, "test_id": "T0277", "benchmark": "medieval_manuscripts", "date": "2026-01-26", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Und die pspel ist bi aoo mil\\ntwit vnd ist der venediger\\npst von kuresun vnz gon\\nmodun sind al mil vnd\\nist och ein giubspst stat vnd\\nein erzbistum vnd ist och\\nder venediger ze modun\\npachtet der rumanier vnd\\nist och do bi ein heilig hesset\\ndo bi ij milen\\nIst von modun bis gon candia\\nsind aoo mil vnd candia\\nist ein grob pspel vnz mil\\nweit vnd pachtet der\\nmaluasier do vnd ist och\\nder venedier\\nIst von candia vnz gon rodus\\nsind aoo mil wir aber\",\"addition1\":\"Candia\\nCreta\",\"addition2\":\"Vrinum\\nMalba\\nnicum\",\"addition3\":\"R(h)odos\"}]}", "model": "gpt-4.1-mini-2025-04-14", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 3023, "output_tokens": 201, "total_tokens": 3224, "input_cost_usd": 0.0012092000000000001, "output_cost_usd": 0.0003216, "estimated_cost_usd": 0.0015308000000000001}, "duration": 4.953597068786621, "timestamp": "2026-01-26T15:05:19.228539", "parsed": {"folios": [{"folio": "", "text": "Und die pspel ist bi aoo mil\ntwit vnd ist der venediger\npst von kuresun vnz gon\nmodun sind al mil vnd\nist och ein giubspst stat vnd\nein erzbistum vnd ist och\nder venediger ze modun\npachtet der rumanier vnd\nist och do bi ein heilig hesset\ndo bi ij milen\nIst von modun bis gon candia\nsind aoo mil vnd candia\nist ein grob pspel vnz mil\nweit vnd pachtet der\nmaluasier do vnd ist och\nder venedier\nIst von candia vnz gon rodus\nsind aoo mil wir aber", "addition1": "Candia\nCreta", "addition2": "Vrinum\nMalba\nnicum", "addition3": "R(h)odos"}]}, "conversation_id": "48dba149-30dc-4be3-a4d0-90ac807b6c92", "score": {"fuzzy": 0.761, "cer": 0.319}, "raw_response": "{\"id\":\"chatcmpl-D2HVLccaVUFfxvtVqm9xmbU99ruCU\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"folios\\\":[{\\\"folio\\\":\\\"\\\",\\\"text\\\":\\\"Und die pspel ist bi aoo mil\\\\ntwit vnd ist der venediger\\\\npst von kuresun vnz gon\\\\nmodun sind al mil vnd\\\\nist och ein giubspst stat vnd\\\\nein erzbistum vnd ist och\\\\nder venediger ze modun\\\\npachtet der rumanier vnd\\\\nist och do bi ein heilig hesset\\\\ndo bi ij milen\\\\nIst von modun bis gon candia\\\\nsind aoo mil vnd candia\\\\nist ein grob pspel vnz mil\\\\nweit vnd pachtet der\\\\nmaluasier do vnd ist och\\\\nder venedier\\\\nIst von candia vnz gon rodus\\\\nsind aoo mil wir aber\\\",\\\"addition1\\\":\\\"Candia\\\\nCreta\\\",\\\"addition2\\\":\\\"Vrinum\\\\nMalba\\\\nnicum\\\",\\\"addition3\\\":\\\"R(h)odos\\\"}]}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"Und die pspel ist bi aoo mil\\ntwit vnd ist der venediger\\npst von kuresun vnz gon\\nmodun sind al mil vnd\\nist och ein giubspst stat vnd\\nein erzbistum vnd ist och\\nder venediger ze modun\\npachtet der rumanier vnd\\nist och do bi ein heilig hesset\\ndo bi ij milen\\nIst von modun bis gon candia\\nsind aoo mil vnd candia\\nist ein grob pspel vnz mil\\nweit vnd pachtet der\\nmaluasier do vnd ist och\\nder venedier\\nIst von candia vnz gon rodus\\nsind aoo mil wir aber\",\"addition1\":\"Candia\\nCreta\",\"addition2\":\"Vrinum\\nMalba\\nnicum\",\"addition3\":\"R(h)odos\"}]}}}],\"created\":1769436315,\"model\":\"gpt-4.1-mini-2025-04-14\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_376a7ccef1\",\"usage\":{\"completion_tokens\":201,\"prompt_tokens\":3023,\"total_tokens\":3224,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.791, "cer": 0.228, "cost_summary": {"total_input_tokens": 36276, "total_output_tokens": 2359, "total_tokens": 38635, "input_cost_usd": 0.014510400000000005, "output_cost_usd": 0.0037744000000000007, "total_cost_usd": 0.018284800000000004}}, "normalized_score": 77.2}, {"_id": {"$oid": "69d4eb12b877594bc5718ba4"}, "test_id": "T0571", "benchmark": "medieval_manuscripts", "date": "2026-01-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "magistral-small-2509", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"12\",\"text\":\"Do man von der stat Basel\\n hort, daz der kunig kame,\\n so voren si sich mit aller\\n macht vnd wapnen. Sie\\n schloszen die pforten vnd\\n warden mit grosser vorsicht.\\n Do der kunig kam, fand er\\n die stat wol verretet.\\n Er belagerte si mit grossem\\n here, aber die burger\\n wehrten sich so kluge,\\n daz er sich wider mussen\\n ziehen, vnd si wurden\\n von im gelassen in friden.\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "magistral-small-2509", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 753, "output_tokens": 169, "total_tokens": 922, "input_cost_usd": 0.0003765, "output_cost_usd": 8.45e-05, "estimated_cost_usd": 0.000461}, "duration": 1.7280292510986328, "timestamp": "2026-01-25T01:54:54.181299", "parsed": {"folios": [{"folio": "12", "text": "Do man von der stat Basel\n hort, daz der kunig kame,\n so voren si sich mit aller\n macht vnd wapnen. Sie\n schloszen die pforten vnd\n warden mit grosser vorsicht.\n Do der kunig kam, fand er\n die stat wol verretet.\n Er belagerte si mit grossem\n here, aber die burger\n wehrten sich so kluge,\n daz er sich wider mussen\n ziehen, vnd si wurden\n von im gelassen in friden.", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "a4d5104d-b079-4af3-84c9-5b5fc47f3061", "score": {"fuzzy": 0.224, "cer": 0.838}, "raw_response": "{\"id\":\"0a029742ba2a4050a4f41860f84cddf3\",\"object\":\"chat.completion\",\"model\":\"magistral-small-2509\",\"usage\":{\"prompt_tokens\":753,\"completion_tokens\":169,\"total_tokens\":922,\"num_cached_tokens\":128},\"created\":1769302492,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"12\\\",\\n      \\\"text\\\": \\\"Do man von der stat Basel\\\\n hort, daz der kunig kame,\\\\n so voren si sich mit aller\\\\n macht vnd wapnen. Sie\\\\n schloszen die pforten vnd\\\\n warden mit grosser vorsicht.\\\\n Do der kunig kam, fand er\\\\n die stat wol verretet.\\\\n Er belagerte si mit grossem\\\\n here, aber die burger\\\\n wehrten sich so kluge,\\\\n daz er sich wider mussen\\\\n ziehen, vnd si wurden\\\\n von im gelassen in friden.\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": null, "normalized_score": null}, {"_id": {"$oid": "69d4eb12b877594bc5718ba7"}, "test_id": "T0301", "benchmark": "medieval_manuscripts", "date": "2026-01-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"90\",\"text\":\"Hainrich von Eilpetten\\nsin dochter\\nJumalus der probst\\n\\nvnd der stift von\\nsant Emonz\\nant Jo\ua75bans ordens ze mong\\nHainrich von wiffengartte\\nkunig smit sin Enckelin\\n\\nVon bury vnd\\nheining von Eicpach\\nGrant von der gruben\\nGrant nemay\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "meta-llama/llama-4-maverick", "provider": "openrouter", "finish_reason": "stop", "usage": {"input_tokens": 2825, "output_tokens": 131, "total_tokens": 2956, "cached_tokens": 496, "input_cost_usd": 0.00042375, "output_cost_usd": 7.86e-05, "estimated_cost_usd": 0.0005023499999999999}, "duration": 3.4048492908477783, "timestamp": "2026-01-25T20:34:17.436808", "parsed": {"folios": [{"folio": "90", "text": "Hainrich von Eilpetten\nsin dochter\nJumalus der probst\n\nvnd der stift von\nsant Emonz\nant Jo\ua75bans ordens ze mong\nHainrich von wiffengartte\nkunig smit sin Enckelin\n\nVon bury vnd\nheining von Eicpach\nGrant von der gruben\nGrant nemay", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "40afc682-bb4d-417a-9fed-c80544d53653", "score": {"fuzzy": 0.81, "cer": 0.223}, "raw_response": "{\"id\":\"gen-1769369654-yQVhqf43FbEY7rAJ4GHv\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"90\\\",\\n      \\\"text\\\": \\\"Hainrich von Eilpetten\\\\nsin dochter\\\\nJumalus der probst\\\\n\\\\nvnd der stift von\\\\nsant Emonz\\\\nant Jo\ua75bans ordens ze mong\\\\nHainrich von wiffengartte\\\\nkunig smit sin Enckelin\\\\n\\\\nVon bury vnd\\\\nheining von Eicpach\\\\nGrant von der gruben\\\\nGrant nemay\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"90\",\"text\":\"Hainrich von Eilpetten\\nsin dochter\\nJumalus der probst\\n\\nvnd der stift von\\nsant Emonz\\nant Jo\ua75bans ordens ze mong\\nHainrich von wiffengartte\\nkunig smit sin Enckelin\\n\\nVon bury vnd\\nheining von Eicpach\\nGrant von der gruben\\nGrant nemay\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]},\"reasoning\":null},\"native_finish_reason\":\"stop\"}],\"created\":1769369654,\"model\":\"meta-llama/llama-4-maverick\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":131,\"prompt_tokens\":2825,\"total_tokens\":2956,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":496,\"video_tokens\":0},\"cost\":0.00111975,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.00098875,\"upstream_inference_completions_cost\":0.000131}},\"provider\":\"Parasail\"}"}, "scoring": {"fuzzy": 0.556, "cer": 0.487, "cost_summary": {"total_input_tokens": 34722, "total_output_tokens": 2563, "total_tokens": 37285, "input_cost_usd": 0.005208299999999999, "output_cost_usd": 0.0015378, "total_cost_usd": 0.0067461}}, "normalized_score": 51.300000000000004}, {"_id": {"$oid": "69d4eb12b877594bc5718bb6"}, "test_id": "T0300", "benchmark": "medieval_manuscripts", "date": "2026-01-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-8b-thinking", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"3\",\"text\":\"In s ein pferit die mir vnd\\\\n minen knyhten uder hulfend\\\\n say so was mency kiny wcy\\\\n same day wir machten\\\\n and vickend die knyhtge oub\\\\n vnd vil in vntz an say anv\\\\n vnd die pferit vntz an die\\\\n pater vnd was ze mot ein groffe\\\\n nebel ez win kum vntz kryge\\\\n vnd also mit grosser arbait kome\\\\n wir ze mittem tag zu pant\\\\n hmpoffel vff say berg so\\\\n so fach ich die bucker so gav\\\\n vnd terry wopay in pom\\\\n die vur stur e dlu gecley band\\\\n so raine minceb vatterb plicuy\\\\n wopay sog in sem smey\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "qwen/qwen3-vl-8b-thinking", "provider": "openrouter", "finish_reason": "stop", "usage": {"input_tokens": 2010, "output_tokens": 319, "total_tokens": 2329}, "duration": 9.418825387954712, "timestamp": "2026-01-25T22:11:00.521677", "parsed": {"folios": [{"folio": "3", "text": "In s ein pferit die mir vnd\\n minen knyhten uder hulfend\\n say so was mency kiny wcy\\n same day wir machten\\n and vickend die knyhtge oub\\n vnd vil in vntz an say anv\\n vnd die pferit vntz an die\\n pater vnd was ze mot ein groffe\\n nebel ez win kum vntz kryge\\n vnd also mit grosser arbait kome\\n wir ze mittem tag zu pant\\n hmpoffel vff say berg so\\n so fach ich die bucker so gav\\n vnd terry wopay in pom\\n die vur stur e dlu gecley band\\n so raine minceb vatterb plicuy\\n wopay sog in sem smey", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "6174e97a-9361-49e3-8f81-75bf818b6a2b", "score": {"fuzzy": 0.866, "cer": 0.161}, "raw_response": "{\"id\":\"gen-1769375451-75zpNY6UmVXC9FjnEpD3\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"folios\\\":[{\\\"addition1\\\":\\\"\\\",\\\"addition2\\\":\\\"\\\",\\\"addition3\\\":\\\"\\\",\\\"folio\\\":\\\"3\\\",\\\"text\\\":\\\"In s ein pferit die mir vnd\\\\\\\\n minen knyhten uder hulfend\\\\\\\\n say so was mency kiny wcy\\\\\\\\n same day wir machten\\\\\\\\n and vickend die knyhtge oub\\\\\\\\n vnd vil in vntz an say anv\\\\\\\\n vnd die pferit vntz an die\\\\\\\\n pater vnd was ze mot ein groffe\\\\\\\\n nebel ez win kum vntz kryge\\\\\\\\n vnd also mit grosser arbait kome\\\\\\\\n wir ze mittem tag zu pant\\\\\\\\n hmpoffel vff say berg so\\\\\\\\n so fach ich die bucker so gav\\\\\\\\n vnd terry wopay in pom\\\\\\\\n die vur stur e dlu gecley band\\\\\\\\n so raine minceb vatterb plicuy\\\\\\\\n wopay sog in sem smey\\\"}]}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"3\",\"text\":\"In s ein pferit die mir vnd\\\\n minen knyhten uder hulfend\\\\n say so was mency kiny wcy\\\\n same day wir machten\\\\n and vickend die knyhtge oub\\\\n vnd vil in vntz an say anv\\\\n vnd die pferit vntz an die\\\\n pater vnd was ze mot ein groffe\\\\n nebel ez win kum vntz kryge\\\\n vnd also mit grosser arbait kome\\\\n wir ze mittem tag zu pant\\\\n hmpoffel vff say berg so\\\\n so fach ich die bucker so gav\\\\n vnd terry wopay in pom\\\\n die vur stur e dlu gecley band\\\\n so raine minceb vatterb plicuy\\\\n wopay sog in sem smey\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]},\"reasoning\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"3\\\",\\n      \\\"text\\\": \\\"In s ein pferit die mir vnd\\\\n minen knyhten uder hulfend\\\\n say so was mency kiny wcy\\\\n same day wir machten\\\\n and vickend die knyhtge oub\\\\n vnd vil in vntz an say anv\\\\n vnd die pferit vntz an die\\\\n pater vnd was ze mot ein groffe\\\\n nebel ez win kum vntz kryge\\\\n vnd also mit grosser arbait kome\\\\n wir ze mittem tag zu pant\\\\n hmpoffel vff say berg so\\\\n so fach ich die bucker so gav\\\\n vnd terry wopay in pom\\\\n die vur stur e dlu gecley band\\\\n so raine minceb vatterb plicuy\\\\n wopay sog in sem smey\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\\n<|im_end\",\"reasoning_details\":[{\"format\":\"unknown\",\"index\":0,\"type\":\"reasoning.text\",\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"3\\\",\\n      \\\"text\\\": \\\"In s ein pferit die mir vnd\\\\n minen knyhten uder hulfend\\\\n say so was mency kiny wcy\\\\n same day wir machten\\\\n and vickend die knyhtge oub\\\\n vnd vil in vntz an say anv\\\\n vnd die pferit vntz an die\\\\n pater vnd was ze mot ein groffe\\\\n nebel ez win kum vntz kryge\\\\n vnd also mit grosser arbait kome\\\\n wir ze mittem tag zu pant\\\\n hmpoffel vff say berg so\\\\n so fach ich die bucker so gav\\\\n vnd terry wopay in pom\\\\n die vur stur e dlu gecley band\\\\n so raine minceb vatterb plicuy\\\\n wopay sog in sem smey\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\\n<|im_end\"}]},\"native_finish_reason\":\"stop\"}],\"created\":1769375451,\"model\":\"qwen/qwen3-vl-8b-thinking\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":319,\"prompt_tokens\":2010,\"total_tokens\":2329,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":170,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0,\"video_tokens\":0},\"cost\":0.0010317,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0003618,\"upstream_inference_completions_cost\":0.0006699}},\"provider\":\"Alibaba\"}"}, "scoring": null, "normalized_score": null}, {"_id": {"$oid": "69d4eb12b877594bc5718bba"}, "test_id": "T0303", "benchmark": "medieval_manuscripts", "date": "2026-01-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-8b-instruct", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Und die pypel ist bi at mil\\n wit und ist der venediger\\n If von kurefun vntz gyn\\n modun und at mil und\\n ist och ein qubstog star und\\n ein sctzbuftum und ist och\\n der venediger ze modun\\n wachfft der rumanier und\\n ist och do bi ein heilig gesset\\n tes bi ij milen\\n Candia. If von modun bib gyn tandia\\n Creta. find at mil und tandia\\n Vinum ist ein grob pypel vy mil\\n Malta wit und wachfft der\\n ricum maluasier do und ist och\\n der venedier\\n Rhody. If von tandia vntz gyn rodib\\n find at mil wer aber\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "qwen/qwen3-vl-8b-instruct", "provider": "openrouter", "finish_reason": "stop", "usage": {"input_tokens": 3281, "output_tokens": 216, "total_tokens": 3497, "input_cost_usd": 0.00026248, "output_cost_usd": 0.000108, "estimated_cost_usd": 0.00037047999999999996}, "duration": 6.690959215164185, "timestamp": "2026-01-25T20:36:42.791134", "parsed": {"folios": [{"folio": "", "text": "Und die pypel ist bi at mil\n wit und ist der venediger\n If von kurefun vntz gyn\n modun und at mil und\n ist och ein qubstog star und\n ein sctzbuftum und ist och\n der venediger ze modun\n wachfft der rumanier und\n ist och do bi ein heilig gesset\n tes bi ij milen\n Candia. If von modun bib gyn tandia\n Creta. find at mil und tandia\n Vinum ist ein grob pypel vy mil\n Malta wit und wachfft der\n ricum maluasier do und ist och\n der venedier\n Rhody. If von tandia vntz gyn rodib\n find at mil wer aber", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "27bb5e5a-e710-4c6c-9540-36c3eed8a1a8", "score": {"fuzzy": 0.206, "cer": 0.811}, "raw_response": "{\"id\":\"gen-1769369796-wQ0eLDvzdA2BYtvz4ivp\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\",\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Und die pypel ist bi at mil\\\\n wit und ist der venediger\\\\n If von kurefun vntz gyn\\\\n modun und at mil und\\\\n ist och ein qubstog star und\\\\n ein sctzbuftum und ist och\\\\n der venediger ze modun\\\\n wachfft der rumanier und\\\\n ist och do bi ein heilig gesset\\\\n tes bi ij milen\\\\n Candia. If von modun bib gyn tandia\\\\n Creta. find at mil und tandia\\\\n Vinum ist ein grob pypel vy mil\\\\n Malta wit und wachfft der\\\\n ricum maluasier do und ist och\\\\n der venedier\\\\n Rhody. If von tandia vntz gyn rodib\\\\n find at mil wer aber\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"Und die pypel ist bi at mil\\n wit und ist der venediger\\n If von kurefun vntz gyn\\n modun und at mil und\\n ist och ein qubstog star und\\n ein sctzbuftum und ist och\\n der venediger ze modun\\n wachfft der rumanier und\\n ist och do bi ein heilig gesset\\n tes bi ij milen\\n Candia. If von modun bib gyn tandia\\n Creta. find at mil und tandia\\n Vinum ist ein grob pypel vy mil\\n Malta wit und wachfft der\\n ricum maluasier do und ist och\\n der venedier\\n Rhody. If von tandia vntz gyn rodib\\n find at mil wer aber\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]},\"reasoning\":null},\"native_finish_reason\":\"stop\"}],\"created\":1769369796,\"model\":\"qwen/qwen3-vl-8b-instruct\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":\"\",\"usage\":{\"completion_tokens\":216,\"prompt_tokens\":3281,\"total_tokens\":3497,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0,\"video_tokens\":0},\"cost\":0.00037048,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.00026248,\"upstream_inference_completions_cost\":0.000108}},\"provider\":\"Novita\"}"}, "scoring": {"fuzzy": 0.689, "cer": 0.34, "cost_summary": {"total_input_tokens": 39327, "total_output_tokens": 2885, "total_tokens": 42212, "input_cost_usd": 0.00314616, "output_cost_usd": 0.0014425, "total_cost_usd": 0.0045886600000000005}}, "normalized_score": 66.0}, {"_id": {"$oid": "69d4eb12b877594bc5718bbd"}, "test_id": "T0302", "benchmark": "medieval_manuscripts", "date": "2026-01-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-30b-a3b-instruct", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"15\",\"text\":\"Oeb selbey tagt gey fahzwin In der wue 2\\n Jey Iff an dem bahntrag Jcrdo wz\\n Der fubenzelchende tag Vitter wu\\n gey felterb Iff an dem adzchende\\n tag gey terfie vnd olibey zwen\\n tag do vnd f\u00fcrend an dem golch\\n Jaufrag Der do wz Der auff vn\\n Zwenziger tag oeb mercey Nf\\n dem waffer No terfie gey Venedig\\n Iff an dem osteroben Der do wz der\\n lep tagim mercey Os folcay wir\\n etwe manigey fithyten vnd gosk\\n piff als medtige gifter sol blic\\n Vm faut marco minster vnd de\\n Vench\\n Ducis\\n palah\\n magnifi\\n cum.\\n geuazgy palap Der yf zensolth\\n vnd Dauif vil gern Viter\\n bnechty kuspen Die dem\\n ze Nencdig gewesen sind 9s yt\\n kiy Cappell an dem minster Jost\\n deir zey in eintalck gewaffet do\\n Vifer bchalter Tto Cpe Nff pas\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "qwen/qwen3-vl-30b-a3b-instruct", "provider": "openrouter", "finish_reason": "stop", "usage": {"input_tokens": 3272, "output_tokens": 328, "total_tokens": 3600, "cached_tokens": 514, "input_cost_usd": 0.00042536000000000006, "output_cost_usd": 0.00017056000000000002, "estimated_cost_usd": 0.0005959200000000001}, "duration": 3.2477400302886963, "timestamp": "2026-01-25T20:36:00.455330", "parsed": {"folios": [{"folio": "15", "text": "Oeb selbey tagt gey fahzwin In der wue 2\n Jey Iff an dem bahntrag Jcrdo wz\n Der fubenzelchende tag Vitter wu\n gey felterb Iff an dem adzchende\n tag gey terfie vnd olibey zwen\n tag do vnd f\u00fcrend an dem golch\n Jaufrag Der do wz Der auff vn\n Zwenziger tag oeb mercey Nf\n dem waffer No terfie gey Venedig\n Iff an dem osteroben Der do wz der\n lep tagim mercey Os folcay wir\n etwe manigey fithyten vnd gosk\n piff als medtige gifter sol blic\n Vm faut marco minster vnd de\n Vench\n Ducis\n palah\n magnifi\n cum.\n geuazgy palap Der yf zensolth\n vnd Dauif vil gern Viter\n bnechty kuspen Die dem\n ze Nencdig gewesen sind 9s yt\n kiy Cappell an dem minster Jost\n deir zey in eintalck gewaffet do\n Vifer bchalter Tto Cpe Nff pas", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "d99ed70d-5541-4157-883f-b005b64c5776", "score": {"fuzzy": 0.244, "cer": 0.772}, "raw_response": "{\"id\":\"gen-1769369757-7Tt5e8KlqBR3qcbYyk3z\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\n\\\"folios\\\": [\\n{\\n\\\"folio\\\": \\\"15\\\",\\n\\\"text\\\": \\\"Oeb selbey tagt gey fahzwin In der wue 2\\\\n Jey Iff an dem bahntrag Jcrdo wz\\\\n Der fubenzelchende tag Vitter wu\\\\n gey felterb Iff an dem adzchende\\\\n tag gey terfie vnd olibey zwen\\\\n tag do vnd f\u00fcrend an dem golch\\\\n Jaufrag Der do wz Der auff vn\\\\n Zwenziger tag oeb mercey Nf\\\\n dem waffer No terfie gey Venedig\\\\n Iff an dem osteroben Der do wz der\\\\n lep tagim mercey Os folcay wir\\\\n etwe manigey fithyten vnd gosk\\\\n piff als medtige gifter sol blic\\\\n Vm faut marco minster vnd de\\\\n Vench\\\\n Ducis\\\\n palah\\\\n magnifi\\\\n cum.\\\\n geuazgy palap Der yf zensolth\\\\n vnd Dauif vil gern Viter\\\\n bnechty kuspen Die dem\\\\n ze Nencdig gewesen sind 9s yt\\\\n kiy Cappell an dem minster Jost\\\\n deir zey in eintalck gewaffet do\\\\n Vifer bchalter Tto Cpe Nff pas\\\",\\n\\\"addition1\\\": \\\"\\\",\\n\\\"addition2\\\": \\\"\\\",\\n\\\"addition3\\\": \\\"\\\"\\n}\\n]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"15\",\"text\":\"Oeb selbey tagt gey fahzwin In der wue 2\\n Jey Iff an dem bahntrag Jcrdo wz\\n Der fubenzelchende tag Vitter wu\\n gey felterb Iff an dem adzchende\\n tag gey terfie vnd olibey zwen\\n tag do vnd f\u00fcrend an dem golch\\n Jaufrag Der do wz Der auff vn\\n Zwenziger tag oeb mercey Nf\\n dem waffer No terfie gey Venedig\\n Iff an dem osteroben Der do wz der\\n lep tagim mercey Os folcay wir\\n etwe manigey fithyten vnd gosk\\n piff als medtige gifter sol blic\\n Vm faut marco minster vnd de\\n Vench\\n Ducis\\n palah\\n magnifi\\n cum.\\n geuazgy palap Der yf zensolth\\n vnd Dauif vil gern Viter\\n bnechty kuspen Die dem\\n ze Nencdig gewesen sind 9s yt\\n kiy Cappell an dem minster Jost\\n deir zey in eintalck gewaffet do\\n Vifer bchalter Tto Cpe Nff pas\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]},\"reasoning\":null},\"native_finish_reason\":\"stop\"}],\"created\":1769369757,\"model\":\"qwen/qwen3-vl-30b-a3b-instruct\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":328,\"prompt_tokens\":3272,\"total_tokens\":3600,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":514,\"video_tokens\":0},\"cost\":0.0006876,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0004908,\"upstream_inference_completions_cost\":0.0001968}},\"provider\":\"Fireworks\"}"}, "scoring": {"fuzzy": 0.621, "cer": 0.4, "cost_summary": {"total_input_tokens": 39309, "total_output_tokens": 2549, "total_tokens": 41858, "input_cost_usd": 0.005110170000000001, "output_cost_usd": 0.00132548, "total_cost_usd": 0.006435650000000001}}, "normalized_score": 60.0}, {"_id": {"$oid": "69d4eb12b877594bc5718bf0"}, "test_id": "T0304", "benchmark": "medieval_manuscripts", "date": "2026-01-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "x-ai/grok-4", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"23\",\"text\":\"H ainrich cz\u00f6pff\\nhf kauf von hainperg von hainftenberg.\\nH graf Beunolt von Mayenfeld\\nH graf heinrich von Montfort\\nH graf hug von Montfort\\nH graf Gotfrit von Montfort\\nH heinrich von Buwenburg\\nH graf Schopff von Wyl\\nH von Quaden Rauten king\\nH heinrich von Thannheim\\nH georg Eselberg von\\nH heinrich quast\",\"addition1\":\"May hewri grafen halfft vo\",\"addition2\":\"B Spannh)\",\"addition3\":\"? Aeschy R\u00fcten\"}]}", "model": "x-ai/grok-4", "provider": "openrouter", "finish_reason": "stop", "usage": {"input_tokens": 1629, "output_tokens": 7499, "total_tokens": 9128, "cached_tokens": 1368, "input_cost_usd": 0.004887, "output_cost_usd": 0.11248499999999999, "estimated_cost_usd": 0.11737199999999999}, "duration": 141.6473731994629, "timestamp": "2026-01-25T20:42:33.747577", "parsed": {"folios": [{"folio": "23", "text": "H ainrich cz\u00f6pff\nhf kauf von hainperg von hainftenberg.\nH graf Beunolt von Mayenfeld\nH graf heinrich von Montfort\nH graf hug von Montfort\nH graf Gotfrit von Montfort\nH heinrich von Buwenburg\nH graf Schopff von Wyl\nH von Quaden Rauten king\nH heinrich von Thannheim\nH georg Eselberg von\nH heinrich quast", "addition1": "May hewri grafen halfft vo", "addition2": "B Spannh)", "addition3": "? Aeschy R\u00fcten"}]}, "conversation_id": "05fd0dec-ccdc-4f9d-ae97-b352aef3514b", "score": {"fuzzy": 0.312, "cer": 0.72}, "raw_response": "{\"id\":\"gen-1769370012-oEyT94QNKQwrzyTgyxT8\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\" {\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"23\\\",\\n      \\\"text\\\": \\\"H ainrich cz\u00f6pff\\\\nhf kauf von hainperg von hainftenberg.\\\\nH graf Beunolt von Mayenfeld\\\\nH graf heinrich von Montfort\\\\nH graf hug von Montfort\\\\nH graf Gotfrit von Montfort\\\\nH heinrich von Buwenburg\\\\nH graf Schopff von Wyl\\\\nH von Quaden Rauten king\\\\nH heinrich von Thannheim\\\\nH georg Eselberg von\\\\nH heinrich quast\\\",\\n      \\\"addition1\\\": \\\"May hewri grafen halfft vo\\\",\\n      \\\"addition2\\\": \\\"B Spannh)\\\",\\n      \\\"addition3\\\": \\\"? Aeschy R\u00fcten\\\"\\n    }\\n  ]\\n}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":null,\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"23\",\"text\":\"H ainrich cz\u00f6pff\\nhf kauf von hainperg von hainftenberg.\\nH graf Beunolt von Mayenfeld\\nH graf heinrich von Montfort\\nH graf hug von Montfort\\nH graf Gotfrit von Montfort\\nH heinrich von Buwenburg\\nH graf Schopff von Wyl\\nH von Quaden Rauten king\\nH heinrich von Thannheim\\nH georg Eselberg von\\nH heinrich quast\",\"addition1\":\"May hewri grafen halfft vo\",\"addition2\":\"B Spannh)\",\"addition3\":\"? Aeschy R\u00fcten\"}]},\"reasoning\":null,\"reasoning_details\":[{\"id\":\"rs_adb7dba2-7afc-f334-5e08-3901bab8c289\",\"format\":\"xai-responses-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"vqfENoRKMsSf7nwuYnWf6cuPFIgR1Gp8LMef/VTUG6NkyUDWkbhKcQbdPYX683OLL0p1b1jhNz3UaqSX6tqGW7Nbm63LXuo6qpQHnHp4i857VqNlD+HuUYuhLvJHw5/kb4GjVoYNZB4nW+WgQotQ6VxANoHyKX6Tvvi+ugws4T8H8jpUN+1b0YAa7hOLI/YyKh6eArGp8RNLDiuAz8HbxR54Xul0oXVHQNtyNZd8D/BHK67SEErXuCblhLrkOXIzmW1R7HIdNeMODZi/cQXdLy4WWhzrjdfCW07Y74dhgSjdzOzp7FCxeAIJdR81uQpLVpcG0/60jBYmUMYxQWhJFWPWUwHN/7UmHDjLzdkiaC0Lq9xzoXql8jldnwqe3Bn1NhiQrrKdfSyXANtgDV1fwcg3z0Ebw8Q6oLsx+NMYnR9ZGtKlqvT8Gbc3Hp+pgDiAi5yQSN+tZ2DT/SrCZCMGH9rALrDqKZuyPxvAJ5+kWCkmrclk7E0Facxrxb0+WFlUDosHOjGcHrQk5S7AuurofVw2V0XWxBAKdKzGJZvwegaIhUHtZn3nIvZSeimin6yc+b4Q9GlGsfKNCyl/lh+Gkht2WTFx6mL5Opk/vYKne5F99vViv659UJhDHQ6102LVymHD0ZvOy3TDRm2pNfTpjg79MxpC2BvtuViHR/YUIm1miw80YGtBazBs9HCSxHmdB77z/9xDqiPytghcKQjhPPQt5/PFkgL53uDC8oa7AP0dmg5jY7txJDJNQ6R0KPq4cYPVViK8YXrLdB77c7v12Ma7mwUlOFLdupmPcx6iywjC1p0xd+BlvPeddz3zj2TeaMsirvNR7ZUKIA/35rKvRFIIcajLCzGPibDsf+dSdEhMDGDstEHo9v/KSNbzp0DmjgnZ+UwG3laCgl9NDXWvqfYtIMHaShjw/CyA5BeFiq204909w36t9r0TVkW/5DA0rxcKatpVy2WLss6A1rJERpevmUqPjysXdZP9lyPgGh8JYmpM/7v1jUmIWiIDglqk9LcJGuar19PMvymcFxCAqf9HVu/lq4WP2CgL5GDw4sRvlRsQs916+jyk7DtZDdneaEDw7uZPEK+UGejcDoxB3LLW8dGr99eUot1e6bJKhn2CvizESlOQVtWBYAP9Yj5Czir+MDd9/2q3oJQpnSrcdx1ZKgsET2W0Ut4bRQ4FTOIe7tUx2LJ/8334XzrYRD/pid64fKCXgKRYb/9iVpDn4YLEQmhZmq5T3AP36WfQP351d2HqwXfdjMi9KD6lK/bUSwuNHcwjMEdUogGU9nu6wik8lJAUTjvmw5Mcxw7PfagDaJVTe/Jzbr8mpP3fDvPsXYOMi8Fw15KILdzZjj+YilBrs9FLqvwT4CuaPRowKLKLr+7AeorBL8fE0GYlvsoUwdE26fAc+llRUx4KU/SpffIynxRAnwtOh36IySHK8IURgi+cHUVDZt+Q8S2d8DcMiB6WCflHHsS/HXxAbpkvJl6Y/DVcRdRnIhmPFQ/4waIluJ0D0Wc+cfJ5EYjvY2rQwYYf6Hy0K4zTJFrOiMRn6tImgPIEuuYZBUhTkY3jmGujLWC5o+yZ/HquDoyIH0vaWnfe8p8c8+QIB2faPe9EMNy8fNy00fugJ9IzKu0W6Z/yjSpgF5FlYtstWWp66nGpNC3Gjta3w2SY6JroLgQv/h1zPnNzqTlWsLw4A1Zze+Ummd4kAKACdDeDzEq1hga4GA4DPPg4lNsPJUt2H25USLAuTaVBfiQFuiRUyA3z4C3ScV2NlsVsymwakgG8Leb+0FTRJ2hwLcKbe0MSenY7bjtjdHYdR0Fq4m5Tn2ZLp+mLVAoBNI396vNy6zQP71un/7oUepl2g5vYKXJgI35xvbJL7qL9ASKmOchAg1D7t/7u/OMd1CBgRApp3iGT6WKhOxNYsCOPnvib/t60LDLMcS2itgG4LeBWsKx+feMnJllKDhuT8Kh/JBEtc19aGB4a6NR6ov3VehUON8IqqVjHp4fvLEEA144f/6ujoqMVyjocUghFjdebqW+UlE3OCmGs7gG8tymc8WuqTGXvEe/Iax1lVcpWqigrxqIlGha1NFeE8h4xHZfAbp6Z9bjOF2Uxd/m8NU96sK2X89nqI1J9XcEjwu3xX/yR9A2aQx5p+t95MTdU1bqQ3heV6IeLOFWltRBgAuo5b1+M24MLEmFVDfWi+7ezSQtZuKdIskQd/G4Sy43ax/meeGBSFQN1nBNqJeipMQJJDIBJ3yAhZld4Sk8IG6hl/Be4A3uKW2XsflIqnCHFIpQfsYVgYQYpCpJFWf3B5j2eLliuO4DiKC7e0zLZVHfpE3RcbtgrwVWCI9l6OEjuWsQgPjcWiA5599jso32JfYjxuhoihmO2JMse2yQdeVyvj1/AW1RGUC5E6EpHkTrq9+EOMEvM990R5/dETjMR2m8DBpmyDHeVIzNClq21IANv3jssb67iTofRLYdwQ4VNB6sXwc0PxDCMWTtXYlNtWXQ0AznIZI8iy9ovzRplSA8SAYyBdadQKouwdTuWp7O3ISFKDSonokhlbti+V1K9wO9tW3nZmDGW0NF0N0lz6XlAnFke5B1kIaXylk0Tvjid85V8ffOXyG1QyE2c8vdMMkVUOtho5qIcrAJQNfPeQCLNnbr3te+HGjKN+bGhuA6Hvsitrt7zdSbzgHImwcl3uPwzlSC6EDlFuGFYD8AidNW2FZyNtGwU2aIgOzH3Fe0EaTWW14uU8jJ8I7cSxyX6Fq2RAgsj+K2mol3J5QUljUpdNkoj7VUQ/9ydDKJvPJ4ZM8hPsVT9gHQNK/ZjPihymM3G33dUmrmPP/7TNzkDvtBaMC2veXqZ/DTDEsPJ7IzzL+njFJfuBrEggvWxl/BCnHVd+X4MGkwfrA+KGB7le2Yb5b74VlOjOR+2YFsyq96FDhaBlOnaNF7DiXne0e5ah44ikVkkukTNkAlMdLXOVxNpxCe061dUzmiwK2M2Hz9cKMFVtVwCMVK6HZNNYmP7JG0w1sSv3snwW2Duhmh5cYYVpL6ZTITPONb/GWCpBYfZMvcaaZLpiSXNwe1MNacEeAh0TMjZtZ2vZj8asvv2tsEGMacNYAC61me3EWspbFJC4zat8/5teXSxrqLRJcY0Rk37s/dyWCaSHpJyI749LL8WGlNpH4DUHBfW2OvaRzGs15KJNbaLIVdll69bowjgRJ2PJADR012DRBYTWt94kr0mF+KuCtWXsSY+3dqi4qNgmhDOD31ENGoRfBJwHYLzSTXQCPdiJUca1/PVyL3xAjcnCzxNv6pyC6vIq56CJm0OCdXavYHBeFS4pbp682yGbwr/aajrbs3UFMrbc3PXAIOPwZ7f4gE4Hs4FMNVycb3lMToZure9GGGninPwf2mREl9Vh5xOpT8JzXoAMERYZHNJn5f3JkQwfz3ETMKM6fkOhJurxL9guOTLIekpzX9QKA7UmzGH0sdy456CUzxwIUxInqbUcL+Z0u4vhHKkD9RKKwPLEC6pyUx5WynvOJrgh8O9vndhtCX4KtAa57iHq2F3Mp66e0FMjLCupt2CHXCYjs8I9kHEZQoyT6mFaZnqBfyTjS0qyCmc9kJ6WyDVRgcQOrv03iG2w/K6hGQG28KlJDLX3FndzbqGbVvhAEAe5LckRuXfvvyCYBh/GUl7AGcRC3W0jpvuu2UCZdtOwjJawOjuYdQ73SkcejflvmbBCyEifmGR8GaW+H5pAG0XG76MSGGHinU7hS+Hxe+SC6i+b09tt9YdG6VJ1t3K5R35E/DciWNAj5T5KYfx4aVpSlhBtLXaXcLzPu9ljWE3pVyThYs0fP6DiN7jL8H2UFjs4RbuRJiRwud/5QDkaDalR+iAd48rVvRRWWD4qwGpOIhas92wB9T2jfAfYJyrwuUs3MmZ+vCkX99+dUCILLscZHwShd4YWgFF1/9LHKmM/F+p+1UYBQXa59xe6wg7H1X8WoA5P0yL+IcRhlmBi1iW/hMSWjD7GIJ5WIb9FCLAI86waE1UBepALnoqR1ZU/usOohmrqYpAerDN+yDHUpHAKc924+gwClV+cCS8Az6YJIAu/7oN+x5A/6hSnRPl3oWDvBS7Aks5o/6vdvgzkBtK/CDyocpUSEFjUfLCxPuZCxB6ywas0we4Woi/2ZeDFAfrUiz3uWMSmP7iYNzEFPE1fkkV3NNdlivlTvcbT+H7j/XgncMncBfHMvP/s+T//XhxsSmz1PuSQVqiOFbMqC7+3Y81wV3xKXMda4t6p88qi6/ilgSXIMOCmFtar3WE9NxwV2ZA+OFsiH6hadb37j4Bywlx0Ei/BHrCUnPcJpBrotmoFXcGSltGfv6eviSpewyBiHQOqHp9DTJRVRzXsvf5yd/V3q2iIjzvwyTk4CHv0XaNgWdM0PZFxtSE54d6oK1MbJhLCPxImhfmpQ3W9aVNEvbfXe5pApSDdWNyaYGi3MDC5g3qVixG7TcK+J5gHUVhNu99OFVHtKh9TF4Yyq+Tc1ZglrgoRIeIdIcdZgwjMUnDB6exSiw1GeWdssTxU5IwecE4Qlz6McQTW97UtDcjbXEBz+XAPhjF12Tg7ExlO7Weu68TYOUUk14jRj/SxUn4+HtKR+EfDwEHrz7u8RTW9ImNE/7ELYDJBHmwJ9xu1l5jbR0BTKvFbq8uABX01xPYg0hCn27VXj7P+GrDlS4Pp8zd/OsQqFX8Z5FWJxzJVZmyH2e8Gxw8ANHMr9CUlxzZ1HKQQD/3l7EGx49IMV8+1PAXHz6EpoBD9uK5YF5Ttp8WVcJLJvPntQT5MibhMW0XVKivsXiKGhafD4vg6c858YnxPgU1Egsq5gOgQhjIFVvzO1nHAT9hXA0q15lMk504jtJpG+PUoXs3F/zpqDcPwwQ+EBQJ3GSv75R+A7fVyR1jLWe7/BCwsi1nm62X5CqajF+E3KNSTZ1PCYsXZu+V0Fz3sR28CpzXF74loBxwQROOXTyHCrLDLAcZBsqY0m6NNXBXFSXugxZzulYSQKtVHWCqOZJNgWqRq/QjubEMHr3Dm1kNzpKXqUzQPrVtpMsJxgBQJCgEFtnbnxJbPn2U3PPiLaTayViPDcKkX9O0zepTATH96Nv5WYmmotAY56KV0VN1omueOVMptGsj3uCYxIXUPZZ5FKcS6aS42IibsMgp646r13estnwoLo31aiMoL1xoKWJ5hQB99ZZs1QcY0sHKHHQCStSGNZ35+VK3NkgbfanW0u1+IvHP1g7cb+Pfb7VjhxxQZREcHa6BGP2U2P1ev9bMDSyDD9/WNFPmZSJysfa2+zfSosV+mZLkib+Oi0cm/XWInGQJPOoL/E7JGM0nIRArrVMV4qGSpnY0dQiYGUSsbPk78bctWvuotXb5W6PAuW1Q1GT5BxzuPRUsVTmPj9B7KNWw7Ji9vdv2Irrgbj4PDKjtYm7Qg68V3hYW3yPtIXrG4FVlE6h3v+J7x7JwzHai+4NeL738diJsZtVBV2V6KrxVmklGxZQuFGHDDZh7CAyp1sfgD3c9WXLeoXSe4TE2B3OUrbFuEiwIuPTQCglCcnXjJPIUp4o8sFXC/KIUY1kDlhTAA3Y/e6vdLfACYW7zHIM0+BFbzbkNqbAiQJZweSxyJhYtfET8g/Vis+e+4BgkOmW1RIgBJx4pVL4ROrsCooMFEFM/In0CTuyb5/QbeduUkpe6TY1Jq6FlyXDg6KqO2/v58uZGMYFuJ7K3irCllBPfuoLITD6m5rnpyHrUmvAzfm+wEnpbXYmlYNl16eLpZwLA6xeaIp8EemFe/+RrIU9iUWY+Xv39neLYm/v5JbW/d8vS4zdP2DvJolDBej45fj2m7bmnF63CLp0ydWHwYx+KbmZeLjw6aWuqLcshQcGQpWMkQrjZXeN8kax3EviwDF5E8vl/7eMVP2N0oKYX1NrGC/WKhmP/HneVPzItDnJETkvKMRxv1nZHcqZNjxFAz3TERnjnxLwH66jUOfqpX6wBjAS2E7OEM4uTnBa1xpIDO37FTaWnWLsYEf3wsgt27TY0aGqgRFNL8h4j2jnJy9pBxtiQF7e6z7Ex2Ne1wwNURTDPfhHTvifmj5UegSFonB/nh0+em+Ke3XVfd+8wTOga7UShiwYKGOvfb2rTZwp14EIQqP0ZJzYu4luT+XcU9e8a6e0qLfwYGp+NB/b6FMW/xYrX+uyiGGlRyEeZsql4HDMbMPJliUc3LQQPkzG8G0G+CVX7zAXGalIxpHhvaHfh9YM8Xm1tCCf8tpvRwY7KsShtqMfZGkeQ9+QAwt0ee7eXmWp/rvEZ1R2a52n09lZWz7K/6+thvb8CVSBWkHJQ/N5ojECTA2djwD3xFUkDHMMRWzNljLliKVW5oG2IFco1Fhx/7K5+VLxE1s3t8BStdR/uUdovki5GQ53pIftXEf83ipvIOuBxNi7gdWnQsu6p2clRxfnFmpGwGy8dcS9W6muUf4OuveLYPflJQyC5DZVf34a9fZ15XCvM4Dl/6p/d9W/yrWgDWzTVJprHMSXjWnNTqJmjdgNaRYXPD1L4i4Y3c0bceTGdqCouKxiwSzB6cK/+Qv1kEwGCmZvtPxhlnKBMJZH6S8ZybRoP0iw65FklbSCT3IhaF7Tyz/hbtYfWSBkiaBKI+suk4LsHsX88ufxkSHTD8rOmvaUwy/pP9Vl1770q0NuH7tgOnoJNyzNx95UhIdpUQf9bRqAs8nnMtaLsHpGpVVBhxPDAZf52sLy8nRYIchBeYdmHO/4/U8kQEE8rp39TUM3H6RJWjGFVoLYRFOJMIqzJoa598hwEllenX0lo7EUr6XDjoO5Uy0F8i46046/I2l+EoQF3O+Um/zV4gPFuOFBtNMP6t49a3u78XzlR3lxga6I34d5bF0Ia4WDnf/UGnK2JPKsipnDHyWMirOqiKYCVX86kgEOflbIOwCyZuTTOdELqd3MBCsLUk/8MWLySQXdx6gWpzLz68H2Va1JvbaLOOVMQdnPuTw4T4G0GQv4Z6xB4z2xA6IVWY7ORjpQ6ibot6+Ube/C4OsGpABUW8fsHXlN16Wi2lh41+GaSjuXWqVnkpSUDAEgVh9oYCZWfp7XJo6//uMsKW/z2J3Ng1f4MLJfONrTry+JqjR7eUSJmIWJG2AMpwwXRx7V4983aybJKixTFtguUvhRk1LXS3CFwCbPdv8ICAl0ZfXWlOV7BReM46dCXENxzRbx2bfIgJy7qz6lJPT1093Mcm/KczBi5Dvdnuh4QepR20OmOOSy61xitZnro0UDKXq2bWRZ/Wup4WTJQWLCRZ7uYI/8pHBJTc5u1uyYtVB7PD8gJTE/6iLW9q++Ng8YrYn2c/k0d7sfVepVyMeMr4Nskk1pSmz591Cs9Td8/sR2aoZJQppB3Sn3f2jewWN98uF7i5rS4FzErX6Q8SyLLRI4bZhTKNRoRx1PFG7Cp2TSVmcRhTX5FIvUMdRoxYCtaNPfJu3z95C+qWc7sUHht4Nb+sUNKNQIhtvP9uc5raDWxeZX0aHDB0wTjsotF17wRFEUux58i9Sqr1+sy9BZV2Hg7f0UnXMgJ7sDf+dJzbKI0JHKZLIwOUJlHBxpHuoUBNzwMHChI0MI+Bo6zkEIZ+A0CQUckZ2dyAllciZlSRxh4l6WQD8xx1TP5H11No7lNgzZL75mztxdAyDizbpzwTep4YSdxStG2EaB7xlUShxDmYCRsWVQk46DxloKNL14ChKhuUroSLDAF4GnZjdS+5odMcz0cjObDxNgGmbBMtR5rrWUzFsXh0GfJhrJPrtqKjDU5eDXChESxjO6EWO0NzBYzFId9fzW6YvyuNS98/Yale9qMh3Cw3yXPcliBpLWQdj/XBkgrr1t5DiBhbVklJrEzSRJ9DlsU4E1n9nzy8hDhUywobsnQz8rAMkPWdOh57n6+MR4o5h8BZUNMJ7uSBvyeu12+MIRb1oM3TpBfS9FByDxRTC8py+ZZRpZR2bUuxxNmL7IdOJbJoBOVrQYJi7z1qq0AG/ME5BPpiQsP/ZBAxLnIBOnK6vmGKappmBaMODm0ZLSSNzWEvf9TAWMUDDdbbWM1dkctzoKtB8Ci0NDduSuFkaDdYYEICXNRIKhWlnPUEzhPOsX8i/c5RH+RQ243zHFaZnLd3R5wQ0a+ycXPAPnA7YWwRGNOVxljXKUvmWAIJkA8yVb3dF5S6/Q1wBa+Fq+FPepg8Qo+iAvaYgSj+dO4Q1lX+1xjL+fnFy0pSb+nKAapMIpC9Bbj3SNIIgUNCLptt0BpttJoVegE2OVTRF8BntqXVkDWjuf9ND+ueEW6K/Gcjbkj+XKE9pM2oNM+5KL9bp4LvukhPC6HrEOwTsu+DBcOZT3/KSOptclgb2JzfRzx7iuDP0r+Gvur0uwuoIfybrf+0G3xPl1pXtidEBWWOuSAg9JQJYyXLWuXVwL4VyXNCb30RsAoAnGqNdo/5Lg2nWElsu/ScWdpQX01aLOoJxO62anJs66qkgueEyO7FmY8dFpxzzIf3acwH7tBTw0RpIuAUOloYDTBdoxTJ1kiM3FOIILZqaYAA/CePnwNZY+cbUhSdAhlVd4ldYVlOgG2ucLU8/3e/yWdfepeacXndjjBzF+HmyCFtFuh7MCpjsRnAHmP2U6IFg65K2jx3pZs+h1LOKWGPKfBB3Uce4bHiW22pv+0Dq4f4LlNSxww8Hn3OB65+LN/Xyseor5cIh5NzKi2wOzrY/3VGp59jILZNGps1Gj/6rKwsk5aWJ5y28+74m6AM3LiKvXtST2K3NSZN7KYM/mDOOr9mKxyMf5jShlxUelFP6kKBCFdFlGpEjV/w4IGGvrjA2sEautsb8U4i9pp9F8ASWoVdk6y2POKLEJGLn8gLFIbs4d+3F5wtM071aPqVJS0xjAL+QKlRCX0uOviAfrvciNKHUn44KHJF2Nz2axxfl8UWEyYQUmwXFp7FBkJl0Yyd51FC8PVODCtoCLQfv28+O+P5l7Jx5DabeOWbwjhIM6wAZLWIMBA+J3FVMwxy+9nkbEgWMALcJrGGoQKUcHURBsDZUlLfEHwdPrWrPP8bdW7DchR36LA5GLQrLlZ/7puxo++8juOZE3kQTGYKcXbb4L+9kTtrHQ+DrHponQBnXl961Q0vhA97HnKfJdoOJaOeIgxI6rM0QRde3uHwz47yzfmqp9fxY/2evH8mu16iJXX35RQlEaysywJ3CUIbH+TqKbqG2YAKqTTTft60SaTEMnh2Dp67/EXF/Iig0EhNl9ep5fCUdGgMoVu/GdBZ4AWfaCIOCuclXx0DbpAbBY1MVSZs7xU2kOOgiQjToHtHYh+e98kv7g6K7XlPy19jRlat/gBsaBbe1y/oFSUMgxMwZvEhXRBy5UPiSa/YR9K2X9uQWWqmSs2It88AWvIp8Kmh9P25lEj1V9hhaUPKurAgio25UcDksLUXD3H6CgWnVS8j9dt3Uf/NQeuFiXqdrfjduNI45zK7O4mqdxMarlMibdoQvbBr096XqRwASUmDQN3vB3I+qQyGuGhRi142/WENjPURMPJEBhOcqe364orHMp9XlITpILkDc1IQFdYAcYxHWvCAabmN53fXGzBzIDnj0gmvTWsyZCfy5HEAhMxG+iUQ1Az4wfUB/U6xeXxv+12z4zNLAJyPUlaqC4yLs5zxwBimfU8sIXaQJlp+hf+PaCGcq1aXLQeD1tzn7KnJJWbwxZo2FkZyDXyMzQ+PahTFBcq4n6/CLtOACo8JBAW4RSjeJt6uzr/PArjQt32mh9Mz8PzTWUEEsViAThNEPu0shXsLY1nFd3P7m0yA30+2LM7x/D4lSJVfIKmdaYnMF4PlAFlsnOqwjTGi1mfKOoaA0JlQ1A+1hiPWYkdSf1NSoASIyMjjuDLz51ftppIdvCa3OScFrNxvtatXGuk0okLdXCBRsLB6hohBUQca0lLcjH2AUWLotjpWlVqcyZDo87+Oi3mR02j6eoFRq9QlzLTf1lxhlo2boBPRpAWTP98u2fhrOL1ROdzQSy0NUjaSiMbva9BylxkjTN8ZAnqZc37fs977i+LbtZC5qiaznW1LWj6FQSxaqXTDxU91wTwCfWx+aHJBrbQAo0btS91k5Xcs0EqpiLjhTBTaYcHewMclkFXKrhSyEwi4sM4dEDYXHLo0Qm9osSrN+Ec37XOTi3MFS0yJTowRMXoglXGR88/ASJfjJzztLbq+222cpjvOjofF5tT1gqEWFvurZSVBOb7V+D+rfVIDIihweanii4Dwdf74i74D5oBr3aTaJKd+JuPVhUcgtvGj/BDSiBNAz7BjT0ToxGboiDqW9KZzmUMVfgyQyqY9SMDDNOttNxAXtYmNGroxEjsjcgprYT5RTa1i2rgNh374EsOHe0ArCCszNtam4bekoRSkv5/QkVTCKVqRrNWoI6OtEYMRlSAX0uyTdlXwJGhvc2Uf/c049wKNCP3VkfKFBmZ7yolYnXgngkcEp2of7KlZWeXvEz/3cPSJA+IkTcWCzSVrpvZotzRDO60u3S0CaZhK0EOG5abINCTCBb8o6xfwi0oTgGrEjFf/UNb/LfdfJ2nEb3xTFYq4cvKp4ItssTrMkLZTnbW+9cL9KvJnRFTflr3iz/qKRPDypep8fXKh/jtfaFO3Yl8h6xuzNTZ8UHU3+2XNDXUG0R/e7H9RG6jgFKlyNo4s3ZrM32Igv0/gjWSEcZ2yNzY41yMmpIzDybf4+hsAcNcTrldVFh+dMawfr4PdijG9mVIe9HamRgEn/UWU1uYq74LBrSARJ9M5l8HKW72wqdOLbkYUSyzyY3CmikjQcvAwBRG5nTurfKdUIF3NJ128rAyNoTDD6ZwPPKM8stmn5LKvPAw+MaAMLdHwHzd/GOjYCfFqMYum2xsYLSX+KxiA2Y/xe0U/N1GgT8VZRYT6ac3JGROIH6MmMPW/ocDJxDWUDCZucoNUzPLHK2ZToBEIHilk5vTVehH7LE9qiwwe/8V3WAui5DNN358xGy5jEd46vjG204LmRtikPsK5jTE5iM+X46aGLZmwuvSSSQlTneJ+FhbI1DgCQYH9NFNr2M9zM+pfpCVgbrep6xycrd41th79kFB90lRI764z//cJ0bOAACdxCA0tWLxirWl8nKnchXFKQfXrF+ds7bONb7IQaKGRZjU0iaBxOsnuLzpqZ1Eq0ZzgY5nAjtjZL1dx8NzYfGSpBXCS9au3J/Ga3irjWbo2rvTya/EzS0ACs1sojXYU7/E/P0+Y9XlF0Obnd/8U2HZFGIiSg1tT5preJzSCzDLC+eWtg3+ztNniC2/jsZav9CV8a+BmRwN5pGbQcaCSMOOCf4RqkS59E8j/XIpzi5s3mNqc3hSY//mKE/3+/ZT1YsCBlX9twuGpdwWfM6QoKtd33VTOiyurEJkZmJTvzleZLsqW/XtdqErlNSrnEsUEVO3yNBvo/1hHBeBg28mGRMOa0ShtEbRlxURgzgNsRCbbRZZARjgEqjidGEIvCMUyGsCIYdSWlyBiaUguduoERJvauD5yor88dlpSKIj4RZ3p+WX3MCwbeAru8gkanPGD1pmpEKam9NKPv6scP46acJ/N4fUiEHQte0CtMVoGGXTwh9gcIbCkknfHqTmVsh0Uy+1AW+hwE2+vOqQ/cSiTrMyeh4aU/Fv17uCDPiQ4w+yxoyUyXXWe8wDsndrE3zYlhUxTh27imr9NDCwFLAZs5CPLg3FpxZudJ1hFa/FZeikzAkjrtUxCBWxRkfIzYoOWxFAz126QtChA7QQp2VtBbNfh9E7sPMbuijQ/NW4CJ4l277F4k4Sg7vt3DMjWKVNiANQA+VGhghReaVtBbOIOxYX5FD4gt1MqkqDMt7CrkN7Sk6DqSG56rYjDAqVMolK7AQ9TIrPMIxOW+0ey69n+2AbO6j/Ns59vsDS6+NyQU9t3vPEuIVRBgC0MGJlnOTj8C4MSZIpe/jXZtQJQkjHfjc/oLg9Hs/BCylVgXMHNhQyNSacPmFfGDgTi2kLrPcRuHuaOg6p2GlqnabX2PHXr14Mid2/6QMDAhPhPbRGDTC2X7Grm7vWHwtwA4vC/7ItnzpDC1Igz/1mrJ9knFLxUXnFd/10KYkM/o+CGFIC9aKYixEYxJnr3JalaDQttagy5JQtlOopDiTM/mSdFbp8ha0LAfHy4xluNsvt/3CS4YpwToYmOeHfKRku2t53jsBXx/7QdF45VTJ1+LvCe45/8kbAD7TbfBdrcytNkmJxinl/jcORnBUrkKr2wgQyGvSET1GtlM+q/OQIGZ5RiovxxJivGd7eh4cR4M30K7TgwpOabrU2JB1KYCafL9qIHvJlOUuPB5QfGX11OZQ0RyTullf+4DAYxUPJX4KurwYnsm02pV0h+wUfbTIccKt7C1jXVuNxzQkEYZcLk68ZDa+tqcpEQ6vAl7HB8eirqwwPFCMhvdK/hiBI1vp1z5k7ySdzsl3ISGfrQ1QvH5646CQoyUh0WNr9lZ3iB5fNIU2H4Hf4TWHc4qsdRxlS8+5qY4lYw95K1XVzDOhlpVtaCfWQzjbwj9D3ND3bGOeGH6yTuYfoOQ/+DaZ/WO7SHzvrRv+wieR8bzr6dphO/4PybwyKqZNnkzdtgbx7gndmnhrFvwGa1wMlsoJ0xaTuFhl98QsZTDKaZ/QO9li8jGr9nukt27/gnGSnfTUgdQwip1WhJfWiSck3kwTH5W4h0aPj2AH5oCUyJsdSw91xVWMyXRSpLWJl9kc14vIPl6TSZBzFbTDQoE7MOv+jiaLn2bW2fUS/L2uwnEYASoWsebZpoTAl6oiOGlitV3rLB0CHPqUpkBqp0jxUCBqOcII45tfOfYWBhfiOVENkgnx7KayugsJfXae8bvECqr1nGDPk0/pwSQe4CSqJ1+BxtRifdNBaY4QC470K5fCOUdOKo2IpWIdypn4rLHRvxh32QTtP6BrBawbLej3AEd2w7iegGQ9+b6r6RFpiWtl3s190IvrkvmqvvTe9OvFP/5wDGGTxDfWgIAaDtmvLZrJPh6E0LEar4Hts855oHELCVoITuzmT9IbJOieHJEGTbgwxheKhgiUIhk/rQRGb/LVKyqs12fFVPzTQ/VcCf71Z7rMKw9kE4q7FvCxVv3BjbuOO8NUgaeg1DGqf10y3Qh/1MjWnhZmw08bJoXuxbOZgJW7ABmsR/2Z+T1Rp/z05ZABgpk1mlXW0W/R4DAQAhKdedfZR1ljKfIoMaIBGZweg8R0W+neB4nBHIHOl2bnUS2ajHcoxCz55jAm0ZgJ0aZO9GfWwRFEVesa5qjk+ysA/L2aZL1kIA3n/t8NDPjag2oZpj2VHIpgy4ti00KjiUpL7KkYsbVydh7YhBMO5eTv2UwwkuyWYmUb32yn+DI6i2FM+48cVXSNBEU4IRtCFQIDDq4KPaOAL2LZw9YGe68Yre82IkAEH2qjQ+gTAjmKJsM4t30iLVX7Gp1UDP8+mNjZMd7Ok1f9CT4diaNpo1RiBEhNcr+H7Uw45Axvp/v/+780MRzs1+Dx3vk1DWE5/4u4HytvsWt4DcAAVCMqrd4fVmr/TgTgehdphNceuIfVqrtUarteVTlXaPN/TqPreiFq5fSisLHmLEHRh2uRu4iZrm6Qy/+IHAMdIyNTcMmjnv24NZs3M4rO5HAsB8RlpjroZhHwacgdnD8jYcaWovRuaHXeNIEO5L1SjiH5I2bo6NVDl+qzW1WZ/sCoUVHmfpoPinky/hQFScQdQS0+28CyzRR4kJRi57F1rGG/m1Wcw7Hvzj9nXn2xKa3CYGTJoe2US/bozdhC1XMrQFy1BKOyyoRdXULXIumO9x6T6DqWGBgDz74v3jbjinifWO3dJsm9ZVbrvTW9XgFqNaHrqWPU0VwZuDzbaX5UZnl1am265MQ82HTfq2B3OEbQd1E2+xAvF3BYjK/vCEax6JHGUa8/ha7RRUmgzlV3fzgLuIXqi1sfiuyDjK6txXnBXg/q5WhaFaK5Mnjs06Qd+uDW4OlEm8y9LHLG8HhMKwlQvSbTyBjI4Q5n0Zg3hemxMLb5svKe7WOd5Z+xi97gQE+mMJZHojuC7gDAgSydcliH5j61hZmZV4j5qJhGQojuStTttOKYv+SIsandG+hYDA2fWEbnOoBXG+KCsQms+TU4bEk5Clob6GLIDl8mIUFAxQp6NKwNCVwWi7mN9okNTZv61Rk8ssGTRxFnIr7xaIbJaN8vlxQVfYKksKdPgdFUSq7jqqKQgnbZS8P/nRv0YfmfKAxQ8GleGZXm0KJ7tdpOgwXe7sxUMQ6nZInCWwcLscOqb2+ahks0WJ9eh51u3sGX7twAGnvZ8NqmkRYXVtYj23MCWg35e6+XTnQoLaAMJicesTe5lJxZmYRb2WEetC3wNwTa7c70BEMXTdc2AN3acmJxDSg/d284RG3G0BjpU+fKVtmTMmFfug5ahPs60Q5vBJtdcZzDMCpkbW0YCRs5FqeDmZDopwjsWA2c0q+feme1j3GPVLR4BpVewqoNK/b2mxbmvds/T6yNFiHoxz1rHFq5KAZUvR2/i7DK5LcUg9kSUrtqwoxxxw9HtyMFPzX7qoz1t+YDnoK+VnKbi/te/EABytkDS4/78wQm2VjdoXnkIaI/5ndOwVzraWDSVuxhfsJ2Th3A38d1bA0LKaIasd9cFDMMN/V7jz6LnDkfYoYKWpb/HOI2fhCN/eAMoRfDIpcEWwFvGZr9OIMYEaAAGJlstxUMkgi62MW3zddFqr29y+uFRUaxImVFJc/lDrBPO5vE1qFfaoTaQFMxHLaKhjrRQZEme3RP2+YWHasq7B6ktU9gVMBtZiKHgE9YEgP2xS2T74rOBLLwa4OwYS/qopGHM4rqxUzeXkD64XIcITXK6Jqs1+85FY71oQZkd+Bd3uvznWTED5IjV9dNjguKo59zekEUj3syMgF8gJkPxd08tUD+b0nUUICK3Tm3FlUBjeQ0ho12a2a6MVqkdoOT9pOPIYB6U+r/co/oEpZjCHMSYA3C+J/l09gggulZ9Nzh4aFd6rr2qMMzLrAbD/WhBdAx9t2u2b61FBu4N7IIIYTp2pMFD2d6ilunMCcxqOcWZhSUez8bV4U2X1LaP0Je8OyICd0OQc79yCf6SiuUkccV4PrQ/6+vCHI8TXdDFYpj8Ant9Ukxt7zxIDNGX5p3PksrjaCgktnmSrnSdYkeeLvHxjZ6PKk5coo6ZwKYLCFILhffJBBb85dxiPD7FCr3SxS1hCnQ0dNCEjTBN+KauEapK6A4mvSdQmNQJqKtRyfDyewPpL3ZPi8IY9N23zg5y0pVY3kU03eF6YKqq0IQbqM/QoDAthtBjA6w5Sc135bcwGq+t73BVNZBwbGRrcpZSAl6VybCMLDaM9QaFEJmBW/5gRepwHW52WR5uWzHRo+kRPc8YzvJG4CjhYsDL+g92cw9zR34b9ylfgMWDWUw/i5PkT/aTZJEacAmXaYEDGJbPblg8VdcKFhIJq0RTsQdCQ7krNi+aFHMEHel5xdNjLgtEH2Ku/2ZaNaaXfL3QV0wpbrTUrCKg1oDG1ADCPAb9yWPGsA965gGnd6EwFQOr2HytMd7MbatMKFqFmGtu1mG3k6aA8mklaS0DDAhBAWI1oa5EyJgST/AoNtGa85hRZzZmdDhDuqulmph+woiXyd2BNzqIWMjcie106ByuhZ300DdXd7+RST2wVaWAOUTQep88jkYEbZjChWDoUefIyP0agC1PiYyKyW5TwSM3YpcRoBIW+NxfGTDafd7fAVXx2qQqpZ6lHiq7sf6+sjv9dPcu5K5WeESOxTGJm57VozDHXZt0xdGu9WLvySlnCtKy/5weSBEAAHjj88mu3zSEV9TZbU1THPTR5CUNVyrF/Nq0axaZCuSbCXg5a/NMFbLyAfanbkV5QS6DjiiWGUwQUN1Y49vPQMf6pdCyffA3xgEw0vm3SsNrUkWJzW49viOriYxJ2+JSXOJ7Twk2SGP1G0R+vj6oSq6rnYIFOLbiLyy2ulc/tM/EeRS8/2n/PoLWq9QoorVseSghKv/xy1zzmcQ/AKv73qOUzIzCpqCu/CM9qW519j0B1dBnQi3vdQYNsGrIJL3zbnuMSUlCm7lBUbtHZdehu64qGCKN2rG15io8WeMoEIriKGCXqvs+Spzjo3SENgmksWx8xKuUrD+EAraI5I6NKJIXI7yuiZydjOqhcFoNEunNRlpj1cMEo9dLee9y4P10nGRKvF+9ZL3aBmuqmfhYPupRczej8pcunNzS/RWg8OgUQfIKx2KrUB2/nLsjFMqw3bzmsWEKvwql3cKKi57mCHqI25Y678xppBBNrhx72vFxUG3Y3bRGiQhnyLI8OWj56rx7PZv7LzT0HddGWrvaHxdOWdEHXrXzmYxieOVIVXwri9FjTRLBsh8ge8sV6bsnYNcXHXqlL3aDNTntrgFM3cHCfk2e32EFOfGwEr8xAj1lR/8vqHTvBBBxEKNTskA9N2sb/AFKIRiv1w9rwmoby60R70jE4XEn5tFIoQDuciNsnqlWj9TaxNDLiPc1tdU7O+3OwhcCGncFWfJmwPedaf7LFHX0fx9x2BBjVmWgQLcDgMOLqYQS77mkhm4lXNnVQKgvL583zc62F94jVk+HTUdL8usjkXXBMIBuwVxV5bOhKfF+uXQVjom+xXhzPmU2v+Ea6cG5gAhUt/wnkcCzxBO14EiGvkIFmCUgxUHBBtvlZTmUoE7Cb++enH3M5egLBWWBOaKjXfn9GOb9qMr8VlH5diBYtn8TCA1uQ0sqfWrPL7jA1wdJ1zOwjjSKt8xACklakr42vDjd18RdYV1OQ+Lroxr7mcqH1UkNlMNaWH1giD0oYRG3Gkgy4IU2uo+iW2UdRR5WTfAlhzmemQxjtPotej+yLLdvmtx8ougQV0MbIhmG2ZWPeNzCaLLF0CRCOlG5vT4saxiNJ7hh6PboWWZQIICy1RyP8aPtkYlVgu0Jawer4fMJyjR0J2agkMaKz6FJLnAOwe5pgST41kJpi9zHGkNy8FAFtQumTX5mMg4XMVSVPDurak1eaSrhyxcnPTJDBNk0XYJNOYyIpODXWLAbn5ipxcfT92tje0OlvSeyoHQXtKML0vJYgXbNxmY9rPwojujDrAdMyb9QSlomQzj296yOphZO0Cvd2f6uQUdn9+Cl/zqpirGC6Xf1xf1x5/GRdrO/hLfE+60mDWepoJDP8Vktq7iyuwjOsdsz6GfCc+JK9Khi8EutaIw9mTi/cEfgyWspmMTi7mA/tHed0SSqN71f2iRxK9Edyp8mvhVXGSUfrtppvYHlA1FP8xU48Vf95rO+pzh7usS1Iu2ycJvLukBJqgWGlQPrmijKGnWKHxpYE19kRctYhYCkt2zjiXJDgKJuUtXBnDYUtHFmvVPy0KW87+BLy2Ay5khrmZFJURx5XCZckTpwJk9taW9NBqj0mctxdPzCehch8tnQdVQMINpy2G3hFCutfr2RbwWjU0pmHSftmu0KlCvQHMYgjyzexGDs47UC5hN+J9uPqmzpRWG5s9ps6GoNt2wQ1syLYs+Z8k5he5OIGshVcJdqWeK6IiW+L5WWS0Duq39LXfl9cEFsfiTMqIwfcwNnasr8v5Gj+NAC+pqSedzReOr4m7pHPnu3dauyg8g64XAS+0a+G9u0aXyNXbksDO2yefR6f7l+vAyzQJcYY/BSucWTuaiJF3JwNRC6GP3xZKRdQURY7UKbXZHE9Koxw5cmNZYIo3HI35OLDnrMPLHakDrqniQhpiCca9B/A5cRqzogbOlpxqtGGuCfQxwxMq+OB5x6c/Hgxeeq12LwhCotnEJfDhNBWAk15cJZxBr3Fwk9eOt6c7DM2GudHbtMCJ6GO1HGPZE0yePtRAj4v8kp6D4OUCMxwIaJ4Ciei/oOvITIu6EN4/D4tZOk5ujlh0dflR0Q0rht/7UHvBsaRVxK9azUKBjV6JVAD+6rGc70xDuGdPT5ercXHSS1IndN2g6Awn4ZsxoM02iZXLD7ko83zwHh1JYkJaqQYRP6IAU5nrnzGX37oeh8tCU0Oxrnk8R7/8gLp3VOSvjJnNhFgKF/Z4oL+3GvVk4SVnpfQGdAadN1LVzDyYMvyZJsjhd6g2570eNkp14Tm0mUeCLCRtwQB5mLZc/vYcMyIreD6xkRmEKSGGdsGXsD+vfCXqm70r3YNL73QkF0H6s6lrqUNhCJ7U43K37RCHlsACIvLP7fElqNzMknL3KQAb7DDMSY1XES5ZEoW7swGquX2zk8eWnVtdRFIkfozH5qNR63HdbysMefgOgxK5G3DAzA/Qja2mQxQZMUvVmmH/VHwQGb1Jj83HpfYiiwa8NGnyAf+OB98hQEn7HsdTNp8eMm7msh6REgSnqSYfA+RfI7REQWoOM4zG8dG+3R87Ot0hBW+RcbX8zBdX3PqUXl08xUMIJZAiOxilrz0X2FPog7XPSPKeV+STm/moIBzJl5qz7vraHwEe1/DCYip0Dl1XCdt+gR1IxDgFPHLuuXboaTgSgBe+22ZEgvHXDKf4zbZKUzTAkrxqCu2EmIO3EEGV8rA41tVlY+Ib+jqhg2P9TaDTokzDLIjSlfP64+125F9gF/g+bsVOdTcMMM00tBBSiKdFAgzD50L1cZe2bjbuobWTdeRR14HIF+K3yyxYYmAbavrQ9PuYJpOJ/BMfGQsdAWWaMOocsE5f4K8w/UNHXKLsg8LzWa3yatNmAb7a2a73nIN+iwCrMwqAsKSTE6Cwt6f1ZcR89m92NZqzTLdpEJWX3KJ3Ppfdq35PRnqT8zfI4ghS1fmcdCgvQb7oT1eFbzE5c9QhRzXjwdGMz5ajw3Y13WTCv+99KiZCFYbr0PFVZ53eH3Adx4VVlC7KSCZVxRWw3Sk6FQoEY+NJe4cGa9uIqCarhaxnoes9mbwNqp1rIS18NocOSVRLdv9CdRZlDfUqqr1Ted8hsnsoU4RBr3m5AkmecEG98NtpVvpLzYiVyKerJDkL+ymP/+xTzl/FlaVELVrB31aS9xcviP2AJif5Z/S5RJ53oBhgOFBHSzIcvrVSOx8o2JxQgdrctPV31tNywaUUZ4QpVu/s/GL3RNUOVfEER9Pe+bZkKdfKbTVedQBc980vdzTHT+3whLdT8SbNEsO1Gz+oU/IQW1fTqCFVK6Vs73rXbkd0h6LcoVhbI9wkABRQ5gXVVQHybtQCpefJZFQE04n7BHOi3HuKOEqyW0Q+BxqM5IenmmmcUv1KNXIKv4uW5FxAvDLCmIH8FFIiFr7K0kzF5GiK5jpU9C6jc5ZzaMQm/9rn8frxaW1rbTw9H4PnCc4NcgtaJn1gSwkwmtIuFaAvm0vdZa3gyu4cTJG/vvHj33QPTfMgfebwMii88bPav3yIzKHWkMGBY0uWQ5mCCMJehymh9DN4XwiPdRgg9poOvYa8w0wNoi/udDPV9OEwb1P1IvpEEjaCDExYuwsYfcefRe5uwR7zkt+57NVXGlTvqQF7BLHQ0A5aUUq+UPkPLJixE8HYTTxZnYG8dguXEIApDhwmYMvxcsSxkgfACTByFFkl71PlW3QGZW6Jw2sURc5NubFJ25Grh8bgYngeQjgQYb5BP6s+LhhQ184lGjM3J21T6jUHI9/Nj12oT8U4Vjw7yoMvXTLwlQkHOdLAR8ju5BJgXAX9VCvyNRJy/vMJGsvPxb84JXjOi3IkF94p3zvZ911DT2Pjd2zkj7yNE6iGVUWUZ5rSek48NtdOeo4Zb1VaNJrO61gDJfMOGuys8iabO6qTBLPWwYmgUYWMNyJmCu62k5C33M4UCJy3bbSk+RwxE1psXw+5eH74BxJMmuWvjgsht/Z7AYF5SxFKXhvIX41AEgWVzNisgB38c6ARYRf3OxX6c7Ped8G8wNeRQA3QdQt78tpAusYLpEqTrmfNEnzL40D0oODGaBM9XQrdc2/6ydurt1iGnVCl1FGKE+muyN7rTJdNZC763i5oXFmnhO5nTzCRIlPtCljocsfh4pKv0z/CV9KUOf4VH+R980cXVoFiqt/8Rzn5jQql++pe5Kz2mM0ubiVwjkOrbWU59moEWH3v9PmCFBG4BqfZn/HRhWsp28MHoEflGIoiOUeukzCJRd26LjmLqyPbipK3VybS3gLASsE3OV2vk5xKGe8Kdger+grNTS3w4BlsGu6qQ1cjf4BgrpTkDJbxuKx5EU+9ByBW9tZ/EQpA7zwafnhvIKN2jvLiQboMpmpsv86ohiZABQ2I71fhn8Q7EkDzhzXLYsWCWGVgdxFuw52Uo6YwOvOye3DOM++1z5iXNUFIVp3lJUHoR7hzuFdYJIHUecH2cBhis0pxBeuz9a3Y4Qnb4AKamIBEVaQCG+V8WuQ8XW100aYkyAuAxGjiMVjyY0xrRHmW+upxJTw1NidXcvZf4H4YWhcHpSRarA0uEpmdTFZArYYKKGCvmLpykrKyemlcxEVnSJXb99cKdjIvVp6USA+sgG+jx3NU5ovNMYypLbjo47yUV6Uq1n+FFkaEvbP/RpkFHlKSmcw/yZKR1t8mffqFm4LCXOlSuWCzH3J7Gkx/gtMCbuN22nDNppqQ2F8fuYL8atGekEYS1NhjRzhocV6tusVd+HtTX18bgS//0+Cc/m/lC6sdD0sy7EwGWnWG7kgOATTSmRKvF5BolJC0QRvKLj9P4cJwvaGHY/HMyQvp1feZpJWkOtoPjHo5iFhi1o9jq77e9JIBr/UZz/Fpqyu2RLMToiqKTqgdYCDJkZx15KV7VmR4rPtf3SVQHaIxWFmGt8ouwPs99092IbJMtbm3wv6C+er7jzZWpspRWZcqAoGJz6RfHnehOOTErRmcmB1oLNuvHu52UF3MeImwbrtoPCwiGa9yFT5VusabA0LhXiVdS+U0+Qp5xASmWdvsWopTkENKA5WKMXbujloVJ1rBvrZiEysYn5hn4IIHIMi2WHcEGYlJeg80NFn+4yxAQFVW4sWcRN5ECxop9Qz9+sC3LRLrgH/N7TWeEzGsE20EdELqhp8jridMXE/eJw0Z5IxkcuSVmwwmXp6LL1CTdjfvsQLar/J46MoPKPXn9axUNHQBDiCYvJv6v8kYrzlZsNl9hKETy/cqVzAKmwxyYs2VeNEuFidsvHiAtafGu/liJ8cLcqA7QUbvZEqKJj+4Z9tstK4uQQZKX7GdFC8I/7dBgITfFUHjNl2YXClV3GhZI2jqVM58ZA1QdU09wGOfiAIqfyH5Sc3LoLyhzfdfHlaeAI4F7fpP8wTnyknO1s/V/I7hHOxnWejWjoW/LsvX7QcOU9kiwgYWU46XIfw8g9bEC/35ER/pipe9AHCdcx/QF0DSkIlGbTUHckD/SymtBtaYQRuqSmdX7izGGcbWiNZwq6xtbeTzV7R44oDS8YV0a7JdRLnyM2kEFY3VKFqaKp5O8F+PoDUlmXmOcLdIT8DPDhqvorDdpf+HnBMb4rC9Rz6eJnJKLOSu5GouZ2tCbJBBsaUq2jLvTzwfVlhF2f1ARL7n/0cDVi+ncB/BQDjxcQRYKC9TTv9RtqhZsqI45YgJ/NpZmgaE1Nddl1czZEZAhyt28pojpV3XjJ6QsSOlc0f597ljoMkKveDdgiGo1PHkas9ffO3mP/xheaPvCrWEKN+0DQi33+VLq9BVtS3cRhBYu7eAlZMmetBD6YLMGNyC6ubdVJp/Vu2T4AEcZUgH2hLitJxqFP5CKbukfki2BO5ihhXX8EKS954d1iAGbuWjA4EEn47MW2eSwK+/qpLpRf29ywwRavFC1UVu6i85tLYbm3BFsaZ7fXnHrCIGKc5tkttv/Z6kDbqqgLPEbcLc2gCWv2Ww3ueEfnN/+33DzUZ0WEQ23sUlaLhz5ImDpoHWJKwfeHorRqNYSFfAdFRTBQCOA2OIpyw2JymbNSUKbL4sBiBPgYM/jaPnYJy2WLM41CXYgKq2Ut+wxD/2Szw9ZQp3SpINQVfSX2xtQyVrzBFfjwv6SrDuKI9ZRGG3hmYMwRfW/t4NxmMNdOS9NPDuUfS5/TCa0/kZ7Z7MKQmkW708JM8HbA6BKZbsrjAU8/10LP1eUpTiZYYTJr+ttErmSr0lBOSo0QnYZ/zhtuY8XiDnPqXRanrmYlZRZOe0foA64sENlzSYgFnmKTmC9ATZ9F2u5pCI9yOJ4IvIQ8BbEWPd1BWUeujidHDsdaWRCpYUp0Oe4XM2ouNC2l6T7Rf4WtgwzBW0aoaDbcXTlUdcg7lohdT9FEDqEhPjk8eG/FLG7eTYXLg7amkBd+hIZJZXNaxGoY77y8SLwfzpJncdXfcCfCDvEWlNZAdCmxg3lVmM+f3LbFl7yQ0STMdn3NmniMnQaC1oTJ/Ycnw+cWFuvMgFelAabk61rWsmQSup0dBZOyhvNSTYAEZwjd8OPKPGVYo+nCULVZolQLXue+Nc5Uled8hQnNlFLhLrWw6dQkbELCFjI24nl9CqKjxkF434tSVisfhoej0oB583omFq1viNEPmNGefesUoLySLjvxXOaqjY6kEpXCkyPUORJ+/UmlGcvcs3F4QRwqNBE+D1tnPhaClivMXHWULtcvRjF+SHZ9I8msIJNS0v/sOJc1TBiCyKZv05tQvkTO98tl5CoPJ8cO5b/HbnGWCrng942OpAHEOswpaEe8a/wynRRaw5jUJAcW3DfEdThIabYpBZ+cWsZLuRzjYj334Izt3o3VbqRNc9WfaWYZik5/RgANSpdUDtDzO4XPTxU2OYJfpHU0qHQCZdx3TULiXX5S4lAGEAOnY1I777CoLCeRiclONEeNzJMwG9TdhnSr67O21vpOGcbJimksM9GaGZG66n/mC3mWcWCcnPde9GB99iBgBUUKGmf5YV0qxL/gTYgoD/L9qIvP1FKBJsdPqtr8etlz5cO++23CqnCuQHtfu+purdZESud1QYHojRL0fKwZJTpoA157CUmTmg2VjOPeZ73DQx+XdINrDY6VsCg8FdXU2ZejBGCDY7Q/A6n7imgCXfhmO/5aR6DgGTqfypdvLLS/P48CkEkVbDHh/ELszdcLaMTIjdeeHNIUEjjXFg9iufCs3nisyEg3uCYYDbS4RKyQj9TRnFoysIDiiCBJ5Tjo7CcMjZ/N/zavEHzNEPZeuwMTMTpYJS93NAlSyBAQ2x+6e8FVAOfbpEO15PVNq0oVXMh55OPtVkUWB8kAlD5lNpbz1CAKIAV1rKo2ftBKYpkY8Ej/b1vFfihfYMfrQlFPvTPbu5N5GUEse0XBcWBSNMxrcLqc72wDqvFLvkvSkuOCV5xeSHd8nw4218ifyxxWjLKi1/LeJGU+YDSj0kUMPvQDMsk+W34t3RoKYB4eqZlokJXi/vFhHCUM0C7TLvEnePKluNqywPozbROfhbMw3np+RH6lpQVoscOdbg60wLq97pGm/CzX5Xnna4zIWLJnnw1omJTdclH/sc83W9QMfFHvzAB6+GZuWUYwuwFO9J5/7kBNmCjDp3QvYr7xWOuelzo3C8aQW6yoJNS9p5W8tklOONaxSUZjoH9JIPblu5xyGXN6eaSy9bavY6IlN/c8onuqtJv1CKCKmvnFC2HexEXpd00+lDfYgmEjmDJFJW3mkZwvphhVdNkTiqbaPUPVrZS+b0MA7SI0wUMrFRZosFhwSyDQRzHnp4wRKk+5NE+CLUgULNmRgpAVRnyzFRpXzHegJDUCAMPtZTShZ+5QqidRK2FWWLeGm1bhX7Lmn8IJ1uCmlJRSaZuMd9rjxR/tUUx6XKcNkCemOPhSbRM897rLFltW21IzCh/yn1yCkLgCc3WE2IdXpGYFetOHWhF0AS4i0M5xFnSI+qWAbbg6xrzqGWLnLhSq/+oi+coAa/0zFyOt26LsgDV3ughI4XxcJ0q3NW72UTyM28gieLJmAd8pJko543ZqG4+CkZnGhSf/y2FUKF9GYIBZR6//pmKN9QwwCwmHjKzZMD3+kyXy6mkFX7inJXozYNu9/OTNplwrk8BHRD8D2VCpbxjyu4RqGHCNDjpkXVQPbyvmSZGRVew5tqNrl7hbD+TjnrMg9K6MlmXzklIJqB8GwmbTFRxQzPGT1SnNA6Ayl88OBSoo0hg4o6NxmdAKvU3kZQFoyz+vmK5rkkVDLq0cJ71q5T7zcp+oDmFn57Y0PcLuvrfkZPEtCtRF8ORluVVffBDyZOf8ie+0fLtnjzEY0jCbC8MCzImSE8e8FVApUSjtLMzCknc7oL+k6NB+G3guwqgLDjZI3lZfnuWgCbcm1gHHnQ23Tb7kGCc6Gz9dHHbUNHoCN0iO0GmMTJElcjI/aTQM2YlVX4KdZjCURRJ8bM6eQnHKtVAYN8B9HYPQOwz8Ozq82uo/NYaIiOnhAmPigdgSaOZgm6zYHzUrb/QYNd2QPCUkawKZLA3bHZw1k7ViKAlKT8pyXEg+B+uj3hwPON79S7+b9gF7qZEiJ0TbIuq4MM87X7R5BxiDXFhYKemsalGeYAxOcKiD+jrmz/KKG5kz569qj9wmHHAOlZrgYufS77rP3BT1Uotbm+9cYe6zI/Wf/+6Uy8Bremee3R9YocqsW6I1TjySaSxs8nfq3Dzbu1hP3y52ZEE/2MV1tmch8Nri9WOY8PMMxNXj/qYzFkuHLxoXId3BRQBApjjpHL2l3NRxFVnpQvOBgoMZyldz9lDkJOC2SVtWydOJ16I71XHA+yE7g6ePll9vJxbm/G9bRa7JCYlra/Qbq+PQrFHL2dBEso2KZjrFyhZeh8vKE690pkiykjUrDdauwaAgsQPz0m8JXfBWKI8Q5MA+ppCgXjFVvQJL9bbggZrmRtEL/nU3xsNHdhOF9Var33IKHby8lrW1jTpqBu1hae4TiO4VY+UE8qAji/LqWDYC18aGvF3JFnyfDy0TTfNKmPoS4p1W1T1pSZYuDe43SlZ36uSRO7doQsCKvavVYKgFMk1sY30KW9HJtA4qhPKyJFHxTzm6d79DpWmvubpOvto+xDmvDxYe6qt4AqGYp6zHzXgqGyxiOmHpvEhX0YZ7nbqCGf21hgk//aynCwx0ZiKat7y4u2MDgyEuQM2B6QBOdJF72SqdNXYohu3IprmDSw2Yu7O9XNHLtsmQ0otASd34ILJYKMWJr+H/HbL4m8I/q6CUnr3g94o3mt1fPSY2rDj35o9GkSyqCguzbw1HhrorpmKeeJ2DQ0yjdEegZq36sEW/G9R0QaBpsuv9ZQzeZzCH/ghktMPz58ruIjHllUaC2iVtRbq3pMmlMRKQe7yvzJqiJCI3robpU5FKLeuUKFTwj+KlMnxh7Adi2Qkc6hA1doD821ngRN0eogxpTG8D8Yps2jm06R90JE/YavYEE0muIQW8lK/8Yz12IaCReHR6aA+YMbFB8c4XWeD9RWP04xaATYwE+jDRg8FyXjMqIjlAyLcC3XHF4x5Ffj70TxiGVqHd5z6n504ijz+A8EgoQaYqscM/QqJ88ZGI0dmBpEkMZvRHZ3gReUC0N9UyPqvrmvnjDSMkLI9bKwrxiXA++GWfz0FXJKAlcO6aJtNq9Bap59CxFimlod3+MtBDN1xH24+qPBRLanubBIVOE8/W5shr6k4j5sgQkqt8Ilz//sHm/eud4hf/L83rESlfUA5LHzpwUYfEb+nDusiW0YUlERL85yoQEu11jmlFmmSKsYlC0U0/MFfONUn7H7n2n1IQVEjQ66WE4c7UHhiElXTuB4mO0uvEso/+rAxtYdlCOz6xKz/NQjctVod2mIMN0aF4m5IyEgs7ro+UUTDXux1g6rAJqbVGvkKZVPHCTPQWI3v5hzQHE8zOGwLgg16bKiLO8bdM0zeGCuBWZCbIXpPGGV7inqTEPONAIBgBDiXgvkKIBHZPvAE4C7bc7qdjCyqDXTPtwcKGH5yLUMBFjaRpt233WHyz4T0cQFe1qVH4St2GSa+J+MQlNbyqjJEQ0LgnvLMZOGQ936dy3nEX+C4TOx4jjuo1qJGOf8ilZM8Y8AH4O6eBRqa7MyYLwCgzIUXKTkRtjI/7yLaHEb/5BTZtMLfncSbIK5/+mMCWUUjqeoUOWcuF2eib2wbbbMoumSmXmZDrRFOi4jCylfYldO3ro0RvEXcJQ3DJ8OpdioodZI5VpQo6mW44kC2QFRsmn1ebSGiiFUz20cZr0IirBFFPladMRa7snx83o1WB9QlhCShKE9jBhZnSpaZoQzUbNUevBZgAEknn3TQrxN+LNn8NmOpNW9vMcRDjby7bLlvOVHHIYPIUdGIdD9m6IHOcQ/WAVW47zCS+dVIL4uNsYj++1A9IjCs3dP9p3Nl3XZQ4OXVZYzLf+I/7YAVd3yMuEbv2WA68dmH5vC8TNGTk2L8M54+/ZTrLuiq7hIN8T7LfWc5Qp0alsJvXi1eAFh4bHtIF6iw9BGLSfaYpnFEvdofzK9LdCSvly6duqKKijytp5DxBRMOvB+rv6KlcovNi4foqdCx4aK9STZhExRzgj+gDwkyuYjboi+TMdsk28rSjj1i/tgqWzg79j3ESE+x/IWZOYbrQlcXUWKFC31gWqji3zNN0YMcJEk4QvfH5kxixbn67uxh6oPk1XLUhiAKKM4HK+DV/U4PnK3mJGSFaxqhLlujDf+Jw/Ah1ctg6RtYzy5qEC3kM9vfIii38Cttxpa4Fu8GlLIx+K1gRTuKWcjiaCl72h6JNP6y5a/alXDdPYA18ktfwOvO9vknaQ3xLW/wQgzfw553PAS0s+Fbo/+yBPs4JFaMDSPvyVs0F/H2mWV6DNVwLU/RdPck+sbhRpX/5OP18lNJg6RXZs5eM0Ued3RkPd8jZmyDCwZ6vUI893UVRtTftlbmZ6XV143T/W1flUJOAbbXnsDi5MmKhS3gNMf+UWtranxQbq3UI8GSJSpXAcV/eaVB1juU/+x80mHEXYGHDU5BzoVVFy+6SfGDINTNaE8P0u0tTR4NrxakjzbFmjOrw6HhH50SXGfeSJviVC8nOOWbXYCS0ZLgSZ/78ciikYlNShlW9dM7bibLp+MqMaSssM7vtMEB8ng+yRkmjWdPUo6ABrVA51s4KRUUscL8gcK6j81niNj2NYUb/TPqcW4AhnMp5lTkMauifaiVIy+mn27VRivGrnHlwJS0Wo2UBL7QObJ4Yky+imMI2/CpxYRO9zZTryB0mUC/V2BnyRbgVdWQHgQ+Jf/7uOqjneR158C1djtiU9VZGmhuygBoAAsgPZMKADMiucpW6/zxEQ8sGZfwFAGC5V/9dJo0Q8y1PqqIIjcfcrGu+iplI3FE1eJ1Ur1yXdVBtmp6Dhk6enJlmKjM/IYX3RtVvMzU4rwcIw7D8sVY4giUnBwe5o20i3L53w5144Qcqf+WtGuFb5xzG+lVECcjkveqxAqp/wtyJd+qsmlwLb762mlPmwtEPoTiFFL2ptmhQ5XHQvbyuelBUV6IkmuehpPRmaUCj6ht3qi1hIpxkdvGOIpGlmnTUasAHygw4cPzO34UYAxMjH/Xgdy1SkHeiZ/cz3WqmYJv3Twvmpa/jXMiCq/IcAmb0wC6WaseW2snGdqULGfExQVdY1sEg6A27b/9ivpAtnR6a05bkQUAn1lDBLS1vxwgG0Pd3uvn5Z7kwP3S9jEkFdMW9WKvzTQFNR6H8E1yhxMjj5bt9rgz/lc3Y2ZdSR1UWg9KTD37vc06hQXBO2VHaDjrZeRKWinCfjaTq1E0ImD+CbhS4t4qDsSVjdx+VgR2OO+2NqwAU6TDcYHXFg0pVJafcN3ph+aGbIwM11BrfGMt5kQc5XJs84ikYZPeZK7CApsMhxMxhMxgiwgJ3gRQL3VlROQM9N6/gwSIK2/4GouCle6FBmAArqd8dz2mqUwI8m3QaN0mixjrIvtBLChcvS+OnrTaDPusxxxlc8boj0sSz7WT0TGBjket/dBrgB8o0DL/bU6FWLX69ULlt4ISUlb++XjpyJRewdE5S3iLQ3ciEjKQcYZDykCbRqw5U/YLPkDEGzWJ/25ppNbUpjr79ErTorf++m17FZD0OxOjI4ycZ53Ref6WZCpi9nhlPBQsR78Kf7dLPiFHe1JKk2X1lY73E57GS78lupoNq2jG0dWgkc4YT9rRuXtsRNBmuNn/CkoKSZ+cJbhzlR+lqFaeEP3yQfYShDHqS2OkEk4SMQk0kdk+cbhqByIzoJkAv36INErKX2QgYCUoUimUTcjLATOh9ZNxYrllP+kmC22J6co+MY8LDPqn+oJtwVYvtjkKnXIQH/vcOrZWbHLh5lB3M2HMp23qPK4BGp5jFS17FlcnAY5HcE6uAbvr9nwcEo5Nc3rmTHgVkW8LBwoGXzr9g6kp8DK2tjRaYr18SF38dssRX+LhRbSu8Aa95rgvj8d4T8lEw2MviQdoT6McWtnoqiPJECK0y8OZSb9s06yEWF4k8DnN1K+CM7xl33ZiTOlPxYKmi0CmUp9vuabylzIJ3NAZBui1HDzqSPrUKlTKVcQozn5Db0664fPXzfuxDkWS1/b7B6B67EsjVegtjv1idAFTP1UUw03hrAfNFvN7adBkS3JbNgRl8PFU/8bbZA3Rs+TlqxFwzsw8tpDCLWtVBfc7+hQTQmKq4BrDY4s6xgmSdLZ56VAOO5ph89Ru6dM3va8wsrLXqK/o7tBU4O+Fm3/YfaNIJkXlZxSDAU9nntH36DpIZ5szdEO+Lj2Y5MYv9jq1mdmf2D/EfeFstL7FBw0PSKAo1wrq7wyewJxT4Xa+m6ToufHYOL70Hbgmuh+rYbwl+OHTZ7kLqt4NuxhAVueIXc5TReMGUJon540jRVLvEwXDV9v11D5kX2HPhBqKVOAfCmqrkOIpc648GUs3gRjXgpnOjAB3O6PBDDVp/OeU53P1/NEsOawKFINP3j+NADcT8k7Ixix1tDJiItCJcu8oGVmDLI/7fUmF7YF4ZEsXVHxcNm8NsIWe0UdC+3832xNImJs6AwSjeEXlp0nCZxmms5+ZXFwy/OOkXEHieXGx4mW8pPw1FHJrp8Z21YyNywIdwikfDmYeW+d65kQoIiod7MOlqdpig9JtaB/xeb46yNceIjVktvzlB83bcQD74YVUj7KCFbnSNtC+uTkWgN+19IrBd4I31l6OnqA8iMrCbfK2tzTB09+lZiDpytScv9Mqldf9eMnh2MsLUvfSZWEZMBmr7tJgNcJWe/BsHaKLG/e4ocPfkiG9KRn44FyocKNEPu7CqcuewNjwRremk55+i9k8KPyUdo8ZMyc+9Q3GcAOAxX+pG6IrPDCXUXrTIinqYB/DEcVlNF9vXYj+38t5LDYe3CHP5w/btIn7oWCrKDhNa2FSBRD7z154QNCCZqqIVKg9+7sx42jkYiNvIznFEkD7BJXTdk9U8jdM9bSyJ96UdcdgRgqhyX7el11zuW69qjS2IW0ogf5prghYqbZI5vcrIlkxgIbMbelHtihq/NYVhQ8nMja4VHBnwsgyJkAhcXHpkkLyzf2XKKYxul01ujWVNSuglZ0wBg6ntDxNFc0q1eJKuPFg4bz+7wtDnyYWVPwks3uyV5lQ5KZGKMdh+wT5Hak7lBeCC2BP8tV9kH8mRKLOniAmq5QNvQeYeJG79N59fDTzi84He1amYt6AgXROYMKlPO11fCwaIgdtqUAzU+EBpOtu5DIalGJBNNfYT+N0SbrbQCE+LFKVkK1szZZ4u902nzHC/n/gNIzKqy1JvTlrrAtHksaNP7VAMxDSo+BXFLoViC/OZuGqs6DszyyDPQAx/+B3XSdkgr7e3/7UxveTBYy6etK5xczvZRkAkXLa+zEPOSKCoMuMkOBz6smNDrE5AIrF8xs5Xh8jy2rbD/OZQgnd2hzttj9VFd3+w/pfnCFDYrxTCskJ81dZGoGZt6ho1Zd1/laNbJkjCEEjwqTF1vdxN7+FWZQCSNp82r6KrX2oQ1+2sg6TOYTKRMgXH0PH7gwe+JtEUIr3Y6PGZNUDiDH6F7J21SkYnl2DRpqVBg+wWjw+NysbeVYDC0Wk8aurWKHyIDccluTF3Sbah0ZNB5aC3TaRfUntvCB3d5OpfMdmvZxeHRGAOkJQ3xkFoGWm/fI0C/KHLyLqBzNYMty45+vDVtxYK9E3Xx3GBYYd7TRRbYBiCK+TgENmqEelk+WfUcphnWkqHwj5opnRJBwSia3gwOPzJn0VlZ8r1nx9VLR6DiDtKKeIcsafk9v/HsxwtTJ2MfBjgld1RtG6PvI72CDde2+eWHxtu/cj1ArDwGJ6168lPuMH5TCr1bzOLs5H9isZPh2VAQv/UAeVDj8HnIh1jTKMh+8Ltx2lCvXGPqT+pB1IPE/BEZ0UtR8xx0Gckx7yldRrUdEOJuWBkLQM3nY+TOgXTLr4kufdwlIOR943lygOEWl++e6JXO9JQW0LFdHPCOZwTDZU4JOzPVw3JdzjSav1CsYmwZXe3V20CskAo+VXfvjXEDD0JPGRk/zasJ/XLqc5nhgN4YWixntVD20nTorq9dl8G89FrC9uD7RbmLjabkoJ6pyg0b4D2cveqPZvsZGVFpTubdyhVne6M0JhppvJOFpzQ7xgefL6wXEC37zh0cvQB8UaHPYN9WWzkiiG7ZaF+/gFiRCbwajBXIfKo7//QfVA4ZDcDTyy6Eego5eog3QTiY8+H4FyD2Rn0fFcDhJE8UoFIo4j6coxO4Lgu5iFyMWMFKKfeFFNWvu2ubZEHAjBmDv7JIqkzksn4+UjD4N5pcNUOg/gSxv8d2xvRCXPrkajCXli5IpLwD8qraBjx6tJZd0Q167imIDMsHJ7h8J2G6B5S9JgXhAZXf55VEGvLAdwDItp2U0TAl/9t2DtKEtBBbRVLWylBA69BnS3cbF+b+ZU0HG9014O6fVxaZWazRH1wJKk4hWhPyIo23jLCmLCgvm7YUUgmlVok96fV9d4GqtMAGcB5LeNZ7RXWpaPxbKyb1Y8EiIDHpWFR25SroraUXZpUq1DokQv5i8qC+c/l4J0nA1Z+o2m1jt3lR7/zaJbRYGfogEguw41l43FAqMFYYek/rAEQaYdhkkx29wdFJfUxKA1V2xYGYXTdhI8hdpkcqz8n97/RPyEjtCdfXSVwpyqwCg+/ekE7nRcqCls4Xy5DFhZoGjn2i3jkbDZjOqX8KjwTHUxmh2mdlPAHcPdDYJtP8ThoFfY3VaqQvfPNWL9OdSSmOqjfmwWywB89bOWg2I3cSfKB7xrAhIaCVb1s5sGF9JN6Vin1yu6XGUToybAF9keSP2PWDYlnDfdG0BIYABZsvf9FcpbI/C8m4ajzMTRnL1jV5z99kIgnzB0BFIH7aRfLcJ7cf4UNfc60fARvHuLRcAwwn4ujB1JdnFOJfCYJ9A08dwNFjEyZ+Xakgbcx3JtYt4oozAH8oS5iTgdpaSvn+nwPmr7u3gqe7lOMNvxHcgkFD2qcz4zo+QZVcILoO43oqFdKerKIquLKMlTMBU4FpsQv746Ey+6XGHu5DaXCsCOc6y+LuLngSrL10OngNUigCjH6sYJd+QEt00uqKw4jklGPDSAyyRhEEPV1jrH2ywyuxa2kPSRzrZJ/Oubn97BqF5BtzqG2lazrBH98Fc2lXI4qhbuumEAdpb1dySCYwBEVPCngThXdMcyAYnZKZtiVxTG26SUZRMgyngUBkQh6dnxXOTGklucqJz9DYtkhdY4pyYw51/KJvIKsMr76HchLzFY7/KiWTlKNesarugKmNdXG4liNb13k72RS2nn1YcVAqDdPOnwjouW8kWcq3lhQYidzO8Id9di1cVmb6ZhG4yu8HuuooLZYymTcdPbuRvkei9I6O2zEkPXtmFxQboAvMhPSOVD6FXr3vNg2ef9TbhJtUywKtBdgSkdtBg8BAo4e94PJBxluqQYgaGBEXksd6MXkILud3ifY0O5eszIOMw70cyaJ1QMExoLtb8VN12SJNUSxkbTvntrqgvonCvi4pOURTyBGvO8ZQsQZ9q2+hhQcXhWWTOFP4wgIC7FpP2lXoRTDyYMzAqD2bgEmaBTwhpIPODeaunrwrtkPgHArzrWs0ZQ7REQR6oBBkrAR2kGD7dDKc/jA+s6eyylOh2sv76MXqdgTshEn1K1PzY8iaNBtaqQKzaeqvJjKdKEck0WvFmeA7dXlm5nLaCsU60qAqeOY4Nh9cFwSv3MzuKo/nrcllUcYXzyc/AesLAfJhrMzG1nY4EaS8LDMHxfbCQMmE8g5kEmPUTvVqvFagbms3jlqntZ1Peu6lvS4tbyuKWECiOrsgz+hzYkvmjzdosHq5FjoYxp6YqaR387YvXVmG7VhLet+MamJ0Zic8Bxk+HwXbfBIK7M4Xym56bhzhGcEIgRi82SwydjyNlLXe8GxzqrRqBZPXC7XNkRldVUclcYfErkT8XwgXxmeNyqwjTmSanJF/guMNxVB7saaMycr+XJxQ1JATpKnNa0Oh0r20tPORJURvcTrkuh3HFkJ4i4F58A02R0Iv+vgqhfvxvvAY7f9d5Ps1VET0QTN6cn25uA111yY14Ec5mBRWo9rSW1+vu6DjVlAVDlsiEYNgxFdCIw/YA0XTFpV/87UAd3sx+rBJ+mImljyvmH/F/eW3g3r87WT6LY/dSwplW3hN4dzGSG/vkdmzs/95KjWwgwVSADQ2iWNCawC3vsXG59K8thz1S5AxeuhJhcScpDGRqibDTbsqgZRryaji3zukGLxjszPLdR5sTqO0N/Zk3JEO4D3o/piCitaqk0ex6961QLHLm1xSnPA94zrHiU/k/gJx2f60b+lQXk/9mBs9xoM9zhy59Owm0K8aVetWy+GuOglkHn7wpiU6jdamcbWgQUHAblmOocWKi6v/xIFpbMSFLUem2yGmhk2s5cA4bAElSxvYOSSsGVmhThiPOpGyct5KXRwqlk5tJDoyc+ldW40i6GXWnLKoCDPp/hJW0jW/XWi15xF/tDN1glO2bXayrZ/wDQrxuQxcqw4qI20W89a9vd4DjEd0lwNSh0lcPBEr1y1R1u5TboPaUxPuIU9dpmXQShrtnwbYHvaf00wEdoqB5Qesy7WbSciJPtNnbej2a9WmazKwfcHLbSPvWV+SP1/Yv/IbJl9MffT34zF8CAIx+ORyMoiWk0Usw0DwCRnq1FzhHngFKTj0tUu67dfMJpkJ1ToMhnxpZVeITdiZhS7J9U5yNzR6TQyWq9C0Z3J80Heox030UGg4efllllv1CJxfHElo/jQphw0fQVXQwn0hzitnnNTod6ync8hWB7RZsWDT01I53Ddm8+hkNCXPhrX7nbH71Mq0bg0Ie9Ja+P2CU3sDlK4/kh/6cQhiwoTJ0G3/BYBNxTYBHwkwPnyHsZCG9dYNMNTw+I5DXl5x9CR0+wdEpWaTCDYWmUcOCzAo64lNlXwFVycx7ZofAkbImjYwHLF6ab7cX+BM5wLw/lMYaFj35rNLTKAqc0bmQv4Ag0/k4vfOO9Wqbp4lFjEZgEhYxqiodRwYSXrGmJz0kwmZ5grSX0RL2vgnqgheJzgodtesxQY9Ov8VbxjhYplhliB1TobJJ5h923I2GPgH32TofAzCzy672zQEepx/yuOBS6O1VaMsA1j9/Iz9qNNa3mGZWG95gEE7mXkZrb7J9h35QvMfa5eCwd/ufGzbMD/WBuzZScA5ZBiNA4SQ7Q8DN2BkGfMrQuSTYIao2F4UtTnN9L93V4JnKKzcSoNX4DlIhcZYoFLdIQGFlJYtTnlt+tak3pGNTYHjVg/V91oWh8JO2ZxdP7KEtZUyq/V9BK8JJzil/JKI/hdIsvwOwU32Ge6DzQ/JB6N2zSiGDFuT1xCZHtGF2VRKdCC3rOhNajikggJsV0fldhKz3Ch/SQq+lQdi1wZjX2Ga96snSipVqBf9MHmvAyjesqOFEL+v6x1WVPHmaBwcfcLYldtlEJ4TJ/6GV9yF2VI5Ml7C/iyJ/HuoypE3Db/PQEaZ2oGy9KJwIj69LotZtuaBFRUjRa0t3ReWoBdVm5Hvpdvi0y+a0y8WFh4GEPqu6JXWm4sqcvRkrUYx4hMR3FASuBG5dvs/3Q0JI/uA5VBfnou6RWcEJZ6Nui1PgvdVe/ek/P+6UmvhNBV+REbIGkOBJ4yTCL7HznIGGRXZ9P5lT12HxK7vzSY62aco3O29Y0f+2UHlCQEXjcDHB/8Cxzp0qMVahTd7LQuEgoUX6yLIuof1uXdnYNNOnhsI6PXhbpNzZe2qWMGhBcY37UeirpVNw87aLq/uUQyydmv4poi0PWGM/ljnsIRAX9Z4Sw5LQx5gBycFcg3Kv+51rYIZDB+Pg8YP1gVp+aFY9e9AsriYRZv685+C9CGFksjeHwLGlIyjoOIDwPhqMZrono1X728NkabN4xosxufR42LRwPD6gkJd5SMz9G9KU8b2o32DmvNS1EvVRmYLBAcdtWMLnZxf3KLbA/6AKBoFNy7THidpR9oFp19FoYaVsT7pVwflSzh4z1Kt50h30FGfIVDka4GpsiAz8oywcFE8cYFMHu0dPoK1Fv5yIkhu82FcDfcorUnzC6/dPxw9+AYUSOoO2KB2+0b2kRJi0gO3rp/B8xVdfe1PvB0rUarm2G19NKpYqP0yvooHGpHhu7EGgSr6NCefZmvswzb9EujooNTzdfjzjEJaUxZja69y9t4pgijW5z+ivNIor9et54wM+qLAadb6h6aRNg5ubiAtbHlyLnVcx1grTKgFnVCTYd3pDkeKhPM7caRbH5F9yhrzccmRt3M/8Jm+9geZiX17FwwBU/74gGNl4V4bodFQXwFo5ib/1dBPgOd90ynYHqJGbN1jaPVaJYG9YvTtp7eK1N4gem/0U7oeclhSEVQFAXtR4ri/1PeF2DTVuSAUAWquxGGJDVb2/v7cXW0h41fZRnRd08bIwwXC2TQxGnYCcqZI0jcG0Nbkm01vTk4/catZH/2E32LYrSfmAK3eJLQAiwPQByRFeQ53lt1++8emFcFTRNtjBiW8NNNnC0hosT8L7n7PVne0E/kE0zomGnkof1DpVFZihesxP7Q+n8dn1m/4VQkDVWQ4hruDprZiADA3HSrYw7P4PLYaLvd9uU4DaAo7Aq34P6jnqKJpkcVhftjMpbl/Mj3vLiH4TQ9hMdZRW6+sw8G/JeKya92/CKFCsltx4741VB83DGFiDwjbxJG2pphi6/DGL9Z9Gl476JrvwQUAOZJdCWPoBbh5S6wfmbZ8/pEgfOKKNS2GCTviENInk8ieKA+CiFIMw02Ik2noNTlWYcNg4yRrjZfdZumm3eUDq/aEPXPPbUgyuM3Qk0oyI18H5wosRW2dYUSBKhicZVi2Ioemgzs+l0S6pRYNHozK2oZQYTfecOWEUjmi9LUcQ8ZkbZ6LeI6fsF2bZQvUW8pZrgdgYnHOoKBUJtbSULB40PUJEfLGGNOV7T9C7SvcsNc/SF4dSLpVMZvCuPFmSRYmGirclZAx9RE2IZxXL76wa1ULLoVr3C4MjOt5UBqSnxhVNmInmFKuQhu3hW4SVoMjFfcJ5ewrFn80HQSRLRxEjEJqxHfJQYav19MWVoaFWffZs6Gwia5dWgn09iDvyIQhjqhm61MyUY6Gac8z39G163soQrpIu8yoIflRMBZFByA9yI2ilf4qj8iAZJ/FxrV+K9gBtOHwhTOrIp8pl/FAJwSZoO/t3jNWATOGpKzcqjKN7tXlPucLog+nUVNOsKjsUeoI9nqaDvPczQIq3vTMI0kn3Lf9Wjjwn1cdD7gHpRreHQS+hKw9vWsLGpCPS/V+3ILzcp5aOSYO91d1tb5IkqRqBd98Q\"}]},\"native_finish_reason\":\"completed\"}],\"created\":1769370012,\"model\":\"x-ai/grok-4\",\"object\":\"chat.completion\",\"service_tier\":null,\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":7499,\"prompt_tokens\":1629,\"total_tokens\":9128,\"completion_tokens_details\":{\"accepted_prediction_tokens\":null,\"audio_tokens\":null,\"reasoning_tokens\":7331,\"rejected_prediction_tokens\":null,\"image_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":1368,\"video_tokens\":0},\"cost\":0.114294,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.001809,\"upstream_inference_completions_cost\":0.112485}},\"provider\":\"xAI\"}"}, "scoring": {"fuzzy": 0.406, "cer": 0.654, "cost_summary": {"total_input_tokens": 19548, "total_output_tokens": 117682, "total_tokens": 137230, "input_cost_usd": 0.05864400000000001, "output_cost_usd": 1.7652299999999999, "total_cost_usd": 1.8238740000000002}}, "normalized_score": 34.599999999999994}, {"_id": {"$oid": "69d4eb12b877594bc5718c04"}, "test_id": "T0505", "benchmark": "medieval_manuscripts", "date": "2026-01-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-3-flash-preview", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"18\",\"text\":\"Der graf von schartzburg\\nvnd sust all ander ritter\\nvnd knecht wolten mit\\ndar den ir was enteilhaft\\nsich v\u00dfgenomen her jacob\\nvon haregurt vnd hans\\nvon herffenberg ritter\\ndie zwen werend gern\\nmit mir gesin aber der\\npatron wolt vnser nit\\nbeiten do der merteil nit\\ndar wolt vnd also m\u00fcsen\\nwir mit grossem vnwillen\\ndanna scheiden den wir\\nmit dem patron vnd\\noch den br\u00fcderen dar\\nvmb hattend ze zipren\\nwachset dz zuker\",\"addition1\":\"Saccharu\u0304\\nin Cypro\\nnascit\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gemini-3-flash-preview", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 1594, "output_tokens": 210, "total_tokens": 6036, "input_cost_usd": 0.000797, "output_cost_usd": 0.00063, "estimated_cost_usd": 0.0014269999999999999}, "duration": 30.540764093399048, "timestamp": "2026-01-25T05:33:06.106334", "parsed": {"folios": [{"folio": "18", "text": "Der graf von schartzburg\nvnd sust all ander ritter\nvnd knecht wolten mit\ndar den ir was enteilhaft\nsich v\u00dfgenomen her jacob\nvon haregurt vnd hans\nvon herffenberg ritter\ndie zwen werend gern\nmit mir gesin aber der\npatron wolt vnser nit\nbeiten do der merteil nit\ndar wolt vnd also m\u00fcsen\nwir mit grossem vnwillen\ndanna scheiden den wir\nmit dem patron vnd\noch den br\u00fcderen dar\nvmb hattend ze zipren\nwachset dz zuker", "addition1": "Saccharu\u0304\nin Cypro\nnascit", "addition2": "", "addition3": ""}]}, "conversation_id": "61682964-df75-49af-8093-d46680c78372", "score": {"fuzzy": 0.957, "cer": 0.032}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sun, 25 Jan 2026 04:33:06 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=30419\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":\"EsxgCslgAXLI2nziSpGrOzuQ24krozaGIwIcYDF1rjcyBpzpBtHV2QDelGWLz5au6Yy-BqHjNwdtRX97ZzKHzQp4flxF03fzQLnJWuHCUEqeUhK8wDdK7NQ_WG-Vq0eawuMeP9QaWuLs1YUP3DJNy8JYFO-WH5pLCJGDs3VzRIa1voGUT73ciloV3LN8p2wsXsasGhhJ-xx7-jJPOE63mf9dUlivDBygX3sbiHPfp3HgvlH-rFpwaj2XgUrBFcvIs_VTog54iLDUcmdHdUCvNNqdKiFrQ0N9mtc8kpNVz2szUeSWHhRrL1QOIBL61G473otPPI6YcfXaKFZvg5vguIrqCPHkiemlB6yoZ-pHv_5rd4kOkxzwKAwYgMbUWVro-UZzjp4GNqtNWtkc4qO9fL1TIT4WZ7HHV6QV594QpxLS4s-xUnA7OpFQs4Ojy_Dd2poyC7AvXbwWoYXnRXGDaOSvfExXi5gu8kJOoTZoyRZzYHZGwxoYk07Mylaj6nzwX-XQjtcNMIX40pm4ys6Q79NTCPwHLgRT4rJr7-l-P0kWDE4LmP1uEuM9pk1s3-ff8LU59IYlIVwCElVW6Um4FmVrLywdblwhd-g5q8Nb9TtFUMA1OjDl1LVamsmxxZcqOfEcHQvJ9sEZ79a5w5HG1cyk8XV6K3owVAc7OPGz-ReCZG_w9I65RNY62BIQLOam3isRO1_UVcmvE92sPhVXMHoQRf6F0Cgngf4UrDteAjFhkYol8AMCY7IoXqcVlSV8ScNo_W44NxqImCyHidR-stCFgIeLTsxyKn81Pg1wfPMX5Ecd5eMBZ64jLlcJMd6HGCS54jkJWvzrb3cpHdf1WBYbFXZC4qRZX0s6caFmJ7yUg8WPHiERgWG1TJQiu7cKqhvUuM-tYSoQj8VdKR58BmpE11PT56rwmU161VN9e9wG-DS3XZCkywqGE3UB6yCJiRN8ONK3bZEy-nNKcyzHPgsJ6D-Zp7F8Bwc9UzIDXDzMtWiuId_FSjqV6t76J3Mad3JgWyGADZDtXnvP0ZGOmeddWCMAcudcF_aE6ZAuLkBNAt8OQsu0A0m_Buo9TV1MrSlROmhLWN6zjDsFPQ_VazZVvR_PJHnVZjYEgDsLWB-5D8p2cpcY5-VlLzKWssQCe29rup7FLQ6tNdKuplCqfQqxAbIlZaGSNqDQ2zRG1UfAt4A0-KIAgRnTjGH1XEQmzDcF7uyUkvbuR0-C0Z_BrItCg1jj6ul0sUJz8iU6t7RkbtmzLIjTOx1oT5uMDxP8Bwa7pyX6I3wY4PxZENb0k7q37gKUFhkN5KFUFMQQ2qht2KiX8R-FvYzoqt5k49lGt1-aUg8ltLQHrDjKyT0XDXxjSBfUYmg582rKCoXgj9SDF-zrxo2LkXdoi7zy9CCqUXAGh7ThR8jPUyJxu7IuqiF2UW2z2DoN3eQ_RGtJ9S_D6vaVguxO8d-DdpOh9Yz8UyIKJAzaXl4kNaPapLjFl-EAvLrp5_ClV1bhOrlvS8KPbmdAbpqzKGqH5j9OMxiu4-OM6-CG7X4vGpLkRBPpCbrm09pPKF0jdg-ws7BSVbe3Uq1DOkXWhRYE4uCuETz_O_r9YUNj9R7uWH1qSibAsKAu1ftxPyb7T_1J0SxRSSsLI1rIUywjNnpYGJ0M8_KKTTh4VWPGpKgvYvQhTCBp18wCU3sva_bRo6UWn_YXdx-W9jietSKfn1zwomERUNlsmojXt82v9TF1VqHHiUWXM_pWBqvjIT6PnyW8YN7tT-OgQSpZyJfhzgAVk3KdV0mDlRlRNJrE-uz7zTuw3O-U5VqGtNucanF7KYeeXeTshEBurrlr6Rd_n7m-DiA9PvrX3DcrDUb8QyIGHVM93rDJvTAbdI0hf7dh2mItOE55e6R9gCYGg-raXR7rtRuTl6pRFgLoX9LbqvdN7UItwukOW0NykmixyMY3fhf0SpmWurgZ4TQxGYsBs1Msi4DJvDfiBauhGS_TGR9GbXQf_S-ZxUf5AQZuRLb2Hp36sCPL-sAyQAR6WJXJ2dxCB_dVYifQY_qw7wJ_xrP6cr-HAdCi3XvkTsb4NjpdH7NvjGflB7f0z5kZc89QmBRAXiQ7YMdEoAieXX2_f3AY8G3XwxL9uNwsCxPhiz8lOSorVartoqPdm4B2ghP0d8fjvqm3Wvm_9xbc8mw7Zt9dsQs5uwBDxekJj1BPcnH2buyU-BnqZlscmVNTfwbcdyKPMYkRo_OcXJ0xDySPf9OHeN8U1U4PRRMM61eVLARRsXmG6XSwq4vaivPytVpNUaZziQz43tc71S7QMoQn4mxpeDJwnAcxD1VR0_9V3JOUqXu4_qRHJi9GOCbOHBGnN-W3mk5xordGM90AhmLoPHJQ3WxaFo-bwndn5J6jEA4VnYasys27foJm3sk_ppQPKAPBPcdTi6mNUx1ILJ_0Hklv7_nDTMMqzBTxp_-K9kh-iZQHq4mUNzcU9xudrLAVa2_R3MuBKWHXl71J2DaLMxXBsV1GkHIUYNPStef2YkpXczgpTXChxKB-VAslwAtalSW4Yr3S1n3zBQCiTY3Fhr1QLv6HLp2sDuD1FX_jUa6yxoGMg0J9evUSl8CKG8LgIj8c8icuHV93S6gIR67k8MXOK1nuWRUW5YyjY8IHNP1pXlTwe_ydy2Yqs5HKQqEda1Nnwrkh5nsUCNOin-uSIv-47BcqbWiDoeLUZPPqDuG_3pDJTlvFgTE4EVWUr-GeopJXENdZLN7xNFP2pECI3riN6w--83fMrQcz6bg5K2kCOR2xvaPw0XSUssy-zwJ4absQyVO29nPzcUL5t_O8mcz7byR8vWD1U8i22O7RYVSngt7j4WplzgaEdUEhKxwfU1p5dHN2zU5KkZFpC-ghR5kWByYSq8rMz-iPl06rqcOQkkQjVsltlBW7KMyvXhAAuOWUPFFBvCJ2bC-OJ-y2TZA3aB53VrRdpQWK-ZBHE7q_4U9_OG7AHWzWENaLvarwGBDtL2QjPx083j31PKQV4r7Nk4ZgfdfadKY7bNcUZo4wAix3MYASKEuj-sCdEFqbQoOt6vdA45SAhaynf0vFJNyoBCFPW7errdy6Osku1za822U4-KgpYMi7s5BjiG150iv-ArvkMD6ssESEO5g3Mg2ReojtBgTn_eImq-aTYpz-uD-LUDrftzRRzcZ9F96e3UsBq44qpriVY3PYdK1OE7UnmYh6k3gteJeJl9Dli3E17Zmm4tCN5lHxML5juHoHzk7mxwPlaUx3D0erkWhZWOz6Gwj_ZM_YafAn4G9jZPQqcSVrr3vth_s-_AEO-qTOI-zRdk85PHBVzr4CRJ07bb0ygCmzEm1aqb6bC4PilqcX9asFcnVYQowhIn9fi8fHTkW3vHTRvhHoplwS_D9n_t1NIrNvp8EcLcV9AWNLdPql9if2XY_MQhooNctTI2bnEn2A6ojhmzN_raPrqAEp0flUqImp7TJf3un5O5Piw6RfcD5RTU1bof5W95k7JdApnirXpUOAqkPmttcE3SP-3diKHNFtWLXApJ6lyvk_4Fcjh_7MDHHo67ZPJOLd5RBG75Vl0b6-LdTB3l07IS6lW9DKYMOT-frKBTxSMaPP6RHe5xF1caWnAEfigus4n38wSyQTkNhaXlQkoHM-BV_EBb_Arugy2rpOjEF3CnukaXDP-9I5CXicxtQbGQsqT5tApO1xYdnpUxm9MvlVZ4PFFsh7TvQmFS1k9e9MCkigbg-PAKXlNEo7fP9DDAsJMhmKkgFAHTjyF1k4Rw0tsssYKIKialh85BGD7h0CliolF9idWI4sqztGlyXoAPvTcmuSrp9_PFWbc_q6zSasC27IXBveN0JEiYYcKKK7_buG5lu5JoA2xnVNVU-TkqMMFG9BFZflaFjblO_Tse7DEEsoy24Y0juoOAqm0mK19-tzvE8LaKkuJLBzT1_q0MoS26XGlQS1FzM4HmmYmzSZpXkn0cUAqZetUU7X4gGiv1RHQTTtaFGmxO1PieUniDZ3k_SA9NzBUqqy-FzAHzNPE590cdgklJ-1DvVleDnRWs7hOeeosHWRoA2Qd_3cd18ZUQ3GZtEXt5YLtb9jev4x2Xs-9UQQdL5RAyJR9AFlt2makbhgJS0Y0_xnc2A_nCEZsMIvxU45epEeIo0rE9ljfA_8LGufY0XMdg_UTY7Ty_5OKMx2iU5n-j1W-JY6q2Z8MreA9r03bhViZdRIqAVlFBaxFCAHbzLKgtw9Lqkvk4NNEXpXHLIc2WR6m-f6mtQFZX-duVFYMF6zPnZAqnZy4nYLXfq-ZzwWfJ9MiukWGfRLCHsEhDcfybj4mtUINrQMskdFfhsb-z2niAlAFGz64RgRjz-EJLjUC3gZP8_Y1ybWftuIEVK-H7s5n33KS03kJVMAqJSSvnyN8jdczuHb6wVgpcN5pZo3FFh-qEuTCfn7VZHnNu_AofeRNv_UigGBKb59gfEFkpd0XTu_U1the2HNjEFnNgIMfYgu_AsiUiBmSe_XpKWFsA32booI2hTsWjZga-KgR2C6_gE-4fSpoXFqEyuB2NX1M0hRgVBMoXL571BfdrbhRMSp7SyMQ12zhCvBV4peb8pPTmZpMYTvHbgT5JXu-oVzYh5OIPXHm2cupJyOpd0Bgm3YsuYh9_Jr512V3hiUXVWxfcDKp6B-gF7DJW1kqbw5gyOq7u8tA9f92aacZAz0VH8wYxUrYN0c9dJaS7C4VzpbzKo1delWL7W48HkZ0gNFAhdaPSpriUcy2i_W7RrWreBecY13uIE8Z0ViuinzthNaMB3UCSkrNapXFLP89N6wJRtIbgoo1DDuc0iuYRqq6JgxDPSoAbCP7o1-zqZrKilp7GGYgkCzcdeFxlTLInCLMajcSNJTCWh9-OvrJJCsC3OOlyKpj1SuI-_TG-Au5au0-4Mod1yd84m83JbL7YhJ7JjTnyPC2dJUeE6wOx0gzX-drtKrukFYJ_abaAeakDftxSdbith1JIKPaUXaWygQXr9BQ3oOMa-Ec6Col_5qsrCNlQWx-RmsP5dcUOjP_jCAKW7lO3dbPTj-zvku7wkmICtNetVX1l8Egh6ygAzXOfPF1rrn5gOjPfwuaxv1jGpf1bVHNoijPnnJY-OJMyL0oP5pFyC_lSxCdPIdnJcyDWt3cvXk0vnRTMI8uaq-NBDMYGYtvdh6gB95zVZIaK17bTKfa3AuLjs67GVW5B5_Bx8kxP84dK2s5E2oUQAjPqLJAi5NwYxQL73Dea65rAD2ywlxSy_8prS_yxpLWx_a2yI_Tklpsd1JQOSZDvy-DbJEBxLNw2CuOFp-oIfQUcnweyk3NGDA8zfvU_kRlfIEakkUokCD5t129VgfN2CjP0U2P9hcvhCjfiABkHKpZwjwXSX3oHDyRvfresVmvUj3VeYm9_amtwtj__O67pYEinB2A31yh_wE296V_RzRpLwA6_79Q17rIlavg57cYBBRyVezH2gwTEbsuZLO4K1O1AJNA-FnAw4zqln0EMNJPJzvG_4H3yRkbdEvNYnLpc2Z7Fisn7JyRdlzseFvRbbTGysYp82kI5p3IFAr3R1s-1ObXhR0gKWSwGKg_CY51nUasboFt9WLpjIygE9Ve0JDzlYXPcLcBZFEXpi7P_2zT2Bo6Iyg5O9XroabaY4ertFcXMxdrBQe1WkfVoAUEE3LN2cmUnMRdMNCGefF0P8XTEONzQQnzk6RLeE8aax0zNbI5oS-G9AfwLQl_1wseMQQ7Ya4LolvOXMsKO2utGXuLJ12Ef2NRj5fBXd_qOqUuWzcaX_faJjpBrZqsjQPzZYCIO1JE3IpW-dFusW0X0JgUkJorzDigabJbd6ZEVmPzf9DQTXzYOv41Aem2eGPO3aAScZChb-pdF_LwGWYCy-ssIgMXkhtNqeqLMYneEtEyDgU3Q6Jnd1sG6TmvcTs_XuDjt6O-di1GYhca1Vcn6eVSlNxoMwNvj2EqaQ7QpzdOCKXFaaYoyQjSpP3U-LOULqjRKbpqYaJ2LQycU4t0KUk_v3zwYQ7lLr45n5zfYbIry--M45aQorIYLFi9I9WYULi5Z4mJQ0YoVmKAokrcL_xAmcozOP5M59Ck93xRId6qNiX4JOmN0VtRiorTUMY7MdON8EON0tVzqeE6KMaxolIBK37TQEbmm1IPa9CAlzx0eaLh6DjrT1ixH8uYnsrJz-d-eHRuuirXvzVOBwS0HrzULGck42CzW4HaK47FoXQ767VDHqC_EZ5D_vrSI-gL_UL9108n3LWe7FmwRL9qaH1iUdxb9Vlsv0P20RJthchUeAiOYndMYDF4RrOLHwxpRtMgt43UUezaWkQygP5n3SUjxFdRBvLYGllyRd7zs39Fxov3oEg6f0q5ATR80dL8dFo7AF_NPlmayBGifZWx9weUPLUjXgqTWH6G9JKXdBNcBt0GXlboJHq1rMeK0SWW4Yad0IFGrl4iLYjknrnmsJtPhOGi1b1Kv0Fe1l1wVE40b78fgE0B-wvOqNedzMbMtrk0RZpRldWmgj2E6ALpkCR4v_3zRvLAQDnh43S4RtgxUtJBOuag_FfYLzigmVnmf7WlTcQBroELL3fEECmeXqYtLacLiDAU3DyUBfnHnC6IVCZ0dkTZDb-d7plDgtzm6vPB6HeLq6DUw2ZQxdh9V5E9ot4yz-8S5W6_rqMiXw-pPy3EO9B46CPjeLBT8fLikACkGoyErw4ka_zXMTIOQ3wlTif35HM_Tyc20uEC3i-nw1CkAUIoc87mvsnaEzoD5-if3JGRSV7nUYIVDnRr82jGHsR1ncE5jIjfn1SmCHzZeANgsoSXgP_w3DVX37vHUmUEwbVqqDt5gXDR8yKoDY3JsLfbxdVZhY35uVn1nhbxfwzYlEjgBFlbdV2VgNFs3F-RQYtRd7FtJ725z7P_oRCPyfIfwnjiPdWvPfyr8AsEJyiIiEeGjpTMVGOzjulI1mnRhmMtJXxOYYRH8e5N0bu1dRytJjL3oqqrGxHYVUuhjbtIWqCvnoO_6IZcjoOi2kEO4Rye9T2OuTKgbCSoyKZLnWyZzVryL2Smlw2ZbirRsQxPBPtuyYHy32HX9D1MQiAWYgv6ntUM5MzLpBS7F3guAy1-sm_Fc9aRyuHKHPOdiLW1Yhnqk6TGPxBBJEaF-3zEDWj6dhXf7_t_K1fwdnuWl5y7nbK8wZiXy9n5-lvKy0S1U2nuUarDcErZ4cvsLOV60Fvu1oL4HmwBXOUzDqgDgiYiBjPPB3uMBzE3o2vjz6dWb4xh8g5asdoTIguCZ36RBe2A7u_1IP7bAW1RpfpzDPhpWgWnLXIFBJUpZwGID7JiNQsosggRDRYAKrdSf3_I0vRn8QYdVpvIoUgfIvaeP5NAXxW-DEXW3iUemDOjKx3VmqHZWtXPC5qVq--YgL-APVF-zZmdMJMiRFVZCh0GDbkEkbGatM8fTosFXsYHWKxJxfNeLPuSzUs6kI0PXQa7vjhFrNWzghHnxML9YoZ53txZoIhQmooR7vGKOID0uarXAlsHNzeYreBcEUe3DCD6x-Y4_tMJ1DUSNGMBEivNuDdMd8DhacAkvZmUA-7V5Dn1xt3UsIRRyNzWIsWGji_7s6DcuLpgYHUlgLKqbuutCcfotSA7MWjVjMSWTJu6d48Y_I8-XBPIT5kQ7SMipZXNN0zuzTGXQQ1RxHK4AeQZ9htZ-NZRJ-WSh7PXskW8fKcyjughzu7UnWMFjh50cyLkkSqxl348SbsoA5NLMhvo7cMkXszhMzuixPcXQ49day2cSStohQcXpQdo8qaRAPxY7U-GU-pBmpj06yL5GYOCcxh4MB5y3uXnLs1vAB3KeqOU1wwsxTQ4L7dj5-sh2bqPHjLsHnqAViXu2ubrEFWg1vpNRdoQRRGLk6JftavXDlcCaoJpxXrnOXOBnQFpOJP8caG33MRC3OV8XvLryTrmrq0KPi0A6IbOuBuyJKmumIshe7Ng8c520ccM8_NPLHEpaREa5ncumd9Fu1MTdDKt3FwCrzEk35X47WeznZWR5aVLYrHut88tsgjzdETS3jN3mkyZEDpI8DBjfNaHW5T0a86JT5iHZnhhg6EUThiXqDDM_q7-k7V13bteOgPzLfy1KHRmLbDzsLpI7BhBF2bjfUNyNZsOKfmb2fpdSWiAGlpWafMNJrv8vII7B0yrsiWbK6cjlXN9Mgggmw7vfruFnfmkxV1wzMBCc0GXejI3VV6-OHeEIqK2zgbLiERVoFTqn4lnFMUvjveyd_Xx5_cLkQI3-jSSBLtIPHdgJaaRvx-F8cR_w6H2NKMPX83ri6jXkGKVR6o_8GVG3FA4yBvukN-bXEQKdl2ofHGAvp-zm8IkRTCu4SCW-bQvky6jm3YKdSAaXiHfc6aUGPD0KI7kSlN73CfeK-kpXmirTS1YYWMnVQqB2hfk_cr9ommDLxXrQh7IzkVKI4mYh5RsnUbr9tah3I9V-004MQwLs5SK_jdcwVwZ280oiiege1HR-RXmEKT2UN7jQELMohs9NKh0iUUxPHhmpOFjpNUaOhx0GMvBusf7uMo0k9RSc2GbOPUYfVy7pHPbgxHZJoWNNHD_pDFTX-lVNkbGeVa-fttkQ9x-OCI6ZDaaiZOKSuufAoNL80yWCTG-QFPeJggwcvhXwRaydstzC2mrlUTIIX0pj-tEgVkU887Uad1CNQKHgAyfGLPaCZey_qLQHy_eUWr8z-zil5W8YZK0_LQqRX86PfnWveItDTZoWRILQRubVHlHd1U8KzKJmOeNAvz8ojDj2uecPc7ZOfRGlDwMc6wCUqXalyINwbFpIVXGh3cwWAWbXb7R1U2i-iFm3niFJNBL9XYBzDErx6rfmgnOvLMgjSDM32WD1_numU5etG31SnpiFqNtkF6Od9OgEwCtWViqq4EEcl07OJTnUp515KAmDenJqcQ93ddzbaTNGVuYUbcMb-l4AXKUJ_nLBsyfmw8D-ynstDyvi6nsBp400YUOlXegr8KcfE32rr6DuYtxFNFUBqNpsAriuZQJkhlbFA7NUCgrP-keKox3H6_cGAnvtfp_e8h_JFLJJQQy5U_5hzRIvspNKDc6heqJUG5hvighvHhXzkHOLoDvCu3v__CsVedO5epOlB4YNhwWnPZeX49kIhqN9dT0GxzPvssiaUyhxlVcHvxqxTzOL7XdI-0Nfb1hCK2sD9lny_2UHpfY-jRgyb1K5NK_r73P_Cbhfhq8PkJr97CHeMxCCvYbeGF63yP8UvIxeDmkq3ScmBH7nC3HFfqWb92n-ePB1E8cqlLGuPaV5sIkcW7FnIxAyV3OBaA0rZpddhFhotL4OwOqu0BT2b8n8tR-fmSsP__LeFVafbLTLBkfVFYklpBz1ogdtGTf2QmHvMfNBAd0XryxxSY2V_PZ1PkhxjrvY4WsumQJtN2qoo2fBBCmcPCKDwvnc_Wh8DsQ_EoAy2_ikK7B4pGZP386wreLSywenpdQhSrEUA2zKfqU66iyEksKxVD4t7eu2opviow0czcyRybJ5aBO0Lk7P-l3fDfhPduyTVR4rKTo1Ti9gb8ynogv_2PH-qeGAQJM7xDWHByh8KvJ9e0z8PzA4nAo1269sf_1fAiX0hZQTpEMLAPct7O6b8Auat-4bZYg_fuEPn7emGj2bUzXnuJAwlvyXGEj14LDTn7-yE0tVqqXENjdO4hbbad033ErGl9nlzEOONzyYn_hTn2N6s6Z4WYp5HEh_qgFlzn9XsfSMXwpQVZqeYoAoXmwEy7zibcYY32iFsB6FQHUGFt-_CiAaJBoqQlv6oR8Em_QIjWDoRVQWH6CFvJJ66ZuVGijzS7oYB91Ux5LumH9oUhTSbbh2rqQFIO3lzIFGcApM2NeRHWU8AKkLA-92cNZUC2qLlwzG82FXyyVza8zpPqrB5GO_9MpHxoSGTIdI7H_w4qxY4wf0HOzdFGCcAl0ocXIb3FbbzFua6IU-C2oQCWrunZJUn6RdzqrOY_tD7IIOdL356-DqGLxPb1lUphtJ8k8M-P14VNCIIBvyijSgrPQbxF_AMRBNhpzE_sNc7RhcFsMqTJiQ4cdc5TLTQAGXht8j6vh41p8lXqbso1WDgqxBzMeLg9-57hcXru-CH8XWTXTHcTLYKJK44JW-2C-jsMcVELSojMPnuWSadAOxKwzU6F8J4lSRYil4VBnwxT1lIKLSXwEZs6dGaehaqPzb1jIS5vtVzPsj2rVowWsHRfb1dUx89R4JRkREKlXY_8mhGHol-AfM55E56ADCWxp2rauaxiFIgh0CjiAa_8J0WiR54gmYyG-j2L5xzNDC2_z9KKs17piFA8mEvC71uVRTbNQ1MR38GZERZMTgoWxBJVTeHQvArJ5OD1dNaeJcLMJUqtx_nII84P9lvFx6ptz4K8Qu1y7rcRMOFx61PMYKgnsqRqgd5Kr9RpMjM4Bx06idmMZiJCCg0IOfr-grFM6KRIVpmEtVxe4w7mK7Ksr8cyOZ4qOZx6e-pcq68qNEcYOSWzHdRpIkIF7ROs1SnSEW5SKJiREp_TlpkJgO7qzNqTJtS--mc5Bk215hhlK2PWXAV6eQ_9tOFAp7gQ-6zPvk3mO6FYEb2N7WvxUnvazpMGPIrdvJ90boSDVtzF0N7FCMLIkGo4vZ-hJELuIpt8KUHU5UGbqC7tedozwc-wc4X_R__eJhKNRL468jtjvBE1aK4iIvRFwyrjLoIS2V57jQoJ4-zNgjj7Ci5e202237N3OF8kpArB_5EAWwx45CxfRQn5T4JcrllKDoswuAaq1vRNdQhDc0RJS475FpI_ShWjrUFk_hfdtAM8S3lIKVMprxPHhvEwlQgHuK-8hFxaaArT42AbhM9Ll3ZZe-rty7Y-z37koItg53vfFqgSEuYLcebnwHYnSZNQ4yCPxdPgWX5ixpWdNym8WQDXv82Y0NphVewagzSL2z3lQpyKN7yLiATtBuODYLdi4ux5AwkuO9qBwPIUtkzE59YTada8sremFs7_gvCQZN92OvirS8m24PhvQKiyV8G_gu8f9NtDhjSVZa6pQiBIPOB5oDTAe1_cJteP0IWpmwrTngX4QUji14Cbis5vrxq3-E47blUmqqSVmTKhUPSoKjB-StoGbHlrtkJywtcwCXHxW39R25p5qVPzMd52hrXR4QDRgaaR6snZxypRMmsqhxUNCUAZU-0BX1et0e3uShEz_0f63UqNYNxA7t3TZ9mKrp5uKUyrx46-KUArvc6LdK8_0mGrAH_V6wnTwayU7kccPvSyFfsAql9WGKrZQnFMmbm1qTxC2FMSw3Bi-o9k8PM6f2NFIqcFi2-oImO5HqhiFEkcTLWDqKdLIneXIKjxT7-rVCf9eBggHvjH6h80DGE9CegPXpmGWWISh6eEB6gu1Ou8Jb_qHZY3_vu7g2ATGxrfAxSlokNJ9QXP5WCtHIBjGhUIxWiKVo9dEdj2YyNl_TAqFHFVD1J0T2DAEjK_lPzOgYHFMszaKzZTNrK2Y6e1Hrg1bufaUsjhHKAKHYEvBBma333TwjzdbSzxVtShETzD_-0AK3guAA7c_Grp1Zrxo2Ux_JkvV-qz3tquJhJBOMRg3kJZZZWw4n0HBP-jQCxIVjw0PPcn06r9pKfna2WPSnQDk_O3dAjdxSionv8jAmAMGfdvHdnPjyrX-AxAtDSy6JjQvo1ethPHQp6-6ync91uIbmjVp8mLfDsqRfeWlG0h0cAz3WHNm6uMZ5vTiSmT9Vdp73oBmRGZVhXgftkcRNPs6u2GKvGpVEj0Crbieii8DQYwzktd2fz7SfzJ1wI0xGoYFI-41jTxbQLFvgcqMGfVelbMdvd4aITJ0oHro1urWPGe2vcmweZstxkTnQGImkhguVaZ0KAERmMsPCKW5hly7E9krm9VYDtz--Xk_cMrdEqUCPThGY84FW0MDquJXE3n80aPabArSou-v6ZFZ89P0MZm_-Tud_7pOWamXbuKhz_6EpIARh-7S13WfW-D5Ty53zDhX49WzgvuMaYPPeKUNWI2s-cmpZr9t3yI3ydCMljB-g07xm7uo1XpTXlwPwQht5ZYKQ7Gn5eYCEYyVf_RqLQkoWd7024uyFD1GgvoHuqmeI1g9SriRHdprLhZjNaMWHp7GNejSpLBFWemGP0D3IO8UR15ZryWRG3SMX14Qn5-yJYG35GLQJdo6xCgAB9gjDx758uTgxvdp93zHvTNeWPDJzmwbIFTgoCDm68wDZTaC2nTMk0-kvRG_AOPq5m9EOyTNjaybImDqpowhDLT6y3-mDU6QjhdXrg_kD3ah0a5frXTW4LCqBwRZjy84L8xHwRroxExFx1iVmZ3n_yjYukG4NAOKWbygq7AuKUSvCH0miiPCnhMv-xwQhI9msMK7OWZwFJj-ikkLlNmlEX-wFk3D9doQW_c1Cat6Zd-qHwf7sN16d8pVHiBm0J4GmjgMueRYVPwELWo3RGIknymdrSJL8flGNF8TuxGqmXrKVnLxVls6tnjU05CmEb12YmavcsFwAJ9uZPiNdAoRj5B5IubhDdigSZKhgithcac4NbgHSKap7U6zyzZq5C4NGgW3eXaUDpVIyDJnLsto3x2GkmOayIic7XiSjll5gioR-iTe9Yjwl8_ItVHp3mY-3VqCV8DsE0qHyJoLCYKFQFQfAUfvMMlyfuKlBKq0UCumWTB4rCKGvQiENKabrjhNMvBQG42avv61A73vMZyptVIaXyihxT8tqQ9JU17vvfxy48yxPoTFTiHvXJ_G9n9TJ0S-o5wfPLmNwpBMSt0iGr4JYw3ADg2oWFDAqE2yx6HbCR_X8c2YA0ijd_ncThU29b_0L0ahbVhB5u6w08jk1_EmKqUyBZ3ooTVLx3WPiphTabqXMLZ8SjXL2y7kWlYfirosiGkP9AfBjBK1jG9gPUGf7IWfQ4A_6R1Ij5euVHJfUFMhWFYy7208IZYqDZf3lo88xjjTWGqOwQ3lCYDSWR0g8KOp0tE2o7eUfDDyrQlfzJHhL0Q3M2LfTu4m4sDaC4eip_lwA3vL38KYya7xooTKAl0hY2tUgYs9XfyIzTzzcNv1Ax4sckxa3SNOOj-2maJktVG3R6ifrirj9EOefjS4ci3K-BEK1921pFq8sSYwAdDaWuxl6BQqHUOD1b_ZuTrj8ZYKwSMczYF-qq2H6yCI9C1mvczfkGFFonpQ-bF8Od1ICVQr0rDG-9SiabK_vtGlbEZ_VzB3WRaeA_kkjevPIWNjU24cEX5ej_nZztXoTKM_QPBkAP3ggzQRTFs_QwmWSCCOI0tKpM_9eAQIxxVV9dlDL8MuUg3H5Amrj8U-dQucV4gk-LFxwYrMgAejxMmGgZlAzp-QkoUWqlEu4xMTO3Qq0-JvAxQwfHWiPTljd4SS0MJPLE6AmMCnMI5H79Xf6dQPaVgSzZFB_ATatmqCEDByexkjNHcU4prJn2fh9dJmmnGXcNKLtYqDMEhIoKv8ddQi6kro65kafdoGAfhXQewBuG7wRrCE060gJ9qleBACtq9fWb-ZFLXJ5TAD2GXzfxuuW1PTRm4wt1q65KWsdtRiOav2mca9J7Lpb7gZCCcVbiKIngTV9XYL_jaVjiS6G9TevY8g_c1B8E7fYv-5tIIKOFQQmUC8kY1SvJ3vJ00OFzeN600yI0-YrvL3fd_5lujJKYSqktDvI7ownlWcFxcfHCNVX4hw4RuYqPG0me7wrVhDCX09AwALv4O5_y-vQ16JOH7MHe9Yxejv-kujmk16ugAI2vBhtV-2t9HPL_l0Q_7krIqIoyEd0c7_mRnMX4-_jR4uDnCcrxvmHHiEuMG-3E_funmHLXL96mCowvD4P22gFo3MhcTLMbpTE1oiAIhs0oJ20J9YJjNILGiw_8GWZrUDzWSjA4mCu7b6OfEUxkbotM7H8HVjCl8ourRiMlWUM1rKWm9wVZBL4VSItk7-zOSp1XmIKec2ouiLiKzmfQ0RC12zwEjXX0pPTzWN6u-AOgdg8fyKh8ZlHVJe3r1nhA7Pu6j_JCR9nnJVzIawyyTGFB7r7hG0AJ2PUdyvS9oCrRC8E-SxxVZjrntkg1Z00xCwJ8vgt2_fJiaz-ZikNRu74k9qWz4DZ_bhWllCb6Z5xGVvsMUzJ7Yr96A5C9kcZYhbxQIAt1vanifiDbW7a7haua9kTS6Mnl4scnVs6slBepK1WD8b6_vSF_VQ0mmTunUa9fCKW5xHpdlKGwW5H8OkwjmUbPWwM69JqT1tSXoAjxhepg777qVgeGHE_9FVWL7chOnpfx3QJbaBoiN4yT9fedI0V3BwKlteaIk2sIZdtVcOCaH1ad7wRfFzStqD4T1ixORo9cNlzff9N0a2ISfNWhjPsqegfBlXLQlERdUqAmMle9Ohj7R75rb-IS_CDEWWLQqZviLvv8zmPCTKnEuvyrGqhBCXMllGelwugbVqN30vWn0-0aLKv_gm-7CgiXBDsVss6SP2omBEGSecUgSMQpwdNsAv4cPUHeAHw8ruEaNJUECSjuPPhvn6-7S5XzEvPocEgvSPC-87MWdEnLrfJm5BWAwxO46ytqYMmaj4_KczRs_ilRtb1VP5gYflivUj73AtcgnEFi2nhrg4T1uBDCAhZEmczXor74HLxO60CkuPyiHhTShHSm95qQvclQ2l8WgpM8GzVKbIOzvN9ahaEJSB5i_-f06klrkvyKLKl6Vaj4r9_UisyWGBr20d8wFIvpEfZm6Lr5jXQX49NpH3siAn_xx6hrbn4SDUHt1Cfvh7ryhSEPIv4AsRP3WEApUD5gSyPZFYqpNJlJped2v22d0zAAyQccl3eTj0_hpgo7-nn0V1ukcN_aLEByHFWQ9cet3rDJV3ErYllh502VaXF2VyOke_HrcN-QV_XLwIZUeCg_lR0_LbKyOJ6m2AILah3RMfxmT7UgwYiWjmb4YxiVQFKYHz3U-97WIK-V-ZFoP6a91kRCH_S1wVI-IA0BsKs2hIduv7lH695Pa8H3PhcAafdWrBU43N2BHzTRQHklRoLXrdDZkWe5iA5BKX3da0W2lr3c9GIUnr2T8CwV90FJ2lnquZVIUBKgbtF7964dPdApQKWwtthcEX96Bhq1OghtDHhgoN4Swp49NYHl3yjKu8g3ZIkSY49n1AVS_g10d3lazYGsxk9k4gSobrv_G6iJm1ldrFRg_17G7Bo94Vpgt-fVNS0eANpSFodyLH2exW_aYlSLavVbZWRRZiclVXqc078OHBgKezG4NgPwWMXa_Q6utOp_HqIhy3aFsRzTvp4U6Xev3Z6SRs3QiR5a1L2P6-Y7cYqRfg8__nSH3Xr4PrVHPVw7qTInQ-P7W-SADeDwI7AGiMsac02KQDOVLtZ5u4nrhOm3dNb-ch647ooTBmg3Js9j4gqJ-ApT7ByMsZl9_cn_jq821MVaJAwVzbf6-UVBF8VGdqwCflGn5-WMfMVKAfv61J7nxP0ncaDouW05jgtcUS5niV-XXacLU4CqDCtOh8wwp2mQRKxz0TLxHtQKb-3ikFi36R3R7zOxUEFZokTQS6fzqCCGiO909BtFzG-0HXoBn0KCYcU8Q7aLQhuTcIS147L2tm0N399dvUuFKp2xLdDKh9cBSxoc5PmSd5S1HLP96DsWvVA1RyqjK1VXZOu3NXB23u6D7SHEbVdgq94FqH-7icF8qt_L2CkLNiuzGtpAk7e4Db_szShFyne3c-FTlNdFpeGwkam1JfkAkA8Nzn70hRsm_2C6U94oP4Rw5H5Is4TSf2OEpsMQK9rp0jx7_-gTUYduYgPMNW6xY5-26m4gJQ-8a1SoHNS-rXZAdKlpp-Wii5tGsq4lyvbH6iv7sqH5-4AgF8KAPqvizgJp5OnuYOn5lel7jMDf3hCSoMh11_XdtwaHONYkH8H_Vs39-1S3LDUQIljdyuFQnnYqJBJD__VJGrvnx-gJ4ac0bVwnqlwRIYcZgyLcSqqmOe0K6R-lveSC0tBHRAj7AJMyZBAnb39kFglNaU4lm5q1LTSlf6fiKy2ORf-W7kdI3JUimWZFMIuhcHOFve3BwZ6nbnM2L5ZvSh0RV2p1pxOXJYGv01HJoLNZM4EUIcLfYv1pZu_p2is8nuvjeaobTL1xJQCXglFClnm-uldX04O0doDxbxp52FKV-2uIBLWqG535zXAiMR5cmCnT996YlFd1jHlc3QeMvfvL1x78dVkkD13wuUFjp4NpXpa0nyrhtFcCQk27HCyNFeP60DSDIXQWy-27SZDW6aOF9P4XITo-Lcm9v_DFzESPwm_uV_rhMWjtMeroDaqbAReXioJoyk3tLP2xo3YTGPFQvtbAPEq-Bg916c0S0Unek-79twnTfWHD70p8xaIr9uijY_fpwN0bIPQT95H6I80mBG5S9kHhuxFGBGlSLJEhtmSoS61qj6OS6tZ-TXMnB30NVA5D3V_x9bGra24ZKUHN9n74Aq9YAA-vpYzymLE6HEpKyWxRSDvPYGeN16ypaTz4yvw==\",\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"18\\\",\\n      \\\"text\\\": \\\"Der graf von schartzburg\\\\nvnd sust all ander ritter\\\\nvnd knecht wolten mit\\\\ndar den ir was enteilhaft\\\\nsich v\u00dfgenomen her jacob\\\\nvon haregurt vnd hans\\\\nvon herffenberg ritter\\\\ndie zwen werend gern\\\\nmit mir gesin aber der\\\\npatron wolt vnser nit\\\\nbeiten do der merteil nit\\\\ndar wolt vnd also m\u00fcsen\\\\nwir mit grossem vnwillen\\\\ndanna scheiden den wir\\\\nmit dem patron vnd\\\\noch den br\u00fcderen dar\\\\nvmb hattend ze zipren\\\\nwachset dz zuker\\\",\\n      \\\"addition1\\\": \\\"Saccharu\u0304\\\\nin Cypro\\\\nnascit\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-3-flash-preview\",\"prompt_feedback\":null,\"response_id\":\"Ap11aeSXA8Dl7M8P5IuNgAI\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":210,\"candidates_tokens_details\":null,\"prompt_token_count\":1594,\"prompt_tokens_details\":[{\"modality\":\"IMAGE\",\"token_count\":1080},{\"modality\":\"TEXT\",\"token_count\":514}],\"thoughts_token_count\":4232,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":6036,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"folios\":[{\"folio\":\"18\",\"text\":\"Der graf von schartzburg\\nvnd sust all ander ritter\\nvnd knecht wolten mit\\ndar den ir was enteilhaft\\nsich v\u00dfgenomen her jacob\\nvon haregurt vnd hans\\nvon herffenberg ritter\\ndie zwen werend gern\\nmit mir gesin aber der\\npatron wolt vnser nit\\nbeiten do der merteil nit\\ndar wolt vnd also m\u00fcsen\\nwir mit grossem vnwillen\\ndanna scheiden den wir\\nmit dem patron vnd\\noch den br\u00fcderen dar\\nvmb hattend ze zipren\\nwachset dz zuker\",\"addition1\":\"Saccharu\u0304\\nin Cypro\\nnascit\",\"addition2\":\"\",\"addition3\":\"\"}]}}"}, "scoring": {"fuzzy": 0.752, "cer": 0.25, "cost_summary": {"total_input_tokens": 19128, "total_output_tokens": 2844, "total_tokens": 21972, "input_cost_usd": 0.009564, "output_cost_usd": 0.008532000000000001, "total_cost_usd": 0.018095999999999998}}, "normalized_score": 75.0}, {"_id": {"$oid": "69d4eb12b877594bc5718ab6"}, "test_id": "T0283", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Belbs in fronaltar Gie pljend\\nwir geantz and ans pinct voy\\nDans Geiligin froy with de\\nhit ie off in brust und ein\\nBeim van pont marien madaline\\nkinggrott of roff pampag noch\\nDem oftertag der Do Was Bor\\nander tay dis abrellen Do\\nfurend wir yoy messay Es\\nlifend in put of reffionB\\ndirt der Emelkin\\nDie Geradio carplug in emem\\nabrar die pljend wir ved\\nfieht man noch beterdentich\\nin hand and f\u00fcs and die\\nnagel darray Mande ay\\nDem funentag frrig de\",\"addition1\":\"Muran\\nglas\\nS\u00fcette\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gemini-2.0-flash", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 2365, "output_tokens": 223, "total_tokens": 2588, "input_cost_usd": 0.0002365, "output_cost_usd": 8.92e-05, "estimated_cost_usd": 0.0003257}, "duration": 3.161621570587158, "timestamp": "2026-01-24T15:16:04.669184", "parsed": {"folios": [{"folio": "", "text": "Belbs in fronaltar Gie pljend\nwir geantz and ans pinct voy\nDans Geiligin froy with de\nhit ie off in brust und ein\nBeim van pont marien madaline\nkinggrott of roff pampag noch\nDem oftertag der Do Was Bor\nander tay dis abrellen Do\nfurend wir yoy messay Es\nlifend in put of reffionB\ndirt der Emelkin\nDie Geradio carplug in emem\nabrar die pljend wir ved\nfieht man noch beterdentich\nin hand and f\u00fcs and die\nnagel darray Mande ay\nDem funentag frrig de", "addition1": "Muran\nglas\nS\u00fcette", "addition2": "", "addition3": ""}]}, "conversation_id": "f47b98e7-b4e7-4351-81c7-ef68a0900fdf", "score": {"fuzzy": 0.776, "cer": 0.228}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sat, 24 Jan 2026 14:16:04 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=3045\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Belbs in fronaltar Gie pljend\\\\nwir geantz and ans pinct voy\\\\nDans Geiligin froy with de\\\\nhit ie off in brust und ein\\\\nBeim van pont marien madaline\\\\nkinggrott of roff pampag noch\\\\nDem oftertag der Do Was Bor\\\\nander tay dis abrellen Do\\\\nfurend wir yoy messay Es\\\\nlifend in put of reffionB\\\\ndirt der Emelkin\\\\nDie Geradio carplug in emem\\\\nabrar die pljend wir ved\\\\nfieht man noch beterdentich\\\\nin hand and f\u00fcs and die\\\\nnagel darray Mande ay\\\\nDem funentag frrig de\\\",\\n      \\\"addition1\\\": \\\"Muran\\\\nglas\\\\nS\u00fcette\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":-0.03821533357081392,\"grounding_metadata\":null,\"index\":null,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.0-flash\",\"prompt_feedback\":null,\"response_id\":\"IdR0aeGULrG0nsEPhcD0kAU\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":223,\"candidates_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":223}],\"prompt_token_count\":2365,\"prompt_tokens_details\":[{\"modality\":\"IMAGE\",\"token_count\":1806},{\"modality\":\"TEXT\",\"token_count\":559}],\"thoughts_token_count\":null,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2588,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"Belbs in fronaltar Gie pljend\\nwir geantz and ans pinct voy\\nDans Geiligin froy with de\\nhit ie off in brust und ein\\nBeim van pont marien madaline\\nkinggrott of roff pampag noch\\nDem oftertag der Do Was Bor\\nander tay dis abrellen Do\\nfurend wir yoy messay Es\\nlifend in put of reffionB\\ndirt der Emelkin\\nDie Geradio carplug in emem\\nabrar die pljend wir ved\\nfieht man noch beterdentich\\nin hand and f\u00fcs and die\\nnagel darray Mande ay\\nDem funentag frrig de\",\"addition1\":\"Muran\\nglas\\nS\u00fcette\",\"addition2\":\"\",\"addition3\":\"\"}]}}"}, "scoring": {"fuzzy": 0.594, "cer": 0.42, "cost_summary": {"total_input_tokens": 28380, "total_output_tokens": 2735, "total_tokens": 31115, "input_cost_usd": 0.0028380000000000002, "output_cost_usd": 0.001094, "total_cost_usd": 0.003932}}, "normalized_score": 58.0}, {"_id": {"$oid": "69d4eb12b877594bc5718ae4"}, "test_id": "T0292", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-1-20250805", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Selbs im fronaltar die schend\\nwir gantz vnd ein stuck von\\ndem heiligen fron truch Sy\\nlit ir vff ir brust vnd ein\\nbein von sant marien madalene\\nruchgrot Jff aff samstag nach\\ndem ostertag der do was der\\nander tag des abrellen do\\nfurend wir gen meyain do\\nligend in sant pfsteffins\\nkilchen bi hundert der kinden\\ndie herodes erschlug in einem\\naltar die schend wir ovch\\nsicht man noch beschedenlich\\nir hend vnd fus vnd die\\nnagel dar an vvundes an\\ndem stinentag frug do\",\"addition1\":\"Meyain\\ndas\\ndritte\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "claude-opus-4-1-20250805", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2718, "output_tokens": 272, "total_tokens": 2990, "input_cost_usd": 0.04077, "output_cost_usd": 0.0204, "estimated_cost_usd": 0.06117}, "duration": 14.905206441879272, "timestamp": "2026-01-24T14:50:21.302987", "parsed": {"folios": [{"folio": "", "text": "Selbs im fronaltar die schend\nwir gantz vnd ein stuck von\ndem heiligen fron truch Sy\nlit ir vff ir brust vnd ein\nbein von sant marien madalene\nruchgrot Jff aff samstag nach\ndem ostertag der do was der\nander tag des abrellen do\nfurend wir gen meyain do\nligend in sant pfsteffins\nkilchen bi hundert der kinden\ndie herodes erschlug in einem\naltar die schend wir ovch\nsicht man noch beschedenlich\nir hend vnd fus vnd die\nnagel dar an vvundes an\ndem stinentag frug do", "addition1": "Meyain\ndas\ndritte", "addition2": "", "addition3": ""}]}, "conversation_id": "04d934c6-7dc6-410c-9059-024fde2fbd23", "score": {"fuzzy": 0.735, "cer": 0.307}, "raw_response": "{\"id\":\"msg_01BauPECNKxw81PTVcgoN47T\",\"content\":[{\"id\":\"toolu_01Q4cgY82NDx6dJAKs5CJz2u\",\"input\":{\"folios\":[{\"folio\":\"\",\"text\":\"Selbs im fronaltar die schend\\nwir gantz vnd ein stuck von\\ndem heiligen fron truch Sy\\nlit ir vff ir brust vnd ein\\nbein von sant marien madalene\\nruchgrot Jff aff samstag nach\\ndem ostertag der do was der\\nander tag des abrellen do\\nfurend wir gen meyain do\\nligend in sant pfsteffins\\nkilchen bi hundert der kinden\\ndie herodes erschlug in einem\\naltar die schend wir ovch\\nsicht man noch beschedenlich\\nir hend vnd fus vnd die\\nnagel dar an vvundes an\\ndem stinentag frug do\",\"addition1\":\"Meyain\\ndas\\ndritte\",\"addition2\":\"\",\"addition3\":\"\"}]},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-opus-4-1-20250805\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2718,\"output_tokens\":272,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"fuzzy": 0.733, "cer": 0.287, "cost_summary": {"total_input_tokens": 32616, "total_output_tokens": 3401, "total_tokens": 36017, "input_cost_usd": 0.4892399999999999, "output_cost_usd": 0.255075, "total_cost_usd": 0.7443150000000001}}, "normalized_score": 71.3}, {"_id": {"$oid": "69d4eb12b877594bc5718aed"}, "test_id": "T0291", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-20250514", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Halbos wir frowalter Sie solhend\\nwir gantz vnd ein fleck dory\\ndem heiligen kron trutz So\\nlit ir vff ir brust vnd ein\\nbem von sant marien madalene\\nrugg rott Jn ost samstag noch\\ndem ostertag Ser So was der\\nander tag des abrellen do\\nsturen furend wir gon meran So\\nglub lichend in sant Steffans\\nkilchen bi hundert der knechten\\ndie herodes erstluog in einem\\naltar die pferd vnd vnd\\nsicht man noch bestandenlich\\nir hend vnd fuos vnd die\\nnadelldar alj vrmels in\\ndem flementag fruog So\",\"addition1\":null,\"addition2\":null,\"addition3\":null}]}", "model": "claude-sonnet-4-20250514", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2718, "output_tokens": 246, "total_tokens": 2964, "input_cost_usd": 0.008154, "output_cost_usd": 0.00369, "estimated_cost_usd": 0.011844}, "duration": 10.981510400772095, "timestamp": "2026-01-24T14:47:42.898156", "parsed": {"folios": [{"folio": "", "text": "Halbos wir frowalter Sie solhend\nwir gantz vnd ein fleck dory\ndem heiligen kron trutz So\nlit ir vff ir brust vnd ein\nbem von sant marien madalene\nrugg rott Jn ost samstag noch\ndem ostertag Ser So was der\nander tag des abrellen do\nsturen furend wir gon meran So\nglub lichend in sant Steffans\nkilchen bi hundert der knechten\ndie herodes erstluog in einem\naltar die pferd vnd vnd\nsicht man noch bestandenlich\nir hend vnd fuos vnd die\nnadelldar alj vrmels in\ndem flementag fruog So"}]}, "conversation_id": "7cc70fe9-9841-44ce-893f-8486a72bd83c", "score": {"fuzzy": 0.421, "cer": 0.59}, "raw_response": "{\"id\":\"msg_01EPVe2gThYW8gbfqpFm4xyH\",\"content\":[{\"id\":\"toolu_01E4ybvjbMZxxwepATWFubYa\",\"input\":{\"folios\":[{\"folio\":\"\",\"text\":\"Halbos wir frowalter Sie solhend\\nwir gantz vnd ein fleck dory\\ndem heiligen kron trutz So\\nlit ir vff ir brust vnd ein\\nbem von sant marien madalene\\nrugg rott Jn ost samstag noch\\ndem ostertag Ser So was der\\nander tag des abrellen do\\nsturen furend wir gon meran So\\nglub lichend in sant Steffans\\nkilchen bi hundert der knechten\\ndie herodes erstluog in einem\\naltar die pferd vnd vnd\\nsicht man noch bestandenlich\\nir hend vnd fuos vnd die\\nnadelldar alj vrmels in\\ndem flementag fruog So\"}]},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-sonnet-4-20250514\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2718,\"output_tokens\":246,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"fuzzy": 0.583, "cer": 0.437, "cost_summary": {"total_input_tokens": 32616, "total_output_tokens": 3517, "total_tokens": 36133, "input_cost_usd": 0.09784799999999998, "output_cost_usd": 0.052754999999999996, "total_cost_usd": 0.15060300000000001}}, "normalized_score": 56.3}, {"_id": {"$oid": "69d4eb12b877594bc5718af6"}, "test_id": "T0516", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-5-20251101", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Jt dem patron iij grossen\\nf\u00fcr sin zerung\\nJt ze ema\u00fcs ein grossen\\nJt ze Jherusalem j gross ze\\nkurti\u00dfg von den eselen\\nJt dem consul v gros\\nJt in den tempel ij gros\\nJt ze betlahem ein gros\\nJt ze sant Johans ein gros\\nJt wider ze Jherusalem ein\\ngros ze curti\u00dfg aber j gros\\nJt wider in den tempel iiij\\ngros\\nJt ze dem Jordan xij gros\\nv\u014d den eselen ze lon\\nJt aber ein gros ze curistet\\nJt aber ze Jherusal\u0113 j gros\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "claude-opus-4-5-20251101", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2991, "output_tokens": 239, "total_tokens": 3230, "input_cost_usd": 0.014955000000000001, "output_cost_usd": 0.005975, "estimated_cost_usd": 0.02093}, "duration": 8.815854549407959, "timestamp": "2026-01-24T17:24:36.517361", "parsed": {"folios": [{"folio": "", "text": "Jt dem patron iij grossen\nf\u00fcr sin zerung\nJt ze ema\u00fcs ein grossen\nJt ze Jherusalem j gross ze\nkurti\u00dfg von den eselen\nJt dem consul v gros\nJt in den tempel ij gros\nJt ze betlahem ein gros\nJt ze sant Johans ein gros\nJt wider ze Jherusalem ein\ngros ze curti\u00dfg aber j gros\nJt wider in den tempel iiij\ngros\nJt ze dem Jordan xij gros\nv\u014d den eselen ze lon\nJt aber ein gros ze curistet\nJt aber ze Jherusal\u0113 j gros", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "d448d954-2699-493f-be93-1f94181768bc", "score": {"fuzzy": 0.892, "cer": 0.115}, "raw_response": "{\"id\":\"msg_01M6Exi7Dcw2BgbJv1RSo4PG\",\"content\":[{\"id\":\"toolu_01VVMLcNSmGkva5QDH69WoX9\",\"input\":{\"folios\":[{\"folio\":\"\",\"text\":\"Jt dem patron iij grossen\\nf\u00fcr sin zerung\\nJt ze ema\u00fcs ein grossen\\nJt ze Jherusalem j gross ze\\nkurti\u00dfg von den eselen\\nJt dem consul v gros\\nJt in den tempel ij gros\\nJt ze betlahem ein gros\\nJt ze sant Johans ein gros\\nJt wider ze Jherusalem ein\\ngros ze curti\u00dfg aber j gros\\nJt wider in den tempel iiij\\ngros\\nJt ze dem Jordan xij gros\\nv\u014d den eselen ze lon\\nJt aber ein gros ze curistet\\nJt aber ze Jherusal\u0113 j gros\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-opus-4-5-20251101\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2991,\"output_tokens\":239,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"fuzzy": 0.855, "cer": 0.151, "cost_summary": {"total_input_tokens": 35892, "total_output_tokens": 3396, "total_tokens": 39288, "input_cost_usd": 0.17945999999999998, "output_cost_usd": 0.08489999999999999, "total_cost_usd": 0.26436}}, "normalized_score": 84.9}, {"_id": {"$oid": "69d4eb12b877594bc5718aff"}, "test_id": "T0441", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-small-2506", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"32\",\"text\":\"Daz ist ein wunderlichez m\u00e6re\\nvon einem ritter der hiez Wolfdietrich\\nund wie er ze Basel kam\\nund wie er da mit den burgern\\nstriet und wie er siegte\\nund wie er die stat eroberte\\nund wie er die burger freite\",\"addition1\":\"Hie hebt sich an die m\u00e6re\\nvon Wolfdietrich\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "mistral-small-2506", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 753, "output_tokens": 130, "total_tokens": 883, "input_cost_usd": 7.53e-05, "output_cost_usd": 3.899999999999999e-05, "estimated_cost_usd": 0.00011429999999999999}, "duration": 1.6013343334197998, "timestamp": "2026-01-24T13:39:50.539246", "parsed": {"folios": [{"folio": "32", "text": "Daz ist ein wunderlichez m\u00e6re\nvon einem ritter der hiez Wolfdietrich\nund wie er ze Basel kam\nund wie er da mit den burgern\nstriet und wie er siegte\nund wie er die stat eroberte\nund wie er die burger freite", "addition1": "Hie hebt sich an die m\u00e6re\nvon Wolfdietrich", "addition2": "", "addition3": ""}]}, "conversation_id": "8f918f6f-2d2b-4f61-b9b6-e4ae9035cf74", "score": {"fuzzy": 0.138, "cer": 0.905}, "raw_response": "{\"id\":\"9a46e3801a4b43c1b6cb48b48cbbbae8\",\"object\":\"chat.completion\",\"model\":\"mistral-small-2506\",\"usage\":{\"prompt_tokens\":753,\"completion_tokens\":130,\"total_tokens\":883,\"num_cached_tokens\":768},\"created\":1769258389,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"32\\\",\\n      \\\"text\\\": \\\"Daz ist ein wunderlichez m\u00e6re\\\\nvon einem ritter der hiez Wolfdietrich\\\\nund wie er ze Basel kam\\\\nund wie er da mit den burgern\\\\nstriet und wie er siegte\\\\nund wie er die stat eroberte\\\\nund wie er die burger freite\\\",\\n      \\\"addition1\\\": \\\"Hie hebt sich an die m\u00e6re\\\\nvon Wolfdietrich\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"fuzzy": 0.181, "cer": 0.916, "cost_summary": {"total_input_tokens": 9036, "total_output_tokens": 1866, "total_tokens": 10902, "input_cost_usd": 0.0009036000000000002, "output_cost_usd": 0.0005598, "total_cost_usd": 0.0014633999999999997}}, "normalized_score": 8.399999999999991}, {"_id": {"$oid": "69d4eb12b877594bc5718b06"}, "test_id": "T0527", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-haiku-4-5-20251001", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Deb pfilley tage orey fabzung Jr dz waz\\ngey Jsfay deng balmutag dz wz\\nDer fubenzefende tag Vutey wur\\ngey pftarb Jsfm dz ey adtzchende\\ntag day terkit Nid bibleyzwey\\ntag dz wnd fwend ay dey Gelden\\nschupag dz dz wz ber dacht wn\\nzwenzwrt tag Deb merget Vff\\ndey dwaffter No terkit gay Nellody\\nJsfay dey ofteroden der dz wz der\\nleft tag iny merget Ds pofey wur\\nethe maunget fitktyen Lund droft\\npfaff als mechtge hufter doff Bde\\nVenck Lwd fant mays munfper wnd dz\\nDueis Deutfzey palaft dz ift zemuft fkity\\npalaft Lem fand Dandf welfgdew Pleter\\nmacmifi dund knechtey Nwppey diz demy\\ncum ze Veneday geweft fulld Dz gt\\nzey Cappelt ay dey munfter doff\\ndew pzin zy ziedalfed gewaffet dz\\nNulfur befaltter geb Cpd Cff faz\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "claude-haiku-4-5-20251001", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2991, "output_tokens": 418, "total_tokens": 3409, "input_cost_usd": 0.002991, "output_cost_usd": 0.0020900000000000003, "estimated_cost_usd": 0.0050810000000000004}, "duration": 8.639029741287231, "timestamp": "2026-01-24T18:32:13.062396", "parsed": {"folios": [{"folio": "", "text": "Deb pfilley tage orey fabzung Jr dz waz\ngey Jsfay deng balmutag dz wz\nDer fubenzefende tag Vutey wur\ngey pftarb Jsfm dz ey adtzchende\ntag day terkit Nid bibleyzwey\ntag dz wnd fwend ay dey Gelden\nschupag dz dz wz ber dacht wn\nzwenzwrt tag Deb merget Vff\ndey dwaffter No terkit gay Nellody\nJsfay dey ofteroden der dz wz der\nleft tag iny merget Ds pofey wur\nethe maunget fitktyen Lund droft\npfaff als mechtge hufter doff Bde\nVenck Lwd fant mays munfper wnd dz\nDueis Deutfzey palaft dz ift zemuft fkity\npalaft Lem fand Dandf welfgdew Pleter\nmacmifi dund knechtey Nwppey diz demy\ncum ze Veneday geweft fulld Dz gt\nzey Cappelt ay dey munfter doff\ndew pzin zy ziedalfed gewaffet dz\nNulfur befaltter geb Cpd Cff faz", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "8132e35b-4b2c-42ca-8312-67e5c2a2bdd9", "score": {"fuzzy": 0.312, "cer": 0.72}, "raw_response": "{\"id\":\"msg_014rDoCga5hNwsb8iCBm3h3M\",\"content\":[{\"id\":\"toolu_01Jn1k9VLZfqFEJHJd3KVwzp\",\"input\":{\"folios\":[{\"folio\":\"\",\"text\":\"Deb pfilley tage orey fabzung Jr dz waz\\ngey Jsfay deng balmutag dz wz\\nDer fubenzefende tag Vutey wur\\ngey pftarb Jsfm dz ey adtzchende\\ntag day terkit Nid bibleyzwey\\ntag dz wnd fwend ay dey Gelden\\nschupag dz dz wz ber dacht wn\\nzwenzwrt tag Deb merget Vff\\ndey dwaffter No terkit gay Nellody\\nJsfay dey ofteroden der dz wz der\\nleft tag iny merget Ds pofey wur\\nethe maunget fitktyen Lund droft\\npfaff als mechtge hufter doff Bde\\nVenck Lwd fant mays munfper wnd dz\\nDueis Deutfzey palaft dz ift zemuft fkity\\npalaft Lem fand Dandf welfgdew Pleter\\nmacmifi dund knechtey Nwppey diz demy\\ncum ze Veneday geweft fulld Dz gt\\nzey Cappelt ay dey munfter doff\\ndew pzin zy ziedalfed gewaffet dz\\nNulfur befaltter geb Cpd Cff faz\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-haiku-4-5-20251001\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2991,\"output_tokens\":418,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"fuzzy": 0.632, "cer": 0.397, "cost_summary": {"total_input_tokens": 35892, "total_output_tokens": 3568, "total_tokens": 39460, "input_cost_usd": 0.035892, "output_cost_usd": 0.01784, "total_cost_usd": 0.053732}}, "normalized_score": 60.3}, {"_id": {"$oid": "69d4eb12b877594bc5718b17"}, "test_id": "T0294", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "pixtral-large-2411", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Item an dem sulchern\\ntage do man den herren\\nsolle loben und ement\\nwie die heiligen pfaffen\\nuns geschriben haben \\nund uns das gnusten  vnd \\nuns zu eime feste machet \\nwir sollen dan des herren\\ngepote volgen und uns \\nkeren zu dem volge der\\nheiligen pfaffen und der gemain\\nwir sollen den herren\\nvolgen und ement\",\"addition1\":\"\",\"addition2\":null,\"addition3\":null}]}", "model": "pixtral-large-2411", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 837, "output_tokens": 179, "total_tokens": 1016, "input_cost_usd": 0.001674, "output_cost_usd": 0.0010739999999999999, "estimated_cost_usd": 0.0027479999999999996}, "duration": 3.7721166610717773, "timestamp": "2026-01-24T12:14:52.789314", "parsed": {"folios": [{"folio": "", "text": "Item an dem sulchern\ntage do man den herren\nsolle loben und ement\nwie die heiligen pfaffen\nuns geschriben haben \nund uns das gnusten  vnd \nuns zu eime feste machet \nwir sollen dan des herren\ngepote volgen und uns \nkeren zu dem volge der\nheiligen pfaffen und der gemain\nwir sollen den herren\nvolgen und ement", "addition1": "", "addition2": null, "addition3": null}]}, "conversation_id": "32e4d07e-c840-428a-ace9-ffb322faef1a", "score": {"fuzzy": 0.213, "cer": 0.931}, "raw_response": "{\"id\":\"4928eb7c5b1d4806945b207f87e3f35d\",\"object\":\"chat.completion\",\"model\":\"pixtral-large-2411\",\"usage\":{\"prompt_tokens\":837,\"completion_tokens\":179,\"total_tokens\":1016,\"num_cached_tokens\":832},\"created\":1769253289,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Item an dem sulchern\\\\ntage do man den herren\\\\nsolle loben und ement\\\\nwie die heiligen pfaffen\\\\nuns geschriben haben \\\\nund uns das gnusten  vnd \\\\nuns zu eime feste machet \\\\nwir sollen dan des herren\\\\ngepote volgen und uns \\\\nkeren zu dem volge der\\\\nheiligen pfaffen und der gemain\\\\nwir sollen den herren\\\\nvolgen und ement\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": null,\\n      \\\"addition3\\\": null\\n    }\\n  ]\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"fuzzy": 0.177, "cer": 0.886, "cost_summary": {"total_input_tokens": 10044, "total_output_tokens": 2238, "total_tokens": 12282, "input_cost_usd": 0.020087999999999995, "output_cost_usd": 0.013427999999999999, "total_cost_usd": 0.033516000000000004}}, "normalized_score": 11.400000000000006}, {"_id": {"$oid": "69d4eb12b877594bc5718b29"}, "test_id": "T0298", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-large-2411", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Dys sind die stucke so vns von\\n den von Solotvrn geleget sint\\n vnd wir habent vns mit gantzem\\n vleis dar wider gesatzt als her\\n nach geschriben stat vnd sint\\n die selbige stucke von vns\\n als her nach geschriben stat\\n angenomen vnd volbracht \ua75buch\\n wir vns dar zu verpflichten habent\\n mit ainander in kraft des eides\\n den wir dar vf geleistet habent\\n vnd wir habent vns mit ainander\\n dar vf vereint das wir die vorgeschri-\\n bene stucke vnd yegliche dar vf\\n volgen sol mit aller vnsr macht\\n vnd kraft vnd vns dar an kenen las-\\n sen vnd den vns vnser burger\\n mit vnser stat gewalt dar zu\\n kenen hinder lag zwingen\\n vnd machen sullen\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "mistral-large-2411", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 837, "output_tokens": 299, "total_tokens": 1136, "input_cost_usd": 0.001674, "output_cost_usd": 0.001794, "estimated_cost_usd": 0.0034679999999999997}, "duration": 4.950856685638428, "timestamp": "2026-01-24T12:16:55.354684", "parsed": {"folios": [{"folio": "", "text": "Dys sind die stucke so vns von\n den von Solotvrn geleget sint\n vnd wir habent vns mit gantzem\n vleis dar wider gesatzt als her\n nach geschriben stat vnd sint\n die selbige stucke von vns\n als her nach geschriben stat\n angenomen vnd volbracht \ua75buch\n wir vns dar zu verpflichten habent\n mit ainander in kraft des eides\n den wir dar vf geleistet habent\n vnd wir habent vns mit ainander\n dar vf vereint das wir die vorgeschri-\n bene stucke vnd yegliche dar vf\n volgen sol mit aller vnsr macht\n vnd kraft vnd vns dar an kenen las-\n sen vnd den vns vnser burger\n mit vnser stat gewalt dar zu\n kenen hinder lag zwingen\n vnd machen sullen", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "3d67d25c-42e8-4ffc-97c5-e390146f39ef", "score": {"fuzzy": 0.203, "cer": 0.95}, "raw_response": "{\"id\":\"62cac56c2c1e42fc8530507800a9cdc6\",\"object\":\"chat.completion\",\"model\":\"mistral-large-2411\",\"usage\":{\"prompt_tokens\":837,\"completion_tokens\":299,\"total_tokens\":1136,\"num_cached_tokens\":0},\"created\":1769253410,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Dys sind die stucke so vns von\\\\n den von Solotvrn geleget sint\\\\n vnd wir habent vns mit gantzem\\\\n vleis dar wider gesatzt als her\\\\n nach geschriben stat vnd sint\\\\n die selbige stucke von vns\\\\n als her nach geschriben stat\\\\n angenomen vnd volbracht \ua75buch\\\\n wir vns dar zu verpflichten habent\\\\n mit ainander in kraft des eides\\\\n den wir dar vf geleistet habent\\\\n vnd wir habent vns mit ainander\\\\n dar vf vereint das wir die vorgeschri-\\\\n bene stucke vnd yegliche dar vf\\\\n volgen sol mit aller vnsr macht\\\\n vnd kraft vnd vns dar an kenen las-\\\\n sen vnd den vns vnser burger\\\\n mit vnser stat gewalt dar zu\\\\n kenen hinder lag zwingen\\\\n vnd machen sullen\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"fuzzy": 0.178, "cer": 0.906, "cost_summary": {"total_input_tokens": 10044, "total_output_tokens": 2501, "total_tokens": 12545, "input_cost_usd": 0.020087999999999995, "output_cost_usd": 0.015006000000000002, "total_cost_usd": 0.035094}}, "normalized_score": 9.399999999999991}, {"_id": {"$oid": "69d4eb12b877594bc5718b2a"}, "test_id": "T0295", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-medium-2508", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"123r\",\"text\":\"Hie vo\u030agt an das bu\u030ach von den\\n gro\u030assen thaten der edeln herren\\n von Basel vnd irer freyheit\\n die sie mit grossem mu\u030at vnd\\n mit go\u030attlicher hilff erka\u030ampft\\n ha\u030abend wider die von O\u030asterreich\\n vnd ir freyheit behalten\\n ha\u030abend bis an disen tag\",\"addition1\":\"Anno domini MCCCCXLV\",\"addition2\":\"Notabene\",\"addition3\":\"\"}]}", "model": "mistral-medium-2508", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 753, "output_tokens": 162, "total_tokens": 915, "input_cost_usd": 0.0003012, "output_cost_usd": 0.000324, "estimated_cost_usd": 0.0006252}, "duration": 2.2565321922302246, "timestamp": "2026-01-24T12:15:48.939190", "parsed": {"folios": [{"folio": "123r", "text": "Hie vo\u030agt an das bu\u030ach von den\n gro\u030assen thaten der edeln herren\n von Basel vnd irer freyheit\n die sie mit grossem mu\u030at vnd\n mit go\u030attlicher hilff erka\u030ampft\n ha\u030abend wider die von O\u030asterreich\n vnd ir freyheit behalten\n ha\u030abend bis an disen tag", "addition1": "Anno domini MCCCCXLV", "addition2": "Notabene", "addition3": ""}]}, "conversation_id": "f49fd165-8adf-4b29-8459-37ed446b8b5e", "score": {"fuzzy": 0.246, "cer": 0.914}, "raw_response": "{\"id\":\"7f82ae1b7ce746e3953c9baf23a9b24d\",\"object\":\"chat.completion\",\"model\":\"mistral-medium-2508\",\"usage\":{\"prompt_tokens\":753,\"completion_tokens\":162,\"total_tokens\":915,\"num_cached_tokens\":0},\"created\":1769253346,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"123r\\\",\\n      \\\"text\\\": \\\"Hie vo\u030agt an das bu\u030ach von den\\\\n gro\u030assen thaten der edeln herren\\\\n von Basel vnd irer freyheit\\\\n die sie mit grossem mu\u030at vnd\\\\n mit go\u030attlicher hilff erka\u030ampft\\\\n ha\u030abend wider die von O\u030asterreich\\\\n vnd ir freyheit behalten\\\\n ha\u030abend bis an disen tag\\\",\\n      \\\"addition1\\\": \\\"Anno domini MCCCCXLV\\\",\\n      \\\"addition2\\\": \\\"Notabene\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"fuzzy": 0.201, "cer": 0.909, "cost_summary": {"total_input_tokens": 9036, "total_output_tokens": 2236, "total_tokens": 11272, "input_cost_usd": 0.0036144000000000007, "output_cost_usd": 0.004472, "total_cost_usd": 0.0080864}}, "normalized_score": 9.099999999999994}, {"_id": {"$oid": "69d4eb12b877594bc5718b3e"}, "test_id": "T0290", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-20250514", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"69\",\"text\":\"Hie nach kome ir pfister vn\u0304\\ntrumeter zu vnnen her\u016bauff\\ndie gallen vnd hofierten do\\nsinen spotten deb pflaff kome\\ndie windgassen pfyffer vnd\\ntrumeter vnd blayssenslaher\\ndie hofierten do och nach we\\nsitten also ward mine gnedige\\nheren vnd allen den sine vil\\nerenerbotten Ist morndeb ge\\ngen tag fuorent wir do danne\\nvnd kamen vff fritag nach sunt\\nOrbanus tag gen do wir det erst\\ntag im kriesmonet gon kurson\\ndo ist ein schone stat vint zwey\\nschlossen woellwrse vn\u0304 ob der\\nstat von vff suntag darnach\\nfuoren wir do danne wie vff\\nkriegtag war sunt vits tag\\nkomen zu vns in gullen vn\u0304 g\u0101g\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "claude-opus-4-20250514", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2718, "output_tokens": 321, "total_tokens": 3039, "input_cost_usd": 0.04077, "output_cost_usd": 0.024075, "estimated_cost_usd": 0.064845}, "duration": 18.010364055633545, "timestamp": "2026-01-24T14:46:25.569480", "parsed": {"folios": [{"folio": "69", "text": "Hie nach kome ir pfister vn\u0304\ntrumeter zu vnnen her\u016bauff\ndie gallen vnd hofierten do\nsinen spotten deb pflaff kome\ndie windgassen pfyffer vnd\ntrumeter vnd blayssenslaher\ndie hofierten do och nach we\nsitten also ward mine gnedige\nheren vnd allen den sine vil\nerenerbotten Ist morndeb ge\ngen tag fuorent wir do danne\nvnd kamen vff fritag nach sunt\nOrbanus tag gen do wir det erst\ntag im kriesmonet gon kurson\ndo ist ein schone stat vint zwey\nschlossen woellwrse vn\u0304 ob der\nstat von vff suntag darnach\nfuoren wir do danne wie vff\nkriegtag war sunt vits tag\nkomen zu vns in gullen vn\u0304 g\u0101g", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "f179db14-a51e-4d23-849c-cacac847a575", "score": {"fuzzy": 0.279, "cer": 0.727}, "raw_response": "{\"id\":\"msg_01AgkSwMAAijFmYTyyFmgNSC\",\"content\":[{\"id\":\"toolu_01VX9UYjAZUHmGCMwyvFU86y\",\"input\":{\"folios\":[{\"folio\":\"69\",\"text\":\"Hie nach kome ir pfister vn\u0304\\ntrumeter zu vnnen her\u016bauff\\ndie gallen vnd hofierten do\\nsinen spotten deb pflaff kome\\ndie windgassen pfyffer vnd\\ntrumeter vnd blayssenslaher\\ndie hofierten do och nach we\\nsitten also ward mine gnedige\\nheren vnd allen den sine vil\\nerenerbotten Ist morndeb ge\\ngen tag fuorent wir do danne\\nvnd kamen vff fritag nach sunt\\nOrbanus tag gen do wir det erst\\ntag im kriesmonet gon kurson\\ndo ist ein schone stat vint zwey\\nschlossen woellwrse vn\u0304 ob der\\nstat von vff suntag darnach\\nfuoren wir do danne wie vff\\nkriegtag war sunt vits tag\\nkomen zu vns in gullen vn\u0304 g\u0101g\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-opus-4-20250514\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2718,\"output_tokens\":321,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"fuzzy": 0.734, "cer": 0.287, "cost_summary": {"total_input_tokens": 32616, "total_output_tokens": 3468, "total_tokens": 36084, "input_cost_usd": 0.4892399999999999, "output_cost_usd": 0.2601, "total_cost_usd": 0.7493400000000001}}, "normalized_score": 71.3}, {"_id": {"$oid": "69d4eb12b877594bc5718b56"}, "test_id": "T0271", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"3\",\"text\":\"Vnd ein pferit Die mir vnd\\nminen knechten vber Rulfend\\nSen So was menen kein weg\\nSanne Das wir machtend\\nvnd ziehend Sie knecht Sind\\nvnd vil in vntz an Sen anb\\nvnd Sie pferit vntz an Die\\npettel vnd was ze mol ein gro\u017f\u017fe\\nnebel dz wir kum gepa\u017f\u017fend\\nvnd als mit gro\u017f\u017fer arbeit kame\\nwir ze mittem tag zuo Sant\\nEinro\u017f\u017fel vff Sen berg So\\nSo sach ich Die bucher Do gar\\nvil herren wopen zuo \u017ftont\\nDie in fuor edlen geben hand\\nSo \u017find mineb vattero Rligen\\nwopen ach in Sem emen\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gemini-2.5-flash", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 772, "output_tokens": 224, "total_tokens": 2944, "input_cost_usd": 0.0002316, "output_cost_usd": 0.00056, "estimated_cost_usd": 0.0007915999999999999}, "duration": 10.915436744689941, "timestamp": "2026-01-24T15:09:34.257597", "parsed": {"folios": [{"folio": "3", "text": "Vnd ein pferit Die mir vnd\nminen knechten vber Rulfend\nSen So was menen kein weg\nSanne Das wir machtend\nvnd ziehend Sie knecht Sind\nvnd vil in vntz an Sen anb\nvnd Sie pferit vntz an Die\npettel vnd was ze mol ein gro\u017f\u017fe\nnebel dz wir kum gepa\u017f\u017fend\nvnd als mit gro\u017f\u017fer arbeit kame\nwir ze mittem tag zuo Sant\nEinro\u017f\u017fel vff Sen berg So\nSo sach ich Die bucher Do gar\nvil herren wopen zuo \u017ftont\nDie in fuor edlen geben hand\nSo \u017find mineb vattero Rligen\nwopen ach in Sem emen", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "fe4a0b2d-157f-4b9e-a682-22cdc87b05cf", "score": {"fuzzy": 0.92, "cer": 0.072}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sat, 24 Jan 2026 14:09:34 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=10757\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"3\\\",\\n      \\\"text\\\": \\\"Vnd ein pferit Die mir vnd\\\\nminen knechten vber Rulfend\\\\nSen So was menen kein weg\\\\nSanne Das wir machtend\\\\nvnd ziehend Sie knecht Sind\\\\nvnd vil in vntz an Sen anb\\\\nvnd Sie pferit vntz an Die\\\\npettel vnd was ze mol ein gro\u017f\u017fe\\\\nnebel dz wir kum gepa\u017f\u017fend\\\\nvnd als mit gro\u017f\u017fer arbeit kame\\\\nwir ze mittem tag zuo Sant\\\\nEinro\u017f\u017fel vff Sen berg So\\\\nSo sach ich Die bucher Do gar\\\\nvil herren wopen zuo \u017ftont\\\\nDie in fuor edlen geben hand\\\\nSo \u017find mineb vattero Rligen\\\\nwopen ach in Sem emen\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.5-flash\",\"prompt_feedback\":null,\"response_id\":\"ntJ0acXADebr7M8P0pnYmAw\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":224,\"candidates_tokens_details\":null,\"prompt_token_count\":772,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":514},{\"modality\":\"IMAGE\",\"token_count\":258}],\"thoughts_token_count\":1948,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2944,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"folios\":[{\"folio\":\"3\",\"text\":\"Vnd ein pferit Die mir vnd\\nminen knechten vber Rulfend\\nSen So was menen kein weg\\nSanne Das wir machtend\\nvnd ziehend Sie knecht Sind\\nvnd vil in vntz an Sen anb\\nvnd Sie pferit vntz an Die\\npettel vnd was ze mol ein gro\u017f\u017fe\\nnebel dz wir kum gepa\u017f\u017fend\\nvnd als mit gro\u017f\u017fer arbeit kame\\nwir ze mittem tag zuo Sant\\nEinro\u017f\u017fel vff Sen berg So\\nSo sach ich Die bucher Do gar\\nvil herren wopen zuo \u017ftont\\nDie in fuor edlen geben hand\\nSo \u017find mineb vattero Rligen\\nwopen ach in Sem emen\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}}"}, "scoring": {"fuzzy": 0.624, "cer": 0.404, "cost_summary": {"total_input_tokens": 9264, "total_output_tokens": 2945, "total_tokens": 12209, "input_cost_usd": 0.0027792000000000008, "output_cost_usd": 0.0073625, "total_cost_usd": 0.0101417}}, "normalized_score": 59.599999999999994}, {"_id": {"$oid": "69d4eb12b877594bc5718b65"}, "test_id": "T0296", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-medium-2505", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"123r\",\"text\":\"Hie vo\u030dn dem heyligen\\n leichnam vnd von der\\n heyligen stat Iherusalem\\n vnd von der heyligen\\n stat Rom vnd von der\\n heyligen stat Constantinopel\\n vnd von der heyligen\\n stat Alexandria vnd von\\n der heyligen stat Antiochia\\n vnd von der heyligen\\n stat Iherusalem\",\"addition1\":\"Notabene\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "mistral-medium-2505", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 753, "output_tokens": 140, "total_tokens": 893, "input_cost_usd": 0.0003012, "output_cost_usd": 0.00028, "estimated_cost_usd": 0.0005812}, "duration": 1.8778116703033447, "timestamp": "2026-01-24T12:16:42.504984", "parsed": {"folios": [{"folio": "123r", "text": "Hie vo\u030dn dem heyligen\n leichnam vnd von der\n heyligen stat Iherusalem\n vnd von der heyligen\n stat Rom vnd von der\n heyligen stat Constantinopel\n vnd von der heyligen\n stat Alexandria vnd von\n der heyligen stat Antiochia\n vnd von der heyligen\n stat Iherusalem", "addition1": "Notabene", "addition2": "", "addition3": ""}]}, "conversation_id": "e0c0e771-468f-47e9-bc67-a8590ab613c2", "score": {"fuzzy": 0.162, "cer": 0.879}, "raw_response": "{\"id\":\"f3114b6d7732417f9bca382d084219ec\",\"object\":\"chat.completion\",\"model\":\"mistral-medium-2505\",\"usage\":{\"prompt_tokens\":753,\"completion_tokens\":140,\"total_tokens\":893,\"num_cached_tokens\":0},\"created\":1769253400,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"123r\\\",\\n      \\\"text\\\": \\\"Hie vo\u030dn dem heyligen\\\\n leichnam vnd von der\\\\n heyligen stat Iherusalem\\\\n vnd von der heyligen\\\\n stat Rom vnd von der\\\\n heyligen stat Constantinopel\\\\n vnd von der heyligen\\\\n stat Alexandria vnd von\\\\n der heyligen stat Antiochia\\\\n vnd von der heyligen\\\\n stat Iherusalem\\\",\\n      \\\"addition1\\\": \\\"Notabene\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"fuzzy": 0.189, "cer": 0.916, "cost_summary": {"total_input_tokens": 9036, "total_output_tokens": 2358, "total_tokens": 11394, "input_cost_usd": 0.0036144000000000007, "output_cost_usd": 0.004716, "total_cost_usd": 0.0083304}}, "normalized_score": 8.399999999999991}, {"_id": {"$oid": "69d4eb12b877594bc5718b6c"}, "test_id": "T0284", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash-lite", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"69\",\"text\":\"Forenarts komme in pfifferen\\nzummery goed off\\nOne galley Lond Roflenten To\\nsimen spidden des gluh kome\\nDie ondertien priffce\\ntruncter and b\u00f6gfenplayex\\nNu hofierten to\\nfittey alps ward moho mohe quedure\\nReed and alley den fine wit\\nLecheebatten Jof midrandes for\\ntoy toy fincant wir Jo Dommo\\nAnd Ricky coff frestag nach fount\\nJak do nos dak ent\\nbag ing benchmanat fon Puchon\\nIf it ciy plane pat heist Birch\\nHeist\\nof dut\\non of\\nFurry meir Da Damne Nu\\nFimpag naar font duty tag\\nFanney zu vids in galley by \u03c7\u03b1\u03c1\",\"addition1\":\"Honores\\nMaschio\\nin exhi\\nbih.\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gemini-2.0-flash-lite", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 2365, "output_tokens": 248, "total_tokens": 2613, "input_cost_usd": 0.000177375, "output_cost_usd": 7.44e-05, "estimated_cost_usd": 0.000251775}, "duration": 3.3797037601470947, "timestamp": "2026-01-24T15:17:02.231286", "parsed": {"folios": [{"folio": "69", "text": "Forenarts komme in pfifferen\nzummery goed off\nOne galley Lond Roflenten To\nsimen spidden des gluh kome\nDie ondertien priffce\ntruncter and b\u00f6gfenplayex\nNu hofierten to\nfittey alps ward moho mohe quedure\nReed and alley den fine wit\nLecheebatten Jof midrandes for\ntoy toy fincant wir Jo Dommo\nAnd Ricky coff frestag nach fount\nJak do nos dak ent\nbag ing benchmanat fon Puchon\nIf it ciy plane pat heist Birch\nHeist\nof dut\non of\nFurry meir Da Damne Nu\nFimpag naar font duty tag\nFanney zu vids in galley by \u03c7\u03b1\u03c1", "addition1": "Honores\nMaschio\nin exhi\nbih.", "addition2": "", "addition3": ""}]}, "conversation_id": "da397046-d00a-458e-8879-6cd78bb324e2", "score": {"fuzzy": 0.459, "cer": 0.551}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sat, 24 Jan 2026 14:17:02 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=3142\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"69\\\",\\n      \\\"text\\\": \\\"Forenarts komme in pfifferen\\\\nzummery goed off\\\\nOne galley Lond Roflenten To\\\\nsimen spidden des gluh kome\\\\nDie ondertien priffce\\\\ntruncter and b\u00f6gfenplayex\\\\nNu hofierten to\\\\nfittey alps ward moho mohe quedure\\\\nReed and alley den fine wit\\\\nLecheebatten Jof midrandes for\\\\ntoy toy fincant wir Jo Dommo\\\\nAnd Ricky coff frestag nach fount\\\\nJak do nos dak ent\\\\nbag ing benchmanat fon Puchon\\\\nIf it ciy plane pat heist Birch\\\\nHeist\\\\nof dut\\\\non of\\\\nFurry meir Da Damne Nu\\\\nFimpag naar font duty tag\\\\nFanney zu vids in galley by \u03c7\u03b1\u03c1\\\",\\n      \\\"addition1\\\": \\\"Honores\\\\nMaschio\\\\nin exhi\\\\nbih.\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":-0.07443549556116905,\"grounding_metadata\":null,\"index\":null,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.0-flash-lite\",\"prompt_feedback\":null,\"response_id\":\"W9R0afj_BNzskdUPoNHsgQY\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":248,\"candidates_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":248}],\"prompt_token_count\":2365,\"prompt_tokens_details\":[{\"modality\":\"IMAGE\",\"token_count\":1806},{\"modality\":\"TEXT\",\"token_count\":559}],\"thoughts_token_count\":null,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":2613,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"folios\":[{\"folio\":\"69\",\"text\":\"Forenarts komme in pfifferen\\nzummery goed off\\nOne galley Lond Roflenten To\\nsimen spidden des gluh kome\\nDie ondertien priffce\\ntruncter and b\u00f6gfenplayex\\nNu hofierten to\\nfittey alps ward moho mohe quedure\\nReed and alley den fine wit\\nLecheebatten Jof midrandes for\\ntoy toy fincant wir Jo Dommo\\nAnd Ricky coff frestag nach fount\\nJak do nos dak ent\\nbag ing benchmanat fon Puchon\\nIf it ciy plane pat heist Birch\\nHeist\\nof dut\\non of\\nFurry meir Da Damne Nu\\nFimpag naar font duty tag\\nFanney zu vids in galley by \u03c7\u03b1\u03c1\",\"addition1\":\"Honores\\nMaschio\\nin exhi\\nbih.\",\"addition2\":\"\",\"addition3\":\"\"}]}}"}, "scoring": {"fuzzy": 0.606, "cer": 0.414, "cost_summary": {"total_input_tokens": 28380, "total_output_tokens": 2763, "total_tokens": 31143, "input_cost_usd": 0.0021284999999999997, "output_cost_usd": 0.0008288999999999999, "total_cost_usd": 0.0029574000000000002}}, "normalized_score": 58.6}, {"_id": {"$oid": "69d4eb12b877594bc5718b79"}, "test_id": "T0272", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-pro", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Des selben tags gan salzburg In der wirtz\\ngan Jheron\u00ff den\u00ff bahurtag der do wz\\nder sibenczehende tag Vitz\u00ff wir\\ngan feltr\u00ff Jof an den achczehende\\ntag gan terfis Nud bliben zwey\\ntag do vnd f\u00farend an den Gossen\\nSontag der do wz der acht vn\\nzwenczigst tag des merczen Vff\\nden wasser Vo terfis gan Venedig\\nJof an den operoben der do wz der\\nlest tag in merczen do sahen wir\\netwe manigey sithencz\u00ff Vnd gross\\npracht als mechtige h\u00faser schloss\\nVnd sant marco munster vnd der\\nHerczogen palast der ist gemost schin\\npalatium vnd daruff vil sch\u00f3n Vuter\\nmagnifi Vnd Encosten knappen die den\u00ff\\neum. ze Venedig gewesen sind do ist\\nein Cappell an den munster So ist\\nder prin in einstalb gewasset do\\nVnser behalter Jhs xps Vff sas\",\"addition1\":\"Venech\\nDucis\\npalatium\\nmagnifi\\ncum.\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gemini-2.5-pro", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 772, "output_tokens": 319, "total_tokens": 3484, "input_cost_usd": 0.000965, "output_cost_usd": 0.00319, "estimated_cost_usd": 0.004155}, "duration": 24.998819589614868, "timestamp": "2026-01-24T15:15:07.092433", "parsed": {"folios": [{"folio": "", "text": "Des selben tags gan salzburg In der wirtz\ngan Jheron\u00ff den\u00ff bahurtag der do wz\nder sibenczehende tag Vitz\u00ff wir\ngan feltr\u00ff Jof an den achczehende\ntag gan terfis Nud bliben zwey\ntag do vnd f\u00farend an den Gossen\nSontag der do wz der acht vn\nzwenczigst tag des merczen Vff\nden wasser Vo terfis gan Venedig\nJof an den operoben der do wz der\nlest tag in merczen do sahen wir\netwe manigey sithencz\u00ff Vnd gross\npracht als mechtige h\u00faser schloss\nVnd sant marco munster vnd der\nHerczogen palast der ist gemost schin\npalatium vnd daruff vil sch\u00f3n Vuter\nmagnifi Vnd Encosten knappen die den\u00ff\neum. ze Venedig gewesen sind do ist\nein Cappell an den munster So ist\nder prin in einstalb gewasset do\nVnser behalter Jhs xps Vff sas", "addition1": "Venech\nDucis\npalatium\nmagnifi\ncum.", "addition2": "", "addition3": ""}]}, "conversation_id": "344c206a-b56c-42ea-8eee-dc9cd332ba32", "score": {"fuzzy": 0.832, "cer": 0.168}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Sat, 24 Jan 2026 14:15:07 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=24878\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"\\\",\\n      \\\"text\\\": \\\"Des selben tags gan salzburg In der wirtz\\\\ngan Jheron\u00ff den\u00ff bahurtag der do wz\\\\nder sibenczehende tag Vitz\u00ff wir\\\\ngan feltr\u00ff Jof an den achczehende\\\\ntag gan terfis Nud bliben zwey\\\\ntag do vnd f\u00farend an den Gossen\\\\nSontag der do wz der acht vn\\\\nzwenczigst tag des merczen Vff\\\\nden wasser Vo terfis gan Venedig\\\\nJof an den operoben der do wz der\\\\nlest tag in merczen do sahen wir\\\\netwe manigey sithencz\u00ff Vnd gross\\\\npracht als mechtige h\u00faser schloss\\\\nVnd sant marco munster vnd der\\\\nHerczogen palast der ist gemost schin\\\\npalatium vnd daruff vil sch\u00f3n Vuter\\\\nmagnifi Vnd Encosten knappen die den\u00ff\\\\neum. ze Venedig gewesen sind do ist\\\\nein Cappell an den munster So ist\\\\nder prin in einstalb gewasset do\\\\nVnser behalter Jhs xps Vff sas\\\",\\n      \\\"addition1\\\": \\\"Venech\\\\nDucis\\\\npalatium\\\\nmagnifi\\\\ncum.\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.5-pro\",\"prompt_feedback\":null,\"response_id\":\"69N0aZVDsbSewQ-FwPSQBQ\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":319,\"candidates_tokens_details\":null,\"prompt_token_count\":772,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":514},{\"modality\":\"IMAGE\",\"token_count\":258}],\"thoughts_token_count\":2393,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":3484,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"folios\":[{\"folio\":\"\",\"text\":\"Des selben tags gan salzburg In der wirtz\\ngan Jheron\u00ff den\u00ff bahurtag der do wz\\nder sibenczehende tag Vitz\u00ff wir\\ngan feltr\u00ff Jof an den achczehende\\ntag gan terfis Nud bliben zwey\\ntag do vnd f\u00farend an den Gossen\\nSontag der do wz der acht vn\\nzwenczigst tag des merczen Vff\\nden wasser Vo terfis gan Venedig\\nJof an den operoben der do wz der\\nlest tag in merczen do sahen wir\\netwe manigey sithencz\u00ff Vnd gross\\npracht als mechtige h\u00faser schloss\\nVnd sant marco munster vnd der\\nHerczogen palast der ist gemost schin\\npalatium vnd daruff vil sch\u00f3n Vuter\\nmagnifi Vnd Encosten knappen die den\u00ff\\neum. ze Venedig gewesen sind do ist\\nein Cappell an den munster So ist\\nder prin in einstalb gewasset do\\nVnser behalter Jhs xps Vff sas\",\"addition1\":\"Venech\\nDucis\\npalatium\\nmagnifi\\ncum.\",\"addition2\":\"\",\"addition3\":\"\"}]}}"}, "scoring": {"fuzzy": 0.677, "cer": 0.335, "cost_summary": {"total_input_tokens": 9264, "total_output_tokens": 2835, "total_tokens": 12099, "input_cost_usd": 0.011580000000000002, "output_cost_usd": 0.02835, "total_cost_usd": 0.03993000000000001}}, "normalized_score": 66.5}, {"_id": {"$oid": "69d4eb12b877594bc5718b81"}, "test_id": "T0293", "benchmark": "medieval_manuscripts", "date": "2026-01-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-5-20250929", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"\",\"text\":\"Felbs im fronaltar die selben\\nnum gantz vnd ein firck von\\ndem heiligen fron trutz So\\nlit ir vff ir brust vnd ein\\nbein von sant marien madalene\\nruggrost vff samstag noch\\ndem ostertag Der bo das dar\\nander tag das abrellen do\\nwaren furend wir von meran do\\nkegend in sant steffens\\nkilchen bi fundat der buchen\\ndie herodes erst\u016fg in eim\\naltar die pherd vnd vnd\\nstet man noch bestandenlich\\nin sand vnd stet bnd die\\nnahelbar an wundes an\\ndem sibmentag frug do\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "claude-sonnet-4-5-20250929", "provider": "anthropic", "finish_reason": "tool_use", "usage": {"input_tokens": 2991, "output_tokens": 251, "total_tokens": 3242, "input_cost_usd": 0.008973, "output_cost_usd": 0.0037649999999999997, "estimated_cost_usd": 0.012738}, "duration": 9.956707954406738, "timestamp": "2026-01-24T14:53:13.022024", "parsed": {"folios": [{"folio": "", "text": "Felbs im fronaltar die selben\nnum gantz vnd ein firck von\ndem heiligen fron trutz So\nlit ir vff ir brust vnd ein\nbein von sant marien madalene\nruggrost vff samstag noch\ndem ostertag Der bo das dar\nander tag das abrellen do\nwaren furend wir von meran do\nkegend in sant steffens\nkilchen bi fundat der buchen\ndie herodes erst\u016fg in eim\naltar die pherd vnd vnd\nstet man noch bestandenlich\nin sand vnd stet bnd die\nnahelbar an wundes an\ndem sibmentag frug do", "addition1": "", "addition2": "", "addition3": ""}]}, "conversation_id": "bd331696-babd-48ec-b5b3-50c4e790c313", "score": {"fuzzy": 0.418, "cer": 0.586}, "raw_response": "{\"id\":\"msg_01CizFGfuzDrwapVHX8PwBwW\",\"content\":[{\"id\":\"toolu_01VA3Prdw1MGQk61hvSwSSw1\",\"input\":{\"folios\":[{\"folio\":\"\",\"text\":\"Felbs im fronaltar die selben\\nnum gantz vnd ein firck von\\ndem heiligen fron trutz So\\nlit ir vff ir brust vnd ein\\nbein von sant marien madalene\\nruggrost vff samstag noch\\ndem ostertag Der bo das dar\\nander tag das abrellen do\\nwaren furend wir von meran do\\nkegend in sant steffens\\nkilchen bi fundat der buchen\\ndie herodes erst\u016fg in eim\\naltar die pherd vnd vnd\\nstet man noch bestandenlich\\nin sand vnd stet bnd die\\nnahelbar an wundes an\\ndem sibmentag frug do\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]},\"name\":\"extract_structured_data\",\"type\":\"tool_use\"}],\"model\":\"claude-sonnet-4-5-20250929\",\"role\":\"assistant\",\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"type\":\"message\",\"usage\":{\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"input_tokens\":2991,\"output_tokens\":251,\"server_tool_use\":null,\"service_tier\":\"standard\"}}"}, "scoring": {"fuzzy": 0.257, "cer": 0.743, "cost_summary": {"total_input_tokens": 35892, "total_output_tokens": 3736, "total_tokens": 39628, "input_cost_usd": 0.10767599999999998, "output_cost_usd": 0.056040000000000006, "total_cost_usd": 0.16371600000000003}}, "normalized_score": 25.700000000000003}, {"_id": {"$oid": "69d4eb12b877594bc5718a4f"}, "test_id": "T0409", "benchmark": "medieval_manuscripts", "date": "2025-11-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5.1-2025-11-13", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"3\",\"text\":\"vnd ein pferit die mir mit\\nminen knechten vber flu\u00dfend\\nsan so was minen knecht weg\\nsame das mir machten\\nund vieland die knecht vnd\\nund vil in vngemeyn arb\\nvnd die pferit vntz an die\\nstetl vnd was ze mol ein groszer\\nnebel dz wir kun ge\u017fahen\\nvnd als mit grosser arbait kamen\\nwir ze mitten tag zu sant\\nkristoffel vff den berg do\\nzo sach ich die bu\u0364rger do gar\\nvil herren wappen zu ro\u017f\u017fe\\nvnd in iren gehurn getzey hant\\nzo fu\u0364nd minen vatter vlagen\\nwoppey och in dem eweg\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gpt-5.1-2025-11-13", "provider": "openai", "finish_reason": "stop", "usage": {"input_tokens": 1570, "output_tokens": 201, "total_tokens": 1771}, "duration": 7.008111238479614, "timestamp": "2025-11-25T15:27:30.984190", "parsed": {"folios": [{"folio": "3", "text": "vnd ein pferit die mir mit\nminen knechten vber flu\u00dfend\nsan so was minen knecht weg\nsame das mir machten\nund vieland die knecht vnd\nund vil in vngemeyn arb\nvnd die pferit vntz an die\nstetl vnd was ze mol ein groszer\nnebel dz wir kun ge\u017fahen\nvnd als mit grosser arbait kamen\nwir ze mitten tag zu sant\nkristoffel vff den berg do\nzo sach ich die bu\u0364rger do gar\nvil herren wappen zu ro\u017f\u017fe\nvnd in iren gehurn getzey hant\nzo fu\u0364nd minen vatter vlagen\nwoppey och in dem eweg", "addition1": "", "addition2": "", "addition3": ""}]}, "score": {"fuzzy": 0.905, "cer": 0.105}, "raw_response": "{\"id\":\"chatcmpl-CfoIncf9J3IJNNwqUd6NMLcBKltfa\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null,\"message\":{\"content\":\"{\\\"folios\\\":[{\\\"folio\\\":\\\"3\\\",\\\"text\\\":\\\"vnd ein pferit die mir mit\\\\nminen knechten vber flu\u00dfend\\\\nsan so was minen knecht weg\\\\nsame das mir machten\\\\nund vieland die knecht vnd\\\\nund vil in vngemeyn arb\\\\nvnd die pferit vntz an die\\\\nstetl vnd was ze mol ein groszer\\\\nnebel dz wir kun ge\u017fahen\\\\nvnd als mit grosser arbait kamen\\\\nwir ze mitten tag zu sant\\\\nkristoffel vff den berg do\\\\nzo sach ich die bu\u0364rger do gar\\\\nvil herren wappen zu ro\u017f\u017fe\\\\nvnd in iren gehurn getzey hant\\\\nzo fu\u0364nd minen vatter vlagen\\\\nwoppey och in dem eweg\\\",\\\"addition1\\\":\\\"\\\",\\\"addition2\\\":\\\"\\\",\\\"addition3\\\":\\\"\\\"}]}\",\"refusal\":null,\"role\":\"assistant\",\"annotations\":[],\"audio\":null,\"function_call\":null,\"tool_calls\":null,\"parsed\":{\"folios\":[{\"folio\":\"3\",\"text\":\"vnd ein pferit die mir mit\\nminen knechten vber flu\u00dfend\\nsan so was minen knecht weg\\nsame das mir machten\\nund vieland die knecht vnd\\nund vil in vngemeyn arb\\nvnd die pferit vntz an die\\nstetl vnd was ze mol ein groszer\\nnebel dz wir kun ge\u017fahen\\nvnd als mit grosser arbait kamen\\nwir ze mitten tag zu sant\\nkristoffel vff den berg do\\nzo sach ich die bu\u0364rger do gar\\nvil herren wappen zu ro\u017f\u017fe\\nvnd in iren gehurn getzey hant\\nzo fu\u0364nd minen vatter vlagen\\nwoppey och in dem eweg\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}}}],\"created\":1764080845,\"model\":\"gpt-5.1-2025-11-13\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"usage\":{\"completion_tokens\":201,\"prompt_tokens\":1570,\"total_tokens\":1771,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":0,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cached_tokens\":0}}}"}, "scoring": {"fuzzy": 0.653, "cer": 0.376, "cost_summary": {"total_input_tokens": 18840, "total_output_tokens": 2484, "total_tokens": 21324, "input_cost_usd": 0.02354999999999999, "output_cost_usd": 0.02484, "total_cost_usd": 0.04838999999999999}}, "normalized_score": 62.4}, {"_id": {"$oid": "69d4eb12b877594bc5718a53"}, "test_id": "T0441", "benchmark": "medieval_manuscripts", "date": "2025-11-25", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-small-2506", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"42\",\"text\":\"Daz ist der wille des herren\\n vnd der gebot des hailigen geistes\\n das wir in aller demut vnd trewe\\n vnsern got vnd herren dienen\\n vnd in aller tugent vnd rehtschaffenheit\\n leben vnd wandeln\\n vnd in aller demut vnd gelassenheit\\n vnsern mitmenschen dienen\\n vnd in aller tugent vnd rehtschaffenheit\\n leben vnd wandeln\",\"addition1\":\"Anno domini 1487\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "mistral-small-2506", "provider": "mistral", "finish_reason": "stop", "usage": {"input_tokens": 753, "output_tokens": 162, "total_tokens": 915, "input_cost_usd": 7.53e-05, "output_cost_usd": 4.86e-05, "estimated_cost_usd": 0.0001239}, "duration": 2.146907091140747, "timestamp": "2025-11-25T11:08:46.214174", "parsed": {"folios": [{"folio": "42", "text": "Daz ist der wille des herren\n vnd der gebot des hailigen geistes\n das wir in aller demut vnd trewe\n vnsern got vnd herren dienen\n vnd in aller tugent vnd rehtschaffenheit\n leben vnd wandeln\n vnd in aller demut vnd gelassenheit\n vnsern mitmenschen dienen\n vnd in aller tugent vnd rehtschaffenheit\n leben vnd wandeln", "addition1": "Anno domini 1487", "addition2": "", "addition3": ""}]}, "score": {"fuzzy": 0.142, "cer": 0.955}, "raw_response": "{\"id\":\"a798fc8e624649c9a180dccf3624fe26\",\"object\":\"chat.completion\",\"model\":\"mistral-small-2506\",\"usage\":{\"prompt_tokens\":753,\"completion_tokens\":162,\"total_tokens\":915},\"created\":1764065324,\"choices\":[{\"index\":0,\"message\":{\"content\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"42\\\",\\n      \\\"text\\\": \\\"Daz ist der wille des herren\\\\n vnd der gebot des hailigen geistes\\\\n das wir in aller demut vnd trewe\\\\n vnsern got vnd herren dienen\\\\n vnd in aller tugent vnd rehtschaffenheit\\\\n leben vnd wandeln\\\\n vnd in aller demut vnd gelassenheit\\\\n vnsern mitmenschen dienen\\\\n vnd in aller tugent vnd rehtschaffenheit\\\\n leben vnd wandeln\\\",\\n      \\\"addition1\\\": \\\"Anno domini 1487\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\",\"tool_calls\":null,\"prefix\":false,\"role\":\"assistant\"},\"finish_reason\":\"stop\"}]}"}, "scoring": {"fuzzy": 0.179, "cer": 0.903, "cost_summary": {"total_input_tokens": 9036, "total_output_tokens": 1956, "total_tokens": 10992, "input_cost_usd": 0.0009036000000000002, "output_cost_usd": 0.0005868, "total_cost_usd": 0.0014904}}, "normalized_score": 9.700000000000003}, {"_id": {"$oid": "69d4eb12b877594bc5718a41"}, "test_id": "T0271", "benchmark": "medieval_manuscripts", "date": "2025-11-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"text": "{\"folios\":[{\"folio\":\"3\",\"text\":\"Vnd ein pferit Die mir vnd\\nminen knechten v\u00df\u00df Kulfend\\nden So was menny kein weg\\ndenne dan wir machend\\nvnd zichend die knecht dich\\nvnd vil in vntz an den anb\\nvnd die pferit vntz an die\\npotel vnd wa\u00df ze mol ein groffe\\nnebel dz wir kum gepassent\\nvnd als mit groffer arbeit kome\\nwir ze mittem tag zu sant\\nkristoffel vff den berg Do\\ndo sach ich die b\u016fcher Do sar\\nvil Gerren wapen in pone\\nDie in sinr edlen geben hand\\nDo fund mincs vatter\u00df fl\u00fc\u00dfen\\nwappen ach in sinem wapp\u00f1\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "model": "gemini-2.5-flash", "provider": "genai", "finish_reason": "STOP", "usage": {"input_tokens": 772, "output_tokens": 218, "total_tokens": 3314, "input_cost_usd": 0.0002316, "output_cost_usd": 0.000545, "estimated_cost_usd": 0.0007766}, "duration": 13.43295693397522, "timestamp": "2025-11-24T21:19:27.174219", "parsed": {"folios": [{"folio": "3", "text": "Vnd ein pferit Die mir vnd\nminen knechten v\u00df\u00df Kulfend\nden So was menny kein weg\ndenne dan wir machend\nvnd zichend die knecht dich\nvnd vil in vntz an den anb\nvnd die pferit vntz an die\npotel vnd wa\u00df ze mol ein groffe\nnebel dz wir kum gepassent\nvnd als mit groffer arbeit kome\nwir ze mittem tag zu sant\nkristoffel vff den berg Do\ndo sach ich die b\u016fcher Do sar\nvil Gerren wapen in pone\nDie in sinr edlen geben hand\nDo fund mincs vatter\u00df fl\u00fc\u00dfen\nwappen ach in sinem wapp\u00f1", "addition1": "", "addition2": "", "addition3": ""}]}, "score": {"fuzzy": 0.925, "cer": 0.072}, "raw_response": "{\"sdk_http_response\":{\"headers\":{\"content-type\":\"application/json; charset=UTF-8\",\"vary\":\"Origin, X-Origin, Referer\",\"content-encoding\":\"gzip\",\"date\":\"Mon, 24 Nov 2025 20:19:27 GMT\",\"server\":\"scaffolding on HTTPServer2\",\"x-xss-protection\":\"0\",\"x-frame-options\":\"SAMEORIGIN\",\"x-content-type-options\":\"nosniff\",\"server-timing\":\"gfet4t7; dur=13231\",\"alt-svc\":\"h3=\\\":443\\\"; ma=2592000,h3-29=\\\":443\\\"; ma=2592000\",\"transfer-encoding\":\"chunked\"},\"body\":null},\"candidates\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"function_call\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_response\":null,\"text\":\"{\\n  \\\"folios\\\": [\\n    {\\n      \\\"folio\\\": \\\"3\\\",\\n      \\\"text\\\": \\\"Vnd ein pferit Die mir vnd\\\\nminen knechten v\u00df\u00df Kulfend\\\\nden So was menny kein weg\\\\ndenne dan wir machend\\\\nvnd zichend die knecht dich\\\\nvnd vil in vntz an den anb\\\\nvnd die pferit vntz an die\\\\npotel vnd wa\u00df ze mol ein groffe\\\\nnebel dz wir kum gepassent\\\\nvnd als mit groffer arbeit kome\\\\nwir ze mittem tag zu sant\\\\nkristoffel vff den berg Do\\\\ndo sach ich die b\u016fcher Do sar\\\\nvil Gerren wapen in pone\\\\nDie in sinr edlen geben hand\\\\nDo fund mincs vatter\u00df fl\u00fc\u00dfen\\\\nwappen ach in sinem wapp\u00f1\\\",\\n      \\\"addition1\\\": \\\"\\\",\\n      \\\"addition2\\\": \\\"\\\",\\n      \\\"addition3\\\": \\\"\\\"\\n    }\\n  ]\\n}\"}],\"role\":\"model\"},\"citation_metadata\":null,\"finish_message\":null,\"token_count\":null,\"finish_reason\":\"STOP\",\"url_context_metadata\":null,\"avg_logprobs\":null,\"grounding_metadata\":null,\"index\":0,\"logprobs_result\":null,\"safety_ratings\":null}],\"create_time\":null,\"model_version\":\"gemini-2.5-flash\",\"prompt_feedback\":null,\"response_id\":\"z70kaeG5Lq2_vdIPgpXOyQw\",\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":218,\"candidates_tokens_details\":null,\"prompt_token_count\":772,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":514},{\"modality\":\"IMAGE\",\"token_count\":258}],\"thoughts_token_count\":2324,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":3314,\"traffic_type\":null},\"automatic_function_calling_history\":[],\"parsed\":{\"folios\":[{\"folio\":\"3\",\"text\":\"Vnd ein pferit Die mir vnd\\nminen knechten v\u00df\u00df Kulfend\\nden So was menny kein weg\\ndenne dan wir machend\\nvnd zichend die knecht dich\\nvnd vil in vntz an den anb\\nvnd die pferit vntz an die\\npotel vnd wa\u00df ze mol ein groffe\\nnebel dz wir kum gepassent\\nvnd als mit groffer arbeit kome\\nwir ze mittem tag zu sant\\nkristoffel vff den berg Do\\ndo sach ich die b\u016fcher Do sar\\nvil Gerren wapen in pone\\nDie in sinr edlen geben hand\\nDo fund mincs vatter\u00df fl\u00fc\u00dfen\\nwappen ach in sinem wapp\u00f1\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}}"}, "scoring": {"fuzzy": 0.645, "cer": 0.399, "cost_summary": {"total_input_tokens": 9264, "total_output_tokens": 3092, "total_tokens": 12356, "input_cost_usd": 0.0027792000000000008, "output_cost_usd": 0.007729999999999999, "total_cost_usd": 0.0105092}}, "normalized_score": 60.099999999999994}, {"_id": {"$oid": "69d4eb12b877594bc57189b7"}, "test_id": "T0279", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openai", "model": "gpt-5", "duration": 126.12644743919373, "timestamp": "2025-10-24T11:13:53.155280", "parsed": {"folios": [{"folio": "", "text": "", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "chatcmpl-CU87m0L21hVVjXqjMcIWx9Ebi6Zge", "model": "gpt-5-2025-08-07", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"folios\":[{\"folio\":\"\",\"text\":\"\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "role": "assistant"}}], "usage": {"prompt_tokens": 1290, "completion_tokens": 3810, "total_tokens": 5100}}, "usage": {"input_tokens": 1290, "output_tokens": 3810, "total_tokens": 5100}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.412, "cer": 0.609, "cost_summary": {"total_input_tokens": 15480, "total_output_tokens": 51481, "total_tokens": 66961, "input_cost_usd": 0.01935, "output_cost_usd": 0.51481, "total_cost_usd": 0.53416, "pricing_date": "2025-10-24", "input_price_per_million": 1.25, "output_price_per_million": 10.0}}, "normalized_score": 39.1}, {"_id": {"$oid": "69d4eb12b877594bc57189b8"}, "test_id": "T0275", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4o", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openai", "model": "gpt-4o", "duration": 9.359649896621704, "timestamp": "2025-10-24T10:17:20.164784", "parsed": {"folios": [{"folio": "73", "text": "It\ua75b Jo dem ersten M\ua75b Hain\ua75b graff Allwa\ua75bt vo\nLantshut\nIt\ua75b her Ludw\u012bg Lantgraff von Hestemberg\nIt\ua75b her Ott von Hermbereg\nIt\ua75b graff Eberhart von N\u016brenberg\nIt\ua75b graff Gottefrid von Hohenlohe\nIt\ua75b graff Albrecht von Mansfelt\nIt\ua75b graff Ludw\u012bg von Oetingen\nIt\ua75b her Gottschalck von Klee + bespannd\nIt\ua75b her Hans von T\u00fcngen\nIt\ua75b her Hauptrich von Stein uz Wachm, Riter\nIt\ua75b lutz von Rotenhan *\nIt\ua75b her von Waldeck\nIt\ua75b hilprand von Th\u00fcngen\nIt\ua75b her von Elach\nIt\ua75b heinin quat", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "chatcmpl-CU7Gxy08QcNqkkfiV39NzCicAgZMT", "model": "gpt-4o-2024-08-06", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"folios\":[{\"folio\":\"73\",\"text\":\"It\ua75b Jo dem ersten M\ua75b Hain\ua75b graff Allwa\ua75bt vo\\nLantshut\\nIt\ua75b her Ludw\u012bg Lantgraff von Hestemberg\\nIt\ua75b her Ott von Hermbereg\\nIt\ua75b graff Eberhart von N\u016brenberg\\nIt\ua75b graff Gottefrid von Hohenlohe\\nIt\ua75b graff Albrecht von Mansfelt\\nIt\ua75b graff Ludw\u012bg von Oetingen\\nIt\ua75b her Gottschalck von Klee + bespannd\\nIt\ua75b her Hans von T\u00fcngen\\nIt\ua75b her Hauptrich von Stein uz Wachm, Riter\\nIt\ua75b lutz von Rotenhan *\\nIt\ua75b her von Waldeck\\nIt\ua75b hilprand von Th\u00fcngen\\nIt\ua75b her von Elach\\nIt\ua75b heinin quat\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "role": "assistant"}}], "usage": {"prompt_tokens": 1429, "completion_tokens": 234, "total_tokens": 1663}}, "usage": {"input_tokens": 1429, "output_tokens": 234, "total_tokens": 1663}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.594, "cer": 0.454, "cost_summary": {"total_input_tokens": 17148, "total_output_tokens": 2371, "total_tokens": 19519, "input_cost_usd": 0.04287, "output_cost_usd": 0.02371, "total_cost_usd": 0.06658, "pricing_date": "2025-10-24", "input_price_per_million": 2.5, "output_price_per_million": 10.0}}, "normalized_score": 54.6}, {"_id": {"$oid": "69d4eb12b877594bc57189b9"}, "test_id": "T0283", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "genai", "model": "gemini-2.0-flash", "duration": 3.0027410984039307, "timestamp": "2025-10-24T11:38:04.764076", "parsed": {"folios": [{"folio": "", "text": "Gelbes im fronaltar Gie Behend\nwir gantz und am fick von\nDem Geiligin fron crits d'\nhit is off is brust und im\nbem voy spent marien madaline\nkinggrott jof roff famstag noch\nDem oftertag der Do was Dix\nander tag Dib abrellen Do\nfurend wir yoy meway Is\nglaus ligend in punt # proffiim\npetitfen bi hundert der Endling\nDie Gerades copflug in emem.\nalwar die pfend\nsieht man noch befterdenlich\nin hand und f\u00fcr und die\nnagel dar ay Mandes ay\nDem funentag frig de", "addition1": "Mura\nfurend wir\ngeus", "addition2": "Puttres", "addition3": ""}]}, "text": "{\n  \"folios\": [\n    {\n      \"folio\": \"\",\n      \"text\": \"Gelbes im fronaltar Gie Behend\\nwir gantz und am fick von\\nDem Geiligin fron crits d'\\nhit is off is brust und im\\nbem voy spent marien madaline\\nkinggrott jof roff famstag noch\\nDem oftertag der Do was Dix\\nander tag Dib abrellen Do\\nfurend wir yoy meway Is\\nglaus ligend in punt # proffiim\\npetitfen bi hundert der Endling\\nDie Gerades copflug in emem.\\nalwar die pfend\\nsieht man noch befterdenlich\\nin hand und f\u00fcr und die\\nnagel dar ay Mandes ay\\nDem funentag frig de\",\n      \"addition1\": \"Mura\\nfurend wir\\ngeus\",\n      \"addition2\": \"Puttres\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "raw_response": {"text": "{\n  \"folios\": [\n    {\n      \"folio\": \"\",\n      \"text\": \"Gelbes im fronaltar Gie Behend\\nwir gantz und am fick von\\nDem Geiligin fron crits d'\\nhit is off is brust und im\\nbem voy spent marien madaline\\nkinggrott jof roff famstag noch\\nDem oftertag der Do was Dix\\nander tag Dib abrellen Do\\nfurend wir yoy meway Is\\nglaus ligend in punt # proffiim\\npetitfen bi hundert der Endling\\nDie Gerades copflug in emem.\\nalwar die pfend\\nsieht man noch befterdenlich\\nin hand und f\u00fcr und die\\nnagel dar ay Mandes ay\\nDem funentag frig de\",\n      \"addition1\": \"Mura\\nfurend wir\\ngeus\",\n      \"addition2\": \"Puttres\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "usage_metadata": {"prompt_token_count": 2348, "candidates_token_count": 218, "total_token_count": 2566}}, "usage": {"input_tokens": 2348, "output_tokens": 218, "total_tokens": 2566}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.567, "cer": 0.457, "cost_summary": {"total_input_tokens": 28176, "total_output_tokens": 10725, "total_tokens": 38901, "input_cost_usd": 0.002818, "output_cost_usd": 0.00429, "total_cost_usd": 0.007108, "pricing_date": "2025-10-24", "input_price_per_million": 0.1, "output_price_per_million": 0.4}}, "normalized_score": 54.3}, {"_id": {"$oid": "69d4eb12b877594bc57189bd"}, "test_id": "T0301", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openrouter", "model": "meta-llama/llama-4-maverick", "duration": 3.409065008163452, "timestamp": "2025-10-24T11:07:22.872768", "parsed": {"folios": [{"folio": "90", "text": "Heinricg Von E\u016flstatten\nfry dar vff\nC\u016fnratz der pfriber\n\nv\u00df dem ftilf von\nhans Egli comtein ze mon-\nant jolans orden\u00df\nheinricg Von vilengert-\nlin\nheinrich fri-\nschin Endeg-\nge\nVon ber-\nny v\u00df:\nher-\nwig Von Diep-\nbach\nhans von der w-\niben\nhans von ma-\ny", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "gen-1761296838-CTKFr5kyE1X5WJrkEfon", "model": "meta-llama/llama-4-maverick", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"folios\": [\n    {\n      \"folio\": \"90\",\n      \"text\": \"Heinricg Von E\u016flstatten\\nfry dar vff\\nC\u016fnratz der pfriber\\n\\nv\u00df dem ftilf von\\nhans Egli comtein ze mon-\\nant jolans orden\u00df\\nheinricg Von vilengert-\\nlin\\nheinrich fri-\\nschin Endeg-\\nge\\nVon ber-\\nny v\u00df:\\nher-\\nwig Von Diep-\\nbach\\nhans von der w-\\niben\\nhans von ma-\\ny\",\n      \"addition1\": \"\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 2456, "completion_tokens": 154, "total_tokens": 2610}}, "usage": {"input_tokens": 2456, "output_tokens": 154, "total_tokens": 2610}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.543, "cer": 0.503, "cost_summary": {"total_input_tokens": 28929, "total_output_tokens": 2839, "total_tokens": 31768, "input_cost_usd": 0.004339, "output_cost_usd": 0.001703, "total_cost_usd": 0.006043, "pricing_date": "2025-10-24", "input_price_per_million": 0.15, "output_price_per_million": 0.6}}, "normalized_score": 49.7}, {"_id": {"$oid": "69d4eb12b877594bc57189be"}, "test_id": "T0276", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4o-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openai", "model": "gpt-4o-mini", "duration": 6.2617433071136475, "timestamp": "2025-10-24T10:23:30.568779", "parsed": {"folios": [{"folio": "18", "text": "Das groft von Pfu\u0304arenburg\nvnd sept all ander ritter\nvnd thoct woltey mit\nvnd ey in was entlich\nvritig vngenomen ga\u0304.\nDer hert vnd gantz\nvnd leht das vicht\nDie zwey wirtend gery\nmit mir ge\u0304n, oder sie\nkleyng, so der me\u0304tcil mit\nDas wolt vnd elp mi\u00dfan\nwir mit gro\u00dfen an willy\nsamm pflegen, der wir\nmit dem harnoy vnd\netg der br\u00fcderin, dar\nund hatz und zippen\nwagtet \u0292 z\u00fcket", "addition1": "Sacchari in Cypro magister", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "chatcmpl-CU7MykevTOMPzFaPrTK7oGCjjwCRH", "model": "gpt-4o-mini-2024-07-18", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"folios\": [\n    {\n      \"folio\": \"18\",\n      \"text\": \"Das groft von Pfu\u0304arenburg\\nvnd sept all ander ritter\\nvnd thoct woltey mit\\nvnd ey in was entlich\\nvritig vngenomen ga\u0304.\\nDer hert vnd gantz\\nvnd leht das vicht\\nDie zwey wirtend gery\\nmit mir ge\u0304n, oder sie\\nkleyng, so der me\u0304tcil mit\\nDas wolt vnd elp mi\u00dfan\\nwir mit gro\u00dfen an willy\\nsamm pflegen, der wir\\nmit dem harnoy vnd\\netg der br\u00fcderin, dar\\nund hatz und zippen\\nwagtet \u0292 z\u00fcket\",\n      \"addition1\": \"Sacchari in Cypro magister\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 26165, "completion_tokens": 209, "total_tokens": 26374}}, "usage": {"input_tokens": 26165, "output_tokens": 209, "total_tokens": 26374}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.569, "cer": 0.486, "cost_summary": {"total_input_tokens": 313980, "total_output_tokens": 2657, "total_tokens": 316637, "input_cost_usd": 0.047097, "output_cost_usd": 0.001594, "total_cost_usd": 0.048691, "pricing_date": "2025-10-24", "input_price_per_million": 0.15, "output_price_per_million": 0.6}}, "normalized_score": 51.4}, {"_id": {"$oid": "69d4eb12b877594bc57189bf"}, "test_id": "T0292", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-1-20250805", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "anthropic", "model": "claude-opus-4-1-20250805", "duration": 22.521444082260132, "timestamp": "2025-10-24T10:36:25.605912", "parsed": {"folios": [{"folio": "", "text": "Ghelbet mij fronalten Sie solhend\nwir gantz dut ein stuk von\ndem heiligen fron trutz So\nlit ir vff ir bruost vnd ein\nbein von sant marien magdalene\nriuogtrot Jst ouch samstag nach\ndem ostentag Der bo was Ar\nander tag des abrellen So\nfurend wir gen magan So\nliezend in sant Jst steffins\nkilchen bi hundert der kinden\ndie heraduo erstluog in einem\naltar die solhend wir vnd\nsicht man noch bescheidenlich\nir hend vnd fuos vnd die\nmachel dar an windes wy\ndem swnendig sung die", "addition1": "Murgan\nglauob\nSuostris", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "msg_014Mf1DpRDdqALbTuHDTyRyw", "model": "claude-opus-4-1-20250805", "role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01MFz4wNa8KFuSHTSVYbkDKk", "name": "extract_structured_data", "input": {"folios": [{"folio": "", "text": "Ghelbet mij fronalten Sie solhend\nwir gantz dut ein stuk von\ndem heiligen fron trutz So\nlit ir vff ir bruost vnd ein\nbein von sant marien magdalene\nriuogtrot Jst ouch samstag nach\ndem ostentag Der bo was Ar\nander tag des abrellen So\nfurend wir gen magan So\nliezend in sant Jst steffins\nkilchen bi hundert der kinden\ndie heraduo erstluog in einem\naltar die solhend wir vnd\nsicht man noch bescheidenlich\nir hend vnd fuos vnd die\nmachel dar an windes wy\ndem swnendig sung die", "addition1": "Murgan\nglauob\nSuostris", "addition2": "", "addition3": ""}]}}], "stop_reason": "tool_use", "usage": {"input_tokens": 2062, "output_tokens": 278}}, "usage": {"input_tokens": 2062, "output_tokens": 278, "total_tokens": 2340}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.671, "cer": 0.373, "cost_summary": {"total_input_tokens": 24744, "total_output_tokens": 3670, "total_tokens": 28414, "input_cost_usd": 0.37116, "output_cost_usd": 0.27525, "total_cost_usd": 0.64641, "pricing_date": "2025-10-24", "input_price_per_million": 15.0, "output_price_per_million": 75.0}}, "normalized_score": 62.7}, {"_id": {"$oid": "69d4eb12b877594bc57189c0"}, "test_id": "T0303", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-8b-instruct", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openrouter", "model": "qwen/qwen3-vl-8b-instruct", "duration": 3.677827835083008, "timestamp": "2025-10-24T11:12:23.443256", "parsed": {"folios": [{"folio": "", "text": "Und die p\u00fcfel ist bi \u00e6t mil\n wit und ist der Venediger\n It von Euchsen vnd g\u00f6n\n modun find \u00e6t mil und\n ist och ein g\u00fcbster star und\n ein extz\u00f6ftum und ist och\n der Venediger ze modun\n wacthet der rumanier und\n ist och do bi ein heilig gesett\n do bi ij miley\n Candia ist von modun bib g\u00f6n candia\n Creta. find \u00e6t mil und candia\n Vinum ist ein grob p\u00fcfel vy mil\n Malze wit und wacthet der\n niemalua\u00dfew. Do und ist och\n der Venedier\n Rhody. ist von candia vnd g\u00f6n podib\n find \u00e6t mil wer aber", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "gen-1761297138-rVQySUgBjTKtLfLs8usJ", "model": "qwen/qwen3-vl-8b-instruct", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"folios\": [\n    {\n      \"folio\": \"\",\n      \"text\": \"Und die p\u00fcfel ist bi \u00e6t mil\\n wit und ist der Venediger\\n It von Euchsen vnd g\u00f6n\\n modun find \u00e6t mil und\\n ist och ein g\u00fcbster star und\\n ein extz\u00f6ftum und ist och\\n der Venediger ze modun\\n wacthet der rumanier und\\n ist och do bi ein heilig gesett\\n do bi ij miley\\n Candia ist von modun bib g\u00f6n candia\\n Creta. find \u00e6t mil und candia\\n Vinum ist ein grob p\u00fcfel vy mil\\n Malze wit und wacthet der\\n niemalua\u00dfew. Do und ist och\\n der Venedier\\n Rhody. ist von candia vnd g\u00f6n podib\\n find \u00e6t mil wer aber\",\n      \"addition1\": \"\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1420, "completion_tokens": 216, "total_tokens": 1636}}, "usage": {"input_tokens": 1420, "output_tokens": 216, "total_tokens": 1636}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.634, "cer": 0.373, "cost_summary": {"total_input_tokens": 15418, "total_output_tokens": 2328, "total_tokens": 17746, "input_cost_usd": 0.001233, "output_cost_usd": 0.001164, "total_cost_usd": 0.002397, "pricing_date": "2025-10-24", "input_price_per_million": 0.08, "output_price_per_million": 0.5}}, "normalized_score": 62.7}, {"_id": {"$oid": "69d4eb12b877594bc57189c2"}, "test_id": "T0278", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openai", "model": "gpt-4.1-nano", "duration": 3.516861915588379, "timestamp": "2025-10-24T10:30:32.816629", "parsed": {"folios": [{"folio": "", "text": "Gottes in prandten die schone\n wir guntz vnd en siet vnd\n zey liching von eris z\n die in uff in brust vnd ein\n hain von sant marien madlne\n vngrot \ua75b off samstag noch\n dem stertag der da was der\n under tayt die abreden do\n\n furent m von mery\n levent in pnt \ua75b stauffn\n sittent in fundt der knelin\n die herdes aufschlug in enen\n altar die solend wir vnd\n plegt may net gelestent in\n fundt vnd f\u00fcr vnd die\n nagel dar ay wrent ay\n dem fuentzig fruog e", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "chatcmpl-CU7Tpl1eFcBQWyKtPzqJCSDkcN1UF", "model": "gpt-4.1-nano-2025-04-14", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"folios\": [{\"folio\": \"\", \"text\": \"Gottes in prandten die schone\\n wir guntz vnd en siet vnd\\n zey liching von eris z\\n die in uff in brust vnd ein\\n hain von sant marien madlne\\n vngrot \ua75b off samstag noch\\n dem stertag der da was der\\n under tayt die abreden do\\n\\n furent m von mery\\n levent in pnt \ua75b stauffn\\n sittent in fundt der knelin\\n die herdes aufschlug in enen\\n altar die solend wir vnd\\n plegt may net gelestent in\\n fundt vnd f\u00fcr vnd die\\n nagel dar ay wrent ay\\n dem fuentzig fruog e\", \"addition1\": \"\", \"addition2\": \"\", \"addition3\": \"\"}]}", "role": "assistant"}}], "usage": {"prompt_tokens": 2397, "completion_tokens": 187, "total_tokens": 2584}}, "usage": {"input_tokens": 2397, "output_tokens": 187, "total_tokens": 2584}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.394, "cer": 0.649, "cost_summary": {"total_input_tokens": 23970, "total_output_tokens": 2036, "total_tokens": 26006, "input_cost_usd": 0.002397, "output_cost_usd": 0.000814, "total_cost_usd": 0.003211, "pricing_date": "2025-10-24", "input_price_per_million": 0.1, "output_price_per_million": 0.4}}, "normalized_score": 35.099999999999994}, {"_id": {"$oid": "69d4eb12b877594bc57189c5"}, "test_id": "T0291", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-20250514", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "anthropic", "model": "claude-sonnet-4-20250514", "duration": 11.634221076965332, "timestamp": "2025-10-24T10:33:23.158144", "parsed": {"folios": [{"folio": "", "text": "Gelibt my frounltev Sie soltend\nvon stuntz vnd em swck vbey\ndem heiligin fron cr\u016btz do\nsit ir off ir brust vnd em\nbem von sant marien magdalene\nringli rot off off sameztag noch\ndem ostertag da\u00df do wa\u00df dev\nandew tag da\u00df alsvell an do\nfr\u016braw sitrend nin thon mergen do\nglawb hesend ny sant \u1ebd stre\u00dfen\u00df\nh\u016bt\u0113 nichten bi hindert der kindn\ndie hannd\u00df enstelligt ny emen\naltar die soltend n\u016br vnd\nsieht man noch heslichem lutzell\nby hend vnd f\u016be drie die\nensollen a\u00df vrimels ay\ndem sidentag fing do", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "msg_01NuX5dhCFLhHrY7oetrWShM", "model": "claude-sonnet-4-20250514", "role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01QqAa957B4j4XRnm1P69U9V", "name": "extract_structured_data", "input": {"folios": [{"folio": "", "text": "Gelibt my frounltev Sie soltend\nvon stuntz vnd em swck vbey\ndem heiligin fron cr\u016btz do\nsit ir off ir brust vnd em\nbem von sant marien magdalene\nringli rot off off sameztag noch\ndem ostertag da\u00df do wa\u00df dev\nandew tag da\u00df alsvell an do\nfr\u016braw sitrend nin thon mergen do\nglawb hesend ny sant \u1ebd stre\u00dfen\u00df\nh\u016bt\u0113 nichten bi hindert der kindn\ndie hannd\u00df enstelligt ny emen\naltar die soltend n\u016br vnd\nsieht man noch heslichem lutzell\nby hend vnd f\u016be drie die\nensollen a\u00df vrimels ay\ndem sidentag fing do", "addition1": "", "addition2": "", "addition3": ""}]}}], "stop_reason": "tool_use", "usage": {"input_tokens": 2062, "output_tokens": 285}}, "usage": {"input_tokens": 2062, "output_tokens": 285, "total_tokens": 2347}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.521, "cer": 0.511, "cost_summary": {"total_input_tokens": 24744, "total_output_tokens": 3748, "total_tokens": 28492, "input_cost_usd": 0.074232, "output_cost_usd": 0.05622, "total_cost_usd": 0.130452, "pricing_date": "2025-10-24", "input_price_per_million": 3.0, "output_price_per_million": 15.0}}, "normalized_score": 48.9}, {"_id": {"$oid": "69d4eb12b877594bc57189c6"}, "test_id": "T0300", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-8b-thinking", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openrouter", "model": "qwen/qwen3-vl-8b-thinking", "duration": 28.596866607666016, "timestamp": "2025-10-24T10:32:16.134084", "text": "", "raw_response": {"id": "gen-1761294706-yDXEvDZhvc0vDcXtDqE4", "model": "qwen/qwen3-vl-8b-thinking", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "", "role": "assistant"}}], "usage": {"prompt_tokens": 1953, "completion_tokens": 197, "total_tokens": 2150}}, "usage": {"input_tokens": 1953, "output_tokens": 197, "total_tokens": 2150}, "error": "JSONDecodeError", "error_message": "Expecting value: line 1 column 1 (char 0)", "finish_reason": "stop"}, "scoring": {"fuzzy": 0.0, "cer": 1.0, "cost_summary": {"total_input_tokens": 21483, "total_output_tokens": 12365, "total_tokens": 33848, "input_cost_usd": 0.003867, "output_cost_usd": 0.025966, "total_cost_usd": 0.029833, "pricing_date": "2025-10-24", "input_price_per_million": 0.18, "output_price_per_million": 2.1}}, "normalized_score": 0}, {"_id": {"$oid": "69d4eb12b877594bc57189c7"}, "test_id": "T0282", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "o3", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openai", "model": "o3", "duration": 53.11458349227905, "timestamp": "2025-10-24T11:29:29.167414", "parsed": {"folios": [{"folio": "93", "text": "V\u00df dem land ze\nmagdeburgt\nCunrat tobe nutt\nDietrich libzow\npeter in barfu\u00df priester\n\nV\u00df L\u00fcttringen\nJobst von Harzkurt (Rett\nSebastian die mistraut priester\nJohan von Basy zu Einet\n\nV\u00df schleszen land\nJakob Nekinger\nHans Nudelt\nHeinric lang\nIn\u00df brosburg\nHans von Genfelt ain priester\nJant Johann durain", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "chatcmpl-CU8O30HCrs3Te2Z5ugH8xdo0DDc9T", "model": "o3-2025-04-16", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"folios\":[{\"folio\":\"93\",\"text\":\"V\u00df dem land ze\\nmagdeburgt\\nCunrat tobe nutt\\nDietrich libzow\\npeter in barfu\u00df priester\\n\\nV\u00df L\u00fcttringen\\nJobst von Harzkurt (Rett\\nSebastian die mistraut priester\\nJohan von Basy zu Einet\\n\\nV\u00df schleszen land\\nJakob Nekinger\\nHans Nudelt\\nHeinric lang\\nIn\u00df brosburg\\nHans von Genfelt ain priester\\nJant Johann durain\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "role": "assistant"}}], "usage": {"prompt_tokens": 1335, "completion_tokens": 3866, "total_tokens": 5201}}, "usage": {"input_tokens": 1335, "output_tokens": 3866, "total_tokens": 5201}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.51, "cer": 0.513, "cost_summary": {"total_input_tokens": 16020, "total_output_tokens": 38619, "total_tokens": 54639, "input_cost_usd": 0.03204, "output_cost_usd": 0.308952, "total_cost_usd": 0.340992, "pricing_date": "2025-10-24", "input_price_per_million": 2.0, "output_price_per_million": 8.0}}, "normalized_score": 48.699999999999996}, {"_id": {"$oid": "69d4eb12b877594bc57189c8"}, "test_id": "T0302", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "qwen/qwen3-vl-30b-a3b-instruct", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openrouter", "model": "qwen/qwen3-vl-30b-a3b-instruct", "duration": 17.64707112312317, "timestamp": "2025-10-24T11:11:56.430808", "parsed": {"folios": [{"folio": "15", "text": "Och felter tagt In der wue\n Der ift ay dem balming der do wiz\n gey felterb ift ay dem adtzelende\n tag do wurd vnd fterch an dem holen\n dawpung der do wiz der adt von\n wenziger tag der mercky\n vnd der waffey do wiz der\n chwe magtig fchreyen vnd gros\n chuff als mechtige ghez der stet fch\n Vench\n Duce\n palati\n cum\n Genracy palat der in zemstfch\n vnd daz vnd daz\n ze nemdig gewert vnd do g\n kyn Cappell an dem minster for\n der frey in einfalt gewaffet do\n vnder befallen THe copf fas", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "gen-1761297098-Ade7pooMRPfBnwDyWwXF", "model": "qwen/qwen3-vl-30b-a3b-instruct", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"Och felter tagt In der wue\\n Der ift ay dem balming der do wiz\\n gey felterb ift ay dem adtzelende\\n tag do wurd vnd fterch an dem holen\\n dawpung der do wiz der adt von\\n wenziger tag der mercky\\n vnd der waffey do wiz der\\n chwe magtig fchreyen vnd gros\\n chuff als mechtige ghez der stet fch\\n Vench\\n Duce\\n palati\\n cum\\n Genracy palat der in zemstfch\\n vnd daz vnd daz\\n ze nemdig gewert vnd do g\\n kyn Cappell an dem minster for\\n der frey in einfalt gewaffet do\\n vnder befallen THe copf fas\",\n      \"addition1\": \"\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1175, "completion_tokens": 238, "total_tokens": 1413}}, "usage": {"input_tokens": 1175, "output_tokens": 238, "total_tokens": 1413}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.504, "cer": 0.517, "cost_summary": {"total_input_tokens": 16305, "total_output_tokens": 6513, "total_tokens": 22818, "input_cost_usd": 0.003261, "output_cost_usd": 0.004559, "total_cost_usd": 0.00782, "pricing_date": "2025-10-24", "input_price_per_million": 0.2, "output_price_per_million": 0.7}}, "normalized_score": 48.3}, {"_id": {"$oid": "69d4eb12b877594bc57189cc"}, "test_id": "T0286", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash-lite-preview-09-2025", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "genai", "model": "gemini-2.5-flash-lite-preview-09-2025", "duration": 1.2044031620025635, "timestamp": "2025-10-24T10:22:11.050670", "parsed": {"folios": [{"folio": "", "text": "Jof Dem patron in groffem\nfiir fiir Zirnund\nJof ze cmad zin groffen\nJof ze Jerufalem ij groff z\ncurtiifig von Sin eplen\nJof Dem conful n grob\nJof in Dem tempel ij grob\nJof ze Betlagem ein grob\nJof ze fant Iohann ein grob\nJof wider ze Jerufalem ein\ngrob ze curtiifig aber ij grob\nJof wider in Den tempel in\ngrob\nJof ze Dem Jordan ij grob\nJof Dey spalen ze lon\nJof aber ein grob ze curtiif\nJof aber ze Jerufalem ij grob.", "addition1": null, "addition2": null, "addition3": null}]}, "text": "{\n  \"folios\": [\n    {\n      \"folio\": \"\",\n      \"text\": \"Jof Dem patron in groffem\\nfiir fiir Zirnund\\nJof ze cmad zin groffen\\nJof ze Jerufalem ij groff z\\ncurtiifig von Sin eplen\\nJof Dem conful n grob\\nJof in Dem tempel ij grob\\nJof ze Betlagem ein grob\\nJof ze fant Iohann ein grob\\nJof wider ze Jerufalem ein\\ngrob ze curtiifig aber ij grob\\nJof wider in Den tempel in\\ngrob\\nJof ze Dem Jordan ij grob\\nJof Dey spalen ze lon\\nJof aber ein grob ze curtiif\\nJof aber ze Jerufalem ij grob.\",\n      \"addition1\": null,\n      \"addition2\": null,\n      \"addition3\": null\n    }\n  ]\n}", "raw_response": {"text": "{\n  \"folios\": [\n    {\n      \"folio\": \"\",\n      \"text\": \"Jof Dem patron in groffem\\nfiir fiir Zirnund\\nJof ze cmad zin groffen\\nJof ze Jerufalem ij groff z\\ncurtiifig von Sin eplen\\nJof Dem conful n grob\\nJof in Dem tempel ij grob\\nJof ze Betlagem ein grob\\nJof ze fant Iohann ein grob\\nJof wider ze Jerufalem ein\\ngrob ze curtiifig aber ij grob\\nJof wider in Den tempel in\\ngrob\\nJof ze Dem Jordan ij grob\\nJof Dey spalen ze lon\\nJof aber ein grob ze curtiif\\nJof aber ze Jerufalem ij grob.\",\n      \"addition1\": null,\n      \"addition2\": null,\n      \"addition3\": null\n    }\n  ]\n}", "usage_metadata": {"prompt_token_count": 759, "candidates_token_count": 227, "total_token_count": 990}}, "usage": {"input_tokens": 759, "output_tokens": 227, "total_tokens": 990}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.623, "cer": 0.393, "cost_summary": {"total_input_tokens": 9108, "total_output_tokens": 3006, "total_tokens": 12114, "input_cost_usd": 0.000911, "output_cost_usd": 0.001202, "total_cost_usd": 0.002113, "pricing_date": "2025-10-24", "input_price_per_million": 0.1, "output_price_per_million": 0.4}}, "normalized_score": 60.699999999999996}, {"_id": {"$oid": "69d4eb12b877594bc57189d5"}, "test_id": "T0299", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "scicore", "model": "GLM-4.5V-FP8", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "scicore", "model": "GLM-4.5V-FP8", "duration": 3.2809674739837646, "timestamp": "2025-10-24T10:31:30.446257", "parsed": {"folios": [{"folio": "1", "text": "ffar noffgot may my dic\n fuyel &c feiligen grubes\n &c ift ein aler namenmel\n feyn gemachf nber &ab geflig\n fynb &o mynper gere gerwra\n got vnd monpf my gtegen\n yt nnd &o noy, wferpandly\n my &cm eritten tag noy\n dem tod\n &c ift my tor &c8 tempel8\n &o ift ein locg vnd ftat &un\n fi cin, flein, ftan, &o fpang\n niper gere &g were &ab mttel\n fec owch\n &f me notgeteftinbay fen&8 dic\n Dic bilgerftet ofwenbig &c8\n tempel8 my &aw ftat\n &f &ab guo &c8 &nffg vndf", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "chatcmpl-8b1a683599de4fee8be857130c037b34", "model": "hosted_vllm/GLM-4.5V-FP8", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"folios\": [{\"folio\": \"1\", \"text\": \"ffar noffgot may my dic\\n fuyel &c feiligen grubes\\n &c ift ein aler namenmel\\n feyn gemachf nber &ab geflig\\n fynb &o mynper gere gerwra\\n got vnd monpf my gtegen\\n yt nnd &o noy, wferpandly\\n my &cm eritten tag noy\\n dem tod\\n &c ift my tor &c8 tempel8\\n &o ift ein locg vnd ftat &un\\n fi cin, flein, ftan, &o fpang\\n niper gere &g were &ab mttel\\n fec owch\\n &f me notgeteftinbay fen&8 dic\\n Dic bilgerftet ofwenbig &c8\\n tempel8 my &aw ftat\\n &f &ab guo &c8 &nffg vndf\", \"addition1\": \"\", \"addition2\": \"\", \"addition3\": \"\"}]}", "role": "assistant"}}], "usage": {"prompt_tokens": 1398, "completion_tokens": 233, "total_tokens": 1631}}, "usage": {"input_tokens": 1398, "output_tokens": 233, "total_tokens": 1631}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.574, "cer": 0.452, "cost_summary": {"total_input_tokens": 16776, "total_output_tokens": 2775, "total_tokens": 19551, "input_cost_usd": 0.0, "output_cost_usd": 0.0, "total_cost_usd": 0.0, "pricing_date": "2025-10-24", "input_price_per_million": 0.0, "output_price_per_million": 0.0}}, "normalized_score": 54.8}, {"_id": {"$oid": "69d4eb12b877594bc57189d6"}, "test_id": "T0294", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "pixtral-large-2411", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "mistral", "model": "pixtral-large-latest", "duration": 11.19400668144226, "timestamp": "2025-10-24T10:42:27.493199", "parsed": {"folios": [{"folio": "90", "text": "Hemrich \ua769oy e\u1eb9l\u1e61pettey\nin daru\ua75b\u00df\nEymnach \ua75b don pre\u0305wibe\n\nV\u030a Do \u1e61u\u1e61 prift v\u030an\nmong\nTran\u00df t\u1eb9ffe romanin zc mong\nant \u1e61pan\u00df u\u1e61dem\nHemrich \ua769oy wi\u1e61lingarde\nin \u1e61\nHemrich \ua769nit\u1e61 \ua75bn \ua75bncoft\n\nV\u030a daru\ua75b w\u1eb9i\nco\u1e61fftan\u00df\n\nLudwife \ua769o\u1e61 Eyd\u1e61backup\nyau\ua75b \ua769o\u1e61 \ua75bu\u1e61 Orliten\nTran\u00df w\u1eb9imar", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "2384d8052c964f13ae8ec872f774784b", "model": "pixtral-large-latest", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"folios\": [\n    {\n      \"folio\": \"90\",\n      \"text\": \"Hemrich \ua769oy e\u1eb9l\u1e61pettey\\nin daru\ua75b\u00df\\nEymnach \ua75b don pre\u0305wibe\\n\\nV\u030a Do \u1e61u\u1e61 prift v\u030an\\nmong\\nTran\u00df t\u1eb9ffe romanin zc mong\\nant \u1e61pan\u00df u\u1e61dem\\nHemrich \ua769oy wi\u1e61lingarde\\nin \u1e61\\nHemrich \ua769nit\u1e61 \ua75bn \ua75bncoft\\n\\nV\u030a daru\ua75b w\u1eb9i\\nco\u1e61fftan\u00df\\n\\nLudwife \ua769o\u1e61 Eyd\u1e61backup\\nyau\ua75b \ua769o\u1e61 \ua75bu\u1e61 Orliten\\nTran\u00df w\u1eb9imar\",\n      \"addition1\": \"\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 3638, "completion_tokens": 291, "total_tokens": 3929}}, "usage": {"input_tokens": 3638, "output_tokens": 291, "total_tokens": 3929}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.447, "cer": 0.604, "cost_summary": {"total_input_tokens": 43656, "total_output_tokens": 5227, "total_tokens": 48883, "input_cost_usd": 0.087312, "output_cost_usd": 0.031362, "total_cost_usd": 0.118674, "pricing_date": "2025-10-24", "input_price_per_million": 2.0, "output_price_per_million": 6.0}}, "normalized_score": 39.6}, {"_id": {"$oid": "69d4eb12b877594bc57189da"}, "test_id": "T0298", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-large-2411", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "mistral", "model": "mistral-large-latest", "duration": 5.562952280044556, "timestamp": "2025-10-24T10:30:03.483949", "parsed": {"folios": [{"folio": "73", "text": "Jf zu\u030a dem er\u017ften My he\u030ar\u017fchgraff Albracht vo\u030a\nO\u030a\u017fter\u017freich\nJf her ludwig lantgraff von he\u017f\u017fenberg\nJf graff \u017f\u017f von hermenberg\nJf graff Bernhart von \u017fegen\nJf graff go\u030atffried von ha\u017f\u017fenlod\nJf graff \u017fchahert von Mamfeld\nJf graff ludwig von \u017ftuburg\nJf her \u017foft\u017fchalk von kle\u017f\u017fe be\u017fpannch\nJf her hans von tu\u030argen\nJf her \u017fappem von Bappenheim\nJf her \u017fuppem von \u017ftein w\u017f vorhin Ritter\nJf huz von Ro\u017fenhan\nJf herg von walden\u017ftett\nJf hulpbrand von thungen\nJf herg von \u017ftuben\nJf he\u030arin qua\u017ft", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "075f7819847349f0a5c47a92a7e4c679", "model": "mistral-large-latest", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"folios\": [\n    {\n      \"folio\": \"73\",\n      \"text\": \"Jf zu\u030a dem er\u017ften My he\u030ar\u017fchgraff Albracht vo\u030a\\nO\u030a\u017fter\u017freich\\nJf her ludwig lantgraff von he\u017f\u017fenberg\\nJf graff \u017f\u017f von hermenberg\\nJf graff Bernhart von \u017fegen\\nJf graff go\u030atffried von ha\u017f\u017fenlod\\nJf graff \u017fchahert von Mamfeld\\nJf graff ludwig von \u017ftuburg\\nJf her \u017foft\u017fchalk von kle\u017f\u017fe be\u017fpannch\\nJf her hans von tu\u030argen\\nJf her \u017fappem von Bappenheim\\nJf her \u017fuppem von \u017ftein w\u017f vorhin Ritter\\nJf huz von Ro\u017fenhan\\nJf herg von walden\u017ftett\\nJf hulpbrand von thungen\\nJf herg von \u017ftuben\\nJf he\u030arin qua\u017ft\",\n      \"addition1\": \"\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1703, "completion_tokens": 290, "total_tokens": 1993}}, "usage": {"input_tokens": 1703, "output_tokens": 290, "total_tokens": 1993}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.618, "cer": 0.405, "cost_summary": {"total_input_tokens": 20436, "total_output_tokens": 2824, "total_tokens": 23260, "input_cost_usd": 0.040872, "output_cost_usd": 0.016944, "total_cost_usd": 0.057816, "pricing_date": "2025-10-24", "input_price_per_million": 2.0, "output_price_per_million": 6.0}}, "normalized_score": 59.5}, {"_id": {"$oid": "69d4eb12b877594bc57189dd"}, "test_id": "T0295", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-medium-2508", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "mistral", "model": "mistral-medium-2508", "duration": 5.29822039604187, "timestamp": "2025-10-24T10:21:30.839728", "parsed": {"folios": [{"folio": "93", "text": "Vb dem land zi megtelinkurt\nEunrat tobe p\ufb01tt\nSietring hitzsow\npeter in barfu\u00df p\ufb01e\ufb05t\n\nVb luttringen\nJakob von hartzkurt p\ufb01e\ufb05t\nDesdering vo micourt p\ufb01ie\ufb05t\nCola von Gayon fin Ence\ufb05t\n\nVb golten land\nJakob N\u00fcwlung\nHans P\u00fc\ufb05ter\nHeinrich Lang\n\nVon stro\u00dfburg\nHans von Benfelt ein p\ufb01ie\ufb05t\nfant hans von orleans", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "5996e4aec9a44a1989fdb1321465a1ca", "model": "mistral-medium-2508", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"folios\": [\n    {\n      \"folio\": \"93\",\n      \"text\": \"Vb dem land zi megtelinkurt\\nEunrat tobe p\ufb01tt\\nSietring hitzsow\\npeter in barfu\u00df p\ufb01e\ufb05t\\n\\nVb luttringen\\nJakob von hartzkurt p\ufb01e\ufb05t\\nDesdering vo micourt p\ufb01ie\ufb05t\\nCola von Gayon fin Ence\ufb05t\\n\\nVb golten land\\nJakob N\u00fcwlung\\nHans P\u00fc\ufb05ter\\nHeinrich Lang\\n\\nVon stro\u00dfburg\\nHans von Benfelt ein p\ufb01ie\ufb05t\\nfant hans von orleans\",\n      \"addition1\": \"\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1703, "completion_tokens": 201, "total_tokens": 1904}}, "usage": {"input_tokens": 1703, "output_tokens": 201, "total_tokens": 1904}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.648, "cer": 0.386, "cost_summary": {"total_input_tokens": 20436, "total_output_tokens": 2966, "total_tokens": 23402, "input_cost_usd": 0.008174, "output_cost_usd": 0.005932, "total_cost_usd": 0.014106, "pricing_date": "2025-10-24", "input_price_per_million": 0.4, "output_price_per_million": 2.0}}, "normalized_score": 61.4}, {"_id": {"$oid": "69d4eb12b877594bc57189de"}, "test_id": "T0281", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5-nano", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openai", "model": "gpt-5-nano", "duration": 15.836687803268433, "timestamp": "2025-10-24T11:24:41.082310", "parsed": {"folios": [{"folio": "18", "text": "", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "chatcmpl-CU8K1AMrcEI221Ex5Vg0dsMPr2ZXr", "model": "gpt-5-nano-2025-08-07", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"folios\":[{\"folio\":\"18\",\"text\":\"\",\"addition1\":\"\",\"addition2\":\"\",\"addition3\":\"\"}]}", "role": "assistant"}}], "usage": {"prompt_tokens": 1717, "completion_tokens": 1382, "total_tokens": 3099}}, "usage": {"input_tokens": 1717, "output_tokens": 1382, "total_tokens": 3099}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.194, "cer": 0.806, "cost_summary": {"total_input_tokens": 20604, "total_output_tokens": 20815, "total_tokens": 41419, "input_cost_usd": 0.00103, "output_cost_usd": 0.008326, "total_cost_usd": 0.009356, "pricing_date": "2025-10-24", "input_price_per_million": 0.05, "output_price_per_million": 0.4}}, "normalized_score": 19.39999999999999}, {"_id": {"$oid": "69d4eb12b877594bc57189e1"}, "test_id": "T0285", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash-lite", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "genai", "model": "gemini-2.5-flash-lite", "duration": 2.7823641300201416, "timestamp": "2025-10-24T10:21:37.077056", "parsed": {"folios": [{"folio": "18", "text": "Der graff von Rurtzburg\n vnd fuff all ander ritter\nvnd vncoff wolten, mit\nDar Sey in was entweyl\nRichy vffgenomen ges. Jacob\nvon Gareguart vnd gant\nvon hertgen\u00dfbeig ritter\nDie zwenx werend gemy\nmit mir gefin aber Sex\npatroy wolt vnfer mit\nBeiten So Sy meltteil mit\nDar walt vnd also mu\u017fen\nwir mit gro\u017f\u017fen Vnwilley\nSamna pgerden Sey wir\nmit dem patroy vnd\noch Sey br\u00fcderen Dar\nvmb garent ze zypren\nzwa\u017f\u017fer Sy zuter", "addition1": "Sarchani\nin Cypro\nnaskiz", "addition2": "", "addition3": ""}]}, "text": "{\n  \"folios\": [\n    {\n      \"folio\": \"18\",\n      \"text\": \"Der graff von Rurtzburg\\n vnd fuff all ander ritter\\nvnd vncoff wolten, mit\\nDar Sey in was entweyl\\nRichy vffgenomen ges. Jacob\\nvon Gareguart vnd gant\\nvon hertgen\u00dfbeig ritter\\nDie zwenx werend gemy\\nmit mir gefin aber Sex\\npatroy wolt vnfer mit\\nBeiten So Sy meltteil mit\\nDar walt vnd also mu\u017fen\\nwir mit gro\u017f\u017fen Vnwilley\\nSamna pgerden Sey wir\\nmit dem patroy vnd\\noch Sey br\u00fcderen Dar\\nvmb garent ze zypren\\nzwa\u017f\u017fer Sy zuter\",\n      \"addition1\": \"Sarchani\\nin Cypro\\nnaskiz\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "raw_response": {"text": "{\n  \"folios\": [\n    {\n      \"folio\": \"18\",\n      \"text\": \"Der graff von Rurtzburg\\n vnd fuff all ander ritter\\nvnd vncoff wolten, mit\\nDar Sey in was entweyl\\nRichy vffgenomen ges. Jacob\\nvon Gareguart vnd gant\\nvon hertgen\u00dfbeig ritter\\nDie zwenx werend gemy\\nmit mir gefin aber Sex\\npatroy wolt vnfer mit\\nBeiten So Sy meltteil mit\\nDar walt vnd also mu\u017fen\\nwir mit gro\u017f\u017fen Vnwilley\\nSamna pgerden Sey wir\\nmit dem patroy vnd\\noch Sey br\u00fcderen Dar\\nvmb garent ze zypren\\nzwa\u017f\u017fer Sy zuter\",\n      \"addition1\": \"Sarchani\\nin Cypro\\nnaskiz\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "usage_metadata": {"prompt_token_count": 759, "candidates_token_count": 226, "total_token_count": 985}}, "usage": {"input_tokens": 759, "output_tokens": 226, "total_tokens": 985}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.582, "cer": 0.43, "cost_summary": {"total_input_tokens": 9108, "total_output_tokens": 2922, "total_tokens": 12030, "input_cost_usd": 0.000911, "output_cost_usd": 0.001169, "total_cost_usd": 0.00208, "pricing_date": "2025-10-24", "input_price_per_million": 0.1, "output_price_per_million": 0.4}}, "normalized_score": 57.0}, {"_id": {"$oid": "69d4eb12b877594bc57189e3"}, "test_id": "T0290", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-opus-4-20250514", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "anthropic", "model": "claude-opus-4-20250514", "duration": 22.24477529525757, "timestamp": "2025-10-24T10:32:04.241787", "parsed": {"folios": [{"folio": "69", "text": "Dar noch kunge ir pfister vn\ntrumetter zu jnnen heim off\ndas ga\u017f\u017fen vnd hoffierten So\njnen gnoden deb g\u017fich konig\ndie vunderthen pfister vnn\ntrumetter vnd berg\u017fenplaher\nvn hie\u017fperten do och nach we\nsitten al\u017fo ward mins gnedigo\nherrn vnd allen den sine wil\nerenierbotten Jch meandeb de\ngen tag furemt wir do danno\nvnd komen off fatag nach \u017fant\nwalburgen dag do wir do e\u017ft\ntagz im kamgmnet ston lagen\nvn\u017fer zu a\u017fhone gat kunig welch\ndie be\u017ften koufflut zu vch der\nhat von off kuntzeff danrnach\nfuren wir do danno vnn off\njm\u017ftag wor \u017fant vrbj tag\nkomen zu vnd \u00f6n gullen by gio", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "msg_01JybKR9duxnUgw1hLptXgLi", "model": "claude-opus-4-20250514", "role": "assistant", "content": [{"type": "tool_use", "id": "toolu_015S1SPWwPKFFreUwqA6o7YR", "name": "extract_structured_data", "input": {"folios": [{"folio": "69", "text": "Dar noch kunge ir pfister vn\ntrumetter zu jnnen heim off\ndas ga\u017f\u017fen vnd hoffierten So\njnen gnoden deb g\u017fich konig\ndie vunderthen pfister vnn\ntrumetter vnd berg\u017fenplaher\nvn hie\u017fperten do och nach we\nsitten al\u017fo ward mins gnedigo\nherrn vnd allen den sine wil\nerenierbotten Jch meandeb de\ngen tag furemt wir do danno\nvnd komen off fatag nach \u017fant\nwalburgen dag do wir do e\u017ft\ntagz im kamgmnet ston lagen\nvn\u017fer zu a\u017fhone gat kunig welch\ndie be\u017ften koufflut zu vch der\nhat von off kuntzeff danrnach\nfuren wir do danno vnn off\njm\u017ftag wor \u017fant vrbj tag\nkomen zu vnd \u00f6n gullen by gio", "addition1": "", "addition2": "", "addition3": ""}]}}], "stop_reason": "tool_use", "usage": {"input_tokens": 2062, "output_tokens": 327}}, "usage": {"input_tokens": 2062, "output_tokens": 327, "total_tokens": 2389}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.648, "cer": 0.391, "cost_summary": {"total_input_tokens": 24744, "total_output_tokens": 3681, "total_tokens": 28425, "input_cost_usd": 0.37116, "output_cost_usd": 0.276075, "total_cost_usd": 0.647235, "pricing_date": "2025-10-24", "input_price_per_million": 15.0, "output_price_per_million": 75.0}}, "normalized_score": 60.9}, {"_id": {"$oid": "69d4eb12b877594bc57189e5"}, "test_id": "T0304", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openrouter", "model": "x-ai/grok-4", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openrouter", "model": "x-ai/grok-4", "duration": 247.2778036594391, "timestamp": "2025-10-24T11:26:21.055296", "parsed": {"folios": [{"folio": "23", "text": "H\ua75b jorg cupen Maj heinrich faltner vo\nH\ua75b frantz von venningen\nH\ua75b jorg freyher\ua75b von hohenburg\nH\ua75b jorg freyher\ua75b von Minfeld\nH\ua75b jorg husman von Minfeld)\nH\ua75b jorg Gotfrid von Bitschoff\nH\ua75b heinrich von Bitschoff)\nH\ua75b fridrich von Bitschoff\nH\ua75b Georg El\u00dfehorn\nH\ua75b heinrich quast", "addition1": "bspand", "addition2": "bspand", "addition3": "w z mochyn"}]}, "text": "", "raw_response": {"id": "gen-1761297732-N9qMMweD8ILCo3K9Alah", "model": "x-ai/grok-4", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": " {\n  \"folios\": [\n    {\n      \"folio\": \"23\",\n      \"text\": \"H\ua75b jorg cupen Maj heinrich faltner vo\\nH\ua75b frantz von venningen\\nH\ua75b jorg freyher\ua75b von hohenburg\\nH\ua75b jorg freyher\ua75b von Minfeld\\nH\ua75b jorg husman von Minfeld)\\nH\ua75b jorg Gotfrid von Bitschoff\\nH\ua75b heinrich von Bitschoff)\\nH\ua75b fridrich von Bitschoff\\nH\ua75b Georg El\u00dfehorn\\nH\ua75b heinrich quast\",\n      \"addition1\": \"bspand\",\n      \"addition2\": \"bspand\",\n      \"addition3\": \"w z mochyn\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1619, "completion_tokens": 10540, "total_tokens": 12159}}, "usage": {"input_tokens": 1619, "output_tokens": 10540, "total_tokens": 12159}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.402, "cer": 0.667, "cost_summary": {"total_input_tokens": 19428, "total_output_tokens": 95682, "total_tokens": 115110, "input_cost_usd": 0.058284, "output_cost_usd": 1.43523, "total_cost_usd": 1.493514, "pricing_date": "2025-10-24", "input_price_per_million": 3.0, "output_price_per_million": 15.0}}, "normalized_score": 33.3}, {"_id": {"$oid": "69d4eb12b877594bc57189e7"}, "test_id": "T0271", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-flash", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "genai", "model": "gemini-2.5-flash", "duration": 11.744013547897339, "timestamp": "2025-10-24T12:54:36.174321", "parsed": {"folios": [{"folio": "3", "text": "Vnd ein pferit Die mir vnd\nminen knechten vber Rulfand\nSen So was mencky kein weg\nDenne den wir machtend\nvnd ziehend Sie knecht die\nvnd vil in vntz an den and\nvnd die pferit vntz an die\nbettel vnd was ze mol ein grosse\nnebel do wir kaum gesahend\nvnd als mit grosser arbeit kame\nwir ze mittem tag zu sant\nkinroffel vff den berg Do\ndo sach ich die buecher do gar\nvil herren wopen zu stonde\ndie in four edlen geley hand\ndo seind mincs vattero pflegen\nwopen ach in dem einem", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "{\n  \"folios\": [\n    {\n      \"folio\": \"3\",\n      \"text\": \"Vnd ein pferit Die mir vnd\\nminen knechten vber Rulfand\\nSen So was mencky kein weg\\nDenne den wir machtend\\nvnd ziehend Sie knecht die\\nvnd vil in vntz an den and\\nvnd die pferit vntz an die\\nbettel vnd was ze mol ein grosse\\nnebel do wir kaum gesahend\\nvnd als mit grosser arbeit kame\\nwir ze mittem tag zu sant\\nkinroffel vff den berg Do\\ndo sach ich die buecher do gar\\nvil herren wopen zu stonde\\ndie in four edlen geley hand\\ndo seind mincs vattero pflegen\\nwopen ach in dem einem\",\n      \"addition1\": \"\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "raw_response": {"text": "{\n  \"folios\": [\n    {\n      \"folio\": \"3\",\n      \"text\": \"Vnd ein pferit Die mir vnd\\nminen knechten vber Rulfand\\nSen So was mencky kein weg\\nDenne den wir machtend\\nvnd ziehend Sie knecht die\\nvnd vil in vntz an den and\\nvnd die pferit vntz an die\\nbettel vnd was ze mol ein grosse\\nnebel do wir kaum gesahend\\nvnd als mit grosser arbeit kame\\nwir ze mittem tag zu sant\\nkinroffel vff den berg Do\\ndo sach ich die buecher do gar\\nvil herren wopen zu stonde\\ndie in four edlen geley hand\\ndo seind mincs vattero pflegen\\nwopen ach in dem einem\",\n      \"addition1\": \"\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "usage_metadata": {"prompt_token_count": 759, "candidates_token_count": 214, "total_token_count": 2265}}, "usage": {"input_tokens": 759, "output_tokens": 214, "total_tokens": 2265}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.69, "cer": 0.34, "cost_summary": {"total_input_tokens": 9108, "total_output_tokens": 3364, "total_tokens": 12472, "input_cost_usd": 0.002732, "output_cost_usd": 0.00841, "total_cost_usd": 0.011142, "pricing_date": "2025-10-24", "input_price_per_million": 0.3, "output_price_per_million": 2.5}}, "normalized_score": 66.0}, {"_id": {"$oid": "69d4eb12b877594bc57189e8"}, "test_id": "T0273", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openai", "model": "gpt-4.1", "duration": 5.71106481552124, "timestamp": "2025-10-24T12:51:52.090097", "parsed": {"folios": [{"folio": "93", "text": "v\u00df dem land ze\nmastel\u00dflingen\nEinhart Tobie pritt\nDietrich liebzan\npeter ein barfu\u00df priester\n\nv\u00df Luttringen\nJakob von hartzwurt pritt\nSebastian de mintrant priester\nJoda von bayern priester\n\nv\u00df gol\u00dfen land\nJakob Melinger\nHans Judenck\nHeinrich Lang\n\nvon pro\u00dfburg\nHans von benfeld ein priester\nSant Johans ordens", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "chatcmpl-CU9gYOU6wQxlmpYqMLXWfFSU1J4Vr", "model": "gpt-4.1-2025-04-14", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"folios\": [\n    {\n      \"folio\": \"93\",\n      \"text\": \"v\u00df dem land ze\\nmastel\u00dflingen\\nEinhart Tobie pritt\\nDietrich liebzan\\npeter ein barfu\u00df priester\\n\\nv\u00df Luttringen\\nJakob von hartzwurt pritt\\nSebastian de mintrant priester\\nJoda von bayern priester\\n\\nv\u00df gol\u00dfen land\\nJakob Melinger\\nHans Judenck\\nHeinrich Lang\\n\\nvon pro\u00dfburg\\nHans von benfeld ein priester\\nSant Johans ordens\",\n      \"addition1\": \"\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1429, "completion_tokens": 151, "total_tokens": 1580}}, "usage": {"input_tokens": 1429, "output_tokens": 151, "total_tokens": 1580}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.648, "cer": 0.398, "cost_summary": {"total_input_tokens": 17148, "total_output_tokens": 2399, "total_tokens": 19547, "input_cost_usd": 0.034296, "output_cost_usd": 0.019192, "total_cost_usd": 0.053488, "pricing_date": "2025-10-24", "input_price_per_million": 2.0, "output_price_per_million": 8.0}}, "normalized_score": 60.199999999999996}, {"_id": {"$oid": "69d4eb12b877594bc57189ee"}, "test_id": "T0296", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "mistral", "model": "mistral-medium-2505", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "mistral", "model": "mistral-medium-2505", "duration": 9.793105125427246, "timestamp": "2025-10-24T10:23:36.817091", "parsed": {"folios": [{"folio": "", "text": "Die felley tags gon falziun in der wirz\n gon ir may den balmung sers do wz\n der subenzelende tag vitten wir\n gon felter gon an den adzchende\n tag gon terzi\u00df vnd blibey zway\n tag do vnd firtend an den golgen\n samptag der do wz der achtzvnzigt tag der merzgen off\ndem wasser No terzi\u00df gon lenedig\n gon an den osteroben der do wz der\n lest tag in merzgen do fohay wir\n erwe manigen fitterkey vnd gesell\n schaft als nechtige husser so fohye\n Lud fawnt marp minster vnd de\n hertzogey palast der ir zemolst phiy\n palation vnd fawnt Saruf vnd hern Viter\n magnifi vnd furchterey woppey die denj\n cum ze Nenedig gewessen find do zt\nein Cappell an den minster fast\n der fritzin einsaltar gewesser da\n wulser belsalter Jhesu Cristi off fas", "addition1": "Venach\nDucis", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "b6fa863c2fbf4a9ba48c7b904bf47f3c", "model": "mistral-medium-2505", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"folios\": [\n    {\n      \"folio\": \"\",\n      \"text\": \"Die felley tags gon falziun in der wirz\\n gon ir may den balmung sers do wz\\n der subenzelende tag vitten wir\\n gon felter gon an den adzchende\\n tag gon terzi\u00df vnd blibey zway\\n tag do vnd firtend an den golgen\\n samptag der do wz der achtzvnzigt tag der merzgen off\\ndem wasser No terzi\u00df gon lenedig\\n gon an den osteroben der do wz der\\n lest tag in merzgen do fohay wir\\n erwe manigen fitterkey vnd gesell\\n schaft als nechtige husser so fohye\\n Lud fawnt marp minster vnd de\\n hertzogey palast der ir zemolst phiy\\n palation vnd fawnt Saruf vnd hern Viter\\n magnifi vnd furchterey woppey die denj\\n cum ze Nenedig gewessen find do zt\\nein Cappell an den minster fast\\n der fritzin einsaltar gewesser da\\n wulser belsalter Jhesu Cristi off fas\",\n      \"addition1\": \"Venach\\nDucis\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1703, "completion_tokens": 309, "total_tokens": 2012}}, "usage": {"input_tokens": 1703, "output_tokens": 309, "total_tokens": 2012}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.601, "cer": 0.42, "cost_summary": {"total_input_tokens": 20436, "total_output_tokens": 5369, "total_tokens": 25805, "input_cost_usd": 0.008174, "output_cost_usd": 0.010738, "total_cost_usd": 0.018912, "pricing_date": "2025-10-24", "input_price_per_million": 0.4, "output_price_per_million": 2.0}}, "normalized_score": 58.0}, {"_id": {"$oid": "69d4eb12b877594bc57189f1"}, "test_id": "T0284", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.0-flash-lite", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "genai", "model": "gemini-2.0-flash-lite", "duration": 3.1598987579345703, "timestamp": "2025-10-24T11:39:48.889742", "parsed": {"folios": [{"folio": "69", "text": "Forenarts komme in pfiffer den\nzummery Horn off\nDie galley Land Roflenten Jall\nfimey gr\u00fcdey Die glut koma\nDie omdeptjen pfiffer vnd\ntxmmmmster cand begefenplayex\nNu hofiertey to ach nach ive\nfittey also waed mune queduc\nReed and alley day fms wit\nxe\ua75b\ua75berbote\ua75by Jff moendeb ge\ntay fucent wir da denne\nAnd Ramiky coff freitag nachts sont\nAxbome tong Sek do nas dat cost\ntag iny\nJff yt ein affone pat hint zuvch\ngeflossen\nfat con\nFiney wir da denne Nu\nNar font Nuty tag\nkachen zu ande in galley by \u03c7\u03b1\u03c1", "addition1": "Honores\nMarchio\nin exhi\nbiti.", "addition2": "7", "addition3": ""}]}, "text": "{\n  \"folios\": [\n    {\n      \"folio\": \"69\",\n      \"text\": \"Forenarts komme in pfiffer den\\nzummery Horn off\\nDie galley Land Roflenten Jall\\nfimey gr\u00fcdey Die glut koma\\nDie omdeptjen pfiffer vnd\\ntxmmmmster cand begefenplayex\\nNu hofiertey to ach nach ive\\nfittey also waed mune queduc\\nReed and alley day fms wit\\nxe\ua75b\ua75berbote\ua75by Jff moendeb ge\\ntay fucent wir da denne\\nAnd Ramiky coff freitag nachts sont\\nAxbome tong Sek do nas dat cost\\ntag iny\\nJff yt ein affone pat hint zuvch\\ngeflossen\\nfat con\\nFiney wir da denne Nu\\nNar font Nuty tag\\nkachen zu ande in galley by \u03c7\u03b1\u03c1\",\n      \"addition1\": \"Honores\\nMarchio\\nin exhi\\nbiti.\",\n      \"addition2\": \"7\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "raw_response": {"text": "{\n  \"folios\": [\n    {\n      \"folio\": \"69\",\n      \"text\": \"Forenarts komme in pfiffer den\\nzummery Horn off\\nDie galley Land Roflenten Jall\\nfimey gr\u00fcdey Die glut koma\\nDie omdeptjen pfiffer vnd\\ntxmmmmster cand begefenplayex\\nNu hofiertey to ach nach ive\\nfittey also waed mune queduc\\nReed and alley day fms wit\\nxe\ua75b\ua75berbote\ua75by Jff moendeb ge\\ntay fucent wir da denne\\nAnd Ramiky coff freitag nachts sont\\nAxbome tong Sek do nas dat cost\\ntag iny\\nJff yt ein affone pat hint zuvch\\ngeflossen\\nfat con\\nFiney wir da denne Nu\\nNar font Nuty tag\\nkachen zu ande in galley by \u03c7\u03b1\u03c1\",\n      \"addition1\": \"Honores\\nMarchio\\nin exhi\\nbiti.\",\n      \"addition2\": \"7\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "usage_metadata": {"prompt_token_count": 2348, "candidates_token_count": 250, "total_token_count": 2598}}, "usage": {"input_tokens": 2348, "output_tokens": 250, "total_tokens": 2598}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.601, "cer": 0.424, "cost_summary": {"total_input_tokens": 28176, "total_output_tokens": 2624, "total_tokens": 30800, "input_cost_usd": 0.002113, "output_cost_usd": 0.000787, "total_cost_usd": 0.0029, "pricing_date": "2025-10-24", "input_price_per_million": 0.075, "output_price_per_million": 0.3}}, "normalized_score": 57.6}, {"_id": {"$oid": "69d4eb12b877594bc57189f2"}, "test_id": "T0280", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-5-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openai", "model": "gpt-5-mini", "duration": 47.921910762786865, "timestamp": "2025-10-24T11:18:06.413897", "parsed": {"folios": [{"folio": "", "text": "Selbes im fronaltar die sohend\nwir gantz vnd em sinckt bey\ndann heiligin frou wirz er\nbit ir vff ir brust vnd ein\nbein von sant marien madalene\nvinggrot ist off samstag noch\nden opfertag das do was dar\nander tag sol abbleyn do\nfuirend wir gen messa do\nliegend in sant steffannb\nkalben bi hundert der kindlin\ndie heredes erslaug in enem\naltar die schent wir vnd\nsicht may noch bechendendlich\nin stund vnd f\u00fcr und die\nnagel das xprinses wy\ndem sunentag frig do", "addition1": "Meuran glauc giessung", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "chatcmpl-CU8D79NbseDaCCRs3e0aesw3bcJpX", "model": "gpt-5-mini-2025-08-07", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\n  \"folios\": [\n    {\n      \"folio\": \"\",\n      \"text\": \"Selbes im fronaltar die sohend\\nwir gantz vnd em sinckt bey\\ndann heiligin frou wirz er\\nbit ir vff ir brust vnd ein\\nbein von sant marien madalene\\nvinggrot ist off samstag noch\\nden opfertag das do was dar\\nander tag sol abbleyn do\\nfuirend wir gen messa do\\nliegend in sant steffannb\\nkalben bi hundert der kindlin\\ndie heredes erslaug in enem\\naltar die schent wir vnd\\nsicht may noch bechendendlich\\nin stund vnd f\u00fcr und die\\nnagel das xprinses wy\\ndem sunentag frig do\",\n      \"addition1\": \"Meuran glauc giessung\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}", "role": "assistant"}}], "usage": {"prompt_tokens": 1506, "completion_tokens": 3361, "total_tokens": 4867}}, "usage": {"input_tokens": 1506, "output_tokens": 3361, "total_tokens": 4867}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.714, "cer": 0.334, "cost_summary": {"total_input_tokens": 18072, "total_output_tokens": 28621, "total_tokens": 46693, "input_cost_usd": 0.004518, "output_cost_usd": 0.057242, "total_cost_usd": 0.06176, "pricing_date": "2025-10-24", "input_price_per_million": 0.25, "output_price_per_million": 2.0}}, "normalized_score": 66.6}, {"_id": {"$oid": "69d4eb12b877594bc57189f3"}, "test_id": "T0277", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "openai", "model": "gpt-4.1-mini", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "openai", "model": "gpt-4.1-mini", "duration": 3.7105448246002197, "timestamp": "2025-10-24T10:24:24.519578", "parsed": {"folios": [{"folio": "", "text": "Und die pspel ist bi ooo mil\nwirt vnd ist der venediger\nIst von kurfurst vns gon\nmodun sind al mil vnd\nist ooch ein giubst stat vnd\nein erczbistum vnd ist ooch\nder venediger ze modun\nwachtet der rumanier vnd\nist ooch do bi ein heilig gesstet\ndo bi ij milen\nIst von modun bis gon candia\nsind tre mil vnd candia\nIst ein grob pspel vj mil\nweit vnd wachtet der\nrumanier do vnd ist ooch\nder venediger\nIst von candia vnz gon rodos\nsind tre mil wer aber", "addition1": "Candia\ncysta.", "addition2": "Virnum\nMalba\nrumania", "addition3": "Rhod\u021d."}]}, "text": "", "raw_response": {"id": "chatcmpl-CU7Ns2yFtWQgTRhL5vz790GPFIIyJ", "model": "gpt-4.1-mini-2025-04-14", "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "{\"folios\":[{\"folio\":\"\",\"text\":\"Und die pspel ist bi ooo mil\\nwirt vnd ist der venediger\\nIst von kurfurst vns gon\\nmodun sind al mil vnd\\nist ooch ein giubst stat vnd\\nein erczbistum vnd ist ooch\\nder venediger ze modun\\nwachtet der rumanier vnd\\nist ooch do bi ein heilig gesstet\\ndo bi ij milen\\nIst von modun bis gon candia\\nsind tre mil vnd candia\\nIst ein grob pspel vj mil\\nweit vnd wachtet der\\nrumanier do vnd ist ooch\\nder venediger\\nIst von candia vnz gon rodos\\nsind tre mil wer aber\",\"addition1\":\"Candia\\ncysta.\",\"addition2\":\"Virnum\\nMalba\\nrumania\",\"addition3\":\"Rhod\u021d.\"}]}", "role": "assistant"}}], "usage": {"prompt_tokens": 1806, "completion_tokens": 202, "total_tokens": 2008}}, "usage": {"input_tokens": 1806, "output_tokens": 202, "total_tokens": 2008}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.798, "cer": 0.231, "cost_summary": {"total_input_tokens": 21672, "total_output_tokens": 2369, "total_tokens": 24041, "input_cost_usd": 0.008669, "output_cost_usd": 0.00379, "total_cost_usd": 0.012459, "pricing_date": "2025-10-24", "input_price_per_million": 0.4, "output_price_per_million": 1.6}}, "normalized_score": 76.9}, {"_id": {"$oid": "69d4eb12b877594bc57189f4"}, "test_id": "T0272", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "genai", "model": "gemini-2.5-pro", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "genai", "model": "gemini-2.5-pro", "duration": 43.08316206932068, "timestamp": "2025-10-24T12:54:59.709080", "parsed": {"folios": [{"folio": "", "text": "Des selben tags gay salzburg In der wue\ngay jhesus dem balmtag der do wz\nder sibenczehende tag Vnd ritten wir\ngay feltkirch jhesus dem achczehende\ntag gay terfis Nid blibey zwey\ntag do vnd furcend an dem Gshen\nSuntag der do wz der achtvn\nzwenczigst tag des merczen vff\ndem wasser Vo terfis gay velledig\nJhesus dem operoben der do wz der\nlest tag im merczen Do sahen wir\netwe manigey sithu\ua75bczey Und grosse\nschiff als mechtige h\u00fcser vff hohe\nW wir sant mar\ua76fo m\u00fcnster vnd des\nhertzogey palast der ist gemalsschin\npalatinu\ua76b vnd daruff vil sch\u00f6n Vnd\nmagnifi vnd luedstey kuppery die denn\nze Venedig gewesey sind Es ist\nein cappell an dem m\u00fcnster So ist\nder prin in einaltar gewasset So\ndiser behalter Jhesus Ope vff sas", "addition1": "Veneh\nDucis\npalatium\nmagnifi\ncum.", "addition2": null, "addition3": null}]}, "text": "{\n  \"folios\": [\n    {\n      \"folio\": \"\",\n      \"text\": \"Des selben tags gay salzburg In der wue\\ngay jhesus dem balmtag der do wz\\nder sibenczehende tag Vnd ritten wir\\ngay feltkirch jhesus dem achczehende\\ntag gay terfis Nid blibey zwey\\ntag do vnd furcend an dem Gshen\\nSuntag der do wz der achtvn\\nzwenczigst tag des merczen vff\\ndem wasser Vo terfis gay velledig\\nJhesus dem operoben der do wz der\\nlest tag im merczen Do sahen wir\\netwe manigey sithu\ua75bczey Und grosse\\nschiff als mechtige h\u00fcser vff hohe\\nW wir sant mar\ua76fo m\u00fcnster vnd des\\nhertzogey palast der ist gemalsschin\\npalatinu\ua76b vnd daruff vil sch\u00f6n Vnd\\nmagnifi vnd luedstey kuppery die denn\\nze Venedig gewesey sind Es ist\\nein cappell an dem m\u00fcnster So ist\\nder prin in einaltar gewasset So\\ndiser behalter Jhesus Ope vff sas\",\n      \"addition1\": \"Veneh\\nDucis\\npalatium\\nmagnifi\\ncum.\",\n      \"addition2\": null,\n      \"addition3\": null\n    }\n  ]\n}", "raw_response": {"text": "{\n  \"folios\": [\n    {\n      \"folio\": \"\",\n      \"text\": \"Des selben tags gay salzburg In der wue\\ngay jhesus dem balmtag der do wz\\nder sibenczehende tag Vnd ritten wir\\ngay feltkirch jhesus dem achczehende\\ntag gay terfis Nid blibey zwey\\ntag do vnd furcend an dem Gshen\\nSuntag der do wz der achtvn\\nzwenczigst tag des merczen vff\\ndem wasser Vo terfis gay velledig\\nJhesus dem operoben der do wz der\\nlest tag im merczen Do sahen wir\\netwe manigey sithu\ua75bczey Und grosse\\nschiff als mechtige h\u00fcser vff hohe\\nW wir sant mar\ua76fo m\u00fcnster vnd des\\nhertzogey palast der ist gemalsschin\\npalatinu\ua76b vnd daruff vil sch\u00f6n Vnd\\nmagnifi vnd luedstey kuppery die denn\\nze Venedig gewesey sind Es ist\\nein cappell an dem m\u00fcnster So ist\\nder prin in einaltar gewasset So\\ndiser behalter Jhesus Ope vff sas\",\n      \"addition1\": \"Veneh\\nDucis\\npalatium\\nmagnifi\\ncum.\",\n      \"addition2\": null,\n      \"addition3\": null\n    }\n  ]\n}", "usage_metadata": {"prompt_token_count": 759, "candidates_token_count": 325, "total_token_count": 5197}}, "usage": {"input_tokens": 759, "output_tokens": 325, "total_tokens": 5197}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.684, "cer": 0.339, "cost_summary": {"total_input_tokens": 9108, "total_output_tokens": 2773, "total_tokens": 11881, "input_cost_usd": 0.011385, "output_cost_usd": 0.02773, "total_cost_usd": 0.039115, "pricing_date": "2025-10-24", "input_price_per_million": 1.25, "output_price_per_million": 10.0}}, "normalized_score": 66.1}, {"_id": {"$oid": "69d4eb12b877594bc57189f6"}, "test_id": "T0293", "benchmark": "medieval_manuscripts", "date": "2025-10-24", "tags": {"document-type": ["manuscript"], "writing": ["handwritten"], "century": [15], "language": ["de"], "layout": ["prose"], "task": ["transcription"]}, "contributors": ["ina_serif", "maximilian_hindermann"], "hidden": false, "config": {"provider": "anthropic", "model": "claude-sonnet-4-5-20250929", "dataclass": "Document", "temperature": 0.0, "role_description": "You are a historian with expertise in medieval manuscripts and paleography", "prompt_file": "prompt.txt", "legacy_test": false}, "prompt": "IDENTITY and PURPOSE:\n\nYou are presented with an image from a 15th century medieval manuscript written in Basel in late medieval German. Your task is to extract the text from the manuscript in the specified JSON format. You must extract all text exactly as it appears in the manuscript, maintaining historical spellings, punctuation, and formatting. Do not resolve abbreviations.\n\nThe manuscript may contain:\n1. A main text body (often in one column as continuous text)\n2. A folio number (if present)\n3. Additional notes or text written in the margins (labeled as addition1, addition2, etc.)\n\nYou must:\n- Identify and transcribe the main text\n- Extract the folio number if visible (use empty string \"\" if not visible)\n- Identify and transcribe any marginal additions separately\n- Preserve line breaks with \\n\n- Maintain all historical spellings and abbreviations exactly as written. If a letter is superscribed, normalize it by writing the superscribed letter after the main letter, e.g. \"u with superscribed o\" is spelled as \"uo\". If special characters are used for abbreviations, do not resolve them, but try to transcribe the special character according to the Medieval Unicode Font Initiative. \ua75b for \"er\" and \ua770 for \"us\" or \"em\" might be the most common special characters.\n- Do not modernize or correct the text\n- Do not use OCR or attempt to interpret unclear text - transcribe what you can see\n- If a field has no content, use an empty string \"\" (not null)\n\nTake a deep breath and think step by step about the layout of the page. First identify the folio number, then the main text area, then any marginal additions. Return only a JSON file with no additional commentary.\n\nEXAMPLE OUTPUT:\n\n{\n  \"folios\": [\n    {\n      \"folio\": \"15\",\n      \"text\": \"In disem jare kam der kunig\\n mit grossem here in daz lant\\n vnd belagerte die stat Basel\\n do wertend sich die burger\\n mit grosser kraft vnd tugent\\n vnd triben den kunig dannen\\n mit schanden vnd verlust\",\n      \"addition1\": \"Anno domini 1444\",\n      \"addition2\": \"\",\n      \"addition3\": \"\"\n    }\n  ]\n}\n", "results": {"provider": "anthropic", "model": "claude-sonnet-4-5-20250929", "duration": 10.938278198242188, "timestamp": "2025-10-24T10:40:24.444878", "parsed": {"folios": [{"folio": "", "text": "Selbe my fronaltar die selhend\nvon hantz vnd ein stuck der y\ndem heiligen frontz itz dz\nsit in vff in brust vnd ein\nbein von sant marien madalena\nruss rot vff ost samstag nach\ndem ostertag daz do was der\nander tag des abrellen do\nkamen f\u00fcrend nun gan merung de\nglaz begend in sant stoffans\nkilchen bl\u00f6ssen bi hundert der br\u00fcder\ndie spraches erstlich in einem\naltar die pflant vin vnd\nsicht man noch beschedentlich\nir sind vnd die brid die\nm\u00fcffend als wymels vn\ndem s\u00fcrnentag frig de", "addition1": "", "addition2": "", "addition3": ""}]}, "text": "", "raw_response": {"id": "msg_01GWqLxojyDXfaVSc82cW9mm", "model": "claude-sonnet-4-5-20250929", "role": "assistant", "content": [{"type": "tool_use", "id": "toolu_017BJ9LFVyujYXLFeb5jgGBX", "name": "extract_structured_data", "input": {"folios": [{"folio": "", "text": "Selbe my fronaltar die selhend\nvon hantz vnd ein stuck der y\ndem heiligen frontz itz dz\nsit in vff in brust vnd ein\nbein von sant marien madalena\nruss rot vff ost samstag nach\ndem ostertag daz do was der\nander tag des abrellen do\nkamen f\u00fcrend nun gan merung de\nglaz begend in sant stoffans\nkilchen bl\u00f6ssen bi hundert der br\u00fcder\ndie spraches erstlich in einem\naltar die pflant vin vnd\nsicht man noch beschedentlich\nir sind vnd die brid die\nm\u00fcffend als wymels vn\ndem s\u00fcrnentag frig de", "addition1": "", "addition2": "", "addition3": ""}]}}], "stop_reason": "tool_use", "usage": {"input_tokens": 2335, "output_tokens": 255}}, "usage": {"input_tokens": 2335, "output_tokens": 255, "total_tokens": 2590}, "finish_reason": "stop"}, "scoring": {"fuzzy": 0.55, "cer": 0.489, "cost_summary": {"total_input_tokens": 27150, "total_output_tokens": 3758, "total_tokens": 30908, "input_cost_usd": 0.08145, "output_cost_usd": 0.05637, "total_cost_usd": 0.13782, "pricing_date": "2025-10-24", "input_price_per_million": 3.0, "output_price_per_million": 15.0}}, "normalized_score": 51.1}]