{ "timestamp_utc": "2026-10-08T20:51:05.789840+00:00", "dataset_version": "synthetic-regression-v1", "metric_definition": "exact normalized field match, amount/date validation; raster OCR subset", "real_inference": true, "fictional_data_only": true, "case_count": 32, "completed": 32, "elapsed_seconds": 80.87, "environment": { "gpu": "NVIDIA GeForce RTX 5060 Ti, 595.84, 16311 MiB", "python": "3.12.3", "model_runtime": "Existing Qwen 3.5 9B GGUF / MiniLM L12-H384 / Whisper small.en / Kokoro af_heart; only applicable capabilities used", "providers_enabled": { "llm": true, "embedding": false, "rerank": false, "asr": false, "tts": false, "ocr": true } }, "metrics": { "all_fields_correct": { "passed": 32, "total": 32 }, "amount_validation_correct": { "passed": 32, "total": 32 }, "date_validation_correct": { "passed": 32, "total": 32 } }, "cases": [ { "id": "doc-01", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-200", "supplier": "", "date": "2026-02-30", "currency": "EUR", "subtotal": "100.00", "tax": "21.00", "total": "1.00", "reference": "ORDER-600" }, "validation_issues": [ "Missing required field: supplier", "Date must be a real calendar date in YYYY-MM-DD format.", "Amount discrepancy: subtotal + tax = 121.00, but total = 1.00." ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.081 }, { "id": "doc-02", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-201", "supplier": "Fictional Vendor 2", "date": "2026-10-08", "currency": "EUR", "subtotal": "137.00", "tax": "28.77", "total": "165.77", "reference": "ORDER-601" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.266 }, { "id": "doc-03", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-202", "supplier": "Fictional Vendor 3", "date": "2026-10-08", "currency": "EUR", "subtotal": "174.00", "tax": "36.54", "total": "210.54", "reference": "ORDER-602" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.29 }, { "id": "doc-04", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-203", "supplier": "Fictional Vendor 4", "date": "2026-10-08", "currency": "EUR", "subtotal": "211.00", "tax": "44.31", "total": "255.31", "reference": "ORDER-603" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.236 }, { "id": "doc-05", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-204", "supplier": "Fictional Vendor 5", "date": "2026-10-08", "currency": "EUR", "subtotal": "248.00", "tax": "52.08", "total": "1.00", "reference": "ORDER-604" }, "validation_issues": [ "Amount discrepancy: subtotal + tax = 300.08, but total = 1.00." ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.231 }, { "id": "doc-06", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-205", "supplier": "Fictional Vendor 6", "date": "2026-02-30", "currency": "EUR", "subtotal": "285.00", "tax": "59.85", "total": "344.85", "reference": "ORDER-605" }, "validation_issues": [ "Date must be a real calendar date in YYYY-MM-DD format." ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.295 }, { "id": "doc-07", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-206", "supplier": "", "date": "2026-10-08", "currency": "EUR", "subtotal": "322.00", "tax": "67.62", "total": "389.62", "reference": "ORDER-606" }, "validation_issues": [ "Missing required field: supplier" ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.109 }, { "id": "doc-08", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-207", "supplier": "Fictional Vendor 8", "date": "2026-10-08", "currency": "EUR", "subtotal": "359.00", "tax": "75.39", "total": "434.39", "reference": "ORDER-607" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.264 }, { "id": "doc-09", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-208", "supplier": "Fictional Vendor 9", "date": "2026-10-08", "currency": "EUR", "subtotal": "396.00", "tax": "83.16", "total": "1.00", "reference": "ORDER-608" }, "validation_issues": [ "Amount discrepancy: subtotal + tax = 479.16, but total = 1.00." ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.253 }, { "id": "doc-10", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-209", "supplier": "Fictional Vendor 10", "date": "2026-10-08", "currency": "EUR", "subtotal": "433.00", "tax": "90.93", "total": "523.93", "reference": "ORDER-609" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.248 }, { "id": "doc-11", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-210", "supplier": "Fictional Vendor 11", "date": "2026-02-30", "currency": "EUR", "subtotal": "470.00", "tax": "98.70", "total": "568.70", "reference": "ORDER-610" }, "validation_issues": [ "Date must be a real calendar date in YYYY-MM-DD format." ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.252 }, { "id": "doc-12", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-211", "supplier": "Fictional Vendor 12", "date": "2026-10-08", "currency": "EUR", "subtotal": "507.00", "tax": "106.47", "total": "613.47", "reference": "ORDER-611" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.315 }, { "id": "doc-13", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-212", "supplier": "", "date": "2026-10-08", "currency": "EUR", "subtotal": "544.00", "tax": "114.24", "total": "1.00", "reference": "ORDER-612" }, "validation_issues": [ "Missing required field: supplier", "Amount discrepancy: subtotal + tax = 658.24, but total = 1.00." ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.141 }, { "id": "doc-14", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-213", "supplier": "Fictional Vendor 14", "date": "2026-10-08", "currency": "EUR", "subtotal": "581.00", "tax": "122.01", "total": "703.01", "reference": "ORDER-613" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.343 }, { "id": "doc-15", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-214", "supplier": "Fictional Vendor 15", "date": "2026-10-08", "currency": "EUR", "subtotal": "618.00", "tax": "129.78", "total": "747.78", "reference": "ORDER-614" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.298 }, { "id": "doc-16", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-215", "supplier": "Fictional Vendor 16", "date": "2026-02-30", "currency": "EUR", "subtotal": "655.00", "tax": "137.55", "total": "792.55", "reference": "ORDER-615" }, "validation_issues": [ "Date must be a real calendar date in YYYY-MM-DD format." ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.278 }, { "id": "doc-17", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-216", "supplier": "Fictional Vendor 17", "date": "2026-10-08", "currency": "EUR", "subtotal": "692.00", "tax": "145.32", "total": "1.00", "reference": "ORDER-616" }, "validation_issues": [ "Amount discrepancy: subtotal + tax = 837.32, but total = 1.00." ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.306 }, { "id": "doc-18", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-217", "supplier": "Fictional Vendor 18", "date": "2026-10-08", "currency": "EUR", "subtotal": "729.00", "tax": "153.09", "total": "882.09", "reference": "ORDER-617" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.33 }, { "id": "doc-19", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-218", "supplier": "", "date": "2026-10-08", "currency": "EUR", "subtotal": "766.00", "tax": "160.86", "total": "926.86", "reference": "ORDER-618" }, "validation_issues": [ "Missing required field: supplier" ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.123 }, { "id": "doc-20", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-219", "supplier": "Fictional Vendor 20", "date": "2026-10-08", "currency": "EUR", "subtotal": "803.00", "tax": "168.63", "total": "971.63", "reference": "ORDER-619" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.307 }, { "id": "doc-21", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-220", "supplier": "Fictional Vendor 21", "date": "2026-02-30", "currency": "EUR", "subtotal": "840.00", "tax": "176.40", "total": "1.00", "reference": "ORDER-620" }, "validation_issues": [ "Date must be a real calendar date in YYYY-MM-DD format.", "Amount discrepancy: subtotal + tax = 1016.40, but total = 1.00." ], "raster_ocr": false, "completed": true, "elapsed_seconds": 2.236 }, { "id": "doc-22", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-221", "supplier": "Fictional Vendor 22", "date": "2026-10-08", "currency": "EUR", "subtotal": "877.00", "tax": "184.17", "total": "1061.17", "reference": "ORDER-621" }, "validation_issues": [], "raster_ocr": true, "completed": true, "elapsed_seconds": 6.569 }, { "id": "doc-23", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-222", "supplier": "Fictional Vendor 23", "date": "2026-10-08", "currency": "EUR", "subtotal": "914.00", "tax": "191.94", "total": "1105.94", "reference": "ORDER-622" }, "validation_issues": [], "raster_ocr": true, "completed": true, "elapsed_seconds": 6.3 }, { "id": "doc-24", "measurements": { "fields_correct": 8, "fields_total": 8, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "EVAL-223", "supplier": "Fictional Vendor 24", "date": "2026-10-08", "currency": "EUR", "subtotal": "951.00", "tax": "199.71", "total": "1150.71", "reference": "ORDER-623" }, "validation_issues": [], "raster_ocr": true, "completed": true, "elapsed_seconds": 6.329 }, { "id": "delivery-01", "measurements": { "fields_correct": 6, "fields_total": 6, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "DEL-100", "supplier": "Fictional Carrier 1", "date": "2026-10-08", "reference": "ORDER-800", "quantity": "4", "recipient": "" }, "validation_issues": [ "Missing required field: recipient" ], "raster_ocr": false, "completed": true, "elapsed_seconds": 1.684 }, { "id": "form-01", "measurements": { "fields_correct": 6, "fields_total": 6, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "FORM-100", "date": "2026-10-08", "requester": "Reviewer 1", "email": "reviewer1@example.invalid", "reference": "ASSET-800", "description": "Inspect access reader" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 1.797 }, { "id": "delivery-02", "measurements": { "fields_correct": 6, "fields_total": 6, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "DEL-101", "supplier": "Fictional Carrier 2", "date": "2026-10-08", "reference": "ORDER-801", "quantity": "5", "recipient": "Reviewer 2" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 1.768 }, { "id": "form-02", "measurements": { "fields_correct": 6, "fields_total": 6, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "FORM-101", "date": "2026-10-08", "requester": "Reviewer 2", "email": "reviewer2@example.invalid", "reference": "ASSET-801", "description": "Inspect access reader" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 1.815 }, { "id": "delivery-03", "measurements": { "fields_correct": 6, "fields_total": 6, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "DEL-102", "supplier": "Fictional Carrier 3", "date": "2026-10-08", "reference": "ORDER-802", "quantity": "6", "recipient": "Reviewer 3" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 1.801 }, { "id": "form-03", "measurements": { "fields_correct": 6, "fields_total": 6, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "FORM-102", "date": "2026-10-08", "requester": "Reviewer 3", "email": "reviewer3@example.invalid", "reference": "ASSET-802", "description": "Inspect access reader" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 1.842 }, { "id": "delivery-04", "measurements": { "fields_correct": 6, "fields_total": 6, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "DEL-103", "supplier": "Fictional Carrier 4", "date": "2026-10-08", "reference": "ORDER-803", "quantity": "7", "recipient": "Reviewer 4" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 1.85 }, { "id": "form-04", "measurements": { "fields_correct": 6, "fields_total": 6, "all_fields_correct": true, "amount_validation_correct": true, "date_validation_correct": true }, "extracted_fields": { "document_number": "FORM-103", "date": "2026-10-08", "requester": "Reviewer 4", "email": "reviewer4@example.invalid", "reference": "ASSET-803", "description": "Inspect access reader" }, "validation_issues": [], "raster_ocr": false, "completed": true, "elapsed_seconds": 1.844 } ], "limits": "Small authored synthetic regression set. Literal comparisons are proxies, not semantic entailment. Synthetic speech is not representative of real-call noise, accents or diarization.", "latency": { "median_case_seconds": 2.25, "p95_case_seconds": 6.329, "includes": "Full HTTP case, including upload/ASR/OCR where applicable; first requests may include lazy loading." } }