{ "timestamp_utc": "2026-10-08T21:06:55.286254+00:00", "dataset_version": "synthetic-regression-v3-english", "metric_definition": "literal expected fact coverage, prohibited phrase occurrences, source reference validity; synthetic audio WER subset", "real_inference": true, "fictional_data_only": true, "case_count": 24, "completed": 24, "elapsed_seconds": 85.66, "environment": { "gpu": "NVIDIA GeForce RTX 5060 Ti, 595.84, 16311 MiB", "python": "3.12.3", "model_runtime": "Existing Qwen 3.5 9B GGUF / MiniLM L12-H384 / Whisper small.en / Kokoro af_heart; only applicable capabilities used", "providers_enabled": { "llm": true, "embedding": false, "rerank": false, "asr": true, "tts": false, "ocr": false } }, "metrics": { "all_expected_facts_found": { "passed": 24, "total": 24 }, "no_prohibited_phrase": { "passed": 24, "total": 24 }, "source_references_valid": { "passed": 24, "total": 24 }, "date_conflict_detection": { "passed": 20, "total": 20 }, "normalized_wer": { "word_errors": 0, "reference_words": 85, "rate": 0.0 } }, "cases": [ { "id": "report-01", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-10 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.063 }, { "id": "report-02", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-11 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.062 }, { "id": "report-03", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-12 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.028 }, { "id": "report-04", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-13 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.111 }, { "id": "report-05", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-14 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.158 }, { "id": "report-06", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-15 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.076 }, { "id": "report-07", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-16 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.022 }, { "id": "report-08", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-17 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.046 }, { "id": "report-09", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-18 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.018 }, { "id": "report-10", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-19 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.059 }, { "id": "report-11", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-20 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 2.992 }, { "id": "report-12", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-21 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.047 }, { "id": "report-13", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-22 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.054 }, { "id": "report-14", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-23 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.026 }, { "id": "report-15", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-24 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.13 }, { "id": "report-16", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-25 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.043 }, { "id": "report-17", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-26 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.097 }, { "id": "report-18", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-27 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.031 }, { "id": "report-19", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-28 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.048 }, { "id": "report-20", "measurements": { "expected_facts_found": 4, "expected_facts_total": 4, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "date_conflict_detection": true }, "generated_content": "[N1] Cable seal has visible wear. [N1] Asset EVAL-29 passed inspection. [N2] Confirm technician availability. [N2] Replace cable seal.", "completed": true, "elapsed_seconds": 3.078 }, { "id": "report-audio-01", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "word_errors": 0, "reference_words": 24 }, "generated_content": "[N1] There is a water leak in the demonstration workshop. [N1] The technician will arrive in the morning. [N1] Inspect the pipe and confirm the visit date.", "transcript": "There is a water leak in the demonstration workshop. Inspect the pipe and confirm the visit date. The technician will arrive in the morning.", "completed": true, "elapsed_seconds": 13.061 }, { "id": "report-audio-02", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "word_errors": 0, "reference_words": 21 }, "generated_content": "[N1] The door in the demonstration building closes slowly. [N1] The inspection is scheduled for Monday. [N1] The mechanism needs to be inspected. [N1] A technician needs to be notified.", "transcript": "The door in the demonstration building closes slowly. Inspect the mechanism and notify the technician. The inspection is scheduled for Monday.", "completed": true, "elapsed_seconds": 3.742 }, { "id": "report-audio-03", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "word_errors": 0, "reference_words": 18 }, "generated_content": "[N1] A spare part is not yet available. [N1] The customer requests an equipment inspection. [N1] The visit still needs confirmation.", "transcript": "The customer requests an equipment inspection. The visit still needs confirmation. A spare part is not yet available.", "completed": true, "elapsed_seconds": 3.13 }, { "id": "report-audio-04", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "source_references_valid": true, "word_errors": 0, "reference_words": 22 }, "generated_content": "[N1] There is a water leak in the demonstration workshop. [N1] The equipment remains switched on. [N1] Inspect the pipe and confirm the visit date. [N1] The visit date needs to be confirmed. [N1] Inspect the pipe.", "transcript": "There is a water leak in the demonstration workshop. Inspect the pipe and confirm the visit date. The equipment remains switched on.", "completed": true, "elapsed_seconds": 4.444 } ], "limits": "Small authored synthetic regression set. Literal comparisons are proxies, not semantic entailment. Synthetic speech is not representative of real-call noise, accents or diarization.", "latency": { "median_case_seconds": 3.061, "p95_case_seconds": 4.444, "includes": "Full HTTP case, including upload/ASR/OCR where applicable; first requests may include lazy loading." } }