{ "timestamp_utc": "2026-10-08T21:02:37.521809+00:00", "dataset_version": "synthetic-regression-v3-english", "metric_definition": "normalized word error rate on synthetic English speech and summary literal fact coverage", "real_inference": true, "fictional_data_only": true, "case_count": 12, "completed": 12, "elapsed_seconds": 33.91, "environment": { "gpu": "NVIDIA GeForce RTX 5060 Ti, 595.84, 16311 MiB", "python": "3.12.3", "model_runtime": "Existing Qwen 3.5 9B GGUF / MiniLM L12-H384 / Whisper small.en / Kokoro af_heart; only applicable capabilities used", "providers_enabled": { "llm": true, "embedding": false, "rerank": false, "asr": true, "tts": false, "ocr": false } }, "metrics": { "all_expected_facts_found": { "passed": 12, "total": 12 }, "no_prohibited_phrase": { "passed": 12, "total": 12 }, "normalized_wer": { "word_errors": 4, "reference_words": 238, "rate": 0.0168 } }, "cases": [ { "id": "audio-01", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 0, "reference_words": 24 }, "generated_content": "A water leak was reported in the demonstration workshop. The pipe was inspected and the visit date was confirmed. A technician is scheduled to arrive in the morning.", "transcript": "There is a water leak in the demonstration workshop. Inspect the pipe and confirm the visit date. The technician will arrive in the morning.", "completed": true, "elapsed_seconds": 12.658 }, { "id": "audio-02", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 0, "reference_words": 21 }, "generated_content": "A door in a demonstration building closes slowly, requiring inspection of the mechanism and notification to a technician, with the inspection scheduled for Monday.", "transcript": "The door in the demonstration building closes slowly. Inspect the mechanism and notify the technician. The inspection is scheduled for Monday.", "completed": true, "elapsed_seconds": 1.898 }, { "id": "audio-03", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 0, "reference_words": 18 }, "generated_content": "The customer requests an equipment inspection, but the visit requires confirmation and a spare part is currently unavailable.", "transcript": "The customer requests an equipment inspection. The visit still needs confirmation. A spare part is not yet available.", "completed": true, "elapsed_seconds": 1.625 }, { "id": "audio-04", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 0, "reference_words": 22 }, "generated_content": "A water leak is reported in the demonstration workshop, requiring inspection of the pipe and confirmation of the visit date, while noting that the equipment remains switched on.", "transcript": "There is a water leak in the demonstration workshop. Inspect the pipe and confirm the visit date. The equipment remains switched on.", "completed": true, "elapsed_seconds": 2.196 }, { "id": "audio-05", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 0, "reference_words": 22 }, "generated_content": "A door in a demonstration building closes slowly, the mechanism is inspected with no damage found to the floor, and a technician is notified.", "transcript": "The door in the demonstration building closes slowly. Inspect the mechanism and notify the technician. There is no damage to the floor.", "completed": true, "elapsed_seconds": 1.91 }, { "id": "audio-06", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 2, "reference_words": 16 }, "generated_content": "A customer has requested an equipment inspection, but the visit date has not yet been confirmed. The next step is to confirm the date via email.", "transcript": "The customer requests an equipment inspection. The visit still needs confirmation. Confirm the date by e-mail.", "completed": true, "elapsed_seconds": 1.744 }, { "id": "audio-07", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 2, "reference_words": 21 }, "generated_content": "A water leak is reported in the demonstration workshop, requiring inspection of the pipe to confirm the visit date and checking of the shut-off valve.", "transcript": "There is a water leak in the demonstration workshop. Inspect the pipe and confirm the visit date. Check the shut-off valve.", "completed": true, "elapsed_seconds": 1.936 }, { "id": "audio-08", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 0, "reference_words": 22 }, "generated_content": "A door in a demonstration building closes slowly, requiring inspection by two people and notification to a technician.", "transcript": "The door in the demonstration building closes slowly. Inspect the mechanism and notify the technician. Two people are needed for the inspection.", "completed": true, "elapsed_seconds": 1.894 }, { "id": "audio-09", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 0, "reference_words": 16 }, "generated_content": "A customer has requested an equipment inspection, but the visit requires confirmation, and the maintenance team is responsible for the task.", "transcript": "The customer requests an equipment inspection. The visit still needs confirmation. The maintenance team is responsible.", "completed": true, "elapsed_seconds": 1.963 }, { "id": "audio-10", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 0, "reference_words": 22 }, "generated_content": "A water leak was reported in the demonstration workshop. An inspection of the pipe was conducted to confirm the visit date, and it was noted that no parts had been replaced.", "transcript": "There is a water leak in the demonstration workshop. Inspect the pipe and confirm the visit date. No parts have been replaced.", "completed": true, "elapsed_seconds": 2.345 }, { "id": "audio-11", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 0, "reference_words": 20 }, "generated_content": "A door in a demonstration building closes slowly, requiring inspection of the mechanism and notification to a technician, though the task remains pending.", "transcript": "The door in the demonstration building closes slowly. Inspect the mechanism and notify the technician. The task is still pending.", "completed": true, "elapsed_seconds": 1.913 }, { "id": "audio-12", "measurements": { "expected_facts_found": 2, "expected_facts_total": 2, "all_expected_facts_found": true, "no_prohibited_phrase": true, "word_errors": 0, "reference_words": 14 }, "generated_content": "A customer requests an equipment inspection, but the visit requires confirmation before proceeding, and the outcome must be documented.", "transcript": "The customer requests an equipment inspection. The visit still needs confirmation. Document the outcome.", "completed": true, "elapsed_seconds": 1.722 } ], "limits": "Small authored synthetic regression set. Literal comparisons are proxies, not semantic entailment. Synthetic speech is not representative of real-call noise, accents or diarization.", "latency": { "median_case_seconds": 1.911, "p95_case_seconds": 12.658, "includes": "Full HTTP case, including upload/ASR/OCR where applicable; first requests may include lazy loading." } }