{
  "timestamp_utc": "2026-10-08T21:09:59.004665+00:00",
  "dataset_version": "synthetic-regression-v3-english",
  "metric_definition": "exact intent/device lookup or refusal; deliberately unresolved requests retained",
  "real_inference": true,
  "fictional_data_only": true,
  "case_count": 12,
  "completed": 12,
  "elapsed_seconds": 1.27,
  "environment": {
    "gpu": "NVIDIA GeForce RTX 5060 Ti, 595.84, 16311 MiB",
    "python": "3.12.3",
    "model_runtime": "Existing Qwen 3.5 9B GGUF / MiniLM L12-H384 / Whisper small.en / Kokoro af_heart; only applicable capabilities used",
    "providers_enabled": {
      "llm": true,
      "embedding": false,
      "rerank": false,
      "asr": true,
      "tts": true,
      "ocr": false
    }
  },
  "metrics": {
    "device_selection_correct": {
      "passed": 12,
      "total": 12
    },
    "intent_correct": {
      "passed": 12,
      "total": 12
    }
  },
  "cases": [
    {
      "id": "assistant-01",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "lookup",
        "device_id": "DEV-202",
        "candidates": []
      },
      "answer": "Temperature sensor (DEV-202) in room 02 is online. The latest reading is 23.4 degrees Celsius. Recommended next step: Continue routine monitoring. This is a fictional inventory snapshot from 2026-09-18 09:12.",
      "completed": true,
      "elapsed_seconds": 0.005
    },
    {
      "id": "assistant-02",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "lookup",
        "device_id": "DEV-301",
        "candidates": []
      },
      "answer": "Access reader (DEV-301) in room 03 is offline. No heartbeat has been recorded since 08:50. Recommended next step: Ask a technician to inspect power and connectivity. This is a fictional inventory snapshot from 2026-09-18 08:50.",
      "completed": true,
      "elapsed_seconds": 0.004
    },
    {
      "id": "assistant-03",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "lookup",
        "device_id": "DEV-201",
        "candidates": []
      },
      "answer": "Cooling unit (DEV-201) in room 02 is warning. Inlet filter inspection is due. Current pressure is 2.1 bar. Recommended next step: Schedule a filter inspection during the next maintenance window. This is a fictional inventory snapshot from 2026-09-18 09:10.",
      "completed": true,
      "elapsed_seconds": 0.003
    },
    {
      "id": "assistant-04",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "lookup",
        "device_id": "DEV-202",
        "candidates": []
      },
      "answer": "Temperature sensor (DEV-202) in room 02 is online. The latest reading is 23.4 degrees Celsius. Recommended next step: Continue routine monitoring. This is a fictional inventory snapshot from 2026-09-18 09:12.",
      "completed": true,
      "elapsed_seconds": 0.004
    },
    {
      "id": "assistant-05",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "unsupported",
        "device_id": null,
        "candidates": []
      },
      "answer": "This assistant can only read inventory. No device was changed. Ask for a device status or latest reading.",
      "completed": true,
      "elapsed_seconds": 0.003
    },
    {
      "id": "assistant-06",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "unsupported",
        "device_id": null,
        "candidates": []
      },
      "answer": "This assistant can only read inventory. No device was changed. Ask for a device status or latest reading.",
      "completed": true,
      "elapsed_seconds": 0.003
    },
    {
      "id": "assistant-07",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "unsupported",
        "device_id": null,
        "candidates": []
      },
      "answer": "This assistant can only read inventory. No device was changed. Ask for a device status or latest reading.",
      "completed": true,
      "elapsed_seconds": 0.004
    },
    {
      "id": "assistant-08",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "unsupported",
        "device_id": null,
        "candidates": []
      },
      "answer": "This assistant can only read inventory. No device was changed. Ask for a device status or latest reading.",
      "completed": true,
      "elapsed_seconds": 0.003
    },
    {
      "id": "assistant-09",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "lookup",
        "device_id": "DEV-999",
        "candidates": []
      },
      "answer": "That device does not exist in the fictional inventory. Try DEV-201, DEV-202 or DEV-301.",
      "completed": true,
      "elapsed_seconds": 0.004
    },
    {
      "id": "assistant-10",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "clarify",
        "device_id": null,
        "candidates": [
          "DEV-201",
          "DEV-202",
          "DEV-301"
        ]
      },
      "answer": "Which device do you mean? DEV-201 · Cooling unit · room 02; DEV-202 · Temperature sensor · room 02; DEV-301 · Access reader · room 03",
      "completed": true,
      "elapsed_seconds": 0.638
    },
    {
      "id": "assistant-11",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "clarify",
        "device_id": null,
        "candidates": [
          "DEV-201",
          "DEV-202",
          "DEV-301"
        ]
      },
      "answer": "Which device do you mean? DEV-201 · Cooling unit · room 02; DEV-202 · Temperature sensor · room 02; DEV-301 · Access reader · room 03",
      "completed": true,
      "elapsed_seconds": 0.527
    },
    {
      "id": "assistant-12",
      "measurements": {
        "device_selection_correct": true,
        "intent_correct": true
      },
      "intent": {
        "action": "lookup",
        "device_id": "DEV-201",
        "candidates": []
      },
      "answer": "Cooling unit (DEV-201) in room 02 is warning. Inlet filter inspection is due. Current pressure is 2.1 bar. Recommended next step: Schedule a filter inspection during the next maintenance window. This is a fictional inventory snapshot from 2026-09-18 09:10.",
      "completed": true,
      "elapsed_seconds": 0.004
    }
  ],
  "limits": "Small authored synthetic regression set. Literal comparisons are proxies, not semantic entailment. Synthetic speech is not representative of real-call noise, accents or diarization.",
  "latency": {
    "median_case_seconds": 0.004,
    "p95_case_seconds": 0.638,
    "includes": "Full HTTP case, including upload/ASR/OCR where applicable; first requests may include lazy loading."
  }
}