{
  "timestamp_utc": "2026-10-08T20:46:56.959609+00:00",
  "dataset_version": "synthetic-regression-v1",
  "metric_definition": "retrieval source/page recall, expected fact coverage and abstention; no semantic judge",
  "real_inference": true,
  "fictional_data_only": true,
  "case_count": 24,
  "completed": 24,
  "elapsed_seconds": 23.68,
  "environment": {
    "gpu": "NVIDIA GeForce RTX 5060 Ti, 595.84, 16311 MiB",
    "python": "3.12.3",
    "model_runtime": "Existing Qwen 3.5 9B GGUF / MiniLM L12-H384 / Whisper small.en / Kokoro af_heart; only applicable capabilities used",
    "providers_enabled": {
      "llm": true,
      "embedding": true,
      "rerank": true,
      "asr": false,
      "tts": false,
      "ocr": true
    }
  },
  "metrics": {
    "expected_source_and_page": {
      "passed": 16,
      "total": 16
    },
    "expected_fact_coverage": {
      "passed": 16,
      "total": 16
    },
    "citation_ids_valid": {
      "passed": 24,
      "total": 24
    },
    "correct_abstention": {
      "passed": 8,
      "total": 8
    }
  },
  "cases": [
    {
      "id": "rag-01",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "Amber opens at 09:00.",
      "sources": [
        {
          "document": "amber-policy.pdf",
          "page": 2,
          "id": "u2"
        },
        {
          "document": "amber-policy.pdf",
          "page": 1,
          "id": "u1"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "haven-policy.pdf",
          "page": 1,
          "id": "u15"
        }
      ],
      "vector_cache_hit": false,
      "completed": true,
      "elapsed_seconds": 4.096
    },
    {
      "id": "rag-02",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "Amber opens at 09:00.",
      "sources": [
        {
          "document": "amber-policy.pdf",
          "page": 2,
          "id": "u2"
        },
        {
          "document": "amber-policy.pdf",
          "page": 1,
          "id": "u1"
        },
        {
          "document": "grove-policy.pdf",
          "page": 1,
          "id": "u13"
        },
        {
          "document": "flint-policy.pdf",
          "page": 1,
          "id": "u11"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.985
    },
    {
      "id": "rag-03",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "Birch closes at 17:30.",
      "sources": [
        {
          "document": "birch-policy.pdf",
          "page": 2,
          "id": "u4"
        },
        {
          "document": "birch-policy.pdf",
          "page": 1,
          "id": "u3"
        },
        {
          "document": "cedar-policy.pdf",
          "page": 1,
          "id": "u5"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.963
    },
    {
      "id": "rag-04",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "17:30",
      "sources": [
        {
          "document": "birch-policy.pdf",
          "page": 2,
          "id": "u4"
        },
        {
          "document": "birch-policy.pdf",
          "page": 1,
          "id": "u3"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "elm-policy.pdf",
          "page": 1,
          "id": "u9"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.856
    },
    {
      "id": "rag-05",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "Cedar inspection interval is 14 days.",
      "sources": [
        {
          "document": "cedar-policy.pdf",
          "page": 2,
          "id": "u6"
        },
        {
          "document": "cedar-policy.pdf",
          "page": 1,
          "id": "u5"
        },
        {
          "document": "elm-policy.pdf",
          "page": 2,
          "id": "u10"
        },
        {
          "document": "haven-policy.pdf",
          "page": 2,
          "id": "u16"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.921
    },
    {
      "id": "rag-06",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "14 days",
      "sources": [
        {
          "document": "cedar-policy.pdf",
          "page": 2,
          "id": "u6"
        },
        {
          "document": "cedar-policy.pdf",
          "page": 1,
          "id": "u5"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "haven-policy.pdf",
          "page": 1,
          "id": "u15"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.819
    },
    {
      "id": "rag-07",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "28 degrees Celsius",
      "sources": [
        {
          "document": "dune-policy.pdf",
          "page": 2,
          "id": "u8"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "elm-policy.pdf",
          "page": 2,
          "id": "u10"
        },
        {
          "document": "haven-policy.pdf",
          "page": 2,
          "id": "u16"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.84
    },
    {
      "id": "rag-08",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "28 degrees Celsius",
      "sources": [
        {
          "document": "dune-policy.pdf",
          "page": 2,
          "id": "u8"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "cedar-policy.pdf",
          "page": 1,
          "id": "u5"
        },
        {
          "document": "haven-policy.pdf",
          "page": 1,
          "id": "u15"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.855
    },
    {
      "id": "rag-09",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "45 minutes",
      "sources": [
        {
          "document": "elm-policy.pdf",
          "page": 2,
          "id": "u10"
        },
        {
          "document": "elm-policy.pdf",
          "page": 1,
          "id": "u9"
        },
        {
          "document": "grove-policy.pdf",
          "page": 2,
          "id": "u14"
        },
        {
          "document": "amber-policy.pdf",
          "page": 1,
          "id": "u1"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.847
    },
    {
      "id": "rag-10",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "45 minutes",
      "sources": [
        {
          "document": "elm-policy.pdf",
          "page": 2,
          "id": "u10"
        },
        {
          "document": "elm-policy.pdf",
          "page": 1,
          "id": "u9"
        },
        {
          "document": "amber-policy.pdf",
          "page": 1,
          "id": "u1"
        },
        {
          "document": "grove-policy.pdf",
          "page": 2,
          "id": "u14"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.851
    },
    {
      "id": "rag-11",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "Morgan Lee",
      "sources": [
        {
          "document": "flint-policy.pdf",
          "page": 2,
          "id": "u12"
        },
        {
          "document": "flint-policy.pdf",
          "page": 1,
          "id": "u11"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "haven-policy.pdf",
          "page": 1,
          "id": "u15"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.832
    },
    {
      "id": "rag-12",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "Morgan Lee",
      "sources": [
        {
          "document": "flint-policy.pdf",
          "page": 2,
          "id": "u12"
        },
        {
          "document": "flint-policy.pdf",
          "page": 1,
          "id": "u11"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "cedar-policy.pdf",
          "page": 1,
          "id": "u5"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.846
    },
    {
      "id": "rag-13",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "DEMO-718",
      "sources": [
        {
          "document": "grove-policy.pdf",
          "page": 2,
          "id": "u14"
        },
        {
          "document": "grove-policy.pdf",
          "page": 1,
          "id": "u13"
        },
        {
          "document": "flint-policy.pdf",
          "page": 1,
          "id": "u11"
        },
        {
          "document": "amber-policy.pdf",
          "page": 1,
          "id": "u1"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.899
    },
    {
      "id": "rag-14",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "DEMO-718",
      "sources": [
        {
          "document": "grove-policy.pdf",
          "page": 2,
          "id": "u14"
        },
        {
          "document": "grove-policy.pdf",
          "page": 1,
          "id": "u13"
        },
        {
          "document": "amber-policy.pdf",
          "page": 1,
          "id": "u1"
        },
        {
          "document": "flint-policy.pdf",
          "page": 1,
          "id": "u11"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.891
    },
    {
      "id": "rag-15",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "GBP",
      "sources": [
        {
          "document": "haven-policy.pdf",
          "page": 2,
          "id": "u16"
        },
        {
          "document": "haven-policy.pdf",
          "page": 1,
          "id": "u15"
        },
        {
          "document": "cedar-policy.pdf",
          "page": 1,
          "id": "u5"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.795
    },
    {
      "id": "rag-16",
      "measurements": {
        "expected_source_and_page": true,
        "expected_fact_coverage": true,
        "correct_abstention": null,
        "citation_ids_valid": true
      },
      "answer": "Haven approved currency is GBP.",
      "sources": [
        {
          "document": "haven-policy.pdf",
          "page": 2,
          "id": "u16"
        },
        {
          "document": "haven-policy.pdf",
          "page": 1,
          "id": "u15"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "elm-policy.pdf",
          "page": 1,
          "id": "u9"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.884
    },
    {
      "id": "rag-17",
      "measurements": {
        "expected_source_and_page": null,
        "expected_fact_coverage": null,
        "correct_abstention": true,
        "citation_ids_valid": true
      },
      "answer": "Insufficient evidence",
      "sources": [
        {
          "document": "amber-policy.pdf",
          "page": 2,
          "id": "u2"
        },
        {
          "document": "amber-policy.pdf",
          "page": 1,
          "id": "u1"
        },
        {
          "document": "grove-policy.pdf",
          "page": 1,
          "id": "u13"
        },
        {
          "document": "elm-policy.pdf",
          "page": 1,
          "id": "u9"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.763
    },
    {
      "id": "rag-18",
      "measurements": {
        "expected_source_and_page": null,
        "expected_fact_coverage": null,
        "correct_abstention": true,
        "citation_ids_valid": true
      },
      "answer": "Insufficient evidence",
      "sources": [
        {
          "document": "birch-policy.pdf",
          "page": 2,
          "id": "u4"
        },
        {
          "document": "birch-policy.pdf",
          "page": 1,
          "id": "u3"
        },
        {
          "document": "cedar-policy.pdf",
          "page": 1,
          "id": "u5"
        },
        {
          "document": "haven-policy.pdf",
          "page": 1,
          "id": "u15"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.762
    },
    {
      "id": "rag-19",
      "measurements": {
        "expected_source_and_page": null,
        "expected_fact_coverage": null,
        "correct_abstention": true,
        "citation_ids_valid": true
      },
      "answer": "Insufficient evidence",
      "sources": [
        {
          "document": "cedar-policy.pdf",
          "page": 1,
          "id": "u5"
        },
        {
          "document": "cedar-policy.pdf",
          "page": 2,
          "id": "u6"
        },
        {
          "document": "elm-policy.pdf",
          "page": 2,
          "id": "u10"
        },
        {
          "document": "haven-policy.pdf",
          "page": 2,
          "id": "u16"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.778
    },
    {
      "id": "rag-20",
      "measurements": {
        "expected_source_and_page": null,
        "expected_fact_coverage": null,
        "correct_abstention": true,
        "citation_ids_valid": true
      },
      "answer": "Insufficient evidence",
      "sources": [
        {
          "document": "dune-policy.pdf",
          "page": 2,
          "id": "u8"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "haven-policy.pdf",
          "page": 2,
          "id": "u16"
        },
        {
          "document": "cedar-policy.pdf",
          "page": 1,
          "id": "u5"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.783
    },
    {
      "id": "rag-21",
      "measurements": {
        "expected_source_and_page": null,
        "expected_fact_coverage": null,
        "correct_abstention": true,
        "citation_ids_valid": true
      },
      "answer": "Insufficient evidence",
      "sources": [
        {
          "document": "elm-policy.pdf",
          "page": 2,
          "id": "u10"
        },
        {
          "document": "elm-policy.pdf",
          "page": 1,
          "id": "u9"
        },
        {
          "document": "haven-policy.pdf",
          "page": 2,
          "id": "u16"
        },
        {
          "document": "cedar-policy.pdf",
          "page": 1,
          "id": "u5"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.777
    },
    {
      "id": "rag-22",
      "measurements": {
        "expected_source_and_page": null,
        "expected_fact_coverage": null,
        "correct_abstention": true,
        "citation_ids_valid": true
      },
      "answer": "Insufficient evidence",
      "sources": [
        {
          "document": "flint-policy.pdf",
          "page": 2,
          "id": "u12"
        },
        {
          "document": "flint-policy.pdf",
          "page": 1,
          "id": "u11"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "haven-policy.pdf",
          "page": 1,
          "id": "u15"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.772
    },
    {
      "id": "rag-23",
      "measurements": {
        "expected_source_and_page": null,
        "expected_fact_coverage": null,
        "correct_abstention": true,
        "citation_ids_valid": true
      },
      "answer": "The administrator password for Grove is not mentioned in the provided passages.",
      "sources": [
        {
          "document": "grove-policy.pdf",
          "page": 2,
          "id": "u14"
        },
        {
          "document": "grove-policy.pdf",
          "page": 1,
          "id": "u13"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "haven-policy.pdf",
          "page": 1,
          "id": "u15"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.92
    },
    {
      "id": "rag-24",
      "measurements": {
        "expected_source_and_page": null,
        "expected_fact_coverage": null,
        "correct_abstention": true,
        "citation_ids_valid": true
      },
      "answer": "Insufficient evidence",
      "sources": [
        {
          "document": "haven-policy.pdf",
          "page": 2,
          "id": "u16"
        },
        {
          "document": "haven-policy.pdf",
          "page": 1,
          "id": "u15"
        },
        {
          "document": "dune-policy.pdf",
          "page": 1,
          "id": "u7"
        },
        {
          "document": "elm-policy.pdf",
          "page": 1,
          "id": "u9"
        }
      ],
      "vector_cache_hit": true,
      "completed": true,
      "elapsed_seconds": 0.796
    }
  ],
  "limits": "Small authored synthetic regression set. Literal comparisons are proxies, not semantic entailment. Synthetic speech is not representative of real-call noise, accents or diarization.",
  "latency": {
    "median_case_seconds": 0.847,
    "p95_case_seconds": 0.985,
    "includes": "Full HTTP case, including upload/ASR/OCR where applicable; first requests may include lazy loading."
  }
}