{
  "protocol_version": "1.0.0",
  "executed_at_utc": "2026-10-21T05:23:48.264716+00:00",
  "environment": {
    "python": "3.12.2",
    "os": "Darwin",
    "architecture": "arm64"
  },
  "scope": "Deterministic lexical retrieval and citation-field/exact-quote validation only. No LLM, embeddings, hosted API, authentication system or semantic entailment model. Query metadata and expected labels are supplied before execution.",
  "protocol_sha256": "7592746f2f18696bbdfdc3625b0a1a93bcc45d314a10e61135ac41fd86269ef8",
  "runner_sha256": "269e3eb77af851b168b2a02f020e4acef2f793e9a4fd0c8568ce34bad8fd66d9",
  "summary": {
    "unfiltered_control": {
      "passed": 3,
      "total": 8
    },
    "metadata_filtered": {
      "passed": 8,
      "total": 8
    },
    "citation_checks": {
      "passed": 6,
      "total": 6
    }
  },
  "retrieval": [
    {
      "case_id": "current_standard",
      "mode": "unfiltered_control",
      "question": "SampleNote standard retention days",
      "as_of": "2026-10-21",
      "role": "public",
      "expected_doc_id": "sample-retention-v2",
      "actual_doc_id": "sample-retention-v1",
      "expected_value": "14 days",
      "actual_value": "30 days",
      "passed": false
    },
    {
      "case_id": "current_standard",
      "mode": "metadata_filtered",
      "question": "SampleNote standard retention days",
      "as_of": "2026-10-21",
      "role": "public",
      "expected_doc_id": "sample-retention-v2",
      "actual_doc_id": "sample-retention-v2",
      "expected_value": "14 days",
      "actual_value": "14 days",
      "passed": true
    },
    {
      "case_id": "historical_standard",
      "mode": "unfiltered_control",
      "question": "SampleNote standard retention days",
      "as_of": "2026-03-01",
      "role": "public",
      "expected_doc_id": "sample-retention-v1",
      "actual_doc_id": "sample-retention-v1",
      "expected_value": "30 days",
      "actual_value": "30 days",
      "passed": true
    },
    {
      "case_id": "historical_standard",
      "mode": "metadata_filtered",
      "question": "SampleNote standard retention days",
      "as_of": "2026-03-01",
      "role": "public",
      "expected_doc_id": "sample-retention-v1",
      "actual_doc_id": "sample-retention-v1",
      "expected_value": "30 days",
      "actual_value": "30 days",
      "passed": true
    },
    {
      "case_id": "invoice_exception",
      "mode": "unfiltered_control",
      "question": "SampleNote invoice retention days",
      "as_of": "2026-10-21",
      "role": "public",
      "expected_doc_id": "sample-invoice-v2",
      "actual_doc_id": "sample-invoice-v1",
      "expected_value": "90 days",
      "actual_value": "60 days",
      "passed": false
    },
    {
      "case_id": "invoice_exception",
      "mode": "metadata_filtered",
      "question": "SampleNote invoice retention days",
      "as_of": "2026-10-21",
      "role": "public",
      "expected_doc_id": "sample-invoice-v2",
      "actual_doc_id": "sample-invoice-v2",
      "expected_value": "90 days",
      "actual_value": "90 days",
      "passed": true
    },
    {
      "case_id": "public_export",
      "mode": "unfiltered_control",
      "question": "SampleNote export batch limit",
      "as_of": "2026-10-21",
      "role": "public",
      "expected_doc_id": "sample-public-export-v2",
      "actual_doc_id": "sample-staff-export-v2",
      "expected_value": "100 records",
      "actual_value": "5000 records",
      "passed": false
    },
    {
      "case_id": "public_export",
      "mode": "metadata_filtered",
      "question": "SampleNote export batch limit",
      "as_of": "2026-10-21",
      "role": "public",
      "expected_doc_id": "sample-public-export-v2",
      "actual_doc_id": "sample-public-export-v2",
      "expected_value": "100 records",
      "actual_value": "100 records",
      "passed": true
    },
    {
      "case_id": "staff_export",
      "mode": "unfiltered_control",
      "question": "SampleNote export batch limit",
      "as_of": "2026-10-21",
      "role": "staff",
      "expected_doc_id": "sample-staff-export-v2",
      "actual_doc_id": "sample-staff-export-v2",
      "expected_value": "5000 records",
      "actual_value": "5000 records",
      "passed": true
    },
    {
      "case_id": "staff_export",
      "mode": "metadata_filtered",
      "question": "SampleNote export batch limit",
      "as_of": "2026-10-21",
      "role": "staff",
      "expected_doc_id": "sample-staff-export-v2",
      "actual_doc_id": "sample-staff-export-v2",
      "expected_value": "5000 records",
      "actual_value": "5000 records",
      "passed": true
    },
    {
      "case_id": "no_answer",
      "mode": "unfiltered_control",
      "question": "Does SampleNote support SAML?",
      "as_of": "2026-10-21",
      "role": "public",
      "expected_doc_id": null,
      "actual_doc_id": "sample-format-v2",
      "expected_value": null,
      "actual_value": "CSV",
      "passed": false
    },
    {
      "case_id": "no_answer",
      "mode": "metadata_filtered",
      "question": "Does SampleNote support SAML?",
      "as_of": "2026-10-21",
      "role": "public",
      "expected_doc_id": null,
      "actual_doc_id": null,
      "expected_value": null,
      "actual_value": null,
      "passed": true
    },
    {
      "case_id": "different_product",
      "mode": "unfiltered_control",
      "question": "OtherNote standard retention days",
      "as_of": "2026-10-21",
      "role": "public",
      "expected_doc_id": "other-retention-v1",
      "actual_doc_id": "other-retention-v1",
      "expected_value": "14 days",
      "actual_value": "14 days",
      "passed": true
    },
    {
      "case_id": "different_product",
      "mode": "metadata_filtered",
      "question": "OtherNote standard retention days",
      "as_of": "2026-10-21",
      "role": "public",
      "expected_doc_id": "other-retention-v1",
      "actual_doc_id": "other-retention-v1",
      "expected_value": "14 days",
      "actual_value": "14 days",
      "passed": true
    },
    {
      "case_id": "future_policy",
      "mode": "unfiltered_control",
      "question": "SampleNote standard retention days",
      "as_of": "2027-02-01",
      "role": "public",
      "expected_doc_id": "sample-retention-v3",
      "actual_doc_id": "sample-retention-v1",
      "expected_value": "7 days",
      "actual_value": "30 days",
      "passed": false
    },
    {
      "case_id": "future_policy",
      "mode": "metadata_filtered",
      "question": "SampleNote standard retention days",
      "as_of": "2027-02-01",
      "role": "public",
      "expected_doc_id": "sample-retention-v3",
      "actual_doc_id": "sample-retention-v3",
      "expected_value": "7 days",
      "actual_value": "7 days",
      "passed": true
    }
  ],
  "citations": [
    {
      "case_id": "valid_current_quote",
      "expected_accepted": true,
      "actual_accepted": true,
      "expected_reason": "accepted",
      "actual_reason": "accepted",
      "passed": true
    },
    {
      "case_id": "altered_quote",
      "expected_accepted": false,
      "actual_accepted": false,
      "expected_reason": "quote_not_in_source",
      "actual_reason": "quote_not_in_source",
      "passed": true
    },
    {
      "case_id": "wrong_value",
      "expected_accepted": false,
      "actual_accepted": false,
      "expected_reason": "claim_value_mismatch",
      "actual_reason": "claim_value_mismatch",
      "passed": true
    },
    {
      "case_id": "expired_source",
      "expected_accepted": false,
      "actual_accepted": false,
      "expected_reason": "source_not_eligible",
      "actual_reason": "source_not_eligible",
      "passed": true
    },
    {
      "case_id": "unauthorized_source",
      "expected_accepted": false,
      "actual_accepted": false,
      "expected_reason": "source_not_eligible",
      "actual_reason": "source_not_eligible",
      "passed": true
    },
    {
      "case_id": "unknown_source",
      "expected_accepted": false,
      "actual_accepted": false,
      "expected_reason": "unknown_source",
      "actual_reason": "unknown_source",
      "passed": true
    }
  ]
}