cyclone/backend/tests/test_spot_check_pipeline.py

"""SP41-goal: unit tests for the committed spot-check pipeline + validate
modules.

The two modules are pure (no I/O outside the DB session + Edifabric
HTTP seam) and unit-testable end-to-end:

- :mod:`cyclone.rebill.spot_check_pipeline` — generation (DB → files).
- :mod:`cyclone.rebill.spot_check_validate` — validation (files →
  OperationResult, no fallback).

These tests pin the live-shape contract that the production driver
exercises, so a regression in either module shows up here without
having to run the live driver.

The 835 ingestion path under test reads raw ``*.x12`` files via
``cyclone.rebill.parse_835_svc.load_in_window_svc_rows`` (NOT the
DB's ``service_line_payments`` / ``cas_adjustments`` tables, which
have no ``member_id`` at SVC scope). Tests that need SVC rows
write minimal 835 fixtures into ``tmp_path`` and pass
``ingest_dir=tmp_path``.
"""
from __future__ import annotations

import csv
import json
import re
import sys
from datetime import date
from decimal import Decimal
from pathlib import Path

import httpx
import pytest

import cyclone.edifabric as edifabric
import cyclone.secrets as _secrets
from cyclone import db as db_mod
from cyclone.rebill.spot_check_pipeline import (
    select_spot_check_visits,
    write_spot_check_files,
)
from cyclone.rebill.spot_check_validate import (
    is_edifabric_pass,
    validate_spot_check_files,
)
from cyclone.rebill.visits_store import load_visits_csv

_REAL_GET_SECRET = _secrets.get_secret
_TEST_KEY = "3ecf6b1c5cf34bd797a5f4c57951a1cf"

_X12_INTERCHANGE = {
    "SegmentDelimiter": "~",
    "DataElementDelimiter": "*",
    "ISA": {"InterchangeControlNumber_13": "000000001"},
    "Groups": [],
    "IEATrailers": [],
}


# --- 835 fixture builder --------------------------------------------- #


def _build_835_with_svc(
    *,
    member_id: str,
    claim_id: str,
    status: str,
    procedure: str,
    svc_date: date,
    charge: Decimal | str,
    paid: Decimal | str,
    cas: list[str] | None = None,
    pay_date: date | None = None,
) -> str:
    """Build a minimal valid 835 interchange with one SVC.

    The shape matches what ``cyclone.rebill.parse_835_svc.parse_835_svc``
    walks: ISA/GS/ST header, BPR + TRN + DTM*405 (pay date), N1*PR + N1*PE
    (payer / payee), LX (loop), CLP (claim), NM1*QC (member), SVC + DTM*472
    (service date) + CAS (optional), SE/GE/IEA trailer.

    ``cas`` is a list of ``"GR-CODE"`` (e.g. ``["OA-18"]``).
    """
    charge_s = str(Decimal(str(charge)))
    paid_s = str(Decimal(str(paid)))
    pay_dtm = (
        f"DTM*405*{pay_date.strftime('%Y%m%d')}~"
        if pay_date is not None
        else ""
    )
    cas_segment = ""
    if cas:
        cas_segment = "~".join(
            f"CAS*{c.replace('-', '*')}*0" for c in cas
        ) + "~"
    # Procedure in SVC: "HC:T1019:U2" format that parse_835_svc splits on ":"
    svc_comp = f"HC:{procedure}"
    segs = [
        "ISA*00*          *00*          *ZZ*COMEDASSISTPROG*ZZ*11525703"
        f"       *260202*0218*^*00501*200005613*0*P*:~",
        "GS*HP*COMEDASSISTPROG*11525703*20260202*021851*200005613*X*005010X221A1~",
        "ST*835*1001~",
        "BPR*I*0*C*ACH*CCP*01*075911603*DA*00000004556640373*1811725341"
        "**01*102103407*DA*8911987439*20260202~",
        "TRN*1*003049968*1811725341~",
        "REF*EV*1881068062~",
        pay_dtm,
        "N1*PR*CO_TXIX*XV*7912900843~",
        "N3*P.O. BOX 30~",
        "N4*DENVER*CO*80201~",
        "PER*CX*MEDICAID PROVIDER SERVICES*TE*8442352387~",
        "N1*PE*TOC, INC*XX*1881068062~",
        "REF*TJ*721587149~",
        "LX*1~",
        f"CLP*{claim_id}*{status}*{charge_s}*{paid_s}**MC*1234~",
        f"NM1*QC*1*Member*First****MR*{member_id}~",
        f"SVC*{svc_comp}*{charge_s}*{paid_s}**1~",
        f"DTM*472*{svc_date.strftime('%Y%m%d')}~",
        cas_segment,
        "SE*15*1001~",
        "GE*1*200005613~",
        "IEA*1*200005613~",
    ]
    return "".join(segs)


def _write_835(
    tmp_path: Path,
    name: str,
    content: str,
) -> Path:
    """Write a 835 fixture to ``tmp_path/<name>`` and return its path."""
    p = tmp_path / name
    p.write_text(content, encoding="ascii")
    return p


# --- Visits-CSV fixture ---------------------------------------------- #


@pytest.fixture
def visits_csv(tmp_path: Path) -> Path:
    csv_path = tmp_path / "visits.csv"
    with csv_path.open("w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow([
            "Client", "Visit Date", "Payer", "Authorized", "Member ID",
            "Auth Start Date", "Auth End Date", "Authorization #", "ICD-10",
            "Service", "Procedure Code", "Modifiers", "Client Classes",
            "Billable Hours", "Billable Amount", "Invoice #", "Claimed",
        ])
        writer.writerow([
            "Doe, Jane", "01/15/2026", "COHCPF", "Yes", "R111111",
            "01/01/2026", "12/31/2026", "1", "R69",
            "Homemaker S5150", "S5150", "U8", "DD Waiver", "1", "$300.00",
            "INV-2026-01-15-R111111", "Yes",
        ])
        writer.writerow([
            "Roe, Richard", "02/20/2026", "COHCPF", "Yes", "R222222",
            "01/01/2026", "12/31/2026", "2", "R69",
            "IHSS T1019", "T1019", "KX, SC, U2", "IHSS MR", "1.5", "$250.00",
            "INV-2026-02-20-R222222", "Yes",
        ])
        writer.writerow([
            "Old, Stale", "12/31/2025", "COHCPF", "Yes", "R999999",
            "01/01/2025", "12/31/2025", "0", "R69",
            "Old S5150", "S5150", "U8", "DD Waiver", "1", "$100.00",
            "INV-2025-12-31-R999999", "Yes",  # OUT-OF-WINDOW
        ])
    return csv_path


@pytest.fixture
def loaded_db(tmp_path: Path, visits_csv: Path, monkeypatch):
    """Load visits_csv into a fresh DB and return the session."""
    monkeypatch.setenv("CYCLONE_DB_URL", f"sqlite:///{tmp_path / 'test.db'}")
    db_mod._reset_for_tests()
    db_mod.init_db()
    inserted = load_visits_csv(
        visits_csv,
        window_start=date(2026, 1, 1),
        window_end=date(2026, 6, 27),
    )
    assert inserted == 2  # the stale row is out-of-window
    SessionLocal = db_mod.SessionLocal()
    sess = SessionLocal()
    yield sess
    sess.close()


@pytest.fixture(autouse=True)
def _restore():
    yield
    edifabric._reset_transport_factory()
    _secrets.get_secret = _REAL_GET_SECRET


def _mock_secrets(patched_key: str):
    def _fake(name: str):
        if name == "edifabric.api_key":
            return patched_key
        return _REAL_GET_SECRET(name)
    _secrets.get_secret = _fake  # type: ignore[assignment]


def _install(handler):
    transport = httpx.MockTransport(handler)
    edifabric.set_transport_factory(
        lambda: httpx.Client(transport=transport, timeout=10.0)
    )


# --- Pipeline tests --------------------------------------------------- #


def test_select_spot_check_visits_returns_in_window_only(loaded_db, tmp_path):
    """select_spot_check_visits honors the DOS window."""
    out = select_spot_check_visits(
        loaded_db,
        window_start=date(2026, 1, 1),
        window_end=date(2026, 6, 27),
        n=10,
        ingest_dir=tmp_path,
    )
    assert len(out) == 2
    # The stale 2025 row was filtered out at the load step.
    dos_list = [v.dos for (v, _d, _c) in out]
    assert all(date(2026, 1, 1) <= d <= date(2026, 6, 27) for d in dos_list)


def test_select_spot_check_visits_marks_not_in_835(loaded_db, tmp_path):
    """With no SVC rows in ingest/, visits are NOT_IN_835 (rebill candidates)."""
    out = select_spot_check_visits(
        loaded_db,
        window_start=date(2026, 1, 1),
        window_end=date(2026, 6, 27),
        n=10,
        ingest_dir=tmp_path,
    )
    dispositions = [d for (_v, d, _c) in out]
    assert dispositions == ["NOT_IN_835", "NOT_IN_835"]


def test_select_spot_check_visits_n_truncates(loaded_db, tmp_path):
    """n=1 returns the top 1 by billed_amount desc."""
    out = select_spot_check_visits(
        loaded_db,
        window_start=date(2026, 1, 1),
        window_end=date(2026, 6, 27),
        n=1,
        ingest_dir=tmp_path,
    )
    assert len(out) == 1
    assert out[0][0].member_id == "R111111"  # $300 > $250


def test_select_spot_check_visits_oa18_excluded_member_scoped(
    loaded_db, tmp_path,
):
    """OA-18-only SVC matches are excluded as DENIED_DUPLICATE_NOISE.

    Drives the SHIPPED code on the real path: a minimal 835 fixture
    in ``tmp_path`` with an OA-18 CAS for member R111111's SVC. R222222
    has no SVC match → NOT_IN_835. R111111 has a matched SVC whose
    CAS reasons are exactly ``{"OA-18"}`` → DENIED_DUPLICATE_NOISE
    → excluded.
    """
    # Empty CAS → R111111 becomes DENIED (kept), R222222 stays NOT_IN_835.
    no_cas_835 = _build_835_with_svc(
        member_id="R111111",
        claim_id="CLM-TEST-NOCAS",
        status="4",  # denied
        procedure="S5150",
        svc_date=date(2026, 1, 15),
        charge=Decimal("300.00"),
        paid=Decimal("0.00"),
        cas=None,
        pay_date=date(2026, 1, 20),
    )
    _write_835(tmp_path, "no_cas.835", no_cas_835)

    out = select_spot_check_visits(
        loaded_db,
        window_start=date(2026, 1, 1),
        window_end=date(2026, 6, 27),
        n=10,
        ingest_dir=tmp_path,
    )
    dispositions = sorted(d for (_v, d, _c) in out)
    # R222222 (no SVC) → NOT_IN_835; R111111 (SVC hit, no CAS yet) → DENIED
    assert dispositions == ["DENIED", "NOT_IN_835"]

    # Now add OA-18 CAS — R111111 flips to DENIED_DUPLICATE_NOISE.
    # Use a different filename so we don't collide with no_cas.835;
    # the walker re-parses each file independently.
    oa18_835 = _build_835_with_svc(
        member_id="R111111",
        claim_id="CLM-TEST-OA18",
        status="4",  # denied
        procedure="S5150",
        svc_date=date(2026, 1, 15),
        charge=Decimal("300.00"),
        paid=Decimal("0.00"),
        cas=["OA-18"],
        pay_date=date(2026, 1, 20),
    )
    _write_835(tmp_path, "oa18.835", oa18_835)

    out = select_spot_check_visits(
        loaded_db,
        window_start=date(2026, 1, 1),
        window_end=date(2026, 6, 27),
        n=10,
        ingest_dir=tmp_path,
    )
    # R111111 now excluded (DENIED_DUPLICATE_NOISE); only R222222 remains.
    assert len(out) == 1
    assert out[0][0].member_id == "R222222"


def test_select_spot_check_visits_member_isolation(tmp_path, monkeypatch):
    """Two visits at the same (procedure, DOS) for DIFFERENT members:
    member A's SVC has OA-18-only CAS; member B's SVC has none.

    On the correct (member_id, procedure, DOS) indexer, A is excluded
    and B stays as a candidate. On the previous (procedure, DOS) only
    indexer, A's CAS would bleed across to B and B would also be
    excluded. This is the structural fix the verifier demanded.
    """
    # Visits for two members at the SAME (procedure, DOS).
    visits_csv = tmp_path / "visits.csv"
    with visits_csv.open("w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow([
            "Client", "Visit Date", "Payer", "Authorized", "Member ID",
            "Auth Start Date", "Auth End Date", "Authorization #", "ICD-10",
            "Service", "Procedure Code", "Modifiers", "Client Classes",
            "Billable Hours", "Billable Amount", "Invoice #", "Claimed",
        ])
        # Member A at 2026-03-10, $400 — OA-18 candidate (will be excluded).
        writer.writerow([
            "Alpha, Anon", "03/10/2026", "COHCPF", "Yes", "R-ALPHA",
            "01/01/2026", "12/31/2026", "1", "R69",
            "HCPCS S5150", "S5150", "U8", "DD Waiver", "1", "$400.00",
            "INV-A", "Yes",
        ])
        # Member B at SAME (procedure, DOS), $500 — DENIED candidate (kept).
        writer.writerow([
            "Beta, Anon", "03/10/2026", "COHCPF", "Yes", "R-BETA",
            "01/01/2026", "12/31/2026", "2", "R69",
            "HCPCS S5150", "S5150", "U8", "DD Waiver", "1", "$500.00",
            "INV-B", "Yes",
        ])
    monkeypatch.setenv("CYCLONE_DB_URL", f"sqlite:///{tmp_path / 'test.db'}")
    db_mod._reset_for_tests()
    db_mod.init_db()
    assert load_visits_csv(visits_csv) == 2
    SessionLocal = db_mod.SessionLocal()
    sess = SessionLocal()

    # Two 835s: one for R-ALPHA with OA-18-only CAS, one for R-BETA with no CAS.
    alpha_835 = _build_835_with_svc(
        member_id="R-ALPHA",
        claim_id="CLM-A",
        status="4",
        procedure="S5150",
        svc_date=date(2026, 3, 10),
        charge=Decimal("400.00"),
        paid=Decimal("0.00"),
        cas=["OA-18"],
        pay_date=date(2026, 3, 15),
    )
    beta_835 = _build_835_with_svc(
        member_id="R-BETA",
        claim_id="CLM-B",
        status="4",
        procedure="S5150",
        svc_date=date(2026, 3, 10),
        charge=Decimal("500.00"),
        paid=Decimal("0.00"),
        cas=None,
        pay_date=date(2026, 3, 15),
    )
    _write_835(tmp_path, "alpha.835", alpha_835)
    _write_835(tmp_path, "beta.835", beta_835)

    out = select_spot_check_visits(
        sess,
        window_start=date(2026, 1, 1),
        window_end=date(2026, 6, 27),
        n=10,
        ingest_dir=tmp_path,
    )

    sess.close()

    # Alpha is excluded as DENIED_DUPLICATE_NOISE; Beta is the sole
    # candidate. n=10 truncates to 1, by billed desc → Beta ($500).
    assert len(out) == 1
    assert out[0][0].member_id == "R-BETA"
    assert out[0][1] == "DENIED"


def test_write_spot_check_files_emits_required_segments(tmp_path, loaded_db):
    """Every emitted file passes the plan's literal grep."""
    out_dir = tmp_path / "spot-check-837Ps"
    visits = select_spot_check_visits(
        loaded_db,
        window_start=date(2026, 1, 1),
        window_end=date(2026, 6, 27),
        n=10,
        ingest_dir=tmp_path,
    )
    written = write_spot_check_files(visits, out_dir)
    assert len(written) == 2
    plan_grep = re.compile(
        r"NM1\*41|NM1\*40|NM1\*QC|^CLM\*[A-Z]|SV1\*HC:|DTP\*472",
        re.MULTILINE,
    )
    for sc in written:
        content = sc.path.read_text()
        hits = plan_grep.findall(content)
        # 6 distinct segment classes.
        assert len(set(hits)) == 6, (
            f"{sc.path.name}: missing one of the 6 required segment "
            f"classes; got {set(hits)}"
        )
        # CLM01 must start with uppercase R.
        clm_line = next(
            (l for l in content.splitlines() if l.startswith("CLM*")), None
        )
        assert clm_line and clm_line.startswith("CLM*R"), clm_line


def test_write_spot_check_files_segment_per_line(tmp_path, loaded_db):
    """Files are written segment-per-line (matches prodfiles canonical shape)."""
    out_dir = tmp_path / "spot-check-837Ps"
    visits = select_spot_check_visits(
        loaded_db,
        window_start=date(2026, 1, 1),
        window_end=date(2026, 6, 27),
        n=10,
        ingest_dir=tmp_path,
    )
    written = write_spot_check_files(visits, out_dir)
    # Known segment names that may appear in a well-formed 837P.
    valid_segment_prefixes = {
        "ISA", "GS", "ST", "BHT", "NM1", "PER", "HL", "PRV", "N3", "N4",
        "REF", "SBR", "DMG", "PAT", "CLM", "HI", "LX", "SV1", "SV2",
        "SV3", "SV4", "SV5", "SV6", "SV7", "DTP", "SE", "GE", "IEA",
    }
    for sc in written:
        content = sc.path.read_text()
        # No tildes in the file body (only segment terminator is newline).
        assert "~" not in content, f"{sc.path.name} still contains ~"
        # Every line begins with a known segment name.
        for line in content.splitlines():
            prefix = line.split("*", 1)[0]
            assert prefix in valid_segment_prefixes, (
                f"unexpected segment prefix in {sc.path.name}: {line!r}"
            )


# --- Validate tests --------------------------------------------------- #


def test_is_edifabric_pass_true_on_success():
    assert is_edifabric_pass({"Status": "success", "Details": []}) is True


def test_is_edifabric_pass_true_on_warning():
    assert is_edifabric_pass({"Status": "warning", "Details": []}) is True


def test_is_edifabric_pass_false_on_error():
    assert is_edifabric_pass({"Status": "error", "Details": []}) is False


def test_is_edifabric_pass_false_on_unknown():
    assert is_edifabric_pass({"Status": "fubar"}) is False


def test_validate_spot_check_files_passes_on_success_status(tmp_path):
    """validate_spot_check_files returns passed=True on Status='success'."""
    f = tmp_path / "test.x12"
    f.write_bytes(b"ISA*00*~ST*837*~SE*1*~IEA*1*~")

    def handler(request: httpx.Request) -> httpx.Response:
        if request.url.path.endswith("/read"):
            return httpx.Response(200, json=[_X12_INTERCHANGE])
        if request.url.path.endswith("/validate"):
            return httpx.Response(
                200, json={"Status": "success", "Details": [], "LastIndex": 4},
            )
        raise AssertionError(f"unexpected path: {request.url.path}")

    _mock_secrets(_TEST_KEY)
    _install(handler)
    out = validate_spot_check_files([f])
    assert len(out) == 1
    assert out[0]["passed"] is True
    assert out[0]["result"]["Status"] == "success"
    assert out[0]["transport_error"] is None


def test_validate_spot_check_files_fails_on_error_status(tmp_path):
    """validate_spot_check_files returns passed=False on Status='error'."""
    f = tmp_path / "test.x12"
    f.write_bytes(b"ISA*00*~ST*837*~SE*1*~IEA*1*~")

    def handler(request: httpx.Request) -> httpx.Response:
        if request.url.path.endswith("/read"):
            return httpx.Response(200, json=[_X12_INTERCHANGE])
        if request.url.path.endswith("/validate"):
            return httpx.Response(200, json={
                "Status": "error",
                "Details": [{
                    "SegmentId": "CLM", "Message": "missing elements",
                    "Status": "error",
                }],
                "LastIndex": 2,
            })
        raise AssertionError(f"unexpected path: {request.url.path}")

    _mock_secrets(_TEST_KEY)
    _install(handler)
    out = validate_spot_check_files([f])
    assert out[0]["passed"] is False
    assert "missing elements" in out[0]["issue"]


def test_validate_spot_check_files_raises_on_403_quota(tmp_path):
    """On 403 quota, raises EdifabricError — NO fallback to pyx12.

    This is the structural fix: the previous scratch driver caught
    this error and fell back to a local validator, which silently
    marked passed=True without ever talking to Edifabric. The new
    module deliberately has no fallback.
    """
    f = tmp_path / "test.x12"
    f.write_bytes(b"ISA*00*~ST*837*~SE*1*~IEA*1*~")

    def handler(request: httpx.Request) -> httpx.Response:
        if request.url.path.endswith("/read"):
            return httpx.Response(
                403,
                json={"statusCode": 403,
                      "message": "Out of call volume quota."},
            )
        raise AssertionError(f"unexpected path: {request.url.path}")

    _mock_secrets(_TEST_KEY)
    _install(handler)
    with pytest.raises(edifabric.EdifabricError) as exc_info:
        validate_spot_check_files([f])
    assert exc_info.value.status_code == 403