cyclone/backend/tests/test_rebill_run.py

"""SP41 Task 13 — Edifabric validation gate on emitted rebill files.

Covers the new emit path in ``cyclone.rebill.run.run_rebill``:

  * Real 837P files are written (not ``Path.touch()`` placeholders).
  * Files pass through ``cyclone.edifabric.validate_edi`` before they
    land in ``<out>/pipeline-{a,b}/``; failures go to
    ``<out>/quarantine/``.
  * When Edifabric is unavailable (no API key, network error, etc.),
    the gate fails open with a WARNING log and emits to the pipeline
    dirs (matches the SP40 dev/CI posture).

Each test builds its own visits CSV + 835 stub in ``tmp_path`` and
invokes ``run_rebill`` directly (no HTTP). The Edifabric API is mocked
via ``unittest.mock.patch`` on
``cyclone.rebill.run._edifabric.validate_edi`` so no live HTTP hits the
network.
"""
from __future__ import annotations

import csv
from datetime import date
from decimal import Decimal
from pathlib import Path
from unittest.mock import patch

from cyclone.rebill.run import run_rebill


# --------------------------------------------------------------------------- #
# Fixtures / helpers
# --------------------------------------------------------------------------- #


def _write_visits_csv(path: Path, rows: list[tuple[str, str, str, str]]) -> Path:
    """rows: list of (dos_mmddyyyy, member, procedure, billed_str)."""
    with path.open("w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["Visit Date", "Member ID", "Procedure Code", "Billable Amount"])
        for r in rows:
            w.writerow(r)
    return path


def _stub_835_with_denied_svc(
    ingest_dir: Path,
    member: str,
    procedure: str,
    svc_date: date,
    claim_id: str = "ORIG-1",
    carc_group: str = "CO",
    carc_reason: str = "29",
) -> Path:
    """An 835 with a single denied SVC so the visit lands in Pipeline A.

    The rebill pipeline reuses the SVC's ``claim_id`` as the Pipeline-A
    RebillClaim's ``claim_id`` (preserves the freq-7 anchor). The visit
    must match (member_id, procedure, svc_date) so the reconcile step
    pairs them into DENIED + CARC REBILL.

    The 835 shape mirrors ``tests/fixtures/835_sample_svc_with_member.txt``:
    CLP02 = 4 (Denied), NM1*QC.NM109 = member_id, then SVC → DTM*472 → CAS.

    Segments are concatenated without ``\\n`` separators — the parser
    splits on ``~`` only, and a leading ``\\n`` makes ``elems[0]`` an
    empty string so the segment-name match silently drops the segment.
    """
    ingest_dir.mkdir(exist_ok=True)
    svc_date_str = svc_date.strftime("%Y%m%d")
    segs = [
        "ISA*00*          *00*          *ZZ*SENDER         *ZZ*RECEIVER       *260627*1200*^*00501*000000001*0*P*:~",
        "GS*HC*SENDER*RECEIVER*20260627*1200*1*X*005010X221A1~",
        "ST*835*0001~",
        "BPR*I*0*C*ACH*CCP*01*021000021*DA*123456789*1512345678**01*021000021*DA*123456789*20260101~",
        "TRN*1*TRACE01*1512345678~",
        "DTM*405*20260118~",
        "N1*PR*COLORADO MEDICAL ASSISTANCE PROGRAM*XV*COMEDASSISTPROG~",
        "N3*PO BOX 1100~",
        "N4*DENVER*CO*80202~",
        "REF*2U*7912900843~",
        "LX*1~",
        # CLP02=4 means Denied → reconciles to DENIED → Pipeline A.
        f"CLP*{claim_id}*4*100*0**MC*2026029105200*11*1~",
        # NM1*QC.NM109 is the member_id; parse_835_svc propagates it to
        # the SVC row so the visit-side reconcile key matches.
        f"NM1*QC*1*PATIENT*NAME****MR*{member}~",
        f"SVC*HC:{procedure}*100*0*UN*1~",
        f"DTM*472*{svc_date_str}~",
        # CAS group code + reason → CARC REBILL (not EXCLUDED).
        f"CAS*{carc_group}*{carc_reason}*100~",
        "SE*14*0001~",
        "GE*1*1~",
        "IEA*1*000000001~",
    ]
    body = "".join(segs)
    p = ingest_dir / "x.835"
    p.write_text(body)
    return p


def _stub_empty_835(ingest_dir: Path) -> Path:
    """Empty 835 → rebill's parse_835_svc emits zero SVCs."""
    ingest_dir.mkdir(exist_ok=True)
    p = ingest_dir / "x.835"
    p.write_text("ST*835*0001~SE*0*0001~")
    return p


# --------------------------------------------------------------------------- #
# Tests
# --------------------------------------------------------------------------- #


def test_run_rebill_emits_real_files_for_pipeline_b(tmp_path):
    """Pipeline-B happy path: 1 in-window visit, 0 SVCs → 1 real 837P
    file in pipeline-b/ (non-empty, contains ISA..IEA), 0 files in
    quarantine/.

    Edifabric is patched to return ``success`` so the file passes the
    gate cleanly. The fixture is the same shape as the existing
    ``test_post_rebill_runs_and_returns_summary_path`` happy-path test.
    """
    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
        ("06/27/2026", "J813715", "T1019", "$2.32"),
    ])
    ingest_dir = _stub_empty_835(tmp_path / "ingest")
    out_dir = tmp_path / "out"

    fake_result = {"Status": "success", "Details": [], "LastIndex": 20}
    with patch(
        "cyclone.rebill.run._edifabric.validate_edi",
        return_value=fake_result,
    ) as mock_validate:
        result = run_rebill(
            window_start=date(2026, 1, 1),
            window_end=date(2026, 6, 27),
            override_filing=False,
            visits_csv_path=str(visits_path),
            ingest_dir=str(ingest_dir),
            out_dir=str(out_dir),
        )

    # Pipeline B emitted exactly one file (matches the existing
    # ``len(body["pipeline_b_files"]) == 1`` assertion in test_api_rebill).
    assert len(result.pipeline_b_files) == 1
    b_path = result.pipeline_b_files[0]
    assert b_path.exists(), b_path
    assert b_path.stat().st_size > 0, "Pipeline-B file must be non-empty"

    # The file is a real 837P envelope (starts with ISA, ends with IEA).
    body = b_path.read_bytes()
    assert body.startswith(b"ISA*"), body[:32]
    assert b"IEA*" in body, body[-32:]

    # Edifabric gate was called exactly once for the one Pipeline-B batch.
    assert mock_validate.call_count == 1

    # HCPF-spec filename prefix is preserved.
    assert b_path.name.startswith("tp11525703-837P-"), b_path.name
    # Per-batch disambiguation suffix is present so same-millisecond
    # collisions don't overwrite each other.
    assert "J813715-2026-W26" in b_path.name, b_path.name

    # Quarantine is empty (no validation failures).
    q_files = list((out_dir / "quarantine").glob("*.837"))
    assert q_files == [], q_files


def test_run_rebill_quarantines_on_edifabric_error(tmp_path):
    """When validate_edi returns Status='error', the file lands in
    ``quarantine/`` (NOT in ``pipeline-b/``).

    Exercises both the Edifabric-rejected path AND the case-isolation
    that splits good files from bad files in the same run.
    """
    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
        ("06/27/2026", "J813715", "T1019", "$2.32"),
        ("06/27/2026", "J999999", "T1019", "$2.32"),  # different member
    ])
    ingest_dir = _stub_empty_835(tmp_path / "ingest")
    out_dir = tmp_path / "out"

    # First call (J813715) returns error → quarantine.
    # Second call (J999999) returns success → pipeline-b.
    fake_results = iter([
        {
            "Status": "error",
            "Details": [
                {"SegmentId": "PER", "Message": "PER-04 is required", "Status": "error"},
            ],
            "LastIndex": 5,
        },
        {"Status": "success", "Details": [], "LastIndex": 20},
    ])

    def _side_effect(_body):
        return next(fake_results)

    with patch(
        "cyclone.rebill.run._edifabric.validate_edi",
        side_effect=_side_effect,
    ) as mock_validate:
        result = run_rebill(
            window_start=date(2026, 1, 1),
            window_end=date(2026, 6, 27),
            override_filing=False,
            visits_csv_path=str(visits_path),
            ingest_dir=str(ingest_dir),
            out_dir=str(out_dir),
        )

    # Only the success batch should land in pipeline-b/.
    assert len(result.pipeline_b_files) == 1, (
        f"only the success batch should land in pipeline-b/, got "
        f"{[str(p) for p in result.pipeline_b_files]}"
    )
    # One file in quarantine (the rejected batch).
    q_files = list((out_dir / "quarantine").glob("*.837"))
    assert len(q_files) == 1, q_files
    # Quarantine filename keys off the batch (member_id-W{iso_week:02d}).
    assert "J813715" in q_files[0].name, q_files[0].name
    assert q_files[0].stat().st_size > 0, "quarantined file must be non-empty"

    # validate_edi called twice — once per Pipeline-B batch.
    assert mock_validate.call_count == 2


def test_run_rebill_fails_open_when_edifabric_unavailable(tmp_path):
    """When validate_edi raises EdifabricError (no API key, network
    error, 5xx), the file lands in the pipeline dir with a WARNING log.

    Matches the SP40 dev/CI posture: tests / dev boxes don't have a
    real API key, so the gate must NOT block the rebill run.
    """
    from cyclone.edifabric import EdifabricError

    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
        ("06/27/2026", "J813715", "T1019", "$2.32"),
    ])
    ingest_dir = _stub_empty_835(tmp_path / "ingest")
    out_dir = tmp_path / "out"

    with patch(
        "cyclone.rebill.run._edifabric.validate_edi",
        side_effect=EdifabricError(0, "API key not configured"),
    ) as mock_validate:
        result = run_rebill(
            window_start=date(2026, 1, 1),
            window_end=date(2026, 6, 27),
            override_filing=False,
            visits_csv_path=str(visits_path),
            ingest_dir=str(ingest_dir),
            out_dir=str(out_dir),
        )

    # File emitted to pipeline-b/ (NOT to quarantine — fail-open).
    assert len(result.pipeline_b_files) == 1
    assert result.pipeline_b_files[0].exists()
    # Quarantine stays empty.
    q_files = list((out_dir / "quarantine").glob("*.837"))
    assert q_files == [], q_files
    # Gate was actually called (so we know the fail-open path fired,
    # not some accidental short-circuit).
    assert mock_validate.call_count == 1


def test_run_rebill_emits_real_files_for_pipeline_a(tmp_path):
    """Pipeline-A happy path: 1 in-window visit, 1 denied SVC matching
    it → 1 real 837P file in pipeline-a/ with frequency-code 7
    preserved.

    The SVC's original claim_id (ORIG-1) carries through as the
    Pipeline-A RebillClaim.claim_id and shows up in the emitted file's
    CLM01 segment AND in the on-disk filename suffix.
    """
    svc_date = date(2026, 6, 27)
    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
        (svc_date.strftime("%m/%d/%Y"), "J813715", "T1019", "$100.00"),
    ])
    ingest_dir = tmp_path / "ingest"
    _stub_835_with_denied_svc(
        ingest_dir,
        member="J813715",
        procedure="T1019",
        svc_date=svc_date,
        claim_id="ORIG-1",
    )
    out_dir = tmp_path / "out"

    fake_result = {"Status": "success", "Details": [], "LastIndex": 30}
    with patch(
        "cyclone.rebill.run._edifabric.validate_edi",
        return_value=fake_result,
    ):
        result = run_rebill(
            window_start=date(2026, 1, 1),
            window_end=date(2026, 6, 27),
            override_filing=False,
            visits_csv_path=str(visits_path),
            ingest_dir=str(ingest_dir),
            out_dir=str(out_dir),
        )

    # Pipeline A emitted exactly one file.
    assert len(result.pipeline_a_files) == 1, result.pipeline_a_files
    a_path = result.pipeline_a_files[0]
    assert a_path.exists(), a_path
    assert a_path.stat().st_size > 0, "Pipeline-A file must be non-empty"

    body = a_path.read_bytes()
    assert body.startswith(b"ISA*"), body[:32]
    assert b"IEA*" in body, body[-32:]

    # Original claim_submit_id is preserved in CLM01 (anchors the
    # frequency-7 replacement).
    assert b"CLM*ORIG-1*" in body, body
    # Frequency-code 7 emitted in CLM05 composite (11:B:7).
    assert b"CLM*ORIG-1*100.00***11:B:7" in body, body

    # Pipeline-A filename uses HCPF-spec prefix + claim_id suffix.
    assert a_path.name.startswith("tp11525703-837P-"), a_path.name
    assert a_path.name.endswith("-ORIG-1.x12"), a_path.name

    # Quarantine empty.
    q_files = list((out_dir / "quarantine").glob("*.837"))
    assert q_files == [], q_files


def test_run_rebill_quarantines_pipeline_a_on_edifabric_error(tmp_path):
    """Pipeline A path on Edifabric Status='error' → quarantine, NOT
    pipeline-a/.

    The existing ``test_run_rebill_quarantines_on_edifabric_error`` uses
    an empty 835 so all visits land in Pipeline B; this one builds a
    real 835 with a denied SVC matching the visit so the claim routes
    to Pipeline A, then asserts the rejected file lands in
    ``quarantine/`` and ``pipeline-a/`` stays empty.
    """
    svc_date = date(2026, 6, 27)
    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
        (svc_date.strftime("%m/%d/%Y"), "J813715", "T1019", "$100.00"),
    ])
    ingest_dir = tmp_path / "ingest"
    _stub_835_with_denied_svc(
        ingest_dir,
        member="J813715",
        procedure="T1019",
        svc_date=svc_date,
        claim_id="ORIG-ERROR-1",
        carc_reason="29",  # CO-29 → REBILL (not EXCLUDED)
    )
    out_dir = tmp_path / "out"

    fake_result = {
        "Status": "error",
        "Details": [
            {"SegmentId": "PER", "Message": "PER-04 is required",
             "Status": "error"},
        ],
        "LastIndex": 5,
    }
    with patch(
        "cyclone.rebill.run._edifabric.validate_edi",
        return_value=fake_result,
    ) as mock_validate:
        result = run_rebill(
            window_start=date(2026, 1, 1),
            window_end=date(2026, 6, 27),
            override_filing=False,
            visits_csv_path=str(visits_path),
            ingest_dir=str(ingest_dir),
            out_dir=str(out_dir),
        )

    # Pipeline-A path rejected → 0 files in pipeline-a/.
    assert result.pipeline_a_files == [], (
        f"Pipeline A rejected file should NOT be in pipeline-a/, got "
        f"{[str(p) for p in result.pipeline_a_files]}"
    )
    a_files = list((out_dir / "pipeline-a").glob("*"))
    assert a_files == [], a_files

    # File quarantined under the original claim_id key.
    q_files = list((out_dir / "quarantine").glob("*.837"))
    assert len(q_files) == 1, q_files
    assert "ORIG-ERROR-1" in q_files[0].name, q_files[0].name
    assert q_files[0].stat().st_size > 0, "quarantined file must be non-empty"

    # Gate was called for the one Pipeline-A claim.
    assert mock_validate.call_count == 1


def test_run_rebill_mixed_pipelines_route_correctly(tmp_path):
    """Mixed-pipeline run: one DENIED visit (Pipeline A) + one
    NOT_IN_835 visit (Pipeline B) in the same window.

    Asserts:
      - pipeline_a_files has exactly 1 entry (the denied visit)
      - pipeline_b_files has exactly 1 entry (the unmatched visit)
      - summary.csv has 2 rows (one per non-PAID visit)
    """
    svc_date = date(2026, 6, 27)
    # Visit 1: matches a denied SVC → Pipeline A.
    # Visit 2: different member entirely → NOT_IN_835 → Pipeline B.
    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
        (svc_date.strftime("%m/%d/%Y"), "J813715", "T1019", "$100.00"),
        (svc_date.strftime("%m/%d/%Y"), "J999999", "T1019", "$2.32"),
    ])
    ingest_dir = tmp_path / "ingest"
    _stub_835_with_denied_svc(
        ingest_dir,
        member="J813715",
        procedure="T1019",
        svc_date=svc_date,
        claim_id="ORIG-MIX-1",
        carc_reason="29",
    )
    out_dir = tmp_path / "out"

    fake_result = {"Status": "success", "Details": [], "LastIndex": 30}
    with patch(
        "cyclone.rebill.run._edifabric.validate_edi",
        return_value=fake_result,
    ) as mock_validate:
        result = run_rebill(
            window_start=date(2026, 1, 1),
            window_end=date(2026, 6, 27),
            override_filing=False,
            visits_csv_path=str(visits_path),
            ingest_dir=str(ingest_dir),
            out_dir=str(out_dir),
        )

    # One file each, the two pipelines route independently.
    assert len(result.pipeline_a_files) == 1, result.pipeline_a_files
    assert len(result.pipeline_b_files) == 1, result.pipeline_b_files

    # Files on disk match the returned paths.
    a_path = result.pipeline_a_files[0]
    b_path = result.pipeline_b_files[0]
    assert a_path.exists() and a_path.stat().st_size > 0
    assert b_path.exists() and b_path.stat().st_size > 0

    # Each gate call hit validate_edi exactly once (one per claim/batch).
    assert mock_validate.call_count == 2

    # summary.csv has one row per non-PAID visit.
    summary_path = out_dir / "summary.csv"
    assert summary_path.exists(), summary_path
    with summary_path.open(newline="") as f:
        rows = list(csv.DictReader(f))
    assert len(rows) == 2, rows
    dispositions = {r["disposition"] for r in rows}
    assert dispositions == {"REBILLED_A", "REBILLED_B"}, dispositions


def test_run_rebill_fail_open_logs_warning(tmp_path, caplog):
    """The Edifabric-unavailable fail-open path MUST log a WARNING so
    the operator knows the validation gate didn't actually run.

    Per SP40/SP41 contract: when ``validate_edi`` raises
    ``EdifabricError``, the file still emits to the pipeline dir but a
    WARNING is logged. This test pins that contract so a future
    refactor doesn't accidentally silence it.
    """
    import logging

    from cyclone.edifabric import EdifabricError

    caplog.set_level(logging.WARNING, logger="cyclone.rebill.run")

    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
        ("06/27/2026", "J813715", "T1019", "$2.32"),
    ])
    ingest_dir = _stub_empty_835(tmp_path / "ingest")
    out_dir = tmp_path / "out"

    with patch(
        "cyclone.rebill.run._edifabric.validate_edi",
        side_effect=EdifabricError(0, "API key not configured"),
    ):
        result = run_rebill(
            window_start=date(2026, 1, 1),
            window_end=date(2026, 6, 27),
            override_filing=False,
            visits_csv_path=str(visits_path),
            ingest_dir=str(ingest_dir),
            out_dir=str(out_dir),
        )

    # File still emits (fail-open semantics).
    assert len(result.pipeline_b_files) == 1

    # And a WARNING was logged about the unrun gate.
    warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
    assert warnings, "expected at least one WARNING record"
    assert any(
        "edifabric" in r.message.lower() or "validation" in r.message.lower()
        for r in warnings
    ), f"expected WARNING mentioning edifabric/validation, got {[r.message for r in warnings]}"