feat(sp41): Edifabric gate on every emitted 837P (quarantine on failure)

Replace the run_rebill() .touch() placeholders for Pipeline A and Pipeline B with real serialize → validate → write paths. Pipeline A: build a ClaimOutput adapter from the canonical RebillClaim fields (claim_id/member_id/procedure/svc_date/charge) — the per-claim envelope context (provider NPI, subscriber name, payer name) comes from the original claim being replaced and isn't carried forward by the rebill pipeline, so the adapter emits safe placeholders and lets the SP40 serializer fallbacks fill PER / SBR09. Then serialize_837 → cyclone.edifabric.validate_edi → write to pipeline-a/ on success or quarantine/ on Status=='error'. Pipeline B: serialize_member_week_batch(b, tpid=tpid) (the Task 14 overload — takes a MemberWeekBatch, not a ClaimOutput) → cyclone.edifabric.validate_edi → write to pipeline-b/ or quarantine/. Filename disambiguation: build_outbound_filename embeds a millisecond timestamp; multiple files emitted in the same millisecond would collide. Append per-claim / per-batch suffix so the audit trail (claim_id ↔ file / batch_key ↔ file) stays one-to-one. Edifabric unavailability (no API key, network error, 5xx) fails OPEN with a WARNING log: the file lands in the pipeline dir, matches the SP40 dev/CI posture for boxes without a paid key. Quarantine only on Edifabric Status=='error'. New tests in backend/tests/test_rebill_run.py cover: real 837P file emission (non-empty, ISA..IEA envelope) for both pipelines, quarantine on Edifabric error, fail-open on Edifabric unavailability, HCPF-spec filename prefix preserved, per-claim / per-batch disambiguation suffix present, original claim_id preserved in Pipeline A CLM01.
2026-07-07 23:25:15 -06:00
parent 9beab5b4fc
commit b0d03bc10e
2 changed files with 505 additions and 25 deletions
@@ -0,0 +1,328 @@
+"""SP41 Task 13 — Edifabric validation gate on emitted rebill files.
+
+Covers the new emit path in ``cyclone.rebill.run.run_rebill``:
+
+  * Real 837P files are written (not ``Path.touch()`` placeholders).
+  * Files pass through ``cyclone.edifabric.validate_edi`` before they
+    land in ``<out>/pipeline-{a,b}/``; failures go to
+    ``<out>/quarantine/``.
+  * When Edifabric is unavailable (no API key, network error, etc.),
+    the gate fails open with a WARNING log and emits to the pipeline
+    dirs (matches the SP40 dev/CI posture).
+
+Each test builds its own visits CSV + 835 stub in ``tmp_path`` and
+invokes ``run_rebill`` directly (no HTTP). The Edifabric API is mocked
+via ``unittest.mock.patch`` on
+``cyclone.rebill.run._edifabric.validate_edi`` so no live HTTP hits the
+network.
+"""
+from __future__ import annotations
+
+import csv
+from datetime import date
+from decimal import Decimal
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from cyclone.rebill.run import RunResult, run_rebill
+
+
+# --------------------------------------------------------------------------- #
+# Fixtures / helpers
+# --------------------------------------------------------------------------- #
+
+
+def _write_visits_csv(path: Path, rows: list[tuple[str, str, str, str]]) -> Path:
+    """rows: list of (dos_mmddyyyy, member, procedure, billed_str)."""
+    with path.open("w", newline="") as f:
+        w = csv.writer(f)
+        w.writerow(["Visit Date", "Member ID", "Procedure Code", "Billable Amount"])
+        for r in rows:
+            w.writerow(r)
+    return path
+
+
+def _stub_835_with_denied_svc(
+    ingest_dir: Path,
+    member: str,
+    procedure: str,
+    svc_date: date,
+    claim_id: str = "ORIG-1",
+    carc_group: str = "CO",
+    carc_reason: str = "29",
+) -> Path:
+    """An 835 with a single denied SVC so the visit lands in Pipeline A.
+
+    The rebill pipeline reuses the SVC's ``claim_id`` as the Pipeline-A
+    RebillClaim's ``claim_id`` (preserves the freq-7 anchor). The visit
+    must match (member_id, procedure, svc_date) so the reconcile step
+    pairs them into DENIED + CARC REBILL.
+
+    The 835 shape mirrors ``tests/fixtures/835_sample_svc_with_member.txt``:
+    CLP02 = 4 (Denied), NM1*QC.NM109 = member_id, then SVC → DTM*472 → CAS.
+
+    Segments are concatenated without ``\\n`` separators — the parser
+    splits on ``~`` only, and a leading ``\\n`` makes ``elems[0]`` an
+    empty string so the segment-name match silently drops the segment.
+    """
+    ingest_dir.mkdir(exist_ok=True)
+    svc_date_str = svc_date.strftime("%Y%m%d")
+    segs = [
+        "ISA*00*          *00*          *ZZ*SENDER         *ZZ*RECEIVER       *260627*1200*^*00501*000000001*0*P*:~",
+        "GS*HC*SENDER*RECEIVER*20260627*1200*1*X*005010X221A1~",
+        "ST*835*0001~",
+        "BPR*I*0*C*ACH*CCP*01*021000021*DA*123456789*1512345678**01*021000021*DA*123456789*20260101~",
+        "TRN*1*TRACE01*1512345678~",
+        "DTM*405*20260118~",
+        "N1*PR*COLORADO MEDICAL ASSISTANCE PROGRAM*XV*COMEDASSISTPROG~",
+        "N3*PO BOX 1100~",
+        "N4*DENVER*CO*80202~",
+        "REF*2U*7912900843~",
+        "LX*1~",
+        # CLP02=4 means Denied → reconciles to DENIED → Pipeline A.
+        f"CLP*{claim_id}*4*100*0**MC*2026029105200*11*1~",
+        # NM1*QC.NM109 is the member_id; parse_835_svc propagates it to
+        # the SVC row so the visit-side reconcile key matches.
+        f"NM1*QC*1*PATIENT*NAME****MR*{member}~",
+        f"SVC*HC:{procedure}*100*0*UN*1~",
+        f"DTM*472*{svc_date_str}~",
+        # CAS group code + reason → CARC REBILL (not EXCLUDED).
+        f"CAS*{carc_group}*{carc_reason}*100~",
+        "SE*14*0001~",
+        "GE*1*1~",
+        "IEA*1*000000001~",
+    ]
+    body = "".join(segs)
+    p = ingest_dir / "x.835"
+    p.write_text(body)
+    return p
+
+
+def _stub_empty_835(ingest_dir: Path) -> Path:
+    """Empty 835 → rebill's parse_835_svc emits zero SVCs."""
+    ingest_dir.mkdir(exist_ok=True)
+    p = ingest_dir / "x.835"
+    p.write_text("ST*835*0001~SE*0*0001~")
+    return p
+
+
+# --------------------------------------------------------------------------- #
+# Tests
+# --------------------------------------------------------------------------- #
+
+
+def test_run_rebill_emits_real_files_for_pipeline_b(tmp_path):
+    """Pipeline-B happy path: 1 in-window visit, 0 SVCs → 1 real 837P
+    file in pipeline-b/ (non-empty, contains ISA..IEA), 0 files in
+    quarantine/.
+
+    Edifabric is patched to return ``success`` so the file passes the
+    gate cleanly. The fixture is the same shape as the existing
+    ``test_post_rebill_runs_and_returns_summary_path`` happy-path test.
+    """
+    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
+        ("06/27/2026", "J813715", "T1019", "$2.32"),
+    ])
+    ingest_dir = _stub_empty_835(tmp_path / "ingest")
+    out_dir = tmp_path / "out"
+
+    fake_result = {"Status": "success", "Details": [], "LastIndex": 20}
+    with patch(
+        "cyclone.rebill.run._edifabric.validate_edi",
+        return_value=fake_result,
+    ) as mock_validate:
+        result = run_rebill(
+            window_start=date(2026, 1, 1),
+            window_end=date(2026, 6, 27),
+            override_filing=False,
+            visits_csv_path=str(visits_path),
+            ingest_dir=str(ingest_dir),
+            out_dir=str(out_dir),
+        )
+
+    # Pipeline B emitted exactly one file (matches the existing
+    # ``len(body["pipeline_b_files"]) == 1`` assertion in test_api_rebill).
+    assert len(result.pipeline_b_files) == 1
+    b_path = result.pipeline_b_files[0]
+    assert b_path.exists(), b_path
+    assert b_path.stat().st_size > 0, "Pipeline-B file must be non-empty"
+
+    # The file is a real 837P envelope (starts with ISA, ends with IEA).
+    body = b_path.read_bytes()
+    assert body.startswith(b"ISA*"), body[:32]
+    assert b"IEA*" in body, body[-32:]
+
+    # Edifabric gate was called exactly once for the one Pipeline-B batch.
+    assert mock_validate.call_count == 1
+
+    # HCPF-spec filename prefix is preserved.
+    assert b_path.name.startswith("tp11525703-837P-"), b_path.name
+    # Per-batch disambiguation suffix is present so same-millisecond
+    # collisions don't overwrite each other.
+    assert "J813715-2026-W26" in b_path.name, b_path.name
+
+    # Quarantine is empty (no validation failures).
+    q_files = list((out_dir / "quarantine").glob("*.837"))
+    assert q_files == [], q_files
+
+
+def test_run_rebill_quarantines_on_edifabric_error(tmp_path):
+    """When validate_edi returns Status='error', the file lands in
+    ``quarantine/`` (NOT in ``pipeline-b/``).
+
+    Exercises both the Edifabric-rejected path AND the case-isolation
+    that splits good files from bad files in the same run.
+    """
+    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
+        ("06/27/2026", "J813715", "T1019", "$2.32"),
+        ("06/27/2026", "J999999", "T1019", "$2.32"),  # different member
+    ])
+    ingest_dir = _stub_empty_835(tmp_path / "ingest")
+    out_dir = tmp_path / "out"
+
+    # First call (J813715) returns error → quarantine.
+    # Second call (J999999) returns success → pipeline-b.
+    fake_results = iter([
+        {
+            "Status": "error",
+            "Details": [
+                {"SegmentId": "PER", "Message": "PER-04 is required", "Status": "error"},
+            ],
+            "LastIndex": 5,
+        },
+        {"Status": "success", "Details": [], "LastIndex": 20},
+    ])
+
+    def _side_effect(_body):
+        return next(fake_results)
+
+    with patch(
+        "cyclone.rebill.run._edifabric.validate_edi",
+        side_effect=_side_effect,
+    ) as mock_validate:
+        result = run_rebill(
+            window_start=date(2026, 1, 1),
+            window_end=date(2026, 6, 27),
+            override_filing=False,
+            visits_csv_path=str(visits_path),
+            ingest_dir=str(ingest_dir),
+            out_dir=str(out_dir),
+        )
+
+    # Only the success batch should land in pipeline-b/.
+    assert len(result.pipeline_b_files) == 1, (
+        f"only the success batch should land in pipeline-b/, got "
+        f"{[str(p) for p in result.pipeline_b_files]}"
+    )
+    # One file in quarantine (the rejected batch).
+    q_files = list((out_dir / "quarantine").glob("*.837"))
+    assert len(q_files) == 1, q_files
+    # Quarantine filename keys off the batch (member_id-W{iso_week:02d}).
+    assert "J813715" in q_files[0].name, q_files[0].name
+    assert q_files[0].stat().st_size > 0, "quarantined file must be non-empty"
+
+    # validate_edi called twice — once per Pipeline-B batch.
+    assert mock_validate.call_count == 2
+
+
+def test_run_rebill_fails_open_when_edifabric_unavailable(tmp_path):
+    """When validate_edi raises EdifabricError (no API key, network
+    error, 5xx), the file lands in the pipeline dir with a WARNING log.
+
+    Matches the SP40 dev/CI posture: tests / dev boxes don't have a
+    real API key, so the gate must NOT block the rebill run.
+    """
+    from cyclone.edifabric import EdifabricError
+
+    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
+        ("06/27/2026", "J813715", "T1019", "$2.32"),
+    ])
+    ingest_dir = _stub_empty_835(tmp_path / "ingest")
+    out_dir = tmp_path / "out"
+
+    with patch(
+        "cyclone.rebill.run._edifabric.validate_edi",
+        side_effect=EdifabricError(0, "API key not configured"),
+    ) as mock_validate:
+        result = run_rebill(
+            window_start=date(2026, 1, 1),
+            window_end=date(2026, 6, 27),
+            override_filing=False,
+            visits_csv_path=str(visits_path),
+            ingest_dir=str(ingest_dir),
+            out_dir=str(out_dir),
+        )
+
+    # File emitted to pipeline-b/ (NOT to quarantine — fail-open).
+    assert len(result.pipeline_b_files) == 1
+    assert result.pipeline_b_files[0].exists()
+    # Quarantine stays empty.
+    q_files = list((out_dir / "quarantine").glob("*.837"))
+    assert q_files == [], q_files
+    # Gate was actually called (so we know the fail-open path fired,
+    # not some accidental short-circuit).
+    assert mock_validate.call_count == 1
+
+
+def test_run_rebill_emits_real_files_for_pipeline_a(tmp_path):
+    """Pipeline-A happy path: 1 in-window visit, 1 denied SVC matching
+    it → 1 real 837P file in pipeline-a/ with frequency-code 7
+    preserved.
+
+    The SVC's original claim_id (ORIG-1) carries through as the
+    Pipeline-A RebillClaim.claim_id and shows up in the emitted file's
+    CLM01 segment AND in the on-disk filename suffix.
+    """
+    svc_date = date(2026, 6, 27)
+    visits_path = _write_visits_csv(tmp_path / "visits.csv", [
+        (svc_date.strftime("%m/%d/%Y"), "J813715", "T1019", "$100.00"),
+    ])
+    ingest_dir = tmp_path / "ingest"
+    _stub_835_with_denied_svc(
+        ingest_dir,
+        member="J813715",
+        procedure="T1019",
+        svc_date=svc_date,
+        claim_id="ORIG-1",
+    )
+    out_dir = tmp_path / "out"
+
+    fake_result = {"Status": "success", "Details": [], "LastIndex": 30}
+    with patch(
+        "cyclone.rebill.run._edifabric.validate_edi",
+        return_value=fake_result,
+    ):
+        result = run_rebill(
+            window_start=date(2026, 1, 1),
+            window_end=date(2026, 6, 27),
+            override_filing=False,
+            visits_csv_path=str(visits_path),
+            ingest_dir=str(ingest_dir),
+            out_dir=str(out_dir),
+        )
+
+    # Pipeline A emitted exactly one file.
+    assert len(result.pipeline_a_files) == 1, result.pipeline_a_files
+    a_path = result.pipeline_a_files[0]
+    assert a_path.exists(), a_path
+    assert a_path.stat().st_size > 0, "Pipeline-A file must be non-empty"
+
+    body = a_path.read_bytes()
+    assert body.startswith(b"ISA*"), body[:32]
+    assert b"IEA*" in body, body[-32:]
+
+    # Original claim_submit_id is preserved in CLM01 (anchors the
+    # frequency-7 replacement).
+    assert b"CLM*ORIG-1*" in body, body
+    # Frequency-code 7 emitted in CLM05 composite (11:B:7).
+    assert b"CLM*ORIG-1*100.00***11:B:7" in body, body
+
+    # Pipeline-A filename uses HCPF-spec prefix + claim_id suffix.
+    assert a_path.name.startswith("tp11525703-837P-"), a_path.name
+    assert a_path.name.endswith("-ORIG-1.x12"), a_path.name
+
+    # Quarantine empty.
+    q_files = list((out_dir / "quarantine").glob("*.837"))
+    assert q_files == [], q_files