diff --git a/backend/src/cyclone/cli.py b/backend/src/cyclone/cli.py index 60c847f..0d00a36 100644 --- a/backend/src/cyclone/cli.py +++ b/backend/src/cyclone/cli.py @@ -610,6 +610,9 @@ def backfill_999_rejections(dry_run: bool, actor: str) -> None: help="Audit-log actor tag for the clearhouse.submitted events.") @click.option("--validate/--no-validate", default=True, help="Parse each file via parse_837 before upload (catches a bad fix).") +@click.option("--skip-edifabric-validate/--no-skip-edifabric-validate", default=False, + help="Skip the SP40 Edifabric pre-upload validation gate. " + "Default: gate runs (any Status='error' aborts the file).") @click.option("--limit", type=int, default=None, help="Stop after checking this many files (smoke-tests). Counts " "all attempts, not just successful uploads.") @@ -619,6 +622,7 @@ def resubmit_rejected_claims( ingest_dir: str, actor: str, validate: bool, + skip_edifabric_validate: bool, limit: int | None, reconnect_every: int, ) -> None: @@ -677,6 +681,8 @@ def resubmit_rejected_claims( skipped = 0 failed = 0 validated = 0 + validate_failed = 0 # SP40: Edifabric gate aborted these + validate_failures: list[tuple[str, list[dict]]] = [] # (filename, [Details...]) payer_cfg = PayerConfig.co_medicaid() start = time.monotonic() last_progress = start @@ -741,6 +747,44 @@ def resubmit_rejected_claims( continue validated += 1 + # SP40: Edifabric pre-upload gate. Fails closed on any + # ``Status == "error"`` item — the file is recorded as + # ``validate_failed`` and the batch continues with the other + # files. Bypass with ``--skip-edifabric-validate`` (dev only). + if not skip_edifabric_validate: + from cyclone import edifabric + try: + op_result = edifabric.validate_edi(content) + except edifabric.EdifabricError as exc: + # Edifabric upstream failure (5xx / 4xx / config) — + # treat as a hard fail for THIS file but don't stop + # the batch. Operator should inspect. + validate_failed += 1 + validate_failures.append((src.name, [ + {"SegmentId": "?", "Message": f"edifabric: {exc}", "Status": "error"}, + ])) + click.echo( + f"EDIFABRIC FAIL {src.name}: {exc.__class__.__name__}: {exc}", + err=True, + ) + continue + if op_result.get("Status") == "error": + validate_failed += 1 + details = op_result.get("Details") or [] + validate_failures.append((src.name, details)) + first = details[:5] + click.echo( + f"EDIFABRIC VALIDATION FAIL {src.name}: {len(details)} error(s)", + err=True, + ) + for d in first: + seg = d.get("SegmentId", "?") + msg = d.get("Message", "") + click.echo(f" [{d.get('Status', '?')}] {seg}: {msg}", err=True) + if len(details) > len(first): + click.echo(f" ... and {len(details) - len(first)} more", err=True) + continue + local_size = len(content) remote_path = f"{remote_root}/{src.name}" @@ -870,9 +914,24 @@ def resubmit_rejected_claims( elapsed = time.monotonic() - start click.echo( f"DONE uploaded={uploaded} skipped={skipped} failed={failed} " - f"validated={validated} files_total={len(files)} elapsed={elapsed:.1f}s" + f"validated={validated} validate_failed={validate_failed} " + f"files_total={len(files)} elapsed={elapsed:.1f}s" ) + # SP40: print Edifabric validation failures as a group so the + # operator can fix and re-run. One block, sorted by file name. + if validate_failures: + click.echo( + f"\n{len(validate_failures)} file(s) blocked by Edifabric gate:", + err=True, + ) + for name, details in sorted(validate_failures): + click.echo(f" {name}", err=True) + for d in details: + seg = d.get("SegmentId", "?") + msg = d.get("Message", "") + click.echo(f" [{d.get('Status', '?')}] {seg}: {msg}", err=True) + @main.command("submit-batch") @click.option("--ingest-dir", default="ingest", diff --git a/backend/tests/test_resubmissions_cli.py b/backend/tests/test_resubmissions_cli.py index 5860a8f..64542aa 100644 --- a/backend/tests/test_resubmissions_cli.py +++ b/backend/tests/test_resubmissions_cli.py @@ -203,6 +203,7 @@ def test_resubmit_rejected_claims_inserts_resubmission_row(tmp_path, monkeypatch "--ingest-dir", str(tmp_path), "--limit", "1", "--no-validate", + "--skip-edifabric-validate", ]) assert result.exit_code == 0, ( f"CLI exited {result.exit_code}, output={result.output!r}, " @@ -279,7 +280,254 @@ def test_resubmit_idempotency_does_not_create_duplicate_rows(tmp_path, monkeypat "--ingest-dir", str(tmp_path), "--limit", "1", "--no-validate", + "--skip-edifabric-validate", ]) # On the skip path, record_resubmission should NOT be called. assert captured == [], f"expected no record_resubmission on skip, got {captured!r}" assert "skipped" in result.output, result.output + + +# --------------------------------------------------------------------------- +# SP40: `cyclone resubmit-rejected-claims` Edifabric pre-upload gate. +# --------------------------------------------------------------------------- + + +_TEST_KEY = "test-edifabric-key-0123456789abcdef" + + +def _install_edifabric_mock(handler): + """Install a mocked Edifabric transport + get_secret stub for the + duration of a single test. Returned teardown callable can be wired + to monkeypatch via the autouse fixture pattern below.""" + import sys + import httpx + from cyclone import edifabric + import cyclone.secrets as _secrets_mod + + real_get_secret = _secrets_mod.get_secret + + def _fake_get_secret(name): + if name == "edifabric.api_key": + return _TEST_KEY + return real_get_secret(name) + + _secrets_mod.get_secret = _fake_get_secret + transport = httpx.MockTransport(handler) + edifabric.set_transport_factory( + lambda: httpx.Client(transport=transport, timeout=10.0) + ) + + def _teardown(): + _secrets_mod.get_secret = real_get_secret + edifabric._reset_transport_factory() + + return _teardown + + +def _two_files_setup(tmp_path): + """Stage two .x12 files in distinct batch dirs so the test can + observe one file's gate failure not blocking the other's upload.""" + src_text = open("tests/fixtures/co_medicaid_837p.txt").read() + a = tmp_path / "batch-aaaaaaaa-1-claims" + a.mkdir() + (a / "tp1-aaaaaaaa-1of1.x12").write_text(src_text) + b = tmp_path / "batch-bbbbbbbb-1-claims" + b.mkdir() + (b / "tp1-bbbbbbbb-1of1.x12").write_text(src_text) + return src_text + + +def test_resubmit_edifabric_gate_aborts_bad_file_continues_batch( + tmp_path, monkeypatch, +): + """SP40: when Edifabric returns Status='error' for one file, that + file is recorded as ``validate_failed`` (not uploaded, no + Resubmission row), but the batch continues with the next file.""" + import sys + import types + from cyclone.cli import resubmit_rejected_claims + + _seed_clearhouse_stub_false(tmp_path) + _two_files_setup(tmp_path) + + seen: list[str] = [] + x12 = { + "SegmentDelimiter": "~", + "DataElementDelimiter": "*", + "ISA": {"InterchangeControlNumber_13": "000000001"}, + "Groups": [], + "IEATrailers": [], + } + + def edifabric_handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/read"): + return httpx.Response(200, json=[x12]) + if request.url.path.endswith("/validate"): + # Fail the FIRST validate call; succeed on the second. + seen.append("validate") + is_bad = len(seen) == 1 + return httpx.Response(200, json={ + "Status": "error" if is_bad else "success", + "Details": ( + [ + {"SegmentId": "PER", + "Message": "PER-04 is required", + "Status": "error"}, + ] + if is_bad else [] + ), + "LastIndex": 5 if is_bad else 46, + }) + raise AssertionError(f"unexpected path: {request.url.path}") + + import httpx + teardown = _install_edifabric_mock(edifabric_handler) + try: + # Fake paramiko so SFTP doesn't actually fire. + fake_paramiko = types.ModuleType("paramiko") + + class _FakeAttr: + def __init__(self, size: int) -> None: + self.st_size = size + + class _FakeSFTP: + def __init__(self): + self.put_calls: list[str] = [] + + def stat(self, path): + raise IOError("not found") + + def put(self, local, remote): + self.put_calls.append(remote) + + fake_sftp = _FakeSFTP() + + class _FakeSSH: + def open_sftp(self): + return fake_sftp + + def set_missing_host_key_policy(self, policy): + return None + + def connect(self, *a, **kw): + return None + + def close(self): + return None + + fake_paramiko.SSHClient = lambda: _FakeSSH() + fake_paramiko.AutoAddPolicy = lambda: None + monkeypatch.setitem(sys.modules, "paramiko", fake_paramiko) + + runner = CliRunner() + result = runner.invoke(resubmit_rejected_claims, [ + "--ingest-dir", str(tmp_path), + "--no-validate", + # default: gate ENABLED + ]) + assert result.exit_code == 0, result.output + + # First file: Edifabric gate aborted it (no SFTP put for it). + # Second file: passed the gate, got uploaded via the SFTP put. + assert len(fake_sftp.put_calls) == 1, fake_sftp.put_calls + assert "bbbbbbbb" in fake_sftp.put_calls[0], fake_sftp.put_calls[0] + + # Output surfaces the failure list. + assert "EDIFABRIC VALIDATION FAIL" in result.output, result.output + assert "1 file(s) blocked by Edifabric gate" in result.output, result.output + assert "aaaaaaa" in result.output, result.output + assert "PER-04 is required" in result.output, result.output + + # DONE summary shows both counters. + assert "validate_failed=1" in result.output, result.output + assert "uploaded=1" in result.output, result.output + finally: + teardown() + + +def test_resubmit_edifabric_skip_flag_bypasses_gate(tmp_path, monkeypatch): + """SP40: ``--skip-edifabric-validate`` bypasses the gate entirely + so the same Edifabric-failing file uploads via SFTP. Used for + dev / hot-fix scenarios.""" + import sys + import types + from cyclone.cli import resubmit_rejected_claims + + _seed_clearhouse_stub_false(tmp_path) + _two_files_setup(tmp_path) + + import httpx + x12 = { + "SegmentDelimiter": "~", + "DataElementDelimiter": "*", + "ISA": {"InterchangeControlNumber_13": "000000001"}, + "Groups": [], + "IEATrailers": [], + } + + validate_calls: list[str] = [] + + def edifabric_handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/read"): + return httpx.Response(200, json=[x12]) + if request.url.path.endswith("/validate"): + validate_calls.append("hit") + return httpx.Response(200, json={ + "Status": "error", + "Details": [{"SegmentId": "PER", + "Message": "PER-04 is required", + "Status": "error"}], + "LastIndex": 5, + }) + raise AssertionError(f"unexpected path: {request.url.path}") + + teardown = _install_edifabric_mock(edifabric_handler) + try: + fake_paramiko = types.ModuleType("paramiko") + + class _FakeSFTP: + def __init__(self): + self.put_calls: list[str] = [] + + def stat(self, path): + raise IOError("not found") + + def put(self, local, remote): + self.put_calls.append(remote) + + fake_sftp = _FakeSFTP() + + class _FakeSSH: + def open_sftp(self): + return fake_sftp + + def set_missing_host_key_policy(self, policy): + return None + + def connect(self, *a, **kw): + return None + + def close(self): + return None + + fake_paramiko.SSHClient = lambda: _FakeSSH() + fake_paramiko.AutoAddPolicy = lambda: None + monkeypatch.setitem(sys.modules, "paramiko", fake_paramiko) + + runner = CliRunner() + result = runner.invoke(resubmit_rejected_claims, [ + "--ingest-dir", str(tmp_path), + "--no-validate", + "--skip-edifabric-validate", + ]) + assert result.exit_code == 0, result.output + + # Both files uploaded; the gate never fired. + assert len(fake_sftp.put_calls) == 2, fake_sftp.put_calls + assert validate_calls == [], ( + "validate must not be called when --skip-edifabric-validate is set" + ) + assert "validate_failed=0" in result.output, result.output + assert "EDIFABRIC VALIDATION FAIL" not in result.output, result.output + finally: + teardown()