diff --git a/backend/src/cyclone/store/batches.py b/backend/src/cyclone/store/batches.py index 75582e8..13bda68 100644 --- a/backend/src/cyclone/store/batches.py +++ b/backend/src/cyclone/store/batches.py @@ -9,7 +9,8 @@ a fresh DB state per-test. from __future__ import annotations -from datetime import timezone +from datetime import date, timezone +from decimal import Decimal from cyclone import db from cyclone.db import ( @@ -20,12 +21,85 @@ from cyclone.db import ( Match, Remittance, ) -from cyclone.parsers.models import ParseResult -from cyclone.parsers.models_835 import ParseResult835 +from cyclone.parsers.models import BatchSummary, Envelope, ParseResult +from cyclone.parsers.models_835 import ( + FinancialInfo, + ParseResult835, + Payee835, + Payer835, + ReassociationTrace, +) from .records import BatchRecord, BatchRecord837, BatchRecord835 +def _stub_parse_result_837(row: Batch) -> ParseResult: + """Synthesize a minimal ``ParseResult`` for a 837p row missing ``raw_result_json``. + + SP25: ``CycloneStore.reconcile_orphan_st02s()`` seeds anchor ``Batch`` + rows for orphan 999 ACK ST02s with ``raw_result_json=NULL`` — no + source 837 was ever parsed for them. Returning a typed stub keeps + the dashboard's recent-batches list and the ``/api/batches`` endpoint + rendering instead of 500'ing on Pydantic validation. + + All counters zero, summary.control_number echoes the row's + ``transaction_set_control_number`` (the SCN the orphan 999 acks + reference), so the batch list widget surfaces the right SCN. + """ + today = row.parsed_at.date() if row.parsed_at else date.today() + return ParseResult( + envelope=None, + claims=[], + summary=BatchSummary( + input_file=row.input_filename, + control_number=row.transaction_set_control_number, + transaction_date=today, + total_claims=0, + passed=0, + failed=0, + ), + ) + + +def _stub_parse_result_835(row: Batch) -> ParseResult835: + """Synthesize a minimal ``ParseResult835`` for an 835 row missing ``raw_result_json``. + + Mirrors ``_stub_parse_result_837``; 835 requires non-None envelope, + financial_info, trace, payer, payee — minimal real-value shells are + the only way Pydantic will accept the stub. + """ + today = row.parsed_at.date() if row.parsed_at else date.today() + return ParseResult835( + envelope=Envelope( + sender_id="", + receiver_id="", + control_number=row.transaction_set_control_number or "", + transaction_date=today, + ), + financial_info=FinancialInfo( + handling_code="", + paid_amount=Decimal("0.00"), + credit_debit_flag="", + ), + trace=ReassociationTrace( + trace_type_code="", + trace_number="", + originating_company_id="", + ), + payer=Payer835(name=""), + payee=Payee835(name="", npi=""), + claims=[], + summary=BatchSummary( + input_file=row.input_filename, + control_number=row.transaction_set_control_number, + transaction_date=today, + total_claims=0, + passed=0, + failed=0, + ), + ) + + class _BatchesShim: """Drop-in replacement for the old in-memory ``_batches`` list. @@ -59,13 +133,26 @@ def _row_to_record(row: Batch) -> BatchRecord: is ``DateTime(timezone=True)``. We re-attach UTC so the ``BatchRecord`` validator (``parsed_at must be tz-aware``) passes. + + SP25: if ``raw_result_json`` is missing or empty (e.g. the row + was seeded by ``reconcile_orphan_st02s()`` and never had a + real parse attached), we hydrate a typed *stub* with empty + claim lists and zero counters so the dashboard's recent-batches + list can still render the row. ``raw_result_json`` is left + untouched — the stub is read-side only. See + ``_stub_parse_result_837`` / ``_stub_parse_result_835``. """ if row.kind == "835": result_cls = ParseResult835 else: result_cls = ParseResult - payload = row.raw_result_json or {} - result = result_cls.model_validate(payload) + payload = row.raw_result_json + if payload: + result = result_cls.model_validate(payload) + elif row.kind == "835": + result = _stub_parse_result_835(row) + else: + result = _stub_parse_result_837(row) parsed_at = row.parsed_at if parsed_at is not None and parsed_at.tzinfo is None: parsed_at = parsed_at.replace(tzinfo=timezone.utc) diff --git a/backend/tests/test_store_batches_synthetic.py b/backend/tests/test_store_batches_synthetic.py new file mode 100644 index 0000000..4e7d77f --- /dev/null +++ b/backend/tests/test_store_batches_synthetic.py @@ -0,0 +1,167 @@ +"""SP25: defensive read-side stub for synthetic ```` +``Batch`` rows. + +Background: ``CycloneStore.reconcile_orphan_st02s()`` (sp38) seeds ``Batch`` +rows for orphan 999 ACK ST02s so the envelope index can resolve +``source_batch_id`` lookups. Those rows are inserted with +``raw_result_json=NULL`` because no source 837 file was ever parsed +for them. The read-side helper ``_row_to_record()`` in +``store/batches.py`` was written assuming the column is always +populated, so it raises a Pydantic ``ValidationError`` on ``summary: +Field required`` the moment it sees a synthetic row — which makes +``GET /api/batches?limit=5`` return HTTP 500. + +This file pins the contract: a synthetic (or otherwise unparseable) +``Batch`` row MUST still hydrate to a typed ``BatchRecord`` with an +empty ``claims`` list and the ST02 echoed on ``result.summary.control_number``, +so the dashboard "Recent batches" widget can render them as +"synthetic: 0 claims" without the whole list blowing up. + +Live state witnessed during the 2026-07-07 dashboard postmortem: +the sp38 reconcile pass on the production DB seeded 4 such rows +(SCN 991102986–989). Once the stub lands, ``/api/batches`` returns 200 +instead of 500 and the dashboard "Recent batches" panel surfaces the +real underlying batches (the test runs). + +Note: the rows inserted here are *ORM-level* fixtures (Batch rows with +NULL raw_result_json, used to exercise the read-path's defensive +deserialize). They are NOT synthetic EDI — the actual claim data for +the recover-ingest path lives under ``ingest/`` and is replayed by the +operator after this defensive stub ships. +""" +from __future__ import annotations + +from datetime import datetime, timezone + +from cyclone import db, store +from cyclone.db import Batch +from cyclone.store.records import BatchRecord837, BatchRecord835 + + +def _now(): + return datetime.now(timezone.utc) + + +def _insert_synthetic(s, *, kind: str, st02: str, filename: str = ""): + bid = f"syn-{st02}" + s.add(Batch( + id=bid, + kind=kind, + input_filename=filename, + parsed_at=_now(), + transaction_set_control_number=st02, + raw_result_json=None, + )) + s.commit() + return bid + + +def test_row_to_record_handles_null_raw_json_837p(): + """837p synthetic row → stub BatchRecord837 with empty claims[] + SCN preserved.""" + s = db.SessionLocal()() + try: + _insert_synthetic(s, kind="837p", st02="991102986") + finally: + s.close() + + # No raw_result_json on the row → _row_to_record must NOT raise. + rows = store.list_batches(limit=50) + + syn = [r for r in rows if r.id == "syn-991102986"] + assert len(syn) == 1 + rec = syn[0] + assert isinstance(rec, BatchRecord837) + assert rec.result.claims == [] + assert rec.result.summary.total_claims == 0 + assert rec.result.summary.control_number == "991102986" + assert rec.result.summary.passed == 0 + assert rec.result.summary.failed == 0 + + +def test_row_to_record_handles_null_raw_json_835(): + """835 synthetic row → stub BatchRecord835 with empty claims[].claims.""" + s = db.SessionLocal()() + try: + _insert_synthetic(s, kind="835", st02="991102987") + finally: + s.close() + + rows = store.list_batches(limit=50) + syn = [r for r in rows if r.id == "syn-991102987"] + assert len(syn) == 1 + rec = syn[0] + assert isinstance(rec, BatchRecord835) + assert rec.result.claims == [] + # summary control_number echoes ST02. + assert rec.result.summary.control_number == "991102987" + + +def test_row_to_record_handles_empty_string_raw_json(): + """raw_result_json='' (empty string) gets the same defensive treatment.""" + s = db.SessionLocal()() + try: + bid = "syn-empty" + s.add(Batch( + id=bid, kind="837p", + input_filename="", + parsed_at=_now(), + transaction_set_control_number="991102988", + raw_result_json="", # empty string instead of None + )) + s.commit() + finally: + s.close() + + rows = store.list_batches(limit=50) + syn = [r for r in rows if r.id == "syn-empty"] + assert len(syn) == 1 + rec = syn[0] + assert rec.result.claims == [] + assert rec.result.summary.total_claims == 0 + + +def test_row_to_record_normal_row_still_parses(): + """Regression guard: a well-formed raw_result_json still deserializes fully. + + Batch.raw_result_json is JSONText-backed, so we store a dict; the type + decorator takes care of the JSON round-trip. + """ + payload = { + "envelope": { + "sender_id": "11525703", + "receiver_id": "RECEIVERID", + "control_number": "000000001", + "transaction_date": "2026-07-07", + "transaction_time": "120000", + "implementation_guide": "005010X222A1", + }, + "claims": [], + "summary": { + "input_file": "single-claim.x12", + "control_number": "000000001", + "transaction_date": "2026-07-07", + "total_claims": 1, + "passed": 1, + "failed": 0, + }, + } + s = db.SessionLocal()() + try: + s.add(Batch( + id="syn-normal", + kind="837p", + input_filename="single-claim.x12", + parsed_at=_now(), + transaction_set_control_number="991102977", + raw_result_json=payload, + )) + s.commit() + finally: + s.close() + + rows = store.list_batches(limit=50) + norm = [r for r in rows if r.id == "syn-normal"] + assert len(norm) == 1 + rec = norm[0] + assert rec.result.summary.total_claims == 1 + assert rec.result.summary.passed == 1 diff --git a/docs/superpowers/plans/2026-07-07-cyclone-orphan-data-recovery.md b/docs/superpowers/plans/2026-07-07-cyclone-orphan-data-recovery.md index d7a26d1..0db29a2 100644 --- a/docs/superpowers/plans/2026-07-07-cyclone-orphan-data-recovery.md +++ b/docs/superpowers/plans/2026-07-07-cyclone-orphan-data-recovery.md @@ -24,12 +24,9 @@ backend/src/cyclone/ submission/ recover.py # B: shared parse-then-store logic (new, small) backend/tests/ - test_store_batches_synthetic.py # A: unit + test_store_batches_synthetic.py # A: unit (the defensive stub needs synthetic rows with NULL raw_result_json — these are ORM-level test fixtures for the read path, not EDI samples) test_api_batches_synthetic.py # A: integration - test_cli_recover_ingest.py # B: cli + integration - fixtures/ - recover_837p_minimal.txt # B: 2-claim 837P fixture (synthetic) - recover_835_minimal.x12 # B: 2-payment 835 fixture (synthetic) + test_cli_recover_ingest.py # B: cli + integration (uses existing co_medicaid_* fixtures; no new minimal_* EDI mirrors) backend/scripts/ verify_dashboard_recovery.py # post-recovery sanity check (operator-only) docs/superpowers/specs/ @@ -61,15 +58,12 @@ docs/superpowers/specs/ - [ ] Restart-free smoke: `curl -sS 'http://127.0.0.1:8000/api/batches?limit=5' | python -m json.tool` — expect 200, `items[].claimCount==0` for the 4 synthetic rows, `inputFilename==""`. -## Task 4: B — copy prodfiles to fixtures + failing test for `recover-ingest` +## Task 4: B — failing test for `recover-ingest` against real co_medicaid fixtures -- [ ] Copy the source 835 into fixtures (operator-only, but commit it as a fixture for hermetic recovery verification): - - `cp ingest/tp11525703-835_M019771179-20260706005516577-1of1.x12 backend/tests/fixtures/recover_835_real.x12`. - - `cp ingest/tp11525703-837P-20260701162932052-1of1.txt backend/tests/fixtures/recover_837p_real_145claims.txt`. -- [ ] Add `backend/tests/test_cli_recover_ingest.py` with three cases: - 1. `test_ingest_835_persists_remittance` — invoke the `recover-ingest` CLI via Click's `CliRunner` with the real 835 fixture; assert `len(store.list_remittances())==1+1`, the row's `total_paid > 0`, and a `processed_inbound_files` row with `sftp_block_name='manual-recover'` was created. - 2. `test_ingest_837p_persists_claims` — invoke with the 145-claim 837P fixture; assert `claims_total_count >= 145` via `SessionLocal().query(Claim).count()` and the corresponding `Batch` row's `claim_count` matches. - 3. `test_ingest_idempotent` — invoke twice with the same file; second run is a no-op (skip messages logged, no extra rows). +- [ ] Add `backend/tests/test_cli_recover_ingest.py` with three cases. Test inputs come from existing `backend/tests/fixtures/co_medicaid_837p.txt` + `co_medicaid_835.txt` — real production EDI already in the repo. **No synthetic EDI fixtures are introduced.** The ingest/ files are reserved for the operator-only recovery run (Task 6), not unit-tested. + 1. `test_ingest_835_persists_remittance` — invoke `recover-ingest` via Click's `CliRunner` with `co_medicaid_835.txt`; assert a Remittance row was created with `total_paid > 0`, plus a `processed_inbound_files` row with `sftp_block_name='manual-recover'`. + 2. `test_ingest_837p_persists_claims` — invoke with `co_medicaid_837p.txt`; assert ≥1 Claim row + a Batch row that references it. + 3. `test_ingest_idempotent` — invoke twice with the same file; second run is a no-op (skip messages, no extra rows). - [ ] Run the test file: must fail (no `recover-ingest` command yet). ## Task 5: B — implement `cyclone recover-ingest`