feat(sp25): read-side defensive stub for synthetic-batch rows

backend/src/cyclone/store/batches.py

@@ -9,7 +9,8 @@ a fresh DB state per-test.
 
 from __future__ import annotations
 
-from datetime import timezone
+from datetime import date, timezone
+from decimal import Decimal
 
 from cyclone import db
 from cyclone.db import (
@@ -20,12 +21,85 @@ from cyclone.db import (
     Match,
     Remittance,
 )
-from cyclone.parsers.models import ParseResult
-from cyclone.parsers.models_835 import ParseResult835
+from cyclone.parsers.models import BatchSummary, Envelope, ParseResult
+from cyclone.parsers.models_835 import (
+    FinancialInfo,
+    ParseResult835,
+    Payee835,
+    Payer835,
+    ReassociationTrace,
+)
 
 from .records import BatchRecord, BatchRecord837, BatchRecord835
 
 
+def _stub_parse_result_837(row: Batch) -> ParseResult:
+    """Synthesize a minimal ``ParseResult`` for a 837p row missing ``raw_result_json``.
+
+    SP25: ``CycloneStore.reconcile_orphan_st02s()`` seeds anchor ``Batch``
+    rows for orphan 999 ACK ST02s with ``raw_result_json=NULL`` — no
+    source 837 was ever parsed for them. Returning a typed stub keeps
+    the dashboard's recent-batches list and the ``/api/batches`` endpoint
+    rendering instead of 500'ing on Pydantic validation.
+
+    All counters zero, summary.control_number echoes the row's
+    ``transaction_set_control_number`` (the SCN the orphan 999 acks
+    reference), so the batch list widget surfaces the right SCN.
+    """
+    today = row.parsed_at.date() if row.parsed_at else date.today()
+    return ParseResult(
+        envelope=None,
+        claims=[],
+        summary=BatchSummary(
+            input_file=row.input_filename,
+            control_number=row.transaction_set_control_number,
+            transaction_date=today,
+            total_claims=0,
+            passed=0,
+            failed=0,
+        ),
+    )
+
+
+def _stub_parse_result_835(row: Batch) -> ParseResult835:
+    """Synthesize a minimal ``ParseResult835`` for an 835 row missing ``raw_result_json``.
+
+    Mirrors ``_stub_parse_result_837``; 835 requires non-None envelope,
+    financial_info, trace, payer, payee — minimal real-value shells are
+    the only way Pydantic will accept the stub.
+    """
+    today = row.parsed_at.date() if row.parsed_at else date.today()
+    return ParseResult835(
+        envelope=Envelope(
+            sender_id="",
+            receiver_id="",
+            control_number=row.transaction_set_control_number or "",
+            transaction_date=today,
+        ),
+        financial_info=FinancialInfo(
+            handling_code="",
+            paid_amount=Decimal("0.00"),
+            credit_debit_flag="",
+        ),
+        trace=ReassociationTrace(
+            trace_type_code="",
+            trace_number="",
+            originating_company_id="",
+        ),
+        payer=Payer835(name=""),
+        payee=Payee835(name="", npi=""),
+        claims=[],
+        summary=BatchSummary(
+            input_file=row.input_filename,
+            control_number=row.transaction_set_control_number,
+            transaction_date=today,
+            total_claims=0,
+            passed=0,
+            failed=0,
+        ),
+    )
+
+
 class _BatchesShim:
     """Drop-in replacement for the old in-memory ``_batches`` list.
 
@@ -59,13 +133,26 @@ def _row_to_record(row: Batch) -> BatchRecord:
     is ``DateTime(timezone=True)``. We re-attach UTC so the
     ``BatchRecord`` validator (``parsed_at must be tz-aware``)
     passes.
+
+    SP25: if ``raw_result_json`` is missing or empty (e.g. the row
+    was seeded by ``reconcile_orphan_st02s()`` and never had a
+    real parse attached), we hydrate a typed *stub* with empty
+    claim lists and zero counters so the dashboard's recent-batches
+    list can still render the row. ``raw_result_json`` is left
+    untouched — the stub is read-side only. See
+    ``_stub_parse_result_837`` / ``_stub_parse_result_835``.
     """
     if row.kind == "835":
         result_cls = ParseResult835
     else:
         result_cls = ParseResult
-    payload = row.raw_result_json or {}
+    payload = row.raw_result_json
+    if payload:
         result = result_cls.model_validate(payload)
+    elif row.kind == "835":
+        result = _stub_parse_result_835(row)
+    else:
+        result = _stub_parse_result_837(row)
     parsed_at = row.parsed_at
     if parsed_at is not None and parsed_at.tzinfo is None:
         parsed_at = parsed_at.replace(tzinfo=timezone.utc)

backend/tests/test_store_batches_synthetic.py

@@ -0,0 +1,167 @@
+"""SP25: defensive read-side stub for synthetic ``<synthetic:orphan-reconcile>``
+``Batch`` rows.
+
+Background: ``CycloneStore.reconcile_orphan_st02s()`` (sp38) seeds ``Batch``
+rows for orphan 999 ACK ST02s so the envelope index can resolve
+``source_batch_id`` lookups. Those rows are inserted with
+``raw_result_json=NULL`` because no source 837 file was ever parsed
+for them. The read-side helper ``_row_to_record()`` in
+``store/batches.py`` was written assuming the column is always
+populated, so it raises a Pydantic ``ValidationError`` on ``summary:
+Field required`` the moment it sees a synthetic row — which makes
+``GET /api/batches?limit=5`` return HTTP 500.
+
+This file pins the contract: a synthetic (or otherwise unparseable)
+``Batch`` row MUST still hydrate to a typed ``BatchRecord`` with an
+empty ``claims`` list and the ST02 echoed on ``result.summary.control_number``,
+so the dashboard "Recent batches" widget can render them as
+"synthetic: 0 claims" without the whole list blowing up.
+
+Live state witnessed during the 2026-07-07 dashboard postmortem:
+the sp38 reconcile pass on the production DB seeded 4 such rows
+(SCN 991102986–989). Once the stub lands, ``/api/batches`` returns 200
+instead of 500 and the dashboard "Recent batches" panel surfaces the
+real underlying batches (the test runs).
+
+Note: the rows inserted here are *ORM-level* fixtures (Batch rows with
+NULL raw_result_json, used to exercise the read-path's defensive
+deserialize). They are NOT synthetic EDI — the actual claim data for
+the recover-ingest path lives under ``ingest/`` and is replayed by the
+operator after this defensive stub ships.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from cyclone import db, store
+from cyclone.db import Batch
+from cyclone.store.records import BatchRecord837, BatchRecord835
+
+
+def _now():
+    return datetime.now(timezone.utc)
+
+
+def _insert_synthetic(s, *, kind: str, st02: str, filename: str = "<synthetic:orphan-reconcile>"):
+    bid = f"syn-{st02}"
+    s.add(Batch(
+        id=bid,
+        kind=kind,
+        input_filename=filename,
+        parsed_at=_now(),
+        transaction_set_control_number=st02,
+        raw_result_json=None,
+    ))
+    s.commit()
+    return bid
+
+
+def test_row_to_record_handles_null_raw_json_837p():
+    """837p synthetic row → stub BatchRecord837 with empty claims[] + SCN preserved."""
+    s = db.SessionLocal()()
+    try:
+        _insert_synthetic(s, kind="837p", st02="991102986")
+    finally:
+        s.close()
+
+    # No raw_result_json on the row → _row_to_record must NOT raise.
+    rows = store.list_batches(limit=50)
+
+    syn = [r for r in rows if r.id == "syn-991102986"]
+    assert len(syn) == 1
+    rec = syn[0]
+    assert isinstance(rec, BatchRecord837)
+    assert rec.result.claims == []
+    assert rec.result.summary.total_claims == 0
+    assert rec.result.summary.control_number == "991102986"
+    assert rec.result.summary.passed == 0
+    assert rec.result.summary.failed == 0
+
+
+def test_row_to_record_handles_null_raw_json_835():
+    """835 synthetic row → stub BatchRecord835 with empty claims[].claims."""
+    s = db.SessionLocal()()
+    try:
+        _insert_synthetic(s, kind="835", st02="991102987")
+    finally:
+        s.close()
+
+    rows = store.list_batches(limit=50)
+    syn = [r for r in rows if r.id == "syn-991102987"]
+    assert len(syn) == 1
+    rec = syn[0]
+    assert isinstance(rec, BatchRecord835)
+    assert rec.result.claims == []
+    # summary control_number echoes ST02.
+    assert rec.result.summary.control_number == "991102987"
+
+
+def test_row_to_record_handles_empty_string_raw_json():
+    """raw_result_json='' (empty string) gets the same defensive treatment."""
+    s = db.SessionLocal()()
+    try:
+        bid = "syn-empty"
+        s.add(Batch(
+            id=bid, kind="837p",
+            input_filename="<synthetic:orphan-reconcile>",
+            parsed_at=_now(),
+            transaction_set_control_number="991102988",
+            raw_result_json="",  # empty string instead of None
+        ))
+        s.commit()
+    finally:
+        s.close()
+
+    rows = store.list_batches(limit=50)
+    syn = [r for r in rows if r.id == "syn-empty"]
+    assert len(syn) == 1
+    rec = syn[0]
+    assert rec.result.claims == []
+    assert rec.result.summary.total_claims == 0
+
+
+def test_row_to_record_normal_row_still_parses():
+    """Regression guard: a well-formed raw_result_json still deserializes fully.
+
+    Batch.raw_result_json is JSONText-backed, so we store a dict; the type
+    decorator takes care of the JSON round-trip.
+    """
+    payload = {
+        "envelope": {
+            "sender_id": "11525703",
+            "receiver_id": "RECEIVERID",
+            "control_number": "000000001",
+            "transaction_date": "2026-07-07",
+            "transaction_time": "120000",
+            "implementation_guide": "005010X222A1",
+        },
+        "claims": [],
+        "summary": {
+            "input_file": "single-claim.x12",
+            "control_number": "000000001",
+            "transaction_date": "2026-07-07",
+            "total_claims": 1,
+            "passed": 1,
+            "failed": 0,
+        },
+    }
+    s = db.SessionLocal()()
+    try:
+        s.add(Batch(
+            id="syn-normal",
+            kind="837p",
+            input_filename="single-claim.x12",
+            parsed_at=_now(),
+            transaction_set_control_number="991102977",
+            raw_result_json=payload,
+        ))
+        s.commit()
+    finally:
+        s.close()
+
+    rows = store.list_batches(limit=50)
+    norm = [r for r in rows if r.id == "syn-normal"]
+    assert len(norm) == 1
+    rec = norm[0]
+    assert rec.result.summary.total_claims == 1
+    assert rec.result.summary.passed == 1

docs/superpowers/plans/2026-07-07-cyclone-orphan-data-recovery.md

@@ -24,12 +24,9 @@ backend/src/cyclone/
   submission/
     recover.py                # B: shared parse-then-store logic (new, small)
 backend/tests/
-  test_store_batches_synthetic.py        # A: unit
+  test_store_batches_synthetic.py        # A: unit (the defensive stub needs synthetic rows with NULL raw_result_json — these are ORM-level test fixtures for the read path, not EDI samples)
   test_api_batches_synthetic.py          # A: integration
-  test_cli_recover_ingest.py             # B: cli + integration
-  fixtures/
-    recover_837p_minimal.txt             # B: 2-claim 837P fixture (synthetic)
-    recover_835_minimal.x12              # B: 2-payment 835 fixture (synthetic)
+  test_cli_recover_ingest.py             # B: cli + integration (uses existing co_medicaid_* fixtures; no new minimal_* EDI mirrors)
 backend/scripts/
   verify_dashboard_recovery.py           # post-recovery sanity check (operator-only)
 docs/superpowers/specs/
@@ -61,15 +58,12 @@ docs/superpowers/specs/
 
 - [ ] Restart-free smoke: `curl -sS 'http://127.0.0.1:8000/api/batches?limit=5' | python -m json.tool` — expect 200, `items[].claimCount==0` for the 4 synthetic rows, `inputFilename=="<synthetic:orphan-reconcile>"`.
 
-## Task 4: B — copy prodfiles to fixtures + failing test for `recover-ingest`
+## Task 4: B — failing test for `recover-ingest` against real co_medicaid fixtures
 
-- [ ] Copy the source 835 into fixtures (operator-only, but commit it as a fixture for hermetic recovery verification):
-  - `cp ingest/tp11525703-835_M019771179-20260706005516577-1of1.x12 backend/tests/fixtures/recover_835_real.x12`.
-  - `cp ingest/tp11525703-837P-20260701162932052-1of1.txt backend/tests/fixtures/recover_837p_real_145claims.txt`.
-- [ ] Add `backend/tests/test_cli_recover_ingest.py` with three cases:
-  1. `test_ingest_835_persists_remittance` — invoke the `recover-ingest` CLI via Click's `CliRunner` with the real 835 fixture; assert `len(store.list_remittances())==1+1`, the row's `total_paid > 0`, and a `processed_inbound_files` row with `sftp_block_name='manual-recover'` was created.
-  2. `test_ingest_837p_persists_claims` — invoke with the 145-claim 837P fixture; assert `claims_total_count >= 145` via `SessionLocal().query(Claim).count()` and the corresponding `Batch` row's `claim_count` matches.
-  3. `test_ingest_idempotent` — invoke twice with the same file; second run is a no-op (skip messages logged, no extra rows).
+- [ ] Add `backend/tests/test_cli_recover_ingest.py` with three cases. Test inputs come from existing `backend/tests/fixtures/co_medicaid_837p.txt` + `co_medicaid_835.txt` — real production EDI already in the repo. **No synthetic EDI fixtures are introduced.** The ingest/ files are reserved for the operator-only recovery run (Task 6), not unit-tested.
+  1. `test_ingest_835_persists_remittance` — invoke `recover-ingest` via Click's `CliRunner` with `co_medicaid_835.txt`; assert a Remittance row was created with `total_paid > 0`, plus a `processed_inbound_files` row with `sftp_block_name='manual-recover'`.
+  2. `test_ingest_837p_persists_claims` — invoke with `co_medicaid_837p.txt`; assert ≥1 Claim row + a Batch row that references it.
+  3. `test_ingest_idempotent` — invoke twice with the same file; second run is a no-op (skip messages, no extra rows).
 - [ ] Run the test file: must fail (no `recover-ingest` command yet).
 
 ## Task 5: B — implement `cyclone recover-ingest`