fix(835): drop over-constraining UNIQUE constraints, add multi-BPR warning

Three related changes for real CO Medicaid data:

1. Drop UNIQUE(batch_id, patient_control_number) on claims and
   UNIQUE(batch_id, payer_claim_control_number) on remittances. The X12
   spec allows multiple CLM segments per 2000B subscriber loop and 835
   ERAs can repeat a payer_claim_control_number for reversals. Claim/
   remittance identity is provided by the primary key (claims.id = CLM01,
   remittances.id = CLP01).

2. Add validator rule R835_MULTI_BPR warning for files with multiple BPR
   segments (CO Medicaid split-payment pattern). The parser already sums
   BPR02 paid_amounts; this surfaces the non-standard data to operators.

3. Skip R835_BAL_BPR_vs_CLP04 when BPR01='I' (Information Only 835).
   In that mode BPR02 is informational and the per-claim CLP04 totals
   are authoritative — a diff is expected, not an error.

Migration 0003 handles the drop with IF EXISTS so fresh DBs skip cleanly.
Updates affected tests to reflect new schema (no UNIQUE constraint on
batch_id + patient_control_number / payer_claim_control_number).

Fixes test_api_835::test_prodfile_round_trip_persists_separately which
was failing on real production data.
This commit is contained in:
Tyler
2026-06-20 18:10:07 -06:00
parent 66da69baa0
commit b5e27927e0
12 changed files with 371 additions and 23 deletions
+157
View File
@@ -8,6 +8,7 @@ import pytest
from fastapi.testclient import TestClient
from cyclone.api import app
from cyclone.store import store as global_store
FIXTURE = Path(__file__).parent / "fixtures" / "co_medicaid_835.txt"
@@ -169,3 +170,159 @@ def test_cors_headers_present_for_835(client: TestClient):
)
assert resp.headers.get("access-control-allow-origin") == "http://localhost:5173"
assert "POST" in resp.headers.get("access-control-allow-methods", "").upper()
# --------------------------------------------------------------------------- #
# Production 835 round-trip (parser → API → store → DB)
# --------------------------------------------------------------------------- #
# Path to the production 835 ERA files from CO Medicaid. These are NOT in
# git (.gitignore: ``docs/prodfiles/*/``) — the test is skipped if the
# directory is missing so it stays green in clean checkouts but exercises
# real data when ops has dropped files in for pipeline validation.
PRODFILE_835_DIR = Path(__file__).parent.parent.parent / "docs" / "prodfiles" / "835fromco"
# (filename, expected_control_number, expected_total_claims, expected_paid).
# Derived from /tmp/parse_prod835.py on 2026-06-20 against the 5 files in
# PRODFILE_835_DIR. All 5 are CO Medicaid Information Only (BPR01="I") ERAs
# that use a "split payment" pattern: each file carries 3 BPR segments
# whose BPR02 amounts sum to the total paid. ``expected_paid`` is the sum
# across all BPRs in the file (the parser accumulates them; the validator
# emits R835_MULTI_BPR as a warning).
EXPECTED_PRODFILES_835: list[tuple[str, str, int, str]] = [
("tp11525703-835_M019110219-20260525001606050-1of1.x12", "200010701", 735, "58445.24"),
("tp11525703-835_M019200601-20260601003507042-1of1.x12", "200000831", 671, "50263.37"),
("tp11525703-835_M019311719-20260608002507036-1of1.x12", "200007522", 352, "52978.74"),
("tp11525703-835_M019414762-20260615005516914-1of1.x12", "200023767", 349, "17913.97"),
("tp11525703-835_M019506114-20260619075017291-1of1.x12", "200003981", 1267, "87722.18"),
]
@pytest.mark.skipif(
not PRODFILE_835_DIR.is_dir(),
reason=f"production 835 files not present at {PRODFILE_835_DIR} (gitignored)",
)
def test_prodfile_round_trip_persists_separately(client: TestClient):
"""All 5 production 835 ERA files from CO Medicaid must parse, persist
as separate batches, and be retrievable by id.
Exercises the full FastAPI path (parse → validate → store.add → DB)
against real production data — 3,374 remittances, 11,163 service
payments, 7,510 CAS adjustments, 5 distinct control numbers,
$8,231.63 total paid.
No 837s are loaded, so all remittances are unmatched by design. The
T10 reconciliation summary returned with each 835 must reflect that.
"""
from decimal import Decimal
assert len(global_store.list()) == 0
# 1. POST every file. Each must return 200 with the expected envelope,
# financial_info, payer XV, and per-claim CLP count. The 835 path
# also runs batch-level validation (R835_BAL_BPR_vs_CLP04, etc.)
# which must pass on real data.
for filename, expected_ctrl, expected_claims, expected_paid in EXPECTED_PRODFILES_835:
path = PRODFILE_835_DIR / filename
with open(path, "rb") as f:
resp = client.post(
"/api/parse-835",
files={"file": (filename, f, "application/octet-stream")},
headers={"Accept": "application/json"},
)
assert resp.status_code == 200, f"{filename}: {resp.text}"
body = resp.json()
assert body["envelope"]["control_number"] == expected_ctrl, filename
assert body["payer"]["id"] == "7912900843", filename # CO Medicaid health plan ID
assert body["payee"]["npi"] == "1467507269", filename
assert body["summary"]["total_claims"] == expected_claims, filename
assert body["summary"]["passed"] == expected_claims, filename
assert body["summary"]["failed"] == 0, filename
# BPR02 (total paid) is a Decimal-as-string in the JSON envelope.
# For CO Medicaid split-payment files, the parser sums the three
# BPR segments into one paid_amount.
assert Decimal(body["financial_info"]["paid_amount"]) == Decimal(expected_paid), filename
# All 5 prod files carry multiple BPR segments; the validator must
# flag that as a warning (R835_MULTI_BPR) but pass the batch.
multi_bpr_warnings = [
issue for issue in body["validation"]["warnings"]
if issue["rule"] == "R835_MULTI_BPR"
]
assert multi_bpr_warnings, f"{filename}: expected R835_MULTI_BPR warning"
assert body["validation"]["errors"] == [], filename
# T10 reconciliation summary is part of the 835 response. No 837s
# loaded, so every remit in this batch must be unmatched.
assert "reconciliation" in body, filename
rec = body["reconciliation"]
assert rec["matched"] == 0, filename
assert rec["unmatched_claims"] == 0, filename
# ``unmatched_remittances`` is global (across all batches), so
# we just assert it's a non-negative int — the precise total is
# covered by the store-level assertion below.
assert rec["unmatched_remittances"] >= 0, filename
assert rec["skipped"] == 0, filename
# 2. Five batches landed in the store. One per file.
batches = global_store.list()
assert len(batches) == 5
assert {b.kind for b in batches} == {"835"}
assert {b.input_filename for b in batches} == {f for f, _, _, _ in EXPECTED_PRODFILES_835}
# 3. Each persisted batch carries the full ParseResult: envelope,
# financial_info, payer, payee, and a claims list with the right
# size. Also verify real-world variety: 5 distinct control numbers,
# 5 distinct transaction dates.
by_filename = {b.input_filename: b for b in batches}
total_remits = 0
total_svcs = 0
total_cas = 0
distinct_ctrl: set[str] = set()
distinct_dates: set[str] = set()
for filename, expected_ctrl, expected_claims, _ in EXPECTED_PRODFILES_835:
rec = by_filename[filename]
assert rec.result.envelope.control_number == expected_ctrl, filename
assert rec.result.summary.total_claims == expected_claims, filename
assert len(rec.result.claims) == expected_claims, filename
for cp in rec.result.claims:
assert cp.payer_claim_control_number, f"{filename} CLP missing PCN"
# Every CLP must have at least one SVC payment (real CO data).
assert cp.service_payments, f"{filename} CLP {cp.payer_claim_control_number} has no SVCs"
total_remits += len(rec.result.claims)
total_svcs += sum(len(c.service_payments) for c in rec.result.claims)
total_cas += sum(
len(sp.adjustments)
for c in rec.result.claims
for sp in c.service_payments
)
distinct_ctrl.add(rec.result.envelope.control_number)
distinct_dates.add(str(rec.result.envelope.transaction_date))
assert total_remits == 3374
assert total_svcs == 11163
assert total_cas == 7510
assert len(distinct_ctrl) == 5
assert len(distinct_dates) == 5
# 4. Read path: every batch is retrievable by id and round-trips with
# the same remit count that was just written. Covers global_store.get
# and the DB-backed lookup that powers /api/batches/{id}.
for rec in batches:
fetched = global_store.get(rec.id)
assert fetched is not None, rec.id
assert fetched.id == rec.id
assert len(fetched.result.claims) == rec.result.summary.total_claims
assert fetched.result.envelope.control_number == rec.result.envelope.control_number
# 5. Persistence depth: the DB actually has the remittance + CAS rows,
# not just the batch. ``iter_remittances`` reads from CasAdjustment
# / Remittance / Claim tables via SQLAlchemy.
from cyclone.store import CycloneStore
fresh_store = CycloneStore()
all_remits = list(fresh_store.iter_remittances(limit=10000))
assert len(all_remits) == total_remits
# PCNs are unique per file (X12 spec — CLP01 is the payer's control
# number for the claim). Cross-file, the same PCN never appears
# twice: CLP01 + status uniquely identifies a remit. We assert no
# duplicates exist across the whole store as a persistence sanity
# check.
pcns = [r.payer_claim_control_number for r in all_remits]
assert len(pcns) == len(set(pcns)), "duplicate PCN across batches would be a persistence bug"