Tyler
de55003484
Captures the modified state of test_api_parse_persists.py so the in-progress test work is preserved on the claims-unique-fix branch as the sp-prefixed worktrees are cleaned up. The .venv and .gitignore_local noise in the working tree are intentionally left untracked.
417 lines
18 KiB
Python
417 lines
18 KiB
Python
"""Successful parses must persist to the store; failed parses must not."""
|
||
|
||
from __future__ import annotations
|
||
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from fastapi.testclient import TestClient
|
||
|
||
from cyclone.api import app
|
||
from cyclone.store import store as global_store
|
||
|
||
FIXTURE = Path(__file__).parent / "fixtures" / "co_medicaid_837p.txt"
|
||
FIXTURE_835 = Path(__file__).parent / "fixtures" / "co_medicaid_835.txt"
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def clear_store():
|
||
"""Reset the module-level store before and after each test."""
|
||
with global_store._lock:
|
||
global_store._batches.clear()
|
||
yield
|
||
with global_store._lock:
|
||
global_store._batches.clear()
|
||
|
||
|
||
@pytest.fixture
|
||
def client() -> TestClient:
|
||
return TestClient(app)
|
||
|
||
|
||
def test_successful_parse_837_creates_batch(client: TestClient):
|
||
assert len(global_store.list()) == 0
|
||
text = FIXTURE.read_text()
|
||
resp = client.post(
|
||
"/api/parse-837",
|
||
files={"file": ("test.txt", text, "text/plain")},
|
||
headers={"Accept": "application/json"},
|
||
)
|
||
assert resp.status_code == 200, resp.text
|
||
assert len(global_store.list()) == 1
|
||
rec = global_store.list()[0]
|
||
assert rec.kind == "837p"
|
||
assert rec.input_filename == "test.txt"
|
||
|
||
|
||
def test_failed_parse_837_does_not_create_batch(client: TestClient):
|
||
"""An empty / garbage file should NOT create a batch."""
|
||
assert len(global_store.list()) == 0
|
||
resp = client.post(
|
||
"/api/parse-837",
|
||
files={"file": ("garbage.txt", "not-edi", "text/plain")},
|
||
headers={"Accept": "application/json"},
|
||
)
|
||
# Either a 4xx from the parser, or a 200 with 0 claims — the contract
|
||
# is that NO batch is added either way.
|
||
assert resp.status_code in (200, 400, 422)
|
||
assert len(global_store.list()) == 0
|
||
|
||
|
||
def test_validation_failed_837_does_not_create_batch(client: TestClient):
|
||
"""A parse that produced claims with validation errors (422) must not be stored.
|
||
|
||
The spec only mandates "do not store on CycloneParseError or unhandled
|
||
exception"; we choose the stricter interpretation that 422 validation
|
||
failures are also excluded so the data on disk is always clean.
|
||
"""
|
||
# Construct a minimal but technically malformed 837P that the parser
|
||
# will accept but validation will reject. We inject a wrong NPI
|
||
# length to trigger the per-claim NPI validation rule.
|
||
text = FIXTURE.read_text()
|
||
# Replace the provider NPI (XX*1881068062) with an invalid 8-digit NPI
|
||
# (the rule is NPI must be 10 digits). This keeps the parse step
|
||
# green while failing claim validation.
|
||
text = text.replace("XX*1881068062", "XX*12345678")
|
||
resp = client.post(
|
||
"/api/parse-837",
|
||
files={"file": ("bad.txt", text, "text/plain")},
|
||
headers={"Accept": "application/json"},
|
||
)
|
||
assert resp.status_code == 422, resp.text
|
||
assert len(global_store.list()) == 0
|
||
|
||
|
||
def test_parse_835_response_includes_reconciliation_summary(
|
||
client: TestClient, tmp_path: Path
|
||
):
|
||
"""A successful 835 parse returns matched/unmatched counts in JSON."""
|
||
fixture = FIXTURE_835.read_text()
|
||
p = tmp_path / "era.txt"
|
||
p.write_text(fixture)
|
||
with open(p, "rb") as f:
|
||
r = client.post(
|
||
"/api/parse-835",
|
||
files={"file": ("era.txt", f, "text/plain")},
|
||
headers={"Accept": "application/json"},
|
||
)
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert "reconciliation" in body, f"missing reconciliation: {list(body.keys())}"
|
||
rec = body["reconciliation"]
|
||
assert "matched" in rec
|
||
assert "unmatched_claims" in rec
|
||
assert "unmatched_remittances" in rec
|
||
assert "skipped" in rec
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Production 837P round-trip (parser → API → store → DB)
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
|
||
# Path to the production 837P files from axiscare. These are NOT in git
|
||
# (.gitignore: ``docs/prodfiles/*/``) — the test is skipped if the directory
|
||
# is missing so it stays green in clean checkouts but exercises real data
|
||
# when ops has dropped files in for pipeline validation.
|
||
PRODFILE_DIR = Path(__file__).parent.parent.parent / "docs" / "prodfiles" / "837p-from-axiscare"
|
||
|
||
# (filename, expected_control_number, expected_total_claims). Derived from
|
||
# /tmp/parse_prodfile.py on 2026-06-20 against the 7 files in PRODFILE_DIR.
|
||
# The first two files share control_number 991102984 — a deliberate replay.
|
||
EXPECTED_PRODFILES: list[tuple[str, str, int]] = [
|
||
("tp11525703-837P-20260618151119397-1of1.txt", "991102984", 141),
|
||
("tp11525703-837P-20260618153339862-1of1.txt", "991102984", 141),
|
||
("tp11525703-837P-20260618153343460-1of1.txt", "991102983", 97),
|
||
("tp11525703-837P-20260618153346107-1of1.txt", "991102982", 28),
|
||
("tp11525703-837P-20260618153349188-1of1.txt", "991102981", 69),
|
||
("tp11525703-837P-20260618153354947-1of1.txt", "991102978", 149),
|
||
("tp11525703-837P-20260618153358831-1of1.txt", "991102977", 99),
|
||
]
|
||
|
||
|
||
@pytest.mark.skipif(
|
||
not PRODFILE_DIR.is_dir(),
|
||
reason=f"production files not present at {PRODFILE_DIR} (gitignored)",
|
||
)
|
||
def test_prodfile_round_trip_persists_separately(client: TestClient):
|
||
"""All 7 production 837P files from axiscare must parse, persist as
|
||
separate batches, and be retrievable by id.
|
||
|
||
Exercises the full FastAPI path (parse → validate → store.add → DB)
|
||
against real production data — 3 distinct billing-provider NPIs, 2
|
||
distinct transaction dates, 6 unique control numbers, 724 claims total.
|
||
"""
|
||
assert len(global_store.list()) == 0
|
||
|
||
# 1. POST every file. Each must return 200 with the expected envelope
|
||
# and summary — proves the parse + payer-config + validation gates
|
||
# work on real EDI, not just the synthetic fixture.
|
||
for filename, expected_ctrl, expected_claims in EXPECTED_PRODFILES:
|
||
path = PRODFILE_DIR / filename
|
||
with open(path, "rb") as f:
|
||
resp = client.post(
|
||
"/api/parse-837",
|
||
files={"file": (filename, f, "text/plain")},
|
||
headers={"Accept": "application/json"},
|
||
)
|
||
assert resp.status_code == 200, f"{filename}: {resp.text}"
|
||
body = resp.json()
|
||
assert body["envelope"]["control_number"] == expected_ctrl, filename
|
||
assert body["envelope"]["sender_id"] == "11525703", filename
|
||
assert body["summary"]["total_claims"] == expected_claims, filename
|
||
assert body["summary"]["passed"] == expected_claims, filename
|
||
assert body["summary"]["failed"] == 0, filename
|
||
|
||
# 2. Seven batches landed in the store (one per file). Confirms the
|
||
# store.add() path was hit for every upload and that the in-process
|
||
# state matches what the response said.
|
||
batches = global_store.list()
|
||
assert len(batches) == 7
|
||
assert {b.kind for b in batches} == {"837p"}
|
||
assert {b.input_filename for b in batches} == {f for f, _, _ in EXPECTED_PRODFILES}
|
||
|
||
# 3. Each persisted batch carries the full ParseResult: envelope,
|
||
# summary, claims. Also sanity-check real-world variety: 3 NPIs,
|
||
# 2 transaction dates (one each for 2026-06-11 vs 2026-06-17).
|
||
by_filename = {b.input_filename: b for b in batches}
|
||
total_claims = 0
|
||
distinct_npis: set[str] = set()
|
||
distinct_dates: set[str] = set()
|
||
for filename, expected_ctrl, expected_claims in EXPECTED_PRODFILES:
|
||
rec = by_filename[filename]
|
||
assert rec.result.envelope.control_number == expected_ctrl, filename
|
||
assert rec.result.summary.total_claims == expected_claims, filename
|
||
assert len(rec.result.claims) == expected_claims, filename
|
||
total_claims += rec.result.summary.total_claims
|
||
for claim in rec.result.claims:
|
||
assert claim.billing_provider.npi, f"{filename} claim {claim.claim_id} missing NPI"
|
||
distinct_npis.add(claim.billing_provider.npi)
|
||
distinct_dates.add(str(claim.transaction_date))
|
||
assert total_claims == 724
|
||
assert len(distinct_npis) >= 3, f"expected ≥3 NPIs across the batch, got {distinct_npis}"
|
||
assert len(distinct_dates) >= 2, f"expected ≥2 dates, got {distinct_dates}"
|
||
|
||
# 4. Read path: every batch is retrievable by id and round-trips with
|
||
# the same claim count that was just written. Covers global_store.get
|
||
# and the DB-backed lookup that powers /api/batches/{id}.
|
||
for rec in batches:
|
||
fetched = global_store.get(rec.id)
|
||
assert fetched is not None, rec.id
|
||
assert fetched.id == rec.id
|
||
assert len(fetched.result.claims) == rec.result.summary.total_claims
|
||
assert fetched.result.envelope.control_number == rec.result.envelope.control_number
|
||
|
||
# 5. Replay sanity: the first two files share control_number 991102984
|
||
# but were persisted as TWO distinct batches (no implicit dedup).
|
||
# Documents the current behavior; flip the assertion if/when a
|
||
# duplicate-control-number policy is added.
|
||
ctrl_991102984 = [b for b in batches if b.result.envelope.control_number == "991102984"]
|
||
assert len(ctrl_991102984) == 2
|
||
assert len({b.id for b in ctrl_991102984}) == 2
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Cross-pipeline reconciliation (837 + 835 together)
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
|
||
PRODFILE_835_DIR_FOR_XP = (
|
||
Path(__file__).parent.parent.parent / "docs" / "prodfiles" / "835fromco"
|
||
)
|
||
|
||
|
||
@pytest.mark.skipif(
|
||
not (PRODFILE_DIR.is_dir() and PRODFILE_835_DIR_FOR_XP.is_dir()),
|
||
reason=f"production 837 or 835 files not present (gitignored): {PRODFILE_DIR}, {PRODFILE_835_DIR_FOR_XP}",
|
||
)
|
||
def test_prodfile_cross_pipeline_reconciles(client: TestClient):
|
||
"""All production 837s + all production 835s end-to-end.
|
||
|
||
Pipeline under test:
|
||
POST /api/parse-837 (×N) → store.add (claim rows) → DB
|
||
POST /api/parse-835 (×M) → store.add (remit rows) → reconcile.run → DB
|
||
|
||
Designed to survive variable 837 / 835 file counts and arbitrary PCN
|
||
overlap. Hard-codes NO match counts. Instead asserts invariants:
|
||
|
||
1. Every 837 file parses (200, total_claims ≥ 1) and lands as one
|
||
837p batch.
|
||
2. Every 835 file parses (200, total_claims ≥ 1) and lands as one
|
||
835 batch; each carries an R835_MULTI_BPR warning.
|
||
3. After all batches: store carries len(837 files) + len(835 files)
|
||
batches with the expected kind split.
|
||
4. After all 835s loaded: store.list_unmatched() returns shapes:
|
||
unmatched_claims = total_unique_837_pcns - total_matched
|
||
unmatched_remittances = total_unique_835_pcns - total_matched
|
||
Both counts are ≥ 0; the matched count is a function of how
|
||
many 837 PCNs happen to appear in the 835 set — that's data-
|
||
dependent and not asserted directly.
|
||
5. Per-835 reconciliation summaries are well-formed (matched,
|
||
unmatched_claims, unmatched_remittances, skipped — all ints).
|
||
|
||
Files are discovered via glob so adding more 837 files (the
|
||
production set grows over time) needs no test edit.
|
||
"""
|
||
from cyclone.store import CycloneStore
|
||
|
||
assert len(global_store.list()) == 0
|
||
|
||
# Discover files dynamically — works for N 837s and M 835s.
|
||
prodfiles_837 = sorted(p for p in PRODFILE_DIR.iterdir() if p.is_file())
|
||
prodfiles_835 = sorted(p for p in PRODFILE_835_DIR_FOR_XP.iterdir() if p.is_file())
|
||
assert prodfiles_837, f"no 837 files at {PRODFILE_DIR}"
|
||
assert prodfiles_835, f"no 835 files at {PRODFILE_835_DIR_FOR_XP}"
|
||
|
||
# 1. Load every 837 first (claims must exist before 835 reconcile runs).
|
||
unique_837_pcns: set[str] = set()
|
||
for path in prodfiles_837:
|
||
with open(path, "rb") as f:
|
||
resp = client.post(
|
||
"/api/parse-837",
|
||
files={"file": (path.name, f, "text/plain")},
|
||
headers={"Accept": "application/json"},
|
||
)
|
||
assert resp.status_code == 200, f"{path.name}: {resp.text}"
|
||
body = resp.json()
|
||
assert body["summary"]["total_claims"] >= 1, path.name
|
||
assert body["summary"]["failed"] == 0, path.name
|
||
for claim in body["claims"]:
|
||
pcn = claim.get("claim_id") or claim.get("patient_control_number")
|
||
if pcn:
|
||
unique_837_pcns.add(pcn)
|
||
|
||
# 2. Load every 835. Each triggers reconcile.run against the existing
|
||
# 837 claims; the per-batch reconciliation summary records how many
|
||
# of THIS batch's remits matched.
|
||
unique_835_pcns: set[str] = set()
|
||
per_835_matched: list[int] = []
|
||
per_835_summary_keys = {"matched", "unmatched_claims", "unmatched_remittances", "skipped"}
|
||
for path in prodfiles_835:
|
||
with open(path, "rb") as f:
|
||
resp = client.post(
|
||
"/api/parse-835",
|
||
files={"file": (path.name, f, "application/octet-stream")},
|
||
headers={"Accept": "application/json"},
|
||
)
|
||
assert resp.status_code == 200, f"{path.name}: {resp.text}"
|
||
body = resp.json()
|
||
assert body["summary"]["total_claims"] >= 1, path.name
|
||
assert body["summary"]["failed"] == 0, path.name
|
||
# CO Medicaid split-payment pattern: validator surfaces the
|
||
# non-standard data as a warning; the batch still passes.
|
||
assert any(
|
||
issue["rule"] == "R835_MULTI_BPR"
|
||
for issue in body["validation"]["warnings"]
|
||
), f"{path.name}: missing R835_MULTI_BPR warning"
|
||
# Per-claim PCN extraction — UI shape is claim_id (not PCN).
|
||
for cp in body["claims"]:
|
||
pcn = cp.get("claim_id") or cp.get("payer_claim_control_number")
|
||
if pcn:
|
||
unique_835_pcns.add(pcn)
|
||
# Reconciliation summary must be well-formed.
|
||
rec = body["reconciliation"]
|
||
assert set(rec.keys()) >= per_835_summary_keys, path.name
|
||
for key in per_835_summary_keys:
|
||
assert isinstance(rec[key], int), (path.name, key, rec[key])
|
||
assert rec["skipped"] == 0, path.name
|
||
per_835_matched.append(rec["matched"])
|
||
|
||
# 3. Store shape: one batch per file, split by kind.
|
||
batches = global_store.list()
|
||
assert len(batches) == len(prodfiles_837) + len(prodfiles_835)
|
||
by_kind: dict[str, set[str]] = {"837p": set(), "835": set()}
|
||
for b in batches:
|
||
by_kind[b.kind].add(b.input_filename)
|
||
assert by_kind["837p"] == {p.name for p in prodfiles_837}
|
||
assert by_kind["835"] == {p.name for p in prodfiles_835}
|
||
|
||
# 4. Reconciliation invariants. matched_count is bounded by the size
|
||
# of the smaller set; unmatched counts are the disjoint remainder.
|
||
fresh_store = CycloneStore()
|
||
unmatched = fresh_store.list_unmatched(kind="both")
|
||
unmatched_claims = len(unmatched["claims"])
|
||
unmatched_remits = len(unmatched["remittances"])
|
||
|
||
total_matched = sum(per_835_matched)
|
||
assert 0 <= total_matched <= len(unique_837_pcns), (
|
||
f"matched {total_matched} outside [0, {len(unique_837_pcns)}]"
|
||
)
|
||
# Every matched pair consumes one claim and one remit.
|
||
assert unmatched_claims == len(unique_837_pcns) - total_matched, (
|
||
f"unmatched_claims {unmatched_claims} != "
|
||
f"{len(unique_837_pcns)} - {total_matched}"
|
||
)
|
||
assert unmatched_remits == len(unique_835_pcns) - total_matched, (
|
||
f"unmatched_remittances {unmatched_remits} != "
|
||
f"{len(unique_835_pcns)} - {total_matched}"
|
||
)
|
||
# The two PCN sets are independent sources of truth: total deduped
|
||
# claim rows + remittance rows in the DB must equal the input totals
|
||
# adjusted for cross-batch dedup. The DB-level ground truth:
|
||
claim_rows = sum(
|
||
len(b.result.claims) for b in batches if b.kind == "837p"
|
||
)
|
||
# Deduped claim rows == unique PCNs (one Claim row per PCN due to
|
||
# the store.add dedup logic, same as 835 PCNs).
|
||
assert claim_rows >= len(unique_837_pcns), (
|
||
f"claim_rows {claim_rows} < unique_837_pcns {len(unique_837_pcns)}"
|
||
)
|
||
|
||
|
||
def test_409_response_includes_existing_batch_id_for_837(client: TestClient) -> None:
|
||
"""When a CLM01 already exists in a prior batch, the 409 body has existing_batch_id."""
|
||
from datetime import datetime, timezone
|
||
|
||
from cyclone import db as _db
|
||
from cyclone.db import Batch, Claim
|
||
|
||
# Seed a prior batch with claim CLM-X.
|
||
with _db.SessionLocal()() as s:
|
||
s.add(Batch(
|
||
id="PRIOR", kind="837p", input_filename="prior.txt",
|
||
parsed_at=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||
raw_result_json={},
|
||
))
|
||
s.add(Claim(id="CLM-X", batch_id="PRIOR", patient_control_number="M"))
|
||
s.commit()
|
||
|
||
# Build a file with two CLM* segments both using CLM-X (forces PK collision).
|
||
text = (
|
||
"ISA*00* *00* *ZZ*SUBMITTERID *ZZ*RECEIVERID "
|
||
"*240101*1200*^*00501*000000001*0*P*:~\n"
|
||
"GS*HC*SUBMITTERID*RECEIVERID*20240101*1200*1*X*005010X222A1~\n"
|
||
"ST*837*0001*005010X222A1~\n"
|
||
"BHT*0019*00*1*20240101*1200*CH~\n"
|
||
"NM1*41*2*SUBMITTER*****46*SUBMITTERID~\n"
|
||
"PER*IC*CONTACT*TE*5555555555~\n"
|
||
"NM1*40*2*RECEIVER*****46*RECEIVERID~\n"
|
||
"HL*1**20*1~\n"
|
||
"NM1*85*2*BILLING*****XX*1881068062~\n"
|
||
"N3*123 MAIN*\nN4*DENVER*CO*80202~\n"
|
||
"REF*EI*123456789~\n"
|
||
"HL*2*1*22*0~\n"
|
||
"SBR*P*18*******CI~\n"
|
||
"NM1*IL*1*DOE*JOHN****MI*M~\n"
|
||
"N3*456 ELM*\nN4*DENVER*CO*80202~\n"
|
||
"DMG*D8*19700101*M~\n"
|
||
"NM1*PR*2*MEDICAID*****PI*MCD~\n"
|
||
"CLM*CLM-X*100***11:B:1*Y*A*Y*Y~\n"
|
||
"LX*1~\nSV1*HC:99213*100*UN*1***1~\n"
|
||
"DTP*472*D8*20240101~\n"
|
||
"CLM*CLM-X*100***11:B:1*Y*A*Y*Y~\n"
|
||
"LX*2~\nSV1*HC:99213*100*UN*1***1~\n"
|
||
"DTP*472*D8*20240101~\n"
|
||
"SE*30*0001~\n"
|
||
"GE*1*1~\n"
|
||
"IEA*1*000000001~\n"
|
||
)
|
||
resp = client.post(
|
||
"/api/parse-837",
|
||
files={"file": ("dup.txt", text, "text/plain")},
|
||
headers={"Accept": "application/json"},
|
||
)
|
||
assert resp.status_code == 409, resp.text
|
||
body = resp.json()
|
||
assert body.get("existing_batch_id") == "PRIOR"
|