feat(ta1): TA1 interchange ACK parser, persistence, and API
Adds full TA1 (Interchange Acknowledgment, X12 envelope-level) support
to mirror the existing 999 transaction-set ACK pipeline.
Parser & models
- parsers/models_ta1.py: Pydantic Ta1Ack + ParseResultTa1 models
(AckCode = Literal['A','E','R'], date serializer mirroring 999)
- parsers/parse_ta1.py: TA1 segment parser
- Tolerant YYMMDD (6-digit) + CCYYMMDD (8-digit) date handling
for CO Medicaid interchange_date / ack_generated_date
- 106-char ISA validation (15-char sender/receiver IDs)
- source_batch_id = 'TA1-<ISA13>'
Persistence
- migrations/0005_create_ta1_acks.sql: ta1_acks table + indexes
on source_batch_id and ack_code
- db.py: Ta1Ack ORM model (flat columns + raw_json, mirrors Ack)
- store.py: add_ta1_ack, list_ta1_acks (returns all rows),
get_ta1_ack
- db version bumped 4 -> 5 (test_acks.py updated)
API (api.py)
- POST /api/parse-ta1: text/file ingest, persists ta1_ack,
returns detail-ready payload
- GET /api/ta1-acks: list with limit + total (mirrors 999 pattern)
- GET /api/ta1-acks/{ack_id}: detail
- _ta1_to_ui / _serialize_ta1 / _serialize_ta1_from_row helpers
Tests (17 new, 508 total passing)
- tests/test_parse_ta1.py (8): CCYYMMDD + YYMMDD acceptance,
E/R codes, source_batch_id, missing ISA/TA1, short defaults
- tests/test_api_ta1.py (9): happy path, rejected persists,
empty/malformed 400, missing file 422, detail regenerates
segment, 404, empty list, newest-first
- tests/test_prodfiles_smoke.py: extended to smoke-test 352
production TA1 files (A=0, R=352, E=0)
- tests/test_api_parse_persists.py: cross-pipeline reconciliation
test asserting invariants across 837P + 835 prod files
Real-data finding: all 352 production TA1s are R (rejected);
operator follow-up warranted.
This commit is contained in:
Tyler
@@ -209,3 +209,151 @@ def test_prodfile_round_trip_persists_separately(client: TestClient):
|
||||
ctrl_991102984 = [b for b in batches if b.result.envelope.control_number == "991102984"]
|
||||
assert len(ctrl_991102984) == 2
|
||||
assert len({b.id for b in ctrl_991102984}) == 2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Cross-pipeline reconciliation (837 + 835 together)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
PRODFILE_835_DIR_FOR_XP = (
|
||||
Path(__file__).parent.parent.parent / "docs" / "prodfiles" / "835fromco"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (PRODFILE_DIR.is_dir() and PRODFILE_835_DIR_FOR_XP.is_dir()),
|
||||
reason=f"production 837 or 835 files not present (gitignored): {PRODFILE_DIR}, {PRODFILE_835_DIR_FOR_XP}",
|
||||
)
|
||||
def test_prodfile_cross_pipeline_reconciles(client: TestClient):
|
||||
"""All production 837s + all production 835s end-to-end.
|
||||
|
||||
Pipeline under test:
|
||||
POST /api/parse-837 (×N) → store.add (claim rows) → DB
|
||||
POST /api/parse-835 (×M) → store.add (remit rows) → reconcile.run → DB
|
||||
|
||||
Designed to survive variable 837 / 835 file counts and arbitrary PCN
|
||||
overlap. Hard-codes NO match counts. Instead asserts invariants:
|
||||
|
||||
1. Every 837 file parses (200, total_claims ≥ 1) and lands as one
|
||||
837p batch.
|
||||
2. Every 835 file parses (200, total_claims ≥ 1) and lands as one
|
||||
835 batch; each carries an R835_MULTI_BPR warning.
|
||||
3. After all batches: store carries len(837 files) + len(835 files)
|
||||
batches with the expected kind split.
|
||||
4. After all 835s loaded: store.list_unmatched() returns shapes:
|
||||
unmatched_claims = total_unique_837_pcns - total_matched
|
||||
unmatched_remittances = total_unique_835_pcns - total_matched
|
||||
Both counts are ≥ 0; the matched count is a function of how
|
||||
many 837 PCNs happen to appear in the 835 set — that's data-
|
||||
dependent and not asserted directly.
|
||||
5. Per-835 reconciliation summaries are well-formed (matched,
|
||||
unmatched_claims, unmatched_remittances, skipped — all ints).
|
||||
|
||||
Files are discovered via glob so adding more 837 files (the
|
||||
production set grows over time) needs no test edit.
|
||||
"""
|
||||
from cyclone.store import CycloneStore
|
||||
|
||||
assert len(global_store.list()) == 0
|
||||
|
||||
# Discover files dynamically — works for N 837s and M 835s.
|
||||
prodfiles_837 = sorted(p for p in PRODFILE_DIR.iterdir() if p.is_file())
|
||||
prodfiles_835 = sorted(p for p in PRODFILE_835_DIR_FOR_XP.iterdir() if p.is_file())
|
||||
assert prodfiles_837, f"no 837 files at {PRODFILE_DIR}"
|
||||
assert prodfiles_835, f"no 835 files at {PRODFILE_835_DIR_FOR_XP}"
|
||||
|
||||
# 1. Load every 837 first (claims must exist before 835 reconcile runs).
|
||||
unique_837_pcns: set[str] = set()
|
||||
for path in prodfiles_837:
|
||||
with open(path, "rb") as f:
|
||||
resp = client.post(
|
||||
"/api/parse-837",
|
||||
files={"file": (path.name, f, "text/plain")},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 200, f"{path.name}: {resp.text}"
|
||||
body = resp.json()
|
||||
assert body["summary"]["total_claims"] >= 1, path.name
|
||||
assert body["summary"]["failed"] == 0, path.name
|
||||
for claim in body["claims"]:
|
||||
pcn = claim.get("claim_id") or claim.get("patient_control_number")
|
||||
if pcn:
|
||||
unique_837_pcns.add(pcn)
|
||||
|
||||
# 2. Load every 835. Each triggers reconcile.run against the existing
|
||||
# 837 claims; the per-batch reconciliation summary records how many
|
||||
# of THIS batch's remits matched.
|
||||
unique_835_pcns: set[str] = set()
|
||||
per_835_matched: list[int] = []
|
||||
per_835_summary_keys = {"matched", "unmatched_claims", "unmatched_remittances", "skipped"}
|
||||
for path in prodfiles_835:
|
||||
with open(path, "rb") as f:
|
||||
resp = client.post(
|
||||
"/api/parse-835",
|
||||
files={"file": (path.name, f, "application/octet-stream")},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 200, f"{path.name}: {resp.text}"
|
||||
body = resp.json()
|
||||
assert body["summary"]["total_claims"] >= 1, path.name
|
||||
assert body["summary"]["failed"] == 0, path.name
|
||||
# CO Medicaid split-payment pattern: validator surfaces the
|
||||
# non-standard data as a warning; the batch still passes.
|
||||
assert any(
|
||||
issue["rule"] == "R835_MULTI_BPR"
|
||||
for issue in body["validation"]["warnings"]
|
||||
), f"{path.name}: missing R835_MULTI_BPR warning"
|
||||
# Per-claim PCN extraction — UI shape is claim_id (not PCN).
|
||||
for cp in body["claims"]:
|
||||
pcn = cp.get("claim_id") or cp.get("payer_claim_control_number")
|
||||
if pcn:
|
||||
unique_835_pcns.add(pcn)
|
||||
# Reconciliation summary must be well-formed.
|
||||
rec = body["reconciliation"]
|
||||
assert set(rec.keys()) >= per_835_summary_keys, path.name
|
||||
for key in per_835_summary_keys:
|
||||
assert isinstance(rec[key], int), (path.name, key, rec[key])
|
||||
assert rec["skipped"] == 0, path.name
|
||||
per_835_matched.append(rec["matched"])
|
||||
|
||||
# 3. Store shape: one batch per file, split by kind.
|
||||
batches = global_store.list()
|
||||
assert len(batches) == len(prodfiles_837) + len(prodfiles_835)
|
||||
by_kind: dict[str, set[str]] = {"837p": set(), "835": set()}
|
||||
for b in batches:
|
||||
by_kind[b.kind].add(b.input_filename)
|
||||
assert by_kind["837p"] == {p.name for p in prodfiles_837}
|
||||
assert by_kind["835"] == {p.name for p in prodfiles_835}
|
||||
|
||||
# 4. Reconciliation invariants. matched_count is bounded by the size
|
||||
# of the smaller set; unmatched counts are the disjoint remainder.
|
||||
fresh_store = CycloneStore()
|
||||
unmatched = fresh_store.list_unmatched(kind="both")
|
||||
unmatched_claims = len(unmatched["claims"])
|
||||
unmatched_remits = len(unmatched["remittances"])
|
||||
|
||||
total_matched = sum(per_835_matched)
|
||||
assert 0 <= total_matched <= len(unique_837_pcns), (
|
||||
f"matched {total_matched} outside [0, {len(unique_837_pcns)}]"
|
||||
)
|
||||
# Every matched pair consumes one claim and one remit.
|
||||
assert unmatched_claims == len(unique_837_pcns) - total_matched, (
|
||||
f"unmatched_claims {unmatched_claims} != "
|
||||
f"{len(unique_837_pcns)} - {total_matched}"
|
||||
)
|
||||
assert unmatched_remits == len(unique_835_pcns) - total_matched, (
|
||||
f"unmatched_remittances {unmatched_remits} != "
|
||||
f"{len(unique_835_pcns)} - {total_matched}"
|
||||
)
|
||||
# The two PCN sets are independent sources of truth: total deduped
|
||||
# claim rows + remittance rows in the DB must equal the input totals
|
||||
# adjusted for cross-batch dedup. The DB-level ground truth:
|
||||
claim_rows = sum(
|
||||
len(b.result.claims) for b in batches if b.kind == "837p"
|
||||
)
|
||||
# Deduped claim rows == unique PCNs (one Claim row per PCN due to
|
||||
# the store.add dedup logic, same as 835 PCNs).
|
||||
assert claim_rows >= len(unique_837_pcns), (
|
||||
f"claim_rows {claim_rows} < unique_837_pcns {len(unique_837_pcns)}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user