feat(ta1): TA1 interchange ACK parser, persistence, and API

Adds full TA1 (Interchange Acknowledgment, X12 envelope-level) support
to mirror the existing 999 transaction-set ACK pipeline.

Parser & models
- parsers/models_ta1.py: Pydantic Ta1Ack + ParseResultTa1 models
  (AckCode = Literal['A','E','R'], date serializer mirroring 999)
- parsers/parse_ta1.py: TA1 segment parser
  - Tolerant YYMMDD (6-digit) + CCYYMMDD (8-digit) date handling
    for CO Medicaid interchange_date / ack_generated_date
  - 106-char ISA validation (15-char sender/receiver IDs)
  - source_batch_id = 'TA1-<ISA13>'

Persistence
- migrations/0005_create_ta1_acks.sql: ta1_acks table + indexes
  on source_batch_id and ack_code
- db.py: Ta1Ack ORM model (flat columns + raw_json, mirrors Ack)
- store.py: add_ta1_ack, list_ta1_acks (returns all rows),
  get_ta1_ack
- db version bumped 4 -> 5 (test_acks.py updated)

API (api.py)
- POST /api/parse-ta1: text/file ingest, persists ta1_ack,
  returns detail-ready payload
- GET /api/ta1-acks: list with limit + total (mirrors 999 pattern)
- GET /api/ta1-acks/{ack_id}: detail
- _ta1_to_ui / _serialize_ta1 / _serialize_ta1_from_row helpers

Tests (17 new, 508 total passing)
- tests/test_parse_ta1.py (8): CCYYMMDD + YYMMDD acceptance,
  E/R codes, source_batch_id, missing ISA/TA1, short defaults
- tests/test_api_ta1.py (9): happy path, rejected persists,
  empty/malformed 400, missing file 422, detail regenerates
  segment, 404, empty list, newest-first
- tests/test_prodfiles_smoke.py: extended to smoke-test 352
  production TA1 files (A=0, R=352, E=0)
- tests/test_api_parse_persists.py: cross-pipeline reconciliation
  test asserting invariants across 837P + 835 prod files

Real-data finding: all 352 production TA1s are R (rejected);
operator follow-up warranted.
This commit is contained in:
Tyler
2026-06-20 18:58:56 -06:00
parent 1db33ef2fa
commit 76278ec9f6
11 changed files with 1285 additions and 4 deletions
+5 -4
View File
@@ -51,16 +51,17 @@ def test_migration_0002_creates_acks_table():
def test_migration_latest_idempotent_on_fresh_db():
"""Re-running the migration on the same DB must be a no-op (PRAGMA
user_version already at the latest version — currently 4 after the
0004 rejection columns + state-history index was added by SP6)."""
user_version already at the latest version — currently 5 after the
0004 rejection columns + state-history index (SP6) and the 0005
ta1_acks table (this PR)."""
with db.engine().begin() as c:
v1 = c.exec_driver_sql("PRAGMA user_version").scalar() or 0
assert v1 == 4
assert v1 == 5
# A second run should not raise and should not bump the version.
db_migrate.run(db.engine())
with db.engine().begin() as c:
v2 = c.exec_driver_sql("PRAGMA user_version").scalar() or 0
assert v2 == 4
assert v2 == 5
def test_add_ack_persists_row():
+148
View File
@@ -209,3 +209,151 @@ def test_prodfile_round_trip_persists_separately(client: TestClient):
ctrl_991102984 = [b for b in batches if b.result.envelope.control_number == "991102984"]
assert len(ctrl_991102984) == 2
assert len({b.id for b in ctrl_991102984}) == 2
# --------------------------------------------------------------------------- #
# Cross-pipeline reconciliation (837 + 835 together)
# --------------------------------------------------------------------------- #
PRODFILE_835_DIR_FOR_XP = (
Path(__file__).parent.parent.parent / "docs" / "prodfiles" / "835fromco"
)
@pytest.mark.skipif(
not (PRODFILE_DIR.is_dir() and PRODFILE_835_DIR_FOR_XP.is_dir()),
reason=f"production 837 or 835 files not present (gitignored): {PRODFILE_DIR}, {PRODFILE_835_DIR_FOR_XP}",
)
def test_prodfile_cross_pipeline_reconciles(client: TestClient):
"""All production 837s + all production 835s end-to-end.
Pipeline under test:
POST /api/parse-837 (×N) → store.add (claim rows) → DB
POST /api/parse-835 (×M) → store.add (remit rows) → reconcile.run → DB
Designed to survive variable 837 / 835 file counts and arbitrary PCN
overlap. Hard-codes NO match counts. Instead asserts invariants:
1. Every 837 file parses (200, total_claims ≥ 1) and lands as one
837p batch.
2. Every 835 file parses (200, total_claims ≥ 1) and lands as one
835 batch; each carries an R835_MULTI_BPR warning.
3. After all batches: store carries len(837 files) + len(835 files)
batches with the expected kind split.
4. After all 835s loaded: store.list_unmatched() returns shapes:
unmatched_claims = total_unique_837_pcns - total_matched
unmatched_remittances = total_unique_835_pcns - total_matched
Both counts are ≥ 0; the matched count is a function of how
many 837 PCNs happen to appear in the 835 set — that's data-
dependent and not asserted directly.
5. Per-835 reconciliation summaries are well-formed (matched,
unmatched_claims, unmatched_remittances, skipped — all ints).
Files are discovered via glob so adding more 837 files (the
production set grows over time) needs no test edit.
"""
from cyclone.store import CycloneStore
assert len(global_store.list()) == 0
# Discover files dynamically — works for N 837s and M 835s.
prodfiles_837 = sorted(p for p in PRODFILE_DIR.iterdir() if p.is_file())
prodfiles_835 = sorted(p for p in PRODFILE_835_DIR_FOR_XP.iterdir() if p.is_file())
assert prodfiles_837, f"no 837 files at {PRODFILE_DIR}"
assert prodfiles_835, f"no 835 files at {PRODFILE_835_DIR_FOR_XP}"
# 1. Load every 837 first (claims must exist before 835 reconcile runs).
unique_837_pcns: set[str] = set()
for path in prodfiles_837:
with open(path, "rb") as f:
resp = client.post(
"/api/parse-837",
files={"file": (path.name, f, "text/plain")},
headers={"Accept": "application/json"},
)
assert resp.status_code == 200, f"{path.name}: {resp.text}"
body = resp.json()
assert body["summary"]["total_claims"] >= 1, path.name
assert body["summary"]["failed"] == 0, path.name
for claim in body["claims"]:
pcn = claim.get("claim_id") or claim.get("patient_control_number")
if pcn:
unique_837_pcns.add(pcn)
# 2. Load every 835. Each triggers reconcile.run against the existing
# 837 claims; the per-batch reconciliation summary records how many
# of THIS batch's remits matched.
unique_835_pcns: set[str] = set()
per_835_matched: list[int] = []
per_835_summary_keys = {"matched", "unmatched_claims", "unmatched_remittances", "skipped"}
for path in prodfiles_835:
with open(path, "rb") as f:
resp = client.post(
"/api/parse-835",
files={"file": (path.name, f, "application/octet-stream")},
headers={"Accept": "application/json"},
)
assert resp.status_code == 200, f"{path.name}: {resp.text}"
body = resp.json()
assert body["summary"]["total_claims"] >= 1, path.name
assert body["summary"]["failed"] == 0, path.name
# CO Medicaid split-payment pattern: validator surfaces the
# non-standard data as a warning; the batch still passes.
assert any(
issue["rule"] == "R835_MULTI_BPR"
for issue in body["validation"]["warnings"]
), f"{path.name}: missing R835_MULTI_BPR warning"
# Per-claim PCN extraction — UI shape is claim_id (not PCN).
for cp in body["claims"]:
pcn = cp.get("claim_id") or cp.get("payer_claim_control_number")
if pcn:
unique_835_pcns.add(pcn)
# Reconciliation summary must be well-formed.
rec = body["reconciliation"]
assert set(rec.keys()) >= per_835_summary_keys, path.name
for key in per_835_summary_keys:
assert isinstance(rec[key], int), (path.name, key, rec[key])
assert rec["skipped"] == 0, path.name
per_835_matched.append(rec["matched"])
# 3. Store shape: one batch per file, split by kind.
batches = global_store.list()
assert len(batches) == len(prodfiles_837) + len(prodfiles_835)
by_kind: dict[str, set[str]] = {"837p": set(), "835": set()}
for b in batches:
by_kind[b.kind].add(b.input_filename)
assert by_kind["837p"] == {p.name for p in prodfiles_837}
assert by_kind["835"] == {p.name for p in prodfiles_835}
# 4. Reconciliation invariants. matched_count is bounded by the size
# of the smaller set; unmatched counts are the disjoint remainder.
fresh_store = CycloneStore()
unmatched = fresh_store.list_unmatched(kind="both")
unmatched_claims = len(unmatched["claims"])
unmatched_remits = len(unmatched["remittances"])
total_matched = sum(per_835_matched)
assert 0 <= total_matched <= len(unique_837_pcns), (
f"matched {total_matched} outside [0, {len(unique_837_pcns)}]"
)
# Every matched pair consumes one claim and one remit.
assert unmatched_claims == len(unique_837_pcns) - total_matched, (
f"unmatched_claims {unmatched_claims} != "
f"{len(unique_837_pcns)} - {total_matched}"
)
assert unmatched_remits == len(unique_835_pcns) - total_matched, (
f"unmatched_remittances {unmatched_remits} != "
f"{len(unique_835_pcns)} - {total_matched}"
)
# The two PCN sets are independent sources of truth: total deduped
# claim rows + remittance rows in the DB must equal the input totals
# adjusted for cross-batch dedup. The DB-level ground truth:
claim_rows = sum(
len(b.result.claims) for b in batches if b.kind == "837p"
)
# Deduped claim rows == unique PCNs (one Claim row per PCN due to
# the store.add dedup logic, same as 835 PCNs).
assert claim_rows >= len(unique_837_pcns), (
f"claim_rows {claim_rows} < unique_837_pcns {len(unique_837_pcns)}"
)
+166
View File
@@ -0,0 +1,166 @@
"""Tests for the /api/parse-ta1 endpoint and the ta1_acks list/detail endpoints."""
from __future__ import annotations
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from cyclone.api import app
from cyclone.store import store as global_store
from cyclone import db
ACCEPTED = (
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
"*260520*1750*^*00501*000000001*0*P*:~"
"TA1*000000001*20260520*1750*A*000*20260520~"
"IEA*1*000000001~"
)
REJECTED = (
"ISA*00* *00* *ZZ*COMEDASSISTPROG*ZZ*TP11525703 "
"*260520*1750*^*00501*000000001*0*P*:~"
"TA1*320293557*260520*2338*R*006~"
"IEA*0*000000001~"
)
@pytest.fixture(autouse=True)
def clear_store():
"""Reset the module-level store before and after each test."""
with global_store._lock:
global_store._batches.clear()
yield
with global_store._lock:
global_store._batches.clear()
@pytest.fixture
def client() -> TestClient:
return TestClient(app)
def test_parse_ta1_endpoint_happy_path(client: TestClient):
"""Accepted TA1 → 200 with parsed envelope + persisted ta1 row."""
resp = client.post(
"/api/parse-ta1",
files={"file": ("accepted.ta1", ACCEPTED, "text/plain")},
headers={"Accept": "application/json"},
)
assert resp.status_code == 200, resp.text
body = resp.json()
ta1 = body["ta1"]
assert ta1["ack_code"] == "A"
assert ta1["control_number"] == "000000001"
assert ta1["note_code"] == "000"
assert ta1["interchange_date"] == "2026-05-20"
assert ta1["source_batch_id"] == "TA1-000000001"
# Round-trip text present and well-formed.
assert ta1["raw_ta1_text"].startswith("TA1*")
assert ta1["raw_ta1_text"].endswith("~")
# Parsed shape mirrors the model.
assert body["parsed"]["ta1"]["ack_code"] == "A"
def test_parse_ta1_endpoint_rejected_persists(client: TestClient):
"""Rejected TA1 (Colorado YYMMDD format) → 200 + R ack_code in DB."""
resp = client.post(
"/api/parse-ta1",
files={"file": ("rejected.ta1", REJECTED, "text/plain")},
headers={"Accept": "application/json"},
)
assert resp.status_code == 200, resp.text
body = resp.json()
assert body["ta1"]["ack_code"] == "R"
assert body["ta1"]["note_code"] == "006"
# Persisted to DB
list_resp = client.get("/api/ta1-acks?limit=10", headers={"Accept": "application/json"})
assert list_resp.status_code == 200, list_resp.text
list_body = list_resp.json()
assert list_body["total"] == 1
assert list_body["items"][0]["ack_code"] == "R"
def test_parse_ta1_endpoint_empty_file_returns_400(client: TestClient):
"""Empty upload → 400 (never 500)."""
resp = client.post(
"/api/parse-ta1",
files={"file": ("empty.ta1", b"", "text/plain")},
headers={"Accept": "application/json"},
)
assert resp.status_code == 400
def test_parse_ta1_endpoint_malformed_returns_400(client: TestClient):
"""No TA1 segment → 400 with parse-error detail."""
text = (
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
"*260520*1750*^*00501*000000001*0*P*:~"
"IEA*0*000000001~"
)
resp = client.post(
"/api/parse-ta1",
files={"file": ("bad.ta1", text, "text/plain")},
headers={"Accept": "application/json"},
)
assert resp.status_code == 400
body = resp.json()
assert "TA1" in body["detail"]
def test_parse_ta1_endpoint_rejects_missing_file(client: TestClient):
"""FastAPI File(...) with no default → 422."""
resp = client.post("/api/parse-ta1")
assert resp.status_code == 422
def test_get_ta1_ack_detail_regenerates_segment(client: TestClient):
"""GET /api/ta1-acks/{id} includes the raw TA1 segment rebuilt from the row."""
resp = client.post(
"/api/parse-ta1",
files={"file": ("accepted.ta1", ACCEPTED, "text/plain")},
headers={"Accept": "application/json"},
)
assert resp.status_code == 200
ta1_id = resp.json()["ta1"]["id"]
detail_resp = client.get(f"/api/ta1-acks/{ta1_id}", headers={"Accept": "application/json"})
assert detail_resp.status_code == 200, detail_resp.text
body = detail_resp.json()
assert body["id"] == ta1_id
assert body["raw_ta1_text"].startswith("TA1*000000001*")
assert body["raw_ta1_text"].rstrip().endswith("~")
# raw_json carries the full ParseResultTa1 for the detail view.
assert body["raw_json"]["ta1"]["ack_code"] == "A"
def test_get_ta1_ack_404_for_missing(client: TestClient):
"""GET /api/ta1-acks/{id} returns 404 for a missing id."""
resp = client.get("/api/ta1-acks/9999", headers={"Accept": "application/json"})
assert resp.status_code == 404
def test_list_ta1_acks_empty(client: TestClient):
"""Empty DB → empty list with total=0."""
resp = client.get("/api/ta1-acks", headers={"Accept": "application/json"})
assert resp.status_code == 200
body = resp.json()
assert body == {"total": 0, "items": []}
def test_list_ta1_acks_newest_first(client: TestClient):
"""Multiple uploads: list returns rows newest first (by auto-increment id desc)."""
for i, payload in enumerate([ACCEPTED, REJECTED]):
resp = client.post(
"/api/parse-ta1",
files={"file": (f"file{i}.ta1", payload, "text/plain")},
headers={"Accept": "application/json"},
)
assert resp.status_code == 200, resp.text
resp = client.get("/api/ta1-acks", headers={"Accept": "application/json"})
items = resp.json()["items"]
assert len(items) == 2
# The REJECTED (uploaded second) is first.
assert items[0]["ack_code"] == "R"
assert items[1]["ack_code"] == "A"
+128
View File
@@ -0,0 +1,128 @@
"""Tests for the TA1 (Interchange Acknowledgment) parser orchestrator."""
from __future__ import annotations
from datetime import date
import pytest
from cyclone.parsers.exceptions import CycloneParseError
from cyclone.parsers.parse_ta1 import parse_ta1_text
# A minimal accepted TA1 in canonical CCYYMMDD form (per X12 spec).
# ISA must be exactly 106 chars (15-char sender/receiver IDs).
ACCEPTED_CCYYMMDD = (
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
"*260520*1750*^*00501*000000001*0*P*:~"
"TA1*000000001*20260520*1750*A*000*20260520~"
"IEA*1*000000001~"
)
# A rejected TA1 with Colorado's YYMMDD date format (6 digits instead of 8).
# This is what FromHPE/tp11525703-837P_M019044596-…_TA1.x12 looks like in
# production — same shape but YYMMDD and ack_code=R, note_code=006.
REJECTED_YYMMDD = (
"ISA*00* *00* *ZZ*COMEDASSISTPROG*ZZ*TP11525703 "
"*260520*1750*^*00501*000000001*0*P*:~"
"TA1*320293557*260520*2338*R*006~"
"IEA*0*000000001~"
)
# Accepted variant with errors (E) — uncommon but spec-legal.
ACCEPTED_WITH_ERRORS = (
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
"*260520*1750*^*00501*000000001*0*P*:~"
"TA1*000000001*20260520*1750*E*007*20260520~"
"IEA*1*000000001~"
)
def test_parse_accepted_ccyymmdd_returns_a():
"""Spec-compliant TA1 with CCYYMMDD dates surfaces ack_code='A'."""
result = parse_ta1_text(ACCEPTED_CCYYMMDD, input_file="accepted.ta1")
assert result.ta1.ack_code == "A"
assert result.ta1.control_number == "000000001"
assert result.ta1.interchange_date == date(2026, 5, 20)
assert result.ta1.interchange_time == "1750"
assert result.ta1.note_code == "000"
assert result.ta1.ack_generated_date == date(2026, 5, 20)
# Envelope mirrors the ISA
assert result.envelope.sender_id == "SENDER"
assert result.envelope.receiver_id == "RECEIVER"
assert result.envelope.control_number == "000000001"
# Summary reflects the single ack
assert result.summary.total_claims == 1
assert result.summary.passed == 1
assert result.summary.failed == 0
def test_parse_rejected_yymmdd_from_production():
"""Colorado's YYMMDD date format (6 digits) must still parse."""
result = parse_ta1_text(REJECTED_YYMMDD, input_file="rejected.ta1")
assert result.ta1.ack_code == "R"
assert result.ta1.note_code == "006"
assert result.ta1.interchange_date == date(2026, 5, 20), (
"YYMMDD must be inferred as 20YY-MM-DD"
)
assert result.ta1.interchange_time == "2338"
assert result.summary.passed == 0
assert result.summary.failed == 1
def test_parse_accepted_with_errors_returns_e():
"""TA1 ack_code='E' (Accepted with errors) is preserved as the literal."""
result = parse_ta1_text(ACCEPTED_WITH_ERRORS, input_file="with_errors.ta1")
assert result.ta1.ack_code == "E"
assert result.ta1.note_code == "007"
def test_source_batch_id_format():
"""source_batch_id must be 'TA1-<ISA13>' so the FK target is stable."""
result = parse_ta1_text(ACCEPTED_CCYYMMDD, input_file="x.ta1")
assert result.source_batch_id == "TA1-000000001"
def test_missing_isa_raises():
"""No ISA envelope at file start → CycloneParseError (caught by tokenize)."""
text = "TA1*000000001*20260520*1750*A*000*20260520~"
with pytest.raises(CycloneParseError, match="ISA"):
parse_ta1_text(text, input_file="bad.ta1")
def test_missing_ta1_raises():
"""ISA but no TA1 segment → CycloneParseError."""
text = (
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
"*260520*1750*^*00501*000000001*0*P*:~"
"IEA*0*000000001~"
)
with pytest.raises(CycloneParseError, match="No TA1 segment"):
parse_ta1_text(text, input_file="no_ta1.ta1")
def test_short_ta1_segment_uses_defaults():
"""A TA1 with fewer than 6 elements still parses with sensible defaults."""
text = (
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
"*260520*1750*^*00501*000000001*0*P*:~"
"TA1*000000001~"
"IEA*0*000000001~"
)
result = parse_ta1_text(text, input_file="short.ta1")
assert result.ta1.ack_code == "R" # default to rejected when missing
assert result.ta1.note_code is None
assert result.ta1.interchange_date is None
assert result.ta1.interchange_time is None
def test_garbage_date_returns_none_not_raises():
"""A garbage date in TA102 doesn't raise — it surfaces as None."""
text = (
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
"*260520*1750*^*00501*000000001*0*P*:~"
"TA1*000000001*not-a-date*1750*A*000~"
"IEA*0*000000001~"
)
result = parse_ta1_text(text, input_file="bad_date.ta1")
assert result.ta1.interchange_date is None
assert result.ta1.ack_code == "A" # the rest still parses
+248
View File
@@ -0,0 +1,248 @@
"""Smoke tests: every prodfiles/* file we have parsers for must parse cleanly.
These tests are data-tolerant: they discover files via ``glob`` and assert
invariants per file rather than hard-coded expected counts. If ops drops
more files into ``docs/prodfiles/``, the tests stay green as long as the
new files parse cleanly.
Scope:
- ``docs/prodfiles/claims/`` → 113 single-claim 837P files (output format reference)
- ``docs/prodfiles/FromHPE/`` 837P .txt → 4 production 837 submissions to CO Medicaid
- ``docs/prodfiles/FromHPE/`` *.999.x12 → ~1012 production transaction-set acks
- ``docs/prodfiles/FromHPE/`` *.TA1.x12 → ~352 production interchange acks
"""
from __future__ import annotations
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from cyclone.api import app
from cyclone.store import store as global_store
DOCS_PRODFILES = Path(__file__).parent.parent.parent / "docs" / "prodfiles"
CLAIMS_DIR = DOCS_PRODFILES / "claims"
FROMHPE_DIR = DOCS_PRODFILES / "FromHPE"
@pytest.fixture(autouse=True)
def clear_store():
"""Reset the module-level store before and after each test."""
with global_store._lock:
global_store._batches.clear()
yield
with global_store._lock:
global_store._batches.clear()
@pytest.fixture
def client() -> TestClient:
return TestClient(app)
def _post(client: TestClient, endpoint: str, path: Path, content_type: str):
with open(path, "rb") as f:
return client.post(
endpoint,
files={"file": (path.name, f, content_type)},
headers={"Accept": "application/json"},
)
# --------------------------------------------------------------------------- #
# 837P — claims/ (output format reference) + FromHPE/ (real submissions)
# --------------------------------------------------------------------------- #
@pytest.mark.skipif(
not CLAIMS_DIR.is_dir(),
reason=f"docs/prodfiles/claims/ not present at {CLAIMS_DIR}",
)
def test_claims_prodfiles_parse_via_837_endpoint(client: TestClient):
"""Every 837P file in docs/prodfiles/claims/ parses cleanly.
These single-claim files represent the output format our writer
needs to produce. If any of them fails to parse, our round-trip is
broken even before submission to the payer.
Note: claims/ PCNs intentionally overlap with the axiscare 837 files
loaded by ``test_prodfile_round_trip_persists_separately`` (verified
100% overlap). That means the store.add() dedup logic skips them
silently — by design, since they represent the same claim shape. We
assert the API response (parse + validate), not persistence count.
"""
paths = sorted(p for p in CLAIMS_DIR.iterdir() if p.is_file() and p.suffix == ".x12")
assert paths, f"no 837P files at {CLAIMS_DIR}"
for path in paths:
resp = _post(client, "/api/parse-837", path, "text/plain")
assert resp.status_code == 200, f"{path.name}: {resp.text}"
body = resp.json()
assert body["summary"]["total_claims"] >= 1, path.name
assert body["summary"]["failed"] == 0, path.name
assert body["summary"]["passed"] == body["summary"]["total_claims"], path.name
# Every claim must have an id (CLM01) — that's what makes it
# addressable in the API and the DB.
for claim in body["claims"]:
assert claim.get("claim_id"), f"{path.name}: missing claim_id"
# Sender / receiver shape must match the axiscare prod data.
assert body["envelope"]["sender_id"] == "11525703", path.name
assert body["envelope"]["receiver_id"] == "COMEDASSISTPROG", path.name
@pytest.mark.skipif(
not FROMHPE_DIR.is_dir(),
reason=f"docs/prodfiles/FromHPE/ not present at {FROMHPE_DIR}",
)
def test_fromhpe_837_prodfiles_parse(client: TestClient):
"""The 4 production 837P submissions in FromHPE/ parse cleanly."""
paths = sorted(FROMHPE_DIR.glob("tp11525703-837P-*.txt"))
assert paths, f"no 837P .txt files at {FROMHPE_DIR}"
for path in paths:
resp = _post(client, "/api/parse-837", path, "text/plain")
assert resp.status_code == 200, f"{path.name}: {resp.text}"
body = resp.json()
assert body["summary"]["total_claims"] >= 1, path.name
assert body["summary"]["failed"] == 0, path.name
# All FromHPE 837s are sender 11525703 → COMEDASSISTPROG (HPE gateway)
assert body["envelope"]["sender_id"] == "11525703", path.name
batches = [b for b in global_store.list() if b.kind == "837p"]
assert len(batches) == len(paths)
# --------------------------------------------------------------------------- #
# 999 — FromHPE/ production transaction-set acks
# --------------------------------------------------------------------------- #
@pytest.mark.skipif(
not FROMHPE_DIR.is_dir(),
reason=f"docs/prodfiles/FromHPE/ not present at {FROMHPE_DIR}",
)
def test_fromhpe_999_prodfiles_parse(client: TestClient):
"""Every 999 ACK file in FromHPE/ parses and persists to /api/acks.
These ~1012 files are real acks Colorado sends back after receiving
our 837 submissions. If any fail to parse, our ingest of payer
responses is broken.
Known data anomaly: Colorado occasionally sends ``AK9*A*1*1*1`` —
1 received, 1 accepted, 1 rejected from a single set, which violates
X12 spec (AK904 should be ``received - accepted``). The parser reports
what the AK9 literally says; if this becomes a real problem, fix
``_ack_count_summary`` in api.py to clamp
``rejected = max(0, received - accepted)``.
"""
paths = sorted(FROMHPE_DIR.glob("*999.x12"))
assert paths, f"no 999 files at {FROMHPE_DIR}"
a_count = 0 # accepted
r_count = 0 # rejected (set-level)
e_count = 0 # partial / error
malformed_ak9 = 0 # AK9 with accepted + rejected > received (CO bug)
for path in paths:
resp = _post(client, "/api/parse-999", path, "application/octet-stream")
assert resp.status_code == 200, f"{path.name}: {resp.text}"
body = resp.json()
ack = body["ack"]
assert ack["ack_code"] in {"A", "E", "R"}, (
f"{path.name}: unexpected ack_code {ack['ack_code']!r}"
)
assert ack["received_count"] >= 1, path.name
# Colorado's real 999s occasionally ship a malformed AK9
# (e.g. ``AK9*A*1*1*1``: 1 received, 1 accepted, 1 rejected,
# which is impossible from a single set). The parser faithfully
# reports what the AK9 says; we count those as anomalies but
# don't fail on them. See follow-up note in test docstring.
if ack["accepted_count"] + ack["rejected_count"] > ack["received_count"]:
malformed_ak9 += 1
if ack["ack_code"] == "A":
a_count += 1
elif ack["ack_code"] == "R":
r_count += 1
else:
e_count += 1
# Persisted as acks rows. Use the store directly to count (GET /api/acks
# is paginated and we want to assert the total).
from cyclone.store import CycloneStore
fresh_store = CycloneStore()
# acks live in the DB via store.add_ack; read via the same store path.
assert _ack_count(fresh_store) == len(paths), (
f"persisted { _ack_count(fresh_store) } acks, expected {len(paths)}"
)
# Sanity: every persisted ack round-trips through GET /api/acks/{id}.
list_resp = client.get("/api/acks?limit=1000", headers={"Accept": "application/json"})
assert list_resp.status_code == 200, list_resp.text
assert list_resp.json()["total"] == len(paths)
# Real-data distribution: surface counts so an anomaly (e.g. all R)
# is visible in test output even when the test passes.
print(
f"\n[999 prodfile distribution] A={a_count} R={r_count} E={e_count} "
f"malformed_AK9={malformed_ak9} total={len(paths)}"
)
# --------------------------------------------------------------------------- #
# TA1 — FromHPE/ production interchange acks
# --------------------------------------------------------------------------- #
@pytest.mark.skipif(
not FROMHPE_DIR.is_dir(),
reason=f"docs/prodfiles/FromHPE/ not present at {FROMHPE_DIR}",
)
def test_fromhpe_ta1_prodfiles_parse(client: TestClient):
"""Every TA1 file in FromHPE/ parses and persists to /api/ta1-acks.
These files are real interchange acks Colorado sends back in response
to our 837 submissions. All real production files are R (rejected)
— that's the dataset ops gave us to test against.
"""
paths = sorted(FROMHPE_DIR.glob("*TA1.x12"))
assert paths, f"no TA1 files at {FROMHPE_DIR}"
a_count = 0
e_count = 0
r_count = 0
for path in paths:
resp = _post(client, "/api/parse-ta1", path, "application/octet-stream")
assert resp.status_code == 200, f"{path.name}: {resp.text}"
body = resp.json()
ta1 = body["ta1"]
assert ta1["ack_code"] in {"A", "E", "R"}, (
f"{path.name}: unexpected ack_code {ta1['ack_code']!r}"
)
assert ta1["control_number"], f"{path.name}: empty TA101"
assert ta1["source_batch_id"].startswith("TA1-"), path.name
if ta1["ack_code"] == "A":
a_count += 1
elif ta1["ack_code"] == "R":
r_count += 1
else:
e_count += 1
# Verify persistence via GET /api/ta1-acks. The endpoint caps at
# limit=1000 but reports total across all rows, so the assertion
# works even with more rows than the cap.
list_resp = client.get("/api/ta1-acks?limit=1000", headers={"Accept": "application/json"})
assert list_resp.status_code == 200, list_resp.text
assert list_resp.json()["total"] == len(paths)
# Real-data distribution: surface counts so an anomaly is visible.
print(
f"\n[TA1 prodfile distribution] A={a_count} R={r_count} E={e_count} "
f"total={len(paths)}"
)
def _ack_count(store: CycloneStore) -> int:
"""Count rows in the acks table."""
from cyclone import db
with db.SessionLocal()() as s:
return s.query(db.Ack).count()