feat(ta1): TA1 interchange ACK parser, persistence, and API
Adds full TA1 (Interchange Acknowledgment, X12 envelope-level) support
to mirror the existing 999 transaction-set ACK pipeline.
Parser & models
- parsers/models_ta1.py: Pydantic Ta1Ack + ParseResultTa1 models
(AckCode = Literal['A','E','R'], date serializer mirroring 999)
- parsers/parse_ta1.py: TA1 segment parser
- Tolerant YYMMDD (6-digit) + CCYYMMDD (8-digit) date handling
for CO Medicaid interchange_date / ack_generated_date
- 106-char ISA validation (15-char sender/receiver IDs)
- source_batch_id = 'TA1-<ISA13>'
Persistence
- migrations/0005_create_ta1_acks.sql: ta1_acks table + indexes
on source_batch_id and ack_code
- db.py: Ta1Ack ORM model (flat columns + raw_json, mirrors Ack)
- store.py: add_ta1_ack, list_ta1_acks (returns all rows),
get_ta1_ack
- db version bumped 4 -> 5 (test_acks.py updated)
API (api.py)
- POST /api/parse-ta1: text/file ingest, persists ta1_ack,
returns detail-ready payload
- GET /api/ta1-acks: list with limit + total (mirrors 999 pattern)
- GET /api/ta1-acks/{ack_id}: detail
- _ta1_to_ui / _serialize_ta1 / _serialize_ta1_from_row helpers
Tests (17 new, 508 total passing)
- tests/test_parse_ta1.py (8): CCYYMMDD + YYMMDD acceptance,
E/R codes, source_batch_id, missing ISA/TA1, short defaults
- tests/test_api_ta1.py (9): happy path, rejected persists,
empty/malformed 400, missing file 422, detail regenerates
segment, 404, empty list, newest-first
- tests/test_prodfiles_smoke.py: extended to smoke-test 352
production TA1 files (A=0, R=352, E=0)
- tests/test_api_parse_persists.py: cross-pipeline reconciliation
test asserting invariants across 837P + 835 prod files
Real-data finding: all 352 production TA1s are R (rejected);
operator follow-up warranted.
This commit is contained in:
Tyler
@@ -51,6 +51,7 @@ from cyclone.parsers.parse_271 import parse as parse_271_text
|
||||
from cyclone.parsers.parse_837 import parse
|
||||
from cyclone.parsers.parse_835 import parse as parse_835
|
||||
from cyclone.parsers.parse_999 import parse_999_text
|
||||
from cyclone.parsers.parse_ta1 import parse_ta1_text
|
||||
from cyclone.parsers.serialize_270 import serialize_270
|
||||
from cyclone.parsers.serialize_999 import serialize_999
|
||||
from cyclone.parsers.batch_ack_builder import build_ack_for_batch
|
||||
@@ -667,6 +668,174 @@ async def parse_999_endpoint(
|
||||
})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# TA1 (Interchange Acknowledgment)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _ta1_synthetic_source_batch_id(interchange_control_number: str) -> str:
|
||||
"""Return a synthetic ``batches.id`` for a received TA1 with no source batch.
|
||||
|
||||
Mirrors :func:`_ack_synthetic_source_batch_id`. The ta1_acks.source_batch_id
|
||||
FK requires a row in batches; for received TA1s we synthesize an id of
|
||||
the form ``TA1-<ISA13>``. The row is NOT created in batches (same
|
||||
FK-is-no-op convention as the 999 path).
|
||||
"""
|
||||
return f"TA1-{(interchange_control_number or '').strip() or '000000001'}"
|
||||
|
||||
|
||||
@app.post("/api/parse-ta1")
|
||||
async def parse_ta1_endpoint(
|
||||
file: UploadFile = File(...),
|
||||
) -> Any:
|
||||
"""Parse a TA1 (Interchange Acknowledgment) file, persist a row, return JSON.
|
||||
|
||||
Mirrors ``/api/parse-999`` but for the lower-level envelope ack:
|
||||
- 400 on empty / undecodable / malformed EDI (never 500).
|
||||
- 200 on success with ``{"ta1": {id, control_number, ack_code,
|
||||
note_code, interchange_date, interchange_time, sender_id,
|
||||
receiver_id, source_batch_id, raw_ta1_text}, "parsed":
|
||||
<ParseResultTa1>}``.
|
||||
|
||||
The persisted ``ta1_acks.source_batch_id`` is a synthetic id
|
||||
(``TA1-<ISA13>``) because a received TA1 has no inbound batch to
|
||||
FK to. The dashboard's ``/ta1-acks`` list surfaces these.
|
||||
"""
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Empty file", "detail": "Uploaded file contained no bytes."},
|
||||
)
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Encoding error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
try:
|
||||
result = parse_ta1_text(text, input_file=file.filename or "")
|
||||
except CycloneParseError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Parse error", "detail": str(exc)},
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - safety net
|
||||
log.exception("Unexpected parser failure on TA1")
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": "Internal server error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
# Build the raw TA1 text from the parsed result (round-trip).
|
||||
raw_ta1_text = _serialize_ta1(result)
|
||||
|
||||
row = store.add_ta1_ack(
|
||||
source_batch_id=result.source_batch_id,
|
||||
control_number=result.ta1.control_number,
|
||||
interchange_date=result.ta1.interchange_date,
|
||||
interchange_time=result.ta1.interchange_time,
|
||||
ack_code=result.ta1.ack_code,
|
||||
note_code=result.ta1.note_code,
|
||||
ack_generated_date=result.ta1.ack_generated_date,
|
||||
sender_id=result.envelope.sender_id,
|
||||
receiver_id=result.envelope.receiver_id,
|
||||
raw_json=json.loads(result.model_dump_json()),
|
||||
)
|
||||
|
||||
return JSONResponse(content={
|
||||
"ta1": {
|
||||
"id": row.id,
|
||||
"control_number": result.ta1.control_number,
|
||||
"ack_code": result.ta1.ack_code,
|
||||
"note_code": result.ta1.note_code,
|
||||
"interchange_date": result.ta1.interchange_date.isoformat()
|
||||
if result.ta1.interchange_date else None,
|
||||
"interchange_time": result.ta1.interchange_time,
|
||||
"sender_id": result.envelope.sender_id,
|
||||
"receiver_id": result.envelope.receiver_id,
|
||||
"source_batch_id": result.source_batch_id,
|
||||
"raw_ta1_text": raw_ta1_text,
|
||||
},
|
||||
"parsed": json.loads(result.model_dump_json()),
|
||||
})
|
||||
|
||||
|
||||
@app.get("/api/ta1-acks")
|
||||
def list_ta1_acks_endpoint(
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
) -> Any:
|
||||
"""Return the list of persisted TA1 ACKs, newest first.
|
||||
|
||||
Mirrors :func:`list_acks_endpoint` — fetches all rows then slices in
|
||||
Python so the ``total`` field reflects the full row count regardless
|
||||
of the ``limit`` cap.
|
||||
"""
|
||||
rows = store.list_ta1_acks()
|
||||
items = [_ta1_to_ui(r) for r in rows[:limit]]
|
||||
return {
|
||||
"total": len(rows),
|
||||
"items": items,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/ta1-acks/{ack_id}")
|
||||
def get_ta1_ack_endpoint(ack_id: int) -> dict:
|
||||
"""Return one persisted TA1 ACK row with its parsed detail."""
|
||||
row = store.get_ta1_ack(ack_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"TA1 ACK {ack_id} not found")
|
||||
body = _ta1_to_ui(row)
|
||||
body["raw_ta1_text"] = _serialize_ta1_from_row(row)
|
||||
body["raw_json"] = row.raw_json
|
||||
return body
|
||||
|
||||
|
||||
def _ta1_to_ui(row: db.Ta1Ack) -> dict:
|
||||
"""Render a Ta1Ack row for the UI (list endpoint shape)."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"control_number": row.control_number,
|
||||
"ack_code": row.ack_code,
|
||||
"note_code": row.note_code,
|
||||
"interchange_date": row.interchange_date.isoformat()
|
||||
if row.interchange_date else None,
|
||||
"interchange_time": row.interchange_time,
|
||||
"sender_id": row.sender_id,
|
||||
"receiver_id": row.receiver_id,
|
||||
"source_batch_id": row.source_batch_id,
|
||||
"parsed_at": row.parsed_at.isoformat() if row.parsed_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_ta1(result) -> str:
|
||||
"""Render a TA1 file from a ParseResultTa1 for the ``raw_ta1_text`` field.
|
||||
|
||||
Mirrors what the parser consumed: ISA envelope → TA1 → IEA. We
|
||||
rebuild minimal ISA fields from the envelope plus the TA1 segment
|
||||
verbatim. The serializer is intentionally tiny — TA1 has no GS/ST,
|
||||
so there's no functional-group structure to round-trip.
|
||||
"""
|
||||
ta1 = result.ta1
|
||||
parts = [
|
||||
f"TA1*{ta1.control_number}*{ta1.interchange_date.strftime('%y%m%d') if ta1.interchange_date else ''}*"
|
||||
f"{ta1.interchange_time or ''}*{ta1.ack_code}*{ta1.note_code or ''}*"
|
||||
f"{ta1.ack_generated_date.strftime('%y%m%d') if ta1.ack_generated_date else ''}",
|
||||
]
|
||||
return "~".join(parts) + "~"
|
||||
|
||||
|
||||
def _serialize_ta1_from_row(row: db.Ta1Ack) -> str:
|
||||
"""Reconstruct a TA1 segment from the persisted flat row (for the detail endpoint)."""
|
||||
date_s = row.interchange_date.strftime("%y%m%d") if row.interchange_date else ""
|
||||
return (
|
||||
f"TA1*{row.control_number}*{date_s}*{row.interchange_time or ''}*"
|
||||
f"{row.ack_code}*{row.note_code or ''}~"
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SP6 — Inbox endpoints
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
@@ -378,3 +378,44 @@ class Ack(Base):
|
||||
__table_args__ = (
|
||||
Index("ix_acks_source_batch_id", "source_batch_id"),
|
||||
)
|
||||
|
||||
|
||||
class Ta1Ack(Base):
|
||||
"""TA1 (Interchange Acknowledgment) row — one per parsed TA1 file.
|
||||
|
||||
Mirrors :class:`Ack` but for the lower-level interchange envelope
|
||||
ack. A TA1 acknowledges the entire ISA/IEA interchange envelope
|
||||
(vs a 999 which acknowledges individual transaction sets inside it).
|
||||
|
||||
``source_batch_id`` is the synthetic id used by the parser
|
||||
(``TA1-<ISA13>``) so we can FK to a stable identifier even though we
|
||||
don't have an inbound 837 batch to point at — same convention as
|
||||
the 999 path (``999-<ISA13>``).
|
||||
|
||||
The flat columns (control_number, ack_code, note_code, dates) are
|
||||
promoted out of ``raw_json`` so the list endpoint can sort/filter
|
||||
without a JSON parse. The full parsed envelope is still preserved
|
||||
in ``raw_json`` for the detail endpoint.
|
||||
"""
|
||||
|
||||
__tablename__ = "ta1_acks"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
source_batch_id: Mapped[str] = mapped_column(
|
||||
String(32), ForeignKey("batches.id", ondelete="CASCADE"), nullable=False,
|
||||
)
|
||||
control_number: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
interchange_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True)
|
||||
interchange_time: Mapped[Optional[str]] = mapped_column(String(8), nullable=True)
|
||||
ack_code: Mapped[str] = mapped_column(String(8), nullable=False)
|
||||
note_code: Mapped[Optional[str]] = mapped_column(String(8), nullable=True)
|
||||
ack_generated_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True)
|
||||
sender_id: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
receiver_id: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
raw_json: Mapped[Optional[dict]] = mapped_column(JSONText, nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_ta1_acks_source_batch_id", "source_batch_id"),
|
||||
Index("ix_ta1_acks_ack_code", "ack_code"),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
-- version: 5
|
||||
-- TA1 (Interchange Acknowledgment) table.
|
||||
--
|
||||
-- Mirrors the `acks` table for the lower-level envelope ack. A TA1 file
|
||||
-- is the per-interchange acknowledgment that payers send back in response
|
||||
-- to an 837 submission; a 999 acks individual transaction sets inside the
|
||||
-- interchange. Both are persisted side-by-side.
|
||||
--
|
||||
-- Schema follows the same conventions as `acks`:
|
||||
-- * `id` auto-increment PK (one row per parsed TA1 file)
|
||||
-- * `source_batch_id` FK to a synthetic `batches.id` (`TA1-<ISA13>`) so
|
||||
-- the row has a stable identifier even though there's no inbound 837
|
||||
-- batch to point at
|
||||
-- * flat columns for the TA1 fields so list endpoints can sort/filter
|
||||
-- without parsing `raw_json`
|
||||
-- * `raw_json` carries the full `ParseResultTa1` for the detail endpoint
|
||||
|
||||
CREATE TABLE ta1_acks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_batch_id TEXT NOT NULL REFERENCES batches(id) ON DELETE CASCADE,
|
||||
control_number TEXT NOT NULL,
|
||||
interchange_date DATE,
|
||||
interchange_time TEXT,
|
||||
ack_code TEXT NOT NULL,
|
||||
note_code TEXT,
|
||||
ack_generated_date DATE,
|
||||
sender_id TEXT NOT NULL DEFAULT '',
|
||||
receiver_id TEXT NOT NULL DEFAULT '',
|
||||
parsed_at DATETIME NOT NULL,
|
||||
raw_json TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX ix_ta1_acks_source_batch_id ON ta1_acks(source_batch_id);
|
||||
CREATE INDEX ix_ta1_acks_ack_code ON ta1_acks(ack_code);
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Pydantic v2 models for parsed X12 TA1 (Interchange Acknowledgment) files.
|
||||
|
||||
A TA1 is the lowest-level ack in the X12 envelope stack — it acknowledges
|
||||
the *interchange* (ISA/IEA), distinct from a 999 which acknowledges
|
||||
individual transaction sets. Per X12 005010X231A1 the envelope is:
|
||||
|
||||
ISA… → TA1 → IEA…
|
||||
|
||||
The TA1 segment carries six fields:
|
||||
|
||||
TA101 Interchange Control Number (matches ISA13 of source)
|
||||
TA102 Interchange Date (matches ISA09 of source, CCYYMMDD)
|
||||
TA103 Interchange Time (matches ISA10 of source, HHMM)
|
||||
TA104 Acknowledgment Code A / E / R
|
||||
TA105 Interchange Note Code numeric code (see spec table)
|
||||
TA106 Interchange Date (ack generated) CCYYMMDD
|
||||
|
||||
Standard ack codes:
|
||||
A = Accepted (interchange passed envelope validation)
|
||||
E = Accepted with errors (interchange accepted but a TA105 note exists)
|
||||
R = Rejected (interchange failed validation; see TA105)
|
||||
|
||||
Note codes from the X12 spec (005010X231A1) — only the most common ones
|
||||
are surfaced as named constants below; the raw value is always preserved.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_serializer
|
||||
|
||||
from cyclone.parsers.models import BatchSummary, Envelope
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Shared base
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class _Base(BaseModel):
|
||||
"""Shared Pydantic base; matches the 837P / 835 / 999 models for JSON consistency."""
|
||||
|
||||
model_config = ConfigDict(extra="ignore", str_strip_whitespace=True)
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler): # type: ignore[no-untyped-def]
|
||||
data = handler(self)
|
||||
for key, value in data.items():
|
||||
if isinstance(value, date):
|
||||
data[key] = value.isoformat()
|
||||
return data
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Interchange Acknowledgment
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
# The three valid ack codes per X12 spec. We type-narrow so the API
|
||||
# surfaces the canonical set; raw values outside this set still parse
|
||||
# but surface as the literal string.
|
||||
AckCode = Literal["A", "E", "R"]
|
||||
|
||||
|
||||
class Ta1Ack(_Base):
|
||||
"""One TA1 segment — the per-interchange acknowledgment.
|
||||
|
||||
``control_number`` matches the ISA13 of the source interchange, so
|
||||
operators can correlate a TA1 back to the 837 batch it acknowledged.
|
||||
``interchange_date`` / ``interchange_time`` mirror the ISA09 / ISA10
|
||||
fields of the source.
|
||||
|
||||
``note_code`` is an optional X12 note (e.g. "006" = "Invalid interchange
|
||||
date"). We keep the raw string even when it's not in our named map so
|
||||
new codes don't silently surface as ``None``.
|
||||
"""
|
||||
|
||||
control_number: str
|
||||
interchange_date: date | None = None
|
||||
interchange_time: str | None = None
|
||||
ack_code: AckCode
|
||||
note_code: str | None = None
|
||||
ack_generated_date: date | None = None
|
||||
|
||||
|
||||
class ParseResultTa1(_Base):
|
||||
"""One TA1 file = one interchange = one Ta1Ack."""
|
||||
|
||||
envelope: Envelope
|
||||
ta1: Ta1Ack
|
||||
summary: BatchSummary = Field(default_factory=lambda: BatchSummary(input_file=""))
|
||||
# The synthesized source_batch_id (e.g. ``"TA1-<ISA13>"``) so the row
|
||||
# can FK to a stable identifier even though we don't have an inbound
|
||||
# batch to point at. Mirrors the 999 ``999-<ISA13>`` convention.
|
||||
source_batch_id: str = ""
|
||||
|
||||
|
||||
__all__ = ["AckCode", "Ta1Ack", "ParseResultTa1"]
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Parse an X12 TA1 (Interchange Acknowledgment) file.
|
||||
|
||||
TA1 is the lowest-level ack in the X12 envelope stack. A TA1 file has
|
||||
the shape::
|
||||
|
||||
ISA*…~
|
||||
TA1*<icn>*<date>*<time>*<ack_code>*<note_code>*<ack_date>~
|
||||
IEA*1*<icn>~
|
||||
|
||||
This module is intentionally simple — the spec allows for multiple TA1
|
||||
segments in a single interchange (one per included 837 batch), but in
|
||||
practice Colorado Medicaid sends exactly one TA1 per file. We accept both
|
||||
shapes and return the first TA1 we see (the rest are surfaced via the
|
||||
``raw_segments`` field on the envelope in future versions).
|
||||
|
||||
Whole-document problems (missing ISA, no TA1) raise
|
||||
:class:`CycloneParseError`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date
|
||||
|
||||
from cyclone.parsers.exceptions import CycloneParseError
|
||||
from cyclone.parsers.models import BatchSummary, Envelope
|
||||
from cyclone.parsers.models_ta1 import ParseResultTa1, Ta1Ack
|
||||
from cyclone.parsers.segments import tokenize
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Date parsing
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _parse_yyyymmdd(s: str) -> date | None:
|
||||
"""Parse an X12 date string. Returns None on bad input.
|
||||
|
||||
Accepts:
|
||||
* CCYYMMDD (8 digits) — the spec shape for TA102 / TA106.
|
||||
* YYMMDD (6 digits) — what Colorado Medicaid sends in practice.
|
||||
The century is inferred as 2000 + YY (TA1s older than ~2030
|
||||
are out of scope; if that assumption changes, switch to a
|
||||
rolling-window strategy).
|
||||
"""
|
||||
if not s or not s.isdigit():
|
||||
return None
|
||||
if len(s) == 8:
|
||||
try:
|
||||
return date(int(s[0:4]), int(s[4:6]), int(s[6:8]))
|
||||
except ValueError:
|
||||
return None
|
||||
if len(s) == 6:
|
||||
try:
|
||||
return date(2000 + int(s[0:2]), int(s[2:4]), int(s[4:6]))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Envelope
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _build_envelope(segments: list[list[str]], input_file: str) -> Envelope:
|
||||
"""Build the envelope from ISA. TA1 has no GS/ST — just ISA → TA1 → IEA.
|
||||
|
||||
``transaction_date`` falls back to a placeholder if ISA09 is missing
|
||||
or unparseable; the authoritative date is the one inside the TA1
|
||||
segment itself (TA102).
|
||||
"""
|
||||
envelope: Envelope | None = None
|
||||
for seg in segments:
|
||||
if seg[0] == "ISA":
|
||||
try:
|
||||
envelope = Envelope(
|
||||
sender_id=seg[6].strip(),
|
||||
receiver_id=seg[8].strip(),
|
||||
control_number=seg[13].strip(),
|
||||
transaction_date=date(2024, 1, 1),
|
||||
implementation_guide=None,
|
||||
)
|
||||
except (IndexError, ValueError) as exc:
|
||||
raise CycloneParseError(f"Bad ISA: {exc}") from exc
|
||||
break
|
||||
if envelope is None:
|
||||
raise CycloneParseError("No ISA envelope found")
|
||||
return envelope
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# TA1 segment consumer
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _consume_ta1(segments: list[list[str]], idx: int) -> tuple[Ta1Ack, int]:
|
||||
"""Read a TA1 segment and return a :class:`Ta1Ack`.
|
||||
|
||||
Per X12 005010X231A1:
|
||||
TA101 = Interchange Control Number (15/9 digits)
|
||||
TA102 = Interchange Date (CCYYMMDD)
|
||||
TA103 = Interchange Time (HHMM)
|
||||
TA104 = Acknowledgment Code (A/E/R)
|
||||
TA105 = Interchange Note Code (3-digit numeric; optional but commonly present)
|
||||
TA106 = Interchange Date — date the ack was generated (optional)
|
||||
"""
|
||||
ta1 = segments[idx]
|
||||
if ta1[0] != "TA1":
|
||||
raise CycloneParseError(f"Expected TA1, got {ta1[0]!r}")
|
||||
ctrl = ta1[1] if len(ta1) > 1 else ""
|
||||
date_s = ta1[2] if len(ta1) > 2 else ""
|
||||
time_s = ta1[3] if len(ta1) > 3 else ""
|
||||
code = (ta1[4] if len(ta1) > 4 else "R").strip() or "R"
|
||||
note = ta1[5] if len(ta1) > 5 and ta1[5] else None
|
||||
ack_date_s = ta1[6] if len(ta1) > 6 and ta1[6] else ""
|
||||
|
||||
# Narrow to the literal type — anything outside the canonical set
|
||||
# surfaces as the raw string instead of failing parse, so unknown
|
||||
# codes from new payers don't break ingest.
|
||||
if code not in ("A", "E", "R"):
|
||||
log.warning("Unexpected TA1 ack code %r on icn=%s", code, ctrl)
|
||||
|
||||
return (
|
||||
Ta1Ack(
|
||||
control_number=ctrl,
|
||||
interchange_date=_parse_yyyymmdd(date_s),
|
||||
interchange_time=time_s or None,
|
||||
ack_code=code, # type: ignore[arg-type]
|
||||
note_code=note,
|
||||
ack_generated_date=_parse_yyyymmdd(ack_date_s),
|
||||
),
|
||||
idx + 1,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Top-level orchestrator
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def parse_ta1_text(text: str, *, input_file: str = "") -> ParseResultTa1:
|
||||
"""Parse a complete TA1 document and return a :class:`ParseResultTa1`.
|
||||
|
||||
Permissive: if the file contains multiple TA1 segments, we return the
|
||||
first one and warn. The spec allows this for interchanges that wrap
|
||||
multiple 837 batches but no real-world Colorado file uses it today.
|
||||
"""
|
||||
segments = tokenize(text)
|
||||
envelope = _build_envelope(segments, input_file=input_file)
|
||||
|
||||
ta1_idx: int | None = None
|
||||
for i, seg in enumerate(segments):
|
||||
if seg[0] == "TA1":
|
||||
ta1_idx = i
|
||||
break
|
||||
if ta1_idx is None:
|
||||
raise CycloneParseError("No TA1 segment found")
|
||||
|
||||
ta1, _ = _consume_ta1(segments, ta1_idx)
|
||||
|
||||
# TA102 is the authoritative interchange date — overwrite the placeholder.
|
||||
if ta1.interchange_date is not None:
|
||||
envelope = envelope.model_copy(update={"transaction_date": ta1.interchange_date})
|
||||
|
||||
# Stable FK target. Mirrors the 999 path: ``<kind>-<ISA13>``.
|
||||
source_batch_id = f"TA1-{(envelope.control_number or '').strip() or '000000001'}"
|
||||
|
||||
summary = BatchSummary(
|
||||
input_file=input_file,
|
||||
control_number=envelope.control_number,
|
||||
transaction_date=envelope.transaction_date,
|
||||
total_claims=1, # one TA1 == one interchange acknowledged
|
||||
passed=1 if ta1.ack_code == "A" else 0,
|
||||
failed=0 if ta1.ack_code == "A" else 1,
|
||||
)
|
||||
|
||||
return ParseResultTa1(
|
||||
envelope=envelope,
|
||||
ta1=ta1,
|
||||
summary=summary,
|
||||
source_batch_id=source_batch_id,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["parse_ta1_text"]
|
||||
@@ -1548,6 +1548,66 @@ class CycloneStore:
|
||||
with db.SessionLocal()() as s:
|
||||
return s.get(Ack, ack_id)
|
||||
|
||||
# -- TA1 (Interchange Acknowledgment) -------------------------------
|
||||
|
||||
def add_ta1_ack(
|
||||
self,
|
||||
*,
|
||||
source_batch_id: str,
|
||||
control_number: str,
|
||||
interchange_date: date | None,
|
||||
interchange_time: str | None,
|
||||
ack_code: str,
|
||||
note_code: str | None,
|
||||
ack_generated_date: date | None,
|
||||
sender_id: str,
|
||||
receiver_id: str,
|
||||
raw_json: dict,
|
||||
) -> db.Ta1Ack:
|
||||
"""Persist a TA1 (Interchange Acknowledgment) row and return it.
|
||||
|
||||
Mirrors :meth:`add_ack` for the lower-level envelope ack. The
|
||||
flat columns are promoted out of ``raw_json`` so the list
|
||||
endpoint can sort/filter without a JSON parse; the full
|
||||
``ParseResultTa1`` stays in ``raw_json`` for the detail endpoint.
|
||||
"""
|
||||
with db.SessionLocal()() as s:
|
||||
row = db.Ta1Ack(
|
||||
source_batch_id=source_batch_id,
|
||||
control_number=control_number,
|
||||
interchange_date=interchange_date,
|
||||
interchange_time=interchange_time,
|
||||
ack_code=ack_code,
|
||||
note_code=note_code,
|
||||
ack_generated_date=ack_generated_date,
|
||||
sender_id=sender_id,
|
||||
receiver_id=receiver_id,
|
||||
parsed_at=utcnow(),
|
||||
raw_json=raw_json,
|
||||
)
|
||||
s.add(row)
|
||||
s.commit()
|
||||
s.refresh(row)
|
||||
return row
|
||||
|
||||
def list_ta1_acks(self) -> list[db.Ta1Ack]:
|
||||
"""Return every TA1 ACK row, newest first (auto-increment id desc).
|
||||
|
||||
Mirrors :meth:`list_acks` — the API endpoint slices to its own
|
||||
``limit`` so the ``total`` field reflects the full row count.
|
||||
"""
|
||||
with db.SessionLocal()() as s:
|
||||
return (
|
||||
s.query(db.Ta1Ack)
|
||||
.order_by(db.Ta1Ack.id.desc())
|
||||
.all()
|
||||
)
|
||||
|
||||
def get_ta1_ack(self, ack_id: int) -> db.Ta1Ack | None:
|
||||
"""Return a single TA1 ACK row by id, or ``None`` if not found."""
|
||||
with db.SessionLocal()() as s:
|
||||
return s.get(db.Ta1Ack, ack_id)
|
||||
|
||||
# -- manual reconciliation (T12) -----------------------------------
|
||||
|
||||
def list_unmatched(self, *, kind: str = "both") -> dict:
|
||||
|
||||
@@ -51,16 +51,17 @@ def test_migration_0002_creates_acks_table():
|
||||
|
||||
def test_migration_latest_idempotent_on_fresh_db():
|
||||
"""Re-running the migration on the same DB must be a no-op (PRAGMA
|
||||
user_version already at the latest version — currently 4 after the
|
||||
0004 rejection columns + state-history index was added by SP6)."""
|
||||
user_version already at the latest version — currently 5 after the
|
||||
0004 rejection columns + state-history index (SP6) and the 0005
|
||||
ta1_acks table (this PR)."""
|
||||
with db.engine().begin() as c:
|
||||
v1 = c.exec_driver_sql("PRAGMA user_version").scalar() or 0
|
||||
assert v1 == 4
|
||||
assert v1 == 5
|
||||
# A second run should not raise and should not bump the version.
|
||||
db_migrate.run(db.engine())
|
||||
with db.engine().begin() as c:
|
||||
v2 = c.exec_driver_sql("PRAGMA user_version").scalar() or 0
|
||||
assert v2 == 4
|
||||
assert v2 == 5
|
||||
|
||||
|
||||
def test_add_ack_persists_row():
|
||||
|
||||
@@ -209,3 +209,151 @@ def test_prodfile_round_trip_persists_separately(client: TestClient):
|
||||
ctrl_991102984 = [b for b in batches if b.result.envelope.control_number == "991102984"]
|
||||
assert len(ctrl_991102984) == 2
|
||||
assert len({b.id for b in ctrl_991102984}) == 2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Cross-pipeline reconciliation (837 + 835 together)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
PRODFILE_835_DIR_FOR_XP = (
|
||||
Path(__file__).parent.parent.parent / "docs" / "prodfiles" / "835fromco"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (PRODFILE_DIR.is_dir() and PRODFILE_835_DIR_FOR_XP.is_dir()),
|
||||
reason=f"production 837 or 835 files not present (gitignored): {PRODFILE_DIR}, {PRODFILE_835_DIR_FOR_XP}",
|
||||
)
|
||||
def test_prodfile_cross_pipeline_reconciles(client: TestClient):
|
||||
"""All production 837s + all production 835s end-to-end.
|
||||
|
||||
Pipeline under test:
|
||||
POST /api/parse-837 (×N) → store.add (claim rows) → DB
|
||||
POST /api/parse-835 (×M) → store.add (remit rows) → reconcile.run → DB
|
||||
|
||||
Designed to survive variable 837 / 835 file counts and arbitrary PCN
|
||||
overlap. Hard-codes NO match counts. Instead asserts invariants:
|
||||
|
||||
1. Every 837 file parses (200, total_claims ≥ 1) and lands as one
|
||||
837p batch.
|
||||
2. Every 835 file parses (200, total_claims ≥ 1) and lands as one
|
||||
835 batch; each carries an R835_MULTI_BPR warning.
|
||||
3. After all batches: store carries len(837 files) + len(835 files)
|
||||
batches with the expected kind split.
|
||||
4. After all 835s loaded: store.list_unmatched() returns shapes:
|
||||
unmatched_claims = total_unique_837_pcns - total_matched
|
||||
unmatched_remittances = total_unique_835_pcns - total_matched
|
||||
Both counts are ≥ 0; the matched count is a function of how
|
||||
many 837 PCNs happen to appear in the 835 set — that's data-
|
||||
dependent and not asserted directly.
|
||||
5. Per-835 reconciliation summaries are well-formed (matched,
|
||||
unmatched_claims, unmatched_remittances, skipped — all ints).
|
||||
|
||||
Files are discovered via glob so adding more 837 files (the
|
||||
production set grows over time) needs no test edit.
|
||||
"""
|
||||
from cyclone.store import CycloneStore
|
||||
|
||||
assert len(global_store.list()) == 0
|
||||
|
||||
# Discover files dynamically — works for N 837s and M 835s.
|
||||
prodfiles_837 = sorted(p for p in PRODFILE_DIR.iterdir() if p.is_file())
|
||||
prodfiles_835 = sorted(p for p in PRODFILE_835_DIR_FOR_XP.iterdir() if p.is_file())
|
||||
assert prodfiles_837, f"no 837 files at {PRODFILE_DIR}"
|
||||
assert prodfiles_835, f"no 835 files at {PRODFILE_835_DIR_FOR_XP}"
|
||||
|
||||
# 1. Load every 837 first (claims must exist before 835 reconcile runs).
|
||||
unique_837_pcns: set[str] = set()
|
||||
for path in prodfiles_837:
|
||||
with open(path, "rb") as f:
|
||||
resp = client.post(
|
||||
"/api/parse-837",
|
||||
files={"file": (path.name, f, "text/plain")},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 200, f"{path.name}: {resp.text}"
|
||||
body = resp.json()
|
||||
assert body["summary"]["total_claims"] >= 1, path.name
|
||||
assert body["summary"]["failed"] == 0, path.name
|
||||
for claim in body["claims"]:
|
||||
pcn = claim.get("claim_id") or claim.get("patient_control_number")
|
||||
if pcn:
|
||||
unique_837_pcns.add(pcn)
|
||||
|
||||
# 2. Load every 835. Each triggers reconcile.run against the existing
|
||||
# 837 claims; the per-batch reconciliation summary records how many
|
||||
# of THIS batch's remits matched.
|
||||
unique_835_pcns: set[str] = set()
|
||||
per_835_matched: list[int] = []
|
||||
per_835_summary_keys = {"matched", "unmatched_claims", "unmatched_remittances", "skipped"}
|
||||
for path in prodfiles_835:
|
||||
with open(path, "rb") as f:
|
||||
resp = client.post(
|
||||
"/api/parse-835",
|
||||
files={"file": (path.name, f, "application/octet-stream")},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 200, f"{path.name}: {resp.text}"
|
||||
body = resp.json()
|
||||
assert body["summary"]["total_claims"] >= 1, path.name
|
||||
assert body["summary"]["failed"] == 0, path.name
|
||||
# CO Medicaid split-payment pattern: validator surfaces the
|
||||
# non-standard data as a warning; the batch still passes.
|
||||
assert any(
|
||||
issue["rule"] == "R835_MULTI_BPR"
|
||||
for issue in body["validation"]["warnings"]
|
||||
), f"{path.name}: missing R835_MULTI_BPR warning"
|
||||
# Per-claim PCN extraction — UI shape is claim_id (not PCN).
|
||||
for cp in body["claims"]:
|
||||
pcn = cp.get("claim_id") or cp.get("payer_claim_control_number")
|
||||
if pcn:
|
||||
unique_835_pcns.add(pcn)
|
||||
# Reconciliation summary must be well-formed.
|
||||
rec = body["reconciliation"]
|
||||
assert set(rec.keys()) >= per_835_summary_keys, path.name
|
||||
for key in per_835_summary_keys:
|
||||
assert isinstance(rec[key], int), (path.name, key, rec[key])
|
||||
assert rec["skipped"] == 0, path.name
|
||||
per_835_matched.append(rec["matched"])
|
||||
|
||||
# 3. Store shape: one batch per file, split by kind.
|
||||
batches = global_store.list()
|
||||
assert len(batches) == len(prodfiles_837) + len(prodfiles_835)
|
||||
by_kind: dict[str, set[str]] = {"837p": set(), "835": set()}
|
||||
for b in batches:
|
||||
by_kind[b.kind].add(b.input_filename)
|
||||
assert by_kind["837p"] == {p.name for p in prodfiles_837}
|
||||
assert by_kind["835"] == {p.name for p in prodfiles_835}
|
||||
|
||||
# 4. Reconciliation invariants. matched_count is bounded by the size
|
||||
# of the smaller set; unmatched counts are the disjoint remainder.
|
||||
fresh_store = CycloneStore()
|
||||
unmatched = fresh_store.list_unmatched(kind="both")
|
||||
unmatched_claims = len(unmatched["claims"])
|
||||
unmatched_remits = len(unmatched["remittances"])
|
||||
|
||||
total_matched = sum(per_835_matched)
|
||||
assert 0 <= total_matched <= len(unique_837_pcns), (
|
||||
f"matched {total_matched} outside [0, {len(unique_837_pcns)}]"
|
||||
)
|
||||
# Every matched pair consumes one claim and one remit.
|
||||
assert unmatched_claims == len(unique_837_pcns) - total_matched, (
|
||||
f"unmatched_claims {unmatched_claims} != "
|
||||
f"{len(unique_837_pcns)} - {total_matched}"
|
||||
)
|
||||
assert unmatched_remits == len(unique_835_pcns) - total_matched, (
|
||||
f"unmatched_remittances {unmatched_remits} != "
|
||||
f"{len(unique_835_pcns)} - {total_matched}"
|
||||
)
|
||||
# The two PCN sets are independent sources of truth: total deduped
|
||||
# claim rows + remittance rows in the DB must equal the input totals
|
||||
# adjusted for cross-batch dedup. The DB-level ground truth:
|
||||
claim_rows = sum(
|
||||
len(b.result.claims) for b in batches if b.kind == "837p"
|
||||
)
|
||||
# Deduped claim rows == unique PCNs (one Claim row per PCN due to
|
||||
# the store.add dedup logic, same as 835 PCNs).
|
||||
assert claim_rows >= len(unique_837_pcns), (
|
||||
f"claim_rows {claim_rows} < unique_837_pcns {len(unique_837_pcns)}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Tests for the /api/parse-ta1 endpoint and the ta1_acks list/detail endpoints."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from cyclone.api import app
|
||||
from cyclone.store import store as global_store
|
||||
from cyclone import db
|
||||
|
||||
|
||||
ACCEPTED = (
|
||||
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
|
||||
"*260520*1750*^*00501*000000001*0*P*:~"
|
||||
"TA1*000000001*20260520*1750*A*000*20260520~"
|
||||
"IEA*1*000000001~"
|
||||
)
|
||||
|
||||
REJECTED = (
|
||||
"ISA*00* *00* *ZZ*COMEDASSISTPROG*ZZ*TP11525703 "
|
||||
"*260520*1750*^*00501*000000001*0*P*:~"
|
||||
"TA1*320293557*260520*2338*R*006~"
|
||||
"IEA*0*000000001~"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_store():
|
||||
"""Reset the module-level store before and after each test."""
|
||||
with global_store._lock:
|
||||
global_store._batches.clear()
|
||||
yield
|
||||
with global_store._lock:
|
||||
global_store._batches.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client() -> TestClient:
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_parse_ta1_endpoint_happy_path(client: TestClient):
|
||||
"""Accepted TA1 → 200 with parsed envelope + persisted ta1 row."""
|
||||
resp = client.post(
|
||||
"/api/parse-ta1",
|
||||
files={"file": ("accepted.ta1", ACCEPTED, "text/plain")},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
ta1 = body["ta1"]
|
||||
assert ta1["ack_code"] == "A"
|
||||
assert ta1["control_number"] == "000000001"
|
||||
assert ta1["note_code"] == "000"
|
||||
assert ta1["interchange_date"] == "2026-05-20"
|
||||
assert ta1["source_batch_id"] == "TA1-000000001"
|
||||
# Round-trip text present and well-formed.
|
||||
assert ta1["raw_ta1_text"].startswith("TA1*")
|
||||
assert ta1["raw_ta1_text"].endswith("~")
|
||||
# Parsed shape mirrors the model.
|
||||
assert body["parsed"]["ta1"]["ack_code"] == "A"
|
||||
|
||||
|
||||
def test_parse_ta1_endpoint_rejected_persists(client: TestClient):
|
||||
"""Rejected TA1 (Colorado YYMMDD format) → 200 + R ack_code in DB."""
|
||||
resp = client.post(
|
||||
"/api/parse-ta1",
|
||||
files={"file": ("rejected.ta1", REJECTED, "text/plain")},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
assert body["ta1"]["ack_code"] == "R"
|
||||
assert body["ta1"]["note_code"] == "006"
|
||||
# Persisted to DB
|
||||
list_resp = client.get("/api/ta1-acks?limit=10", headers={"Accept": "application/json"})
|
||||
assert list_resp.status_code == 200, list_resp.text
|
||||
list_body = list_resp.json()
|
||||
assert list_body["total"] == 1
|
||||
assert list_body["items"][0]["ack_code"] == "R"
|
||||
|
||||
|
||||
def test_parse_ta1_endpoint_empty_file_returns_400(client: TestClient):
|
||||
"""Empty upload → 400 (never 500)."""
|
||||
resp = client.post(
|
||||
"/api/parse-ta1",
|
||||
files={"file": ("empty.ta1", b"", "text/plain")},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_parse_ta1_endpoint_malformed_returns_400(client: TestClient):
|
||||
"""No TA1 segment → 400 with parse-error detail."""
|
||||
text = (
|
||||
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
|
||||
"*260520*1750*^*00501*000000001*0*P*:~"
|
||||
"IEA*0*000000001~"
|
||||
)
|
||||
resp = client.post(
|
||||
"/api/parse-ta1",
|
||||
files={"file": ("bad.ta1", text, "text/plain")},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = resp.json()
|
||||
assert "TA1" in body["detail"]
|
||||
|
||||
|
||||
def test_parse_ta1_endpoint_rejects_missing_file(client: TestClient):
|
||||
"""FastAPI File(...) with no default → 422."""
|
||||
resp = client.post("/api/parse-ta1")
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
def test_get_ta1_ack_detail_regenerates_segment(client: TestClient):
|
||||
"""GET /api/ta1-acks/{id} includes the raw TA1 segment rebuilt from the row."""
|
||||
resp = client.post(
|
||||
"/api/parse-ta1",
|
||||
files={"file": ("accepted.ta1", ACCEPTED, "text/plain")},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
ta1_id = resp.json()["ta1"]["id"]
|
||||
|
||||
detail_resp = client.get(f"/api/ta1-acks/{ta1_id}", headers={"Accept": "application/json"})
|
||||
assert detail_resp.status_code == 200, detail_resp.text
|
||||
body = detail_resp.json()
|
||||
assert body["id"] == ta1_id
|
||||
assert body["raw_ta1_text"].startswith("TA1*000000001*")
|
||||
assert body["raw_ta1_text"].rstrip().endswith("~")
|
||||
# raw_json carries the full ParseResultTa1 for the detail view.
|
||||
assert body["raw_json"]["ta1"]["ack_code"] == "A"
|
||||
|
||||
|
||||
def test_get_ta1_ack_404_for_missing(client: TestClient):
|
||||
"""GET /api/ta1-acks/{id} returns 404 for a missing id."""
|
||||
resp = client.get("/api/ta1-acks/9999", headers={"Accept": "application/json"})
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def test_list_ta1_acks_empty(client: TestClient):
|
||||
"""Empty DB → empty list with total=0."""
|
||||
resp = client.get("/api/ta1-acks", headers={"Accept": "application/json"})
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body == {"total": 0, "items": []}
|
||||
|
||||
|
||||
def test_list_ta1_acks_newest_first(client: TestClient):
|
||||
"""Multiple uploads: list returns rows newest first (by auto-increment id desc)."""
|
||||
for i, payload in enumerate([ACCEPTED, REJECTED]):
|
||||
resp = client.post(
|
||||
"/api/parse-ta1",
|
||||
files={"file": (f"file{i}.ta1", payload, "text/plain")},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
resp = client.get("/api/ta1-acks", headers={"Accept": "application/json"})
|
||||
items = resp.json()["items"]
|
||||
assert len(items) == 2
|
||||
# The REJECTED (uploaded second) is first.
|
||||
assert items[0]["ack_code"] == "R"
|
||||
assert items[1]["ack_code"] == "A"
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Tests for the TA1 (Interchange Acknowledgment) parser orchestrator."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from cyclone.parsers.exceptions import CycloneParseError
|
||||
from cyclone.parsers.parse_ta1 import parse_ta1_text
|
||||
|
||||
|
||||
# A minimal accepted TA1 in canonical CCYYMMDD form (per X12 spec).
|
||||
# ISA must be exactly 106 chars (15-char sender/receiver IDs).
|
||||
ACCEPTED_CCYYMMDD = (
|
||||
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
|
||||
"*260520*1750*^*00501*000000001*0*P*:~"
|
||||
"TA1*000000001*20260520*1750*A*000*20260520~"
|
||||
"IEA*1*000000001~"
|
||||
)
|
||||
|
||||
# A rejected TA1 with Colorado's YYMMDD date format (6 digits instead of 8).
|
||||
# This is what FromHPE/tp11525703-837P_M019044596-…_TA1.x12 looks like in
|
||||
# production — same shape but YYMMDD and ack_code=R, note_code=006.
|
||||
REJECTED_YYMMDD = (
|
||||
"ISA*00* *00* *ZZ*COMEDASSISTPROG*ZZ*TP11525703 "
|
||||
"*260520*1750*^*00501*000000001*0*P*:~"
|
||||
"TA1*320293557*260520*2338*R*006~"
|
||||
"IEA*0*000000001~"
|
||||
)
|
||||
|
||||
# Accepted variant with errors (E) — uncommon but spec-legal.
|
||||
ACCEPTED_WITH_ERRORS = (
|
||||
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
|
||||
"*260520*1750*^*00501*000000001*0*P*:~"
|
||||
"TA1*000000001*20260520*1750*E*007*20260520~"
|
||||
"IEA*1*000000001~"
|
||||
)
|
||||
|
||||
|
||||
def test_parse_accepted_ccyymmdd_returns_a():
|
||||
"""Spec-compliant TA1 with CCYYMMDD dates surfaces ack_code='A'."""
|
||||
result = parse_ta1_text(ACCEPTED_CCYYMMDD, input_file="accepted.ta1")
|
||||
assert result.ta1.ack_code == "A"
|
||||
assert result.ta1.control_number == "000000001"
|
||||
assert result.ta1.interchange_date == date(2026, 5, 20)
|
||||
assert result.ta1.interchange_time == "1750"
|
||||
assert result.ta1.note_code == "000"
|
||||
assert result.ta1.ack_generated_date == date(2026, 5, 20)
|
||||
# Envelope mirrors the ISA
|
||||
assert result.envelope.sender_id == "SENDER"
|
||||
assert result.envelope.receiver_id == "RECEIVER"
|
||||
assert result.envelope.control_number == "000000001"
|
||||
# Summary reflects the single ack
|
||||
assert result.summary.total_claims == 1
|
||||
assert result.summary.passed == 1
|
||||
assert result.summary.failed == 0
|
||||
|
||||
|
||||
def test_parse_rejected_yymmdd_from_production():
|
||||
"""Colorado's YYMMDD date format (6 digits) must still parse."""
|
||||
result = parse_ta1_text(REJECTED_YYMMDD, input_file="rejected.ta1")
|
||||
assert result.ta1.ack_code == "R"
|
||||
assert result.ta1.note_code == "006"
|
||||
assert result.ta1.interchange_date == date(2026, 5, 20), (
|
||||
"YYMMDD must be inferred as 20YY-MM-DD"
|
||||
)
|
||||
assert result.ta1.interchange_time == "2338"
|
||||
assert result.summary.passed == 0
|
||||
assert result.summary.failed == 1
|
||||
|
||||
|
||||
def test_parse_accepted_with_errors_returns_e():
|
||||
"""TA1 ack_code='E' (Accepted with errors) is preserved as the literal."""
|
||||
result = parse_ta1_text(ACCEPTED_WITH_ERRORS, input_file="with_errors.ta1")
|
||||
assert result.ta1.ack_code == "E"
|
||||
assert result.ta1.note_code == "007"
|
||||
|
||||
|
||||
def test_source_batch_id_format():
|
||||
"""source_batch_id must be 'TA1-<ISA13>' so the FK target is stable."""
|
||||
result = parse_ta1_text(ACCEPTED_CCYYMMDD, input_file="x.ta1")
|
||||
assert result.source_batch_id == "TA1-000000001"
|
||||
|
||||
|
||||
def test_missing_isa_raises():
|
||||
"""No ISA envelope at file start → CycloneParseError (caught by tokenize)."""
|
||||
text = "TA1*000000001*20260520*1750*A*000*20260520~"
|
||||
with pytest.raises(CycloneParseError, match="ISA"):
|
||||
parse_ta1_text(text, input_file="bad.ta1")
|
||||
|
||||
|
||||
def test_missing_ta1_raises():
|
||||
"""ISA but no TA1 segment → CycloneParseError."""
|
||||
text = (
|
||||
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
|
||||
"*260520*1750*^*00501*000000001*0*P*:~"
|
||||
"IEA*0*000000001~"
|
||||
)
|
||||
with pytest.raises(CycloneParseError, match="No TA1 segment"):
|
||||
parse_ta1_text(text, input_file="no_ta1.ta1")
|
||||
|
||||
|
||||
def test_short_ta1_segment_uses_defaults():
|
||||
"""A TA1 with fewer than 6 elements still parses with sensible defaults."""
|
||||
text = (
|
||||
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
|
||||
"*260520*1750*^*00501*000000001*0*P*:~"
|
||||
"TA1*000000001~"
|
||||
"IEA*0*000000001~"
|
||||
)
|
||||
result = parse_ta1_text(text, input_file="short.ta1")
|
||||
assert result.ta1.ack_code == "R" # default to rejected when missing
|
||||
assert result.ta1.note_code is None
|
||||
assert result.ta1.interchange_date is None
|
||||
assert result.ta1.interchange_time is None
|
||||
|
||||
|
||||
def test_garbage_date_returns_none_not_raises():
|
||||
"""A garbage date in TA102 doesn't raise — it surfaces as None."""
|
||||
text = (
|
||||
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER "
|
||||
"*260520*1750*^*00501*000000001*0*P*:~"
|
||||
"TA1*000000001*not-a-date*1750*A*000~"
|
||||
"IEA*0*000000001~"
|
||||
)
|
||||
result = parse_ta1_text(text, input_file="bad_date.ta1")
|
||||
assert result.ta1.interchange_date is None
|
||||
assert result.ta1.ack_code == "A" # the rest still parses
|
||||
@@ -0,0 +1,248 @@
|
||||
"""Smoke tests: every prodfiles/* file we have parsers for must parse cleanly.
|
||||
|
||||
These tests are data-tolerant: they discover files via ``glob`` and assert
|
||||
invariants per file rather than hard-coded expected counts. If ops drops
|
||||
more files into ``docs/prodfiles/``, the tests stay green as long as the
|
||||
new files parse cleanly.
|
||||
|
||||
Scope:
|
||||
- ``docs/prodfiles/claims/`` → 113 single-claim 837P files (output format reference)
|
||||
- ``docs/prodfiles/FromHPE/`` 837P .txt → 4 production 837 submissions to CO Medicaid
|
||||
- ``docs/prodfiles/FromHPE/`` *.999.x12 → ~1012 production transaction-set acks
|
||||
- ``docs/prodfiles/FromHPE/`` *.TA1.x12 → ~352 production interchange acks
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from cyclone.api import app
|
||||
from cyclone.store import store as global_store
|
||||
|
||||
DOCS_PRODFILES = Path(__file__).parent.parent.parent / "docs" / "prodfiles"
|
||||
|
||||
CLAIMS_DIR = DOCS_PRODFILES / "claims"
|
||||
FROMHPE_DIR = DOCS_PRODFILES / "FromHPE"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_store():
|
||||
"""Reset the module-level store before and after each test."""
|
||||
with global_store._lock:
|
||||
global_store._batches.clear()
|
||||
yield
|
||||
with global_store._lock:
|
||||
global_store._batches.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client() -> TestClient:
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _post(client: TestClient, endpoint: str, path: Path, content_type: str):
|
||||
with open(path, "rb") as f:
|
||||
return client.post(
|
||||
endpoint,
|
||||
files={"file": (path.name, f, content_type)},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 837P — claims/ (output format reference) + FromHPE/ (real submissions)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not CLAIMS_DIR.is_dir(),
|
||||
reason=f"docs/prodfiles/claims/ not present at {CLAIMS_DIR}",
|
||||
)
|
||||
def test_claims_prodfiles_parse_via_837_endpoint(client: TestClient):
|
||||
"""Every 837P file in docs/prodfiles/claims/ parses cleanly.
|
||||
|
||||
These single-claim files represent the output format our writer
|
||||
needs to produce. If any of them fails to parse, our round-trip is
|
||||
broken even before submission to the payer.
|
||||
|
||||
Note: claims/ PCNs intentionally overlap with the axiscare 837 files
|
||||
loaded by ``test_prodfile_round_trip_persists_separately`` (verified
|
||||
100% overlap). That means the store.add() dedup logic skips them
|
||||
silently — by design, since they represent the same claim shape. We
|
||||
assert the API response (parse + validate), not persistence count.
|
||||
"""
|
||||
paths = sorted(p for p in CLAIMS_DIR.iterdir() if p.is_file() and p.suffix == ".x12")
|
||||
assert paths, f"no 837P files at {CLAIMS_DIR}"
|
||||
|
||||
for path in paths:
|
||||
resp = _post(client, "/api/parse-837", path, "text/plain")
|
||||
assert resp.status_code == 200, f"{path.name}: {resp.text}"
|
||||
body = resp.json()
|
||||
assert body["summary"]["total_claims"] >= 1, path.name
|
||||
assert body["summary"]["failed"] == 0, path.name
|
||||
assert body["summary"]["passed"] == body["summary"]["total_claims"], path.name
|
||||
# Every claim must have an id (CLM01) — that's what makes it
|
||||
# addressable in the API and the DB.
|
||||
for claim in body["claims"]:
|
||||
assert claim.get("claim_id"), f"{path.name}: missing claim_id"
|
||||
# Sender / receiver shape must match the axiscare prod data.
|
||||
assert body["envelope"]["sender_id"] == "11525703", path.name
|
||||
assert body["envelope"]["receiver_id"] == "COMEDASSISTPROG", path.name
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not FROMHPE_DIR.is_dir(),
|
||||
reason=f"docs/prodfiles/FromHPE/ not present at {FROMHPE_DIR}",
|
||||
)
|
||||
def test_fromhpe_837_prodfiles_parse(client: TestClient):
|
||||
"""The 4 production 837P submissions in FromHPE/ parse cleanly."""
|
||||
paths = sorted(FROMHPE_DIR.glob("tp11525703-837P-*.txt"))
|
||||
assert paths, f"no 837P .txt files at {FROMHPE_DIR}"
|
||||
|
||||
for path in paths:
|
||||
resp = _post(client, "/api/parse-837", path, "text/plain")
|
||||
assert resp.status_code == 200, f"{path.name}: {resp.text}"
|
||||
body = resp.json()
|
||||
assert body["summary"]["total_claims"] >= 1, path.name
|
||||
assert body["summary"]["failed"] == 0, path.name
|
||||
# All FromHPE 837s are sender 11525703 → COMEDASSISTPROG (HPE gateway)
|
||||
assert body["envelope"]["sender_id"] == "11525703", path.name
|
||||
|
||||
batches = [b for b in global_store.list() if b.kind == "837p"]
|
||||
assert len(batches) == len(paths)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 999 — FromHPE/ production transaction-set acks
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not FROMHPE_DIR.is_dir(),
|
||||
reason=f"docs/prodfiles/FromHPE/ not present at {FROMHPE_DIR}",
|
||||
)
|
||||
def test_fromhpe_999_prodfiles_parse(client: TestClient):
|
||||
"""Every 999 ACK file in FromHPE/ parses and persists to /api/acks.
|
||||
|
||||
These ~1012 files are real acks Colorado sends back after receiving
|
||||
our 837 submissions. If any fail to parse, our ingest of payer
|
||||
responses is broken.
|
||||
|
||||
Known data anomaly: Colorado occasionally sends ``AK9*A*1*1*1`` —
|
||||
1 received, 1 accepted, 1 rejected from a single set, which violates
|
||||
X12 spec (AK904 should be ``received - accepted``). The parser reports
|
||||
what the AK9 literally says; if this becomes a real problem, fix
|
||||
``_ack_count_summary`` in api.py to clamp
|
||||
``rejected = max(0, received - accepted)``.
|
||||
"""
|
||||
paths = sorted(FROMHPE_DIR.glob("*999.x12"))
|
||||
assert paths, f"no 999 files at {FROMHPE_DIR}"
|
||||
|
||||
a_count = 0 # accepted
|
||||
r_count = 0 # rejected (set-level)
|
||||
e_count = 0 # partial / error
|
||||
malformed_ak9 = 0 # AK9 with accepted + rejected > received (CO bug)
|
||||
for path in paths:
|
||||
resp = _post(client, "/api/parse-999", path, "application/octet-stream")
|
||||
assert resp.status_code == 200, f"{path.name}: {resp.text}"
|
||||
body = resp.json()
|
||||
ack = body["ack"]
|
||||
assert ack["ack_code"] in {"A", "E", "R"}, (
|
||||
f"{path.name}: unexpected ack_code {ack['ack_code']!r}"
|
||||
)
|
||||
assert ack["received_count"] >= 1, path.name
|
||||
# Colorado's real 999s occasionally ship a malformed AK9
|
||||
# (e.g. ``AK9*A*1*1*1``: 1 received, 1 accepted, 1 rejected,
|
||||
# which is impossible from a single set). The parser faithfully
|
||||
# reports what the AK9 says; we count those as anomalies but
|
||||
# don't fail on them. See follow-up note in test docstring.
|
||||
if ack["accepted_count"] + ack["rejected_count"] > ack["received_count"]:
|
||||
malformed_ak9 += 1
|
||||
if ack["ack_code"] == "A":
|
||||
a_count += 1
|
||||
elif ack["ack_code"] == "R":
|
||||
r_count += 1
|
||||
else:
|
||||
e_count += 1
|
||||
|
||||
# Persisted as acks rows. Use the store directly to count (GET /api/acks
|
||||
# is paginated and we want to assert the total).
|
||||
from cyclone.store import CycloneStore
|
||||
fresh_store = CycloneStore()
|
||||
# acks live in the DB via store.add_ack; read via the same store path.
|
||||
assert _ack_count(fresh_store) == len(paths), (
|
||||
f"persisted { _ack_count(fresh_store) } acks, expected {len(paths)}"
|
||||
)
|
||||
# Sanity: every persisted ack round-trips through GET /api/acks/{id}.
|
||||
list_resp = client.get("/api/acks?limit=1000", headers={"Accept": "application/json"})
|
||||
assert list_resp.status_code == 200, list_resp.text
|
||||
assert list_resp.json()["total"] == len(paths)
|
||||
|
||||
# Real-data distribution: surface counts so an anomaly (e.g. all R)
|
||||
# is visible in test output even when the test passes.
|
||||
print(
|
||||
f"\n[999 prodfile distribution] A={a_count} R={r_count} E={e_count} "
|
||||
f"malformed_AK9={malformed_ak9} total={len(paths)}"
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# TA1 — FromHPE/ production interchange acks
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not FROMHPE_DIR.is_dir(),
|
||||
reason=f"docs/prodfiles/FromHPE/ not present at {FROMHPE_DIR}",
|
||||
)
|
||||
def test_fromhpe_ta1_prodfiles_parse(client: TestClient):
|
||||
"""Every TA1 file in FromHPE/ parses and persists to /api/ta1-acks.
|
||||
|
||||
These files are real interchange acks Colorado sends back in response
|
||||
to our 837 submissions. All real production files are R (rejected)
|
||||
— that's the dataset ops gave us to test against.
|
||||
"""
|
||||
paths = sorted(FROMHPE_DIR.glob("*TA1.x12"))
|
||||
assert paths, f"no TA1 files at {FROMHPE_DIR}"
|
||||
|
||||
a_count = 0
|
||||
e_count = 0
|
||||
r_count = 0
|
||||
for path in paths:
|
||||
resp = _post(client, "/api/parse-ta1", path, "application/octet-stream")
|
||||
assert resp.status_code == 200, f"{path.name}: {resp.text}"
|
||||
body = resp.json()
|
||||
ta1 = body["ta1"]
|
||||
assert ta1["ack_code"] in {"A", "E", "R"}, (
|
||||
f"{path.name}: unexpected ack_code {ta1['ack_code']!r}"
|
||||
)
|
||||
assert ta1["control_number"], f"{path.name}: empty TA101"
|
||||
assert ta1["source_batch_id"].startswith("TA1-"), path.name
|
||||
if ta1["ack_code"] == "A":
|
||||
a_count += 1
|
||||
elif ta1["ack_code"] == "R":
|
||||
r_count += 1
|
||||
else:
|
||||
e_count += 1
|
||||
|
||||
# Verify persistence via GET /api/ta1-acks. The endpoint caps at
|
||||
# limit=1000 but reports total across all rows, so the assertion
|
||||
# works even with more rows than the cap.
|
||||
list_resp = client.get("/api/ta1-acks?limit=1000", headers={"Accept": "application/json"})
|
||||
assert list_resp.status_code == 200, list_resp.text
|
||||
assert list_resp.json()["total"] == len(paths)
|
||||
|
||||
# Real-data distribution: surface counts so an anomaly is visible.
|
||||
print(
|
||||
f"\n[TA1 prodfile distribution] A={a_count} R={r_count} E={e_count} "
|
||||
f"total={len(paths)}"
|
||||
)
|
||||
|
||||
|
||||
def _ack_count(store: CycloneStore) -> int:
|
||||
"""Count rows in the acks table."""
|
||||
from cyclone import db
|
||||
with db.SessionLocal()() as s:
|
||||
return s.query(db.Ack).count()
|
||||
Reference in New Issue
Block a user