feat(ta1): TA1 interchange ACK parser, persistence, and API

Adds full TA1 (Interchange Acknowledgment, X12 envelope-level) support
to mirror the existing 999 transaction-set ACK pipeline.

Parser & models
- parsers/models_ta1.py: Pydantic Ta1Ack + ParseResultTa1 models
  (AckCode = Literal['A','E','R'], date serializer mirroring 999)
- parsers/parse_ta1.py: TA1 segment parser
  - Tolerant YYMMDD (6-digit) + CCYYMMDD (8-digit) date handling
    for CO Medicaid interchange_date / ack_generated_date
  - 106-char ISA validation (15-char sender/receiver IDs)
  - source_batch_id = 'TA1-<ISA13>'

Persistence
- migrations/0005_create_ta1_acks.sql: ta1_acks table + indexes
  on source_batch_id and ack_code
- db.py: Ta1Ack ORM model (flat columns + raw_json, mirrors Ack)
- store.py: add_ta1_ack, list_ta1_acks (returns all rows),
  get_ta1_ack
- db version bumped 4 -> 5 (test_acks.py updated)

API (api.py)
- POST /api/parse-ta1: text/file ingest, persists ta1_ack,
  returns detail-ready payload
- GET /api/ta1-acks: list with limit + total (mirrors 999 pattern)
- GET /api/ta1-acks/{ack_id}: detail
- _ta1_to_ui / _serialize_ta1 / _serialize_ta1_from_row helpers

Tests (17 new, 508 total passing)
- tests/test_parse_ta1.py (8): CCYYMMDD + YYMMDD acceptance,
  E/R codes, source_batch_id, missing ISA/TA1, short defaults
- tests/test_api_ta1.py (9): happy path, rejected persists,
  empty/malformed 400, missing file 422, detail regenerates
  segment, 404, empty list, newest-first
- tests/test_prodfiles_smoke.py: extended to smoke-test 352
  production TA1 files (A=0, R=352, E=0)
- tests/test_api_parse_persists.py: cross-pipeline reconciliation
  test asserting invariants across 837P + 835 prod files

Real-data finding: all 352 production TA1s are R (rejected);
operator follow-up warranted.
This commit is contained in:
Tyler
2026-06-20 18:58:56 -06:00
parent 1db33ef2fa
commit 76278ec9f6
11 changed files with 1285 additions and 4 deletions
+169
View File
@@ -51,6 +51,7 @@ from cyclone.parsers.parse_271 import parse as parse_271_text
from cyclone.parsers.parse_837 import parse
from cyclone.parsers.parse_835 import parse as parse_835
from cyclone.parsers.parse_999 import parse_999_text
from cyclone.parsers.parse_ta1 import parse_ta1_text
from cyclone.parsers.serialize_270 import serialize_270
from cyclone.parsers.serialize_999 import serialize_999
from cyclone.parsers.batch_ack_builder import build_ack_for_batch
@@ -667,6 +668,174 @@ async def parse_999_endpoint(
})
# --------------------------------------------------------------------------- #
# TA1 (Interchange Acknowledgment)
# --------------------------------------------------------------------------- #
def _ta1_synthetic_source_batch_id(interchange_control_number: str) -> str:
"""Return a synthetic ``batches.id`` for a received TA1 with no source batch.
Mirrors :func:`_ack_synthetic_source_batch_id`. The ta1_acks.source_batch_id
FK requires a row in batches; for received TA1s we synthesize an id of
the form ``TA1-<ISA13>``. The row is NOT created in batches (same
FK-is-no-op convention as the 999 path).
"""
return f"TA1-{(interchange_control_number or '').strip() or '000000001'}"
@app.post("/api/parse-ta1")
async def parse_ta1_endpoint(
file: UploadFile = File(...),
) -> Any:
"""Parse a TA1 (Interchange Acknowledgment) file, persist a row, return JSON.
Mirrors ``/api/parse-999`` but for the lower-level envelope ack:
- 400 on empty / undecodable / malformed EDI (never 500).
- 200 on success with ``{"ta1": {id, control_number, ack_code,
note_code, interchange_date, interchange_time, sender_id,
receiver_id, source_batch_id, raw_ta1_text}, "parsed":
<ParseResultTa1>}``.
The persisted ``ta1_acks.source_batch_id`` is a synthetic id
(``TA1-<ISA13>``) because a received TA1 has no inbound batch to
FK to. The dashboard's ``/ta1-acks`` list surfaces these.
"""
raw = await file.read()
if not raw:
return JSONResponse(
status_code=400,
content={"error": "Empty file", "detail": "Uploaded file contained no bytes."},
)
try:
text = raw.decode("utf-8")
except UnicodeDecodeError as exc:
return JSONResponse(
status_code=400,
content={"error": "Encoding error", "detail": str(exc)},
)
try:
result = parse_ta1_text(text, input_file=file.filename or "")
except CycloneParseError as exc:
return JSONResponse(
status_code=400,
content={"error": "Parse error", "detail": str(exc)},
)
except Exception as exc: # pragma: no cover - safety net
log.exception("Unexpected parser failure on TA1")
return JSONResponse(
status_code=500,
content={"error": "Internal server error", "detail": str(exc)},
)
# Build the raw TA1 text from the parsed result (round-trip).
raw_ta1_text = _serialize_ta1(result)
row = store.add_ta1_ack(
source_batch_id=result.source_batch_id,
control_number=result.ta1.control_number,
interchange_date=result.ta1.interchange_date,
interchange_time=result.ta1.interchange_time,
ack_code=result.ta1.ack_code,
note_code=result.ta1.note_code,
ack_generated_date=result.ta1.ack_generated_date,
sender_id=result.envelope.sender_id,
receiver_id=result.envelope.receiver_id,
raw_json=json.loads(result.model_dump_json()),
)
return JSONResponse(content={
"ta1": {
"id": row.id,
"control_number": result.ta1.control_number,
"ack_code": result.ta1.ack_code,
"note_code": result.ta1.note_code,
"interchange_date": result.ta1.interchange_date.isoformat()
if result.ta1.interchange_date else None,
"interchange_time": result.ta1.interchange_time,
"sender_id": result.envelope.sender_id,
"receiver_id": result.envelope.receiver_id,
"source_batch_id": result.source_batch_id,
"raw_ta1_text": raw_ta1_text,
},
"parsed": json.loads(result.model_dump_json()),
})
@app.get("/api/ta1-acks")
def list_ta1_acks_endpoint(
limit: int = Query(100, ge=1, le=1000),
) -> Any:
"""Return the list of persisted TA1 ACKs, newest first.
Mirrors :func:`list_acks_endpoint` — fetches all rows then slices in
Python so the ``total`` field reflects the full row count regardless
of the ``limit`` cap.
"""
rows = store.list_ta1_acks()
items = [_ta1_to_ui(r) for r in rows[:limit]]
return {
"total": len(rows),
"items": items,
}
@app.get("/api/ta1-acks/{ack_id}")
def get_ta1_ack_endpoint(ack_id: int) -> dict:
"""Return one persisted TA1 ACK row with its parsed detail."""
row = store.get_ta1_ack(ack_id)
if row is None:
raise HTTPException(status_code=404, detail=f"TA1 ACK {ack_id} not found")
body = _ta1_to_ui(row)
body["raw_ta1_text"] = _serialize_ta1_from_row(row)
body["raw_json"] = row.raw_json
return body
def _ta1_to_ui(row: db.Ta1Ack) -> dict:
"""Render a Ta1Ack row for the UI (list endpoint shape)."""
return {
"id": row.id,
"control_number": row.control_number,
"ack_code": row.ack_code,
"note_code": row.note_code,
"interchange_date": row.interchange_date.isoformat()
if row.interchange_date else None,
"interchange_time": row.interchange_time,
"sender_id": row.sender_id,
"receiver_id": row.receiver_id,
"source_batch_id": row.source_batch_id,
"parsed_at": row.parsed_at.isoformat() if row.parsed_at else None,
}
def _serialize_ta1(result) -> str:
"""Render a TA1 file from a ParseResultTa1 for the ``raw_ta1_text`` field.
Mirrors what the parser consumed: ISA envelope → TA1 → IEA. We
rebuild minimal ISA fields from the envelope plus the TA1 segment
verbatim. The serializer is intentionally tiny — TA1 has no GS/ST,
so there's no functional-group structure to round-trip.
"""
ta1 = result.ta1
parts = [
f"TA1*{ta1.control_number}*{ta1.interchange_date.strftime('%y%m%d') if ta1.interchange_date else ''}*"
f"{ta1.interchange_time or ''}*{ta1.ack_code}*{ta1.note_code or ''}*"
f"{ta1.ack_generated_date.strftime('%y%m%d') if ta1.ack_generated_date else ''}",
]
return "~".join(parts) + "~"
def _serialize_ta1_from_row(row: db.Ta1Ack) -> str:
"""Reconstruct a TA1 segment from the persisted flat row (for the detail endpoint)."""
date_s = row.interchange_date.strftime("%y%m%d") if row.interchange_date else ""
return (
f"TA1*{row.control_number}*{date_s}*{row.interchange_time or ''}*"
f"{row.ack_code}*{row.note_code or ''}~"
)
# --------------------------------------------------------------------------- #
# SP6 — Inbox endpoints
# --------------------------------------------------------------------------- #
+41
View File
@@ -378,3 +378,44 @@ class Ack(Base):
__table_args__ = (
Index("ix_acks_source_batch_id", "source_batch_id"),
)
class Ta1Ack(Base):
"""TA1 (Interchange Acknowledgment) row — one per parsed TA1 file.
Mirrors :class:`Ack` but for the lower-level interchange envelope
ack. A TA1 acknowledges the entire ISA/IEA interchange envelope
(vs a 999 which acknowledges individual transaction sets inside it).
``source_batch_id`` is the synthetic id used by the parser
(``TA1-<ISA13>``) so we can FK to a stable identifier even though we
don't have an inbound 837 batch to point at — same convention as
the 999 path (``999-<ISA13>``).
The flat columns (control_number, ack_code, note_code, dates) are
promoted out of ``raw_json`` so the list endpoint can sort/filter
without a JSON parse. The full parsed envelope is still preserved
in ``raw_json`` for the detail endpoint.
"""
__tablename__ = "ta1_acks"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
source_batch_id: Mapped[str] = mapped_column(
String(32), ForeignKey("batches.id", ondelete="CASCADE"), nullable=False,
)
control_number: Mapped[str] = mapped_column(String(32), nullable=False)
interchange_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True)
interchange_time: Mapped[Optional[str]] = mapped_column(String(8), nullable=True)
ack_code: Mapped[str] = mapped_column(String(8), nullable=False)
note_code: Mapped[Optional[str]] = mapped_column(String(8), nullable=True)
ack_generated_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True)
sender_id: Mapped[str] = mapped_column(String(64), nullable=False, default="")
receiver_id: Mapped[str] = mapped_column(String(64), nullable=False, default="")
parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
raw_json: Mapped[Optional[dict]] = mapped_column(JSONText, nullable=True)
__table_args__ = (
Index("ix_ta1_acks_source_batch_id", "source_batch_id"),
Index("ix_ta1_acks_ack_code", "ack_code"),
)
@@ -0,0 +1,34 @@
-- version: 5
-- TA1 (Interchange Acknowledgment) table.
--
-- Mirrors the `acks` table for the lower-level envelope ack. A TA1 file
-- is the per-interchange acknowledgment that payers send back in response
-- to an 837 submission; a 999 acks individual transaction sets inside the
-- interchange. Both are persisted side-by-side.
--
-- Schema follows the same conventions as `acks`:
-- * `id` auto-increment PK (one row per parsed TA1 file)
-- * `source_batch_id` FK to a synthetic `batches.id` (`TA1-<ISA13>`) so
-- the row has a stable identifier even though there's no inbound 837
-- batch to point at
-- * flat columns for the TA1 fields so list endpoints can sort/filter
-- without parsing `raw_json`
-- * `raw_json` carries the full `ParseResultTa1` for the detail endpoint
CREATE TABLE ta1_acks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_batch_id TEXT NOT NULL REFERENCES batches(id) ON DELETE CASCADE,
control_number TEXT NOT NULL,
interchange_date DATE,
interchange_time TEXT,
ack_code TEXT NOT NULL,
note_code TEXT,
ack_generated_date DATE,
sender_id TEXT NOT NULL DEFAULT '',
receiver_id TEXT NOT NULL DEFAULT '',
parsed_at DATETIME NOT NULL,
raw_json TEXT
);
CREATE INDEX ix_ta1_acks_source_batch_id ON ta1_acks(source_batch_id);
CREATE INDEX ix_ta1_acks_ack_code ON ta1_acks(ack_code);
+99
View File
@@ -0,0 +1,99 @@
"""Pydantic v2 models for parsed X12 TA1 (Interchange Acknowledgment) files.
A TA1 is the lowest-level ack in the X12 envelope stack — it acknowledges
the *interchange* (ISA/IEA), distinct from a 999 which acknowledges
individual transaction sets. Per X12 005010X231A1 the envelope is:
ISA… → TA1 → IEA…
The TA1 segment carries six fields:
TA101 Interchange Control Number (matches ISA13 of source)
TA102 Interchange Date (matches ISA09 of source, CCYYMMDD)
TA103 Interchange Time (matches ISA10 of source, HHMM)
TA104 Acknowledgment Code A / E / R
TA105 Interchange Note Code numeric code (see spec table)
TA106 Interchange Date (ack generated) CCYYMMDD
Standard ack codes:
A = Accepted (interchange passed envelope validation)
E = Accepted with errors (interchange accepted but a TA105 note exists)
R = Rejected (interchange failed validation; see TA105)
Note codes from the X12 spec (005010X231A1) — only the most common ones
are surfaced as named constants below; the raw value is always preserved.
"""
from __future__ import annotations
from datetime import date
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, model_serializer
from cyclone.parsers.models import BatchSummary, Envelope
# --------------------------------------------------------------------------- #
# Shared base
# --------------------------------------------------------------------------- #
class _Base(BaseModel):
"""Shared Pydantic base; matches the 837P / 835 / 999 models for JSON consistency."""
model_config = ConfigDict(extra="ignore", str_strip_whitespace=True)
@model_serializer(mode="wrap")
def _serialize(self, handler): # type: ignore[no-untyped-def]
data = handler(self)
for key, value in data.items():
if isinstance(value, date):
data[key] = value.isoformat()
return data
# --------------------------------------------------------------------------- #
# Interchange Acknowledgment
# --------------------------------------------------------------------------- #
# The three valid ack codes per X12 spec. We type-narrow so the API
# surfaces the canonical set; raw values outside this set still parse
# but surface as the literal string.
AckCode = Literal["A", "E", "R"]
class Ta1Ack(_Base):
"""One TA1 segment — the per-interchange acknowledgment.
``control_number`` matches the ISA13 of the source interchange, so
operators can correlate a TA1 back to the 837 batch it acknowledged.
``interchange_date`` / ``interchange_time`` mirror the ISA09 / ISA10
fields of the source.
``note_code`` is an optional X12 note (e.g. "006" = "Invalid interchange
date"). We keep the raw string even when it's not in our named map so
new codes don't silently surface as ``None``.
"""
control_number: str
interchange_date: date | None = None
interchange_time: str | None = None
ack_code: AckCode
note_code: str | None = None
ack_generated_date: date | None = None
class ParseResultTa1(_Base):
"""One TA1 file = one interchange = one Ta1Ack."""
envelope: Envelope
ta1: Ta1Ack
summary: BatchSummary = Field(default_factory=lambda: BatchSummary(input_file=""))
# The synthesized source_batch_id (e.g. ``"TA1-<ISA13>"``) so the row
# can FK to a stable identifier even though we don't have an inbound
# batch to point at. Mirrors the 999 ``999-<ISA13>`` convention.
source_batch_id: str = ""
__all__ = ["AckCode", "Ta1Ack", "ParseResultTa1"]
+187
View File
@@ -0,0 +1,187 @@
"""Parse an X12 TA1 (Interchange Acknowledgment) file.
TA1 is the lowest-level ack in the X12 envelope stack. A TA1 file has
the shape::
ISA*…~
TA1*<icn>*<date>*<time>*<ack_code>*<note_code>*<ack_date>~
IEA*1*<icn>~
This module is intentionally simple — the spec allows for multiple TA1
segments in a single interchange (one per included 837 batch), but in
practice Colorado Medicaid sends exactly one TA1 per file. We accept both
shapes and return the first TA1 we see (the rest are surfaced via the
``raw_segments`` field on the envelope in future versions).
Whole-document problems (missing ISA, no TA1) raise
:class:`CycloneParseError`.
"""
from __future__ import annotations
import logging
from datetime import date
from cyclone.parsers.exceptions import CycloneParseError
from cyclone.parsers.models import BatchSummary, Envelope
from cyclone.parsers.models_ta1 import ParseResultTa1, Ta1Ack
from cyclone.parsers.segments import tokenize
log = logging.getLogger(__name__)
# --------------------------------------------------------------------------- #
# Date parsing
# --------------------------------------------------------------------------- #
def _parse_yyyymmdd(s: str) -> date | None:
"""Parse an X12 date string. Returns None on bad input.
Accepts:
* CCYYMMDD (8 digits) — the spec shape for TA102 / TA106.
* YYMMDD (6 digits) — what Colorado Medicaid sends in practice.
The century is inferred as 2000 + YY (TA1s older than ~2030
are out of scope; if that assumption changes, switch to a
rolling-window strategy).
"""
if not s or not s.isdigit():
return None
if len(s) == 8:
try:
return date(int(s[0:4]), int(s[4:6]), int(s[6:8]))
except ValueError:
return None
if len(s) == 6:
try:
return date(2000 + int(s[0:2]), int(s[2:4]), int(s[4:6]))
except ValueError:
return None
return None
# --------------------------------------------------------------------------- #
# Envelope
# --------------------------------------------------------------------------- #
def _build_envelope(segments: list[list[str]], input_file: str) -> Envelope:
"""Build the envelope from ISA. TA1 has no GS/ST — just ISA → TA1 → IEA.
``transaction_date`` falls back to a placeholder if ISA09 is missing
or unparseable; the authoritative date is the one inside the TA1
segment itself (TA102).
"""
envelope: Envelope | None = None
for seg in segments:
if seg[0] == "ISA":
try:
envelope = Envelope(
sender_id=seg[6].strip(),
receiver_id=seg[8].strip(),
control_number=seg[13].strip(),
transaction_date=date(2024, 1, 1),
implementation_guide=None,
)
except (IndexError, ValueError) as exc:
raise CycloneParseError(f"Bad ISA: {exc}") from exc
break
if envelope is None:
raise CycloneParseError("No ISA envelope found")
return envelope
# --------------------------------------------------------------------------- #
# TA1 segment consumer
# --------------------------------------------------------------------------- #
def _consume_ta1(segments: list[list[str]], idx: int) -> tuple[Ta1Ack, int]:
"""Read a TA1 segment and return a :class:`Ta1Ack`.
Per X12 005010X231A1:
TA101 = Interchange Control Number (15/9 digits)
TA102 = Interchange Date (CCYYMMDD)
TA103 = Interchange Time (HHMM)
TA104 = Acknowledgment Code (A/E/R)
TA105 = Interchange Note Code (3-digit numeric; optional but commonly present)
TA106 = Interchange Date — date the ack was generated (optional)
"""
ta1 = segments[idx]
if ta1[0] != "TA1":
raise CycloneParseError(f"Expected TA1, got {ta1[0]!r}")
ctrl = ta1[1] if len(ta1) > 1 else ""
date_s = ta1[2] if len(ta1) > 2 else ""
time_s = ta1[3] if len(ta1) > 3 else ""
code = (ta1[4] if len(ta1) > 4 else "R").strip() or "R"
note = ta1[5] if len(ta1) > 5 and ta1[5] else None
ack_date_s = ta1[6] if len(ta1) > 6 and ta1[6] else ""
# Narrow to the literal type — anything outside the canonical set
# surfaces as the raw string instead of failing parse, so unknown
# codes from new payers don't break ingest.
if code not in ("A", "E", "R"):
log.warning("Unexpected TA1 ack code %r on icn=%s", code, ctrl)
return (
Ta1Ack(
control_number=ctrl,
interchange_date=_parse_yyyymmdd(date_s),
interchange_time=time_s or None,
ack_code=code, # type: ignore[arg-type]
note_code=note,
ack_generated_date=_parse_yyyymmdd(ack_date_s),
),
idx + 1,
)
# --------------------------------------------------------------------------- #
# Top-level orchestrator
# --------------------------------------------------------------------------- #
def parse_ta1_text(text: str, *, input_file: str = "") -> ParseResultTa1:
"""Parse a complete TA1 document and return a :class:`ParseResultTa1`.
Permissive: if the file contains multiple TA1 segments, we return the
first one and warn. The spec allows this for interchanges that wrap
multiple 837 batches but no real-world Colorado file uses it today.
"""
segments = tokenize(text)
envelope = _build_envelope(segments, input_file=input_file)
ta1_idx: int | None = None
for i, seg in enumerate(segments):
if seg[0] == "TA1":
ta1_idx = i
break
if ta1_idx is None:
raise CycloneParseError("No TA1 segment found")
ta1, _ = _consume_ta1(segments, ta1_idx)
# TA102 is the authoritative interchange date — overwrite the placeholder.
if ta1.interchange_date is not None:
envelope = envelope.model_copy(update={"transaction_date": ta1.interchange_date})
# Stable FK target. Mirrors the 999 path: ``<kind>-<ISA13>``.
source_batch_id = f"TA1-{(envelope.control_number or '').strip() or '000000001'}"
summary = BatchSummary(
input_file=input_file,
control_number=envelope.control_number,
transaction_date=envelope.transaction_date,
total_claims=1, # one TA1 == one interchange acknowledged
passed=1 if ta1.ack_code == "A" else 0,
failed=0 if ta1.ack_code == "A" else 1,
)
return ParseResultTa1(
envelope=envelope,
ta1=ta1,
summary=summary,
source_batch_id=source_batch_id,
)
__all__ = ["parse_ta1_text"]
+60
View File
@@ -1548,6 +1548,66 @@ class CycloneStore:
with db.SessionLocal()() as s:
return s.get(Ack, ack_id)
# -- TA1 (Interchange Acknowledgment) -------------------------------
def add_ta1_ack(
self,
*,
source_batch_id: str,
control_number: str,
interchange_date: date | None,
interchange_time: str | None,
ack_code: str,
note_code: str | None,
ack_generated_date: date | None,
sender_id: str,
receiver_id: str,
raw_json: dict,
) -> db.Ta1Ack:
"""Persist a TA1 (Interchange Acknowledgment) row and return it.
Mirrors :meth:`add_ack` for the lower-level envelope ack. The
flat columns are promoted out of ``raw_json`` so the list
endpoint can sort/filter without a JSON parse; the full
``ParseResultTa1`` stays in ``raw_json`` for the detail endpoint.
"""
with db.SessionLocal()() as s:
row = db.Ta1Ack(
source_batch_id=source_batch_id,
control_number=control_number,
interchange_date=interchange_date,
interchange_time=interchange_time,
ack_code=ack_code,
note_code=note_code,
ack_generated_date=ack_generated_date,
sender_id=sender_id,
receiver_id=receiver_id,
parsed_at=utcnow(),
raw_json=raw_json,
)
s.add(row)
s.commit()
s.refresh(row)
return row
def list_ta1_acks(self) -> list[db.Ta1Ack]:
"""Return every TA1 ACK row, newest first (auto-increment id desc).
Mirrors :meth:`list_acks` — the API endpoint slices to its own
``limit`` so the ``total`` field reflects the full row count.
"""
with db.SessionLocal()() as s:
return (
s.query(db.Ta1Ack)
.order_by(db.Ta1Ack.id.desc())
.all()
)
def get_ta1_ack(self, ack_id: int) -> db.Ta1Ack | None:
"""Return a single TA1 ACK row by id, or ``None`` if not found."""
with db.SessionLocal()() as s:
return s.get(db.Ta1Ack, ack_id)
# -- manual reconciliation (T12) -----------------------------------
def list_unmatched(self, *, kind: str = "both") -> dict: