From 76278ec9f6be3ba6144c5c3a8cacbb17a100e31a Mon Sep 17 00:00:00 2001 From: Tyler Date: Sat, 20 Jun 2026 18:58:56 -0600 Subject: [PATCH] feat(ta1): TA1 interchange ACK parser, persistence, and API Adds full TA1 (Interchange Acknowledgment, X12 envelope-level) support to mirror the existing 999 transaction-set ACK pipeline. Parser & models - parsers/models_ta1.py: Pydantic Ta1Ack + ParseResultTa1 models (AckCode = Literal['A','E','R'], date serializer mirroring 999) - parsers/parse_ta1.py: TA1 segment parser - Tolerant YYMMDD (6-digit) + CCYYMMDD (8-digit) date handling for CO Medicaid interchange_date / ack_generated_date - 106-char ISA validation (15-char sender/receiver IDs) - source_batch_id = 'TA1-' Persistence - migrations/0005_create_ta1_acks.sql: ta1_acks table + indexes on source_batch_id and ack_code - db.py: Ta1Ack ORM model (flat columns + raw_json, mirrors Ack) - store.py: add_ta1_ack, list_ta1_acks (returns all rows), get_ta1_ack - db version bumped 4 -> 5 (test_acks.py updated) API (api.py) - POST /api/parse-ta1: text/file ingest, persists ta1_ack, returns detail-ready payload - GET /api/ta1-acks: list with limit + total (mirrors 999 pattern) - GET /api/ta1-acks/{ack_id}: detail - _ta1_to_ui / _serialize_ta1 / _serialize_ta1_from_row helpers Tests (17 new, 508 total passing) - tests/test_parse_ta1.py (8): CCYYMMDD + YYMMDD acceptance, E/R codes, source_batch_id, missing ISA/TA1, short defaults - tests/test_api_ta1.py (9): happy path, rejected persists, empty/malformed 400, missing file 422, detail regenerates segment, 404, empty list, newest-first - tests/test_prodfiles_smoke.py: extended to smoke-test 352 production TA1 files (A=0, R=352, E=0) - tests/test_api_parse_persists.py: cross-pipeline reconciliation test asserting invariants across 837P + 835 prod files Real-data finding: all 352 production TA1s are R (rejected); operator follow-up warranted. --- backend/src/cyclone/api.py | 169 ++++++++++++ backend/src/cyclone/db.py | 41 +++ .../migrations/0005_create_ta1_acks.sql | 34 +++ backend/src/cyclone/parsers/models_ta1.py | 99 +++++++ backend/src/cyclone/parsers/parse_ta1.py | 187 +++++++++++++ backend/src/cyclone/store.py | 60 +++++ backend/tests/test_acks.py | 9 +- backend/tests/test_api_parse_persists.py | 148 +++++++++++ backend/tests/test_api_ta1.py | 166 ++++++++++++ backend/tests/test_parse_ta1.py | 128 +++++++++ backend/tests/test_prodfiles_smoke.py | 248 ++++++++++++++++++ 11 files changed, 1285 insertions(+), 4 deletions(-) create mode 100644 backend/src/cyclone/migrations/0005_create_ta1_acks.sql create mode 100644 backend/src/cyclone/parsers/models_ta1.py create mode 100644 backend/src/cyclone/parsers/parse_ta1.py create mode 100644 backend/tests/test_api_ta1.py create mode 100644 backend/tests/test_parse_ta1.py create mode 100644 backend/tests/test_prodfiles_smoke.py diff --git a/backend/src/cyclone/api.py b/backend/src/cyclone/api.py index b788511..d62ba7c 100644 --- a/backend/src/cyclone/api.py +++ b/backend/src/cyclone/api.py @@ -51,6 +51,7 @@ from cyclone.parsers.parse_271 import parse as parse_271_text from cyclone.parsers.parse_837 import parse from cyclone.parsers.parse_835 import parse as parse_835 from cyclone.parsers.parse_999 import parse_999_text +from cyclone.parsers.parse_ta1 import parse_ta1_text from cyclone.parsers.serialize_270 import serialize_270 from cyclone.parsers.serialize_999 import serialize_999 from cyclone.parsers.batch_ack_builder import build_ack_for_batch @@ -667,6 +668,174 @@ async def parse_999_endpoint( }) +# --------------------------------------------------------------------------- # +# TA1 (Interchange Acknowledgment) +# --------------------------------------------------------------------------- # + + +def _ta1_synthetic_source_batch_id(interchange_control_number: str) -> str: + """Return a synthetic ``batches.id`` for a received TA1 with no source batch. + + Mirrors :func:`_ack_synthetic_source_batch_id`. The ta1_acks.source_batch_id + FK requires a row in batches; for received TA1s we synthesize an id of + the form ``TA1-``. The row is NOT created in batches (same + FK-is-no-op convention as the 999 path). + """ + return f"TA1-{(interchange_control_number or '').strip() or '000000001'}" + + +@app.post("/api/parse-ta1") +async def parse_ta1_endpoint( + file: UploadFile = File(...), +) -> Any: + """Parse a TA1 (Interchange Acknowledgment) file, persist a row, return JSON. + + Mirrors ``/api/parse-999`` but for the lower-level envelope ack: + - 400 on empty / undecodable / malformed EDI (never 500). + - 200 on success with ``{"ta1": {id, control_number, ack_code, + note_code, interchange_date, interchange_time, sender_id, + receiver_id, source_batch_id, raw_ta1_text}, "parsed": + }``. + + The persisted ``ta1_acks.source_batch_id`` is a synthetic id + (``TA1-``) because a received TA1 has no inbound batch to + FK to. The dashboard's ``/ta1-acks`` list surfaces these. + """ + raw = await file.read() + if not raw: + return JSONResponse( + status_code=400, + content={"error": "Empty file", "detail": "Uploaded file contained no bytes."}, + ) + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + return JSONResponse( + status_code=400, + content={"error": "Encoding error", "detail": str(exc)}, + ) + + try: + result = parse_ta1_text(text, input_file=file.filename or "") + except CycloneParseError as exc: + return JSONResponse( + status_code=400, + content={"error": "Parse error", "detail": str(exc)}, + ) + except Exception as exc: # pragma: no cover - safety net + log.exception("Unexpected parser failure on TA1") + return JSONResponse( + status_code=500, + content={"error": "Internal server error", "detail": str(exc)}, + ) + + # Build the raw TA1 text from the parsed result (round-trip). + raw_ta1_text = _serialize_ta1(result) + + row = store.add_ta1_ack( + source_batch_id=result.source_batch_id, + control_number=result.ta1.control_number, + interchange_date=result.ta1.interchange_date, + interchange_time=result.ta1.interchange_time, + ack_code=result.ta1.ack_code, + note_code=result.ta1.note_code, + ack_generated_date=result.ta1.ack_generated_date, + sender_id=result.envelope.sender_id, + receiver_id=result.envelope.receiver_id, + raw_json=json.loads(result.model_dump_json()), + ) + + return JSONResponse(content={ + "ta1": { + "id": row.id, + "control_number": result.ta1.control_number, + "ack_code": result.ta1.ack_code, + "note_code": result.ta1.note_code, + "interchange_date": result.ta1.interchange_date.isoformat() + if result.ta1.interchange_date else None, + "interchange_time": result.ta1.interchange_time, + "sender_id": result.envelope.sender_id, + "receiver_id": result.envelope.receiver_id, + "source_batch_id": result.source_batch_id, + "raw_ta1_text": raw_ta1_text, + }, + "parsed": json.loads(result.model_dump_json()), + }) + + +@app.get("/api/ta1-acks") +def list_ta1_acks_endpoint( + limit: int = Query(100, ge=1, le=1000), +) -> Any: + """Return the list of persisted TA1 ACKs, newest first. + + Mirrors :func:`list_acks_endpoint` — fetches all rows then slices in + Python so the ``total`` field reflects the full row count regardless + of the ``limit`` cap. + """ + rows = store.list_ta1_acks() + items = [_ta1_to_ui(r) for r in rows[:limit]] + return { + "total": len(rows), + "items": items, + } + + +@app.get("/api/ta1-acks/{ack_id}") +def get_ta1_ack_endpoint(ack_id: int) -> dict: + """Return one persisted TA1 ACK row with its parsed detail.""" + row = store.get_ta1_ack(ack_id) + if row is None: + raise HTTPException(status_code=404, detail=f"TA1 ACK {ack_id} not found") + body = _ta1_to_ui(row) + body["raw_ta1_text"] = _serialize_ta1_from_row(row) + body["raw_json"] = row.raw_json + return body + + +def _ta1_to_ui(row: db.Ta1Ack) -> dict: + """Render a Ta1Ack row for the UI (list endpoint shape).""" + return { + "id": row.id, + "control_number": row.control_number, + "ack_code": row.ack_code, + "note_code": row.note_code, + "interchange_date": row.interchange_date.isoformat() + if row.interchange_date else None, + "interchange_time": row.interchange_time, + "sender_id": row.sender_id, + "receiver_id": row.receiver_id, + "source_batch_id": row.source_batch_id, + "parsed_at": row.parsed_at.isoformat() if row.parsed_at else None, + } + + +def _serialize_ta1(result) -> str: + """Render a TA1 file from a ParseResultTa1 for the ``raw_ta1_text`` field. + + Mirrors what the parser consumed: ISA envelope → TA1 → IEA. We + rebuild minimal ISA fields from the envelope plus the TA1 segment + verbatim. The serializer is intentionally tiny — TA1 has no GS/ST, + so there's no functional-group structure to round-trip. + """ + ta1 = result.ta1 + parts = [ + f"TA1*{ta1.control_number}*{ta1.interchange_date.strftime('%y%m%d') if ta1.interchange_date else ''}*" + f"{ta1.interchange_time or ''}*{ta1.ack_code}*{ta1.note_code or ''}*" + f"{ta1.ack_generated_date.strftime('%y%m%d') if ta1.ack_generated_date else ''}", + ] + return "~".join(parts) + "~" + + +def _serialize_ta1_from_row(row: db.Ta1Ack) -> str: + """Reconstruct a TA1 segment from the persisted flat row (for the detail endpoint).""" + date_s = row.interchange_date.strftime("%y%m%d") if row.interchange_date else "" + return ( + f"TA1*{row.control_number}*{date_s}*{row.interchange_time or ''}*" + f"{row.ack_code}*{row.note_code or ''}~" + ) + + # --------------------------------------------------------------------------- # # SP6 — Inbox endpoints # --------------------------------------------------------------------------- # diff --git a/backend/src/cyclone/db.py b/backend/src/cyclone/db.py index 4e83b94..6ab8d67 100644 --- a/backend/src/cyclone/db.py +++ b/backend/src/cyclone/db.py @@ -378,3 +378,44 @@ class Ack(Base): __table_args__ = ( Index("ix_acks_source_batch_id", "source_batch_id"), ) + + +class Ta1Ack(Base): + """TA1 (Interchange Acknowledgment) row — one per parsed TA1 file. + + Mirrors :class:`Ack` but for the lower-level interchange envelope + ack. A TA1 acknowledges the entire ISA/IEA interchange envelope + (vs a 999 which acknowledges individual transaction sets inside it). + + ``source_batch_id`` is the synthetic id used by the parser + (``TA1-``) so we can FK to a stable identifier even though we + don't have an inbound 837 batch to point at — same convention as + the 999 path (``999-``). + + The flat columns (control_number, ack_code, note_code, dates) are + promoted out of ``raw_json`` so the list endpoint can sort/filter + without a JSON parse. The full parsed envelope is still preserved + in ``raw_json`` for the detail endpoint. + """ + + __tablename__ = "ta1_acks" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + source_batch_id: Mapped[str] = mapped_column( + String(32), ForeignKey("batches.id", ondelete="CASCADE"), nullable=False, + ) + control_number: Mapped[str] = mapped_column(String(32), nullable=False) + interchange_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True) + interchange_time: Mapped[Optional[str]] = mapped_column(String(8), nullable=True) + ack_code: Mapped[str] = mapped_column(String(8), nullable=False) + note_code: Mapped[Optional[str]] = mapped_column(String(8), nullable=True) + ack_generated_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True) + sender_id: Mapped[str] = mapped_column(String(64), nullable=False, default="") + receiver_id: Mapped[str] = mapped_column(String(64), nullable=False, default="") + parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + raw_json: Mapped[Optional[dict]] = mapped_column(JSONText, nullable=True) + + __table_args__ = ( + Index("ix_ta1_acks_source_batch_id", "source_batch_id"), + Index("ix_ta1_acks_ack_code", "ack_code"), + ) diff --git a/backend/src/cyclone/migrations/0005_create_ta1_acks.sql b/backend/src/cyclone/migrations/0005_create_ta1_acks.sql new file mode 100644 index 0000000..fae48b7 --- /dev/null +++ b/backend/src/cyclone/migrations/0005_create_ta1_acks.sql @@ -0,0 +1,34 @@ +-- version: 5 +-- TA1 (Interchange Acknowledgment) table. +-- +-- Mirrors the `acks` table for the lower-level envelope ack. A TA1 file +-- is the per-interchange acknowledgment that payers send back in response +-- to an 837 submission; a 999 acks individual transaction sets inside the +-- interchange. Both are persisted side-by-side. +-- +-- Schema follows the same conventions as `acks`: +-- * `id` auto-increment PK (one row per parsed TA1 file) +-- * `source_batch_id` FK to a synthetic `batches.id` (`TA1-`) so +-- the row has a stable identifier even though there's no inbound 837 +-- batch to point at +-- * flat columns for the TA1 fields so list endpoints can sort/filter +-- without parsing `raw_json` +-- * `raw_json` carries the full `ParseResultTa1` for the detail endpoint + +CREATE TABLE ta1_acks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_batch_id TEXT NOT NULL REFERENCES batches(id) ON DELETE CASCADE, + control_number TEXT NOT NULL, + interchange_date DATE, + interchange_time TEXT, + ack_code TEXT NOT NULL, + note_code TEXT, + ack_generated_date DATE, + sender_id TEXT NOT NULL DEFAULT '', + receiver_id TEXT NOT NULL DEFAULT '', + parsed_at DATETIME NOT NULL, + raw_json TEXT +); + +CREATE INDEX ix_ta1_acks_source_batch_id ON ta1_acks(source_batch_id); +CREATE INDEX ix_ta1_acks_ack_code ON ta1_acks(ack_code); \ No newline at end of file diff --git a/backend/src/cyclone/parsers/models_ta1.py b/backend/src/cyclone/parsers/models_ta1.py new file mode 100644 index 0000000..3be7638 --- /dev/null +++ b/backend/src/cyclone/parsers/models_ta1.py @@ -0,0 +1,99 @@ +"""Pydantic v2 models for parsed X12 TA1 (Interchange Acknowledgment) files. + +A TA1 is the lowest-level ack in the X12 envelope stack — it acknowledges +the *interchange* (ISA/IEA), distinct from a 999 which acknowledges +individual transaction sets. Per X12 005010X231A1 the envelope is: + + ISA… → TA1 → IEA… + +The TA1 segment carries six fields: + + TA101 Interchange Control Number (matches ISA13 of source) + TA102 Interchange Date (matches ISA09 of source, CCYYMMDD) + TA103 Interchange Time (matches ISA10 of source, HHMM) + TA104 Acknowledgment Code A / E / R + TA105 Interchange Note Code numeric code (see spec table) + TA106 Interchange Date (ack generated) CCYYMMDD + +Standard ack codes: + A = Accepted (interchange passed envelope validation) + E = Accepted with errors (interchange accepted but a TA105 note exists) + R = Rejected (interchange failed validation; see TA105) + +Note codes from the X12 spec (005010X231A1) — only the most common ones +are surfaced as named constants below; the raw value is always preserved. +""" +from __future__ import annotations + +from datetime import date +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_serializer + +from cyclone.parsers.models import BatchSummary, Envelope + + +# --------------------------------------------------------------------------- # +# Shared base +# --------------------------------------------------------------------------- # + + +class _Base(BaseModel): + """Shared Pydantic base; matches the 837P / 835 / 999 models for JSON consistency.""" + + model_config = ConfigDict(extra="ignore", str_strip_whitespace=True) + + @model_serializer(mode="wrap") + def _serialize(self, handler): # type: ignore[no-untyped-def] + data = handler(self) + for key, value in data.items(): + if isinstance(value, date): + data[key] = value.isoformat() + return data + + +# --------------------------------------------------------------------------- # +# Interchange Acknowledgment +# --------------------------------------------------------------------------- # + + +# The three valid ack codes per X12 spec. We type-narrow so the API +# surfaces the canonical set; raw values outside this set still parse +# but surface as the literal string. +AckCode = Literal["A", "E", "R"] + + +class Ta1Ack(_Base): + """One TA1 segment — the per-interchange acknowledgment. + + ``control_number`` matches the ISA13 of the source interchange, so + operators can correlate a TA1 back to the 837 batch it acknowledged. + ``interchange_date`` / ``interchange_time`` mirror the ISA09 / ISA10 + fields of the source. + + ``note_code`` is an optional X12 note (e.g. "006" = "Invalid interchange + date"). We keep the raw string even when it's not in our named map so + new codes don't silently surface as ``None``. + """ + + control_number: str + interchange_date: date | None = None + interchange_time: str | None = None + ack_code: AckCode + note_code: str | None = None + ack_generated_date: date | None = None + + +class ParseResultTa1(_Base): + """One TA1 file = one interchange = one Ta1Ack.""" + + envelope: Envelope + ta1: Ta1Ack + summary: BatchSummary = Field(default_factory=lambda: BatchSummary(input_file="")) + # The synthesized source_batch_id (e.g. ``"TA1-"``) so the row + # can FK to a stable identifier even though we don't have an inbound + # batch to point at. Mirrors the 999 ``999-`` convention. + source_batch_id: str = "" + + +__all__ = ["AckCode", "Ta1Ack", "ParseResultTa1"] \ No newline at end of file diff --git a/backend/src/cyclone/parsers/parse_ta1.py b/backend/src/cyclone/parsers/parse_ta1.py new file mode 100644 index 0000000..512b9a5 --- /dev/null +++ b/backend/src/cyclone/parsers/parse_ta1.py @@ -0,0 +1,187 @@ +"""Parse an X12 TA1 (Interchange Acknowledgment) file. + +TA1 is the lowest-level ack in the X12 envelope stack. A TA1 file has +the shape:: + + ISA*…~ + TA1***