diff --git a/CLAUDE.md b/CLAUDE.md index 8dfdba6..8c7cb9e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -125,7 +125,7 @@ Frontend triplet for any live page: `use(params)` (initial fetch) + `useTailS ## Backend at a glance -`backend/src/cyclone/` is a single namespace. The two largest files are `api.py` (~3,548 LOC, the only large file) and `store.py` (~2,423 LOC, the `CycloneStore` facade — SP21 is in flight to split it into a `cyclone/store/` subpackage; the public API stays unchanged). Subpackages: `api_routers/` (acks, admin, health, ta1_acks), `clearhouse/` (Clearhouse + SftpClient), `edi/` (filenames), `parsers/` (X12 transaction parsers + models + validators + serializers), `workflow/` (placeholder for future sub-project 6). +`backend/src/cyclone/` is a single namespace. The two largest files are `api.py` (~3,548 LOC, the only large file) and `store.py` (~2,423 LOC, the `CycloneStore` facade — SP21 is in flight to split it into a `cyclone/store/` subpackage; the public API stays unchanged). Subpackages: `api_routers/` (acks, admin, health, ta1_acks), `clearhouse/` (Clearhouse + SftpClient), `edi/` (filenames), `parsers/` (X12 transaction parsers + models + validators + serializers), `submission/` (SP37 — canonical `submit_file` helper shared by `cyclone submit-batch` CLI + `POST /api/submit-batch` HTTP endpoint; owns parse → DB-write → SFTP-upload → audit per file), `workflow/` (placeholder for future sub-project 6). The store is the only read/write surface for the database; every mutating endpoint goes through it. All persistence flows through SQLAlchemy sessions via `db.SessionLocal()()`. SQLAlchemy ORM models live in `db.py`; 12 SQL migrations under `migrations/` (0001_initial through 0012_backups) are walked in order by `db_migrate.py`. diff --git a/backend/src/cyclone/api_routers/__init__.py b/backend/src/cyclone/api_routers/__init__.py index 23cd9ef..c82b181 100644 --- a/backend/src/cyclone/api_routers/__init__.py +++ b/backend/src/cyclone/api_routers/__init__.py @@ -28,6 +28,7 @@ from cyclone.api_routers import ( providers, reconciliation, remittances, + submission, ta1_acks, ) @@ -49,6 +50,7 @@ routers: list[APIRouter] = [ providers.router, # gated reconciliation.router, # gated remittances.router, # gated + submission.router, # gated ta1_acks.router, # gated ] diff --git a/backend/src/cyclone/api_routers/submission.py b/backend/src/cyclone/api_routers/submission.py new file mode 100644 index 0000000..938f246 --- /dev/null +++ b/backend/src/cyclone/api_routers/submission.py @@ -0,0 +1,186 @@ +"""SP37 Task 6: HTTP endpoint for the canonical submit-batch flow. + +Thin wrapper around ``cyclone.submission.submit_file`` — same logic as +the ``cyclone submit-batch`` CLI (SP37 Task 5), just framed as JSON in +/ JSON out and gated by ``matrix_gate``. The walker pattern, ``._*`` +AppleDouble skip, ``limit`` semantics, and per-file outcomes all match +the CLI byte-for-byte so a batch run via the CLI and the same batch +run via this endpoint produce identical DB + SFTP state. + +The endpoint deliberately does NOT inject an ``sftp_client_factory``: +``submit_file`` defaults to its paramiko-based factory so SKIPPED is +reachable in production (the ``SftpClient`` wrapper has no ``stat()``). +Tests monkey-patch ``cyclone.api_routers.submission.submit_file`` +itself; that avoids the paramiko factory entirely without touching +the helper's contract. + +Status code contract (per Task 6 spec §4): + - 200: completed run. Per-file failures live in the JSON body. + - 401: not authenticated (matrix_gate). + - 404: no clearhouse seeded (config-level "missing" → 4xx, not 5xx). + - 409: clearhouse SFTP block is in stub mode (refuses to upload). + - 422: ``ingest_dir`` missing on disk OR Pydantic body validation + failed (missing fields, wrong types). + - 5xx: truly unexpected exceptions propagate (do not swallow). +""" +from __future__ import annotations + +import logging +from pathlib import Path + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, ConfigDict, Field + +from cyclone.auth.deps import matrix_gate +from cyclone.store import store as cycl_store +from cyclone.submission.core import submit_file +from cyclone.submission.result import SubmitOutcome, SubmitResult + +log = logging.getLogger(__name__) + +# No `prefix=` here — every other gated router in this package declares +# the full path in the decorator (clearhouse.py uses "/api/clearhouse", +# parse.py uses "/api/parse-837", etc.). The decorator sets the full URL. +router = APIRouter( + tags=["submission"], + dependencies=[Depends(matrix_gate)], +) + + +class SubmitBatchRequest(BaseModel): + """Body schema for ``POST /api/submit-batch``. + + ``ingest_dir`` has no default so Pydantic raises 422 when it's + missing — better UX than letting the walker crash on a missing + path. ``validate_files`` and ``actor`` default so a minimal client + can skip them. ``limit`` truncates the file list after the walker + collects it (mirrors the CLI's post-collection ``if i > limit: + break`` semantics, but applied as a slice since the HTTP body + model is type-checked up-front). + + The ``validate_files`` field is aliased to ``validate`` in the JSON + body to mirror the CLI's ``--validate`` flag and avoid the + hardcoded Pydantic warning about ``validate`` shadowing + ``BaseModel.validate``. ``populate_by_name=True`` lets tests + construct the model with either key. + """ + model_config = ConfigDict(populate_by_name=True, protected_namespaces=()) + + ingest_dir: str + validate_files: bool = Field(default=True, alias="validate") + actor: str = "api-submit-batch" + limit: int | None = None + + +@router.post("/api/submit-batch") +def submit_batch(body: SubmitBatchRequest): + """Submit every ``batch-*-claims/*.x12`` under ``ingest_dir``. + + Walks ``ingest_dir`` for any directory matching ``batch-*-claims``, + collects each one's ``*.x12`` files (sorted, with ``._*`` + AppleDouble files skipped), truncates to ``limit`` if set, then + calls :func:`cyclone.submission.submit_file` per file with the + seeded clearhouse's ``sftp_block`` and ``actor`` from the body. + + Returns counts (``submitted`` / ``skipped`` / ``failed``) plus a + per-file ``results`` array. Per-file failures NEVER change the + HTTP status code — the response is 200 whenever the run itself + completed. + """ + + # 1. Config-level guards. Order matters: a missing clearhouse is a + # 404 (config-level "missing"), but if it IS present and in stub + # mode the operator's request is a 409 (configured-but-wrong). + clearhouse = cycl_store.get_clearhouse() + if clearhouse is None: + raise HTTPException( + status_code=404, detail="no clearhouse seeded", + ) + sftp_block = clearhouse.sftp_block + if sftp_block.stub: + raise HTTPException( + status_code=409, detail="clearhouse SFTP block is in stub mode", + ) + + # 2. Resolve + validate the ingest dir. Use ``resolve()`` so a + # symlink-relative path still produces a stable error message. + root = Path(body.ingest_dir).resolve() + if not root.exists(): + raise HTTPException( + status_code=422, + detail=f"ingest_dir does not exist: {root}", + ) + + # 3. Walker — must match the CLI EXACTLY. Same sort, same ``._*`` + # AppleDouble skip. Any drift here is a quiet split between the + # two surfaces and silently produces different batch outcomes. + files: list[Path] = [] + for batch_dir in sorted(root.glob("batch-*-claims")): + files.extend(sorted( + p for p in batch_dir.glob("*.x12") + if not p.name.startswith("._") + )) + + # 4. ``limit`` truncates after collection. The CLI uses an inline + # ``if i > limit: break``; we slice instead because the HTTP + # body model validates ``limit`` up-front (Pydantic-level int + # check) and slicing keeps the walker branchless. + if body.limit is not None: + files = files[: body.limit] + + if not files: + raise HTTPException( + status_code=422, + detail=f"no batch-*-claims/*.x12 files found under {root}", + ) + + # 5. Per-file submit. Wrap the helper call in try/except so an + # unexpected exception in submit_file surfaces as a per-file + # failure (outcome="unexpected") instead of crashing the whole + # run. The helper's own SubmitOutcome enum covers every typed + # failure path; an uncaught exception here is a true + # surprise (bug or service outage mid-loop). + results: list[dict] = [] + submitted = skipped = failed = 0 + for src in files: + try: + r = submit_file( + src, + sftp_block=sftp_block, + actor=body.actor, + validate=body.validate_files, + # No ``sftp_client_factory`` — submit_file's default + # paramiko factory opens the real MFT. Tests + # monkey-patch submit_file itself instead of wiring a + # factory here. + ) + except Exception as exc: # noqa: BLE001 + log.exception( + "submit-batch unexpected error on %s", src.name, + ) + r = SubmitResult( + file=src.name, + outcome=SubmitOutcome.SFTP_FAILED, + error=f"{exc.__class__.__name__}: {exc}", + ) + + if r.outcome == SubmitOutcome.SUBMITTED: + submitted += 1 + elif r.outcome == SubmitOutcome.SKIPPED: + skipped += 1 + else: + failed += 1 + + results.append({ + "file": r.file, + "outcome": r.outcome.value, + "batch_id": r.batch_id, + "error": r.error, + }) + + return { + "submitted": submitted, + "skipped": skipped, + "failed": failed, + "results": results, + } \ No newline at end of file diff --git a/backend/src/cyclone/auth/permissions.py b/backend/src/cyclone/auth/permissions.py index 917e3f3..0c2de12 100644 --- a/backend/src/cyclone/auth/permissions.py +++ b/backend/src/cyclone/auth/permissions.py @@ -86,6 +86,7 @@ PERMISSIONS: dict[tuple[str, str], set[Role]] = { ("POST", "/api/acks"): WRITE_ROLES, ("POST", "/api/batches"): WRITE_ROLES, # /export-837 regenerates X12 from DB rows ("POST", "/api/eligibility"): WRITE_ROLES, + ("POST", "/api/submit-batch"): WRITE_ROLES, # SP37: canonical outbound path (mirrors CLI) # CSV export — read-only. ("GET", "/api/export.csv"): ALL_ROLES, diff --git a/backend/src/cyclone/cli.py b/backend/src/cyclone/cli.py index e9f3b9e..7e6b358 100644 --- a/backend/src/cyclone/cli.py +++ b/backend/src/cyclone/cli.py @@ -768,6 +768,99 @@ def resubmit_rejected_claims( ) +@main.command("submit-batch") +@click.option("--ingest-dir", default="ingest", + show_default=True, + help="Root dir holding batch-*-claims/ subfolders of .x12 files.") +@click.option("--actor", default="cli-submit-batch", + show_default=True, + help="Audit-log actor tag for the clearhouse.submitted events.") +@click.option("--validate/--no-validate", default=True, + help="Parse each file via parse_837 before upload.") +@click.option("--limit", type=int, default=None, + help="Stop after N files (smoke-tests).") +def submit_batch( + ingest_dir: str, + actor: str, + validate: bool, + limit: int | None, +) -> None: + """Parse → DB-write → SFTP-upload each batch-*-claims/*.x12 (SP37). + + Canonical outbound path. Mirrors ``resubmit-rejected-claims`` but + routes through ``cyclone.submission.submit_file``, which writes to + the DB before uploading. Use this as the preferred outbound path; + ``resubmit-rejected-claims`` remains for one-off cases where you + don't want a DB row (dzinesco-generated fixes not ready for + canonical tracking). + + Exit codes: 0 = run completed (even with per-file failures), 1 = + unexpected exception, 2 = config-level failure (no clearhouse, + SFTP block in stub mode, ingest-dir missing). + """ + from cyclone import db as db_mod + from cyclone.submission.core import submit_file + from cyclone.submission.result import SubmitOutcome + from cyclone.store import store as cycl_store + + db_mod.init_db() + cycl_store.ensure_clearhouse_seeded() + clearhouse = cycl_store.get_clearhouse() + if clearhouse is None: + click.echo("No clearhouse seeded; cannot resolve SFTP block.", err=True) + sys.exit(2) + sftp_block = clearhouse.sftp_block + if sftp_block.stub: + click.echo("Clearhouse SFTP block is in stub mode; refusing to upload.", err=True) + sys.exit(2) + + root = Path(ingest_dir).resolve() + if not root.exists(): + click.echo(f"--ingest-dir does not exist: {root}", err=True) + sys.exit(2) + + files: list[Path] = [] + for batch_dir in sorted(root.glob("batch-*-claims")): + files.extend(sorted(p for p in batch_dir.glob("*.x12") + if not p.name.startswith("._"))) + if not files: + click.echo(f"No .x12 files under {root}", err=True) + sys.exit(2) + + click.echo(f"found {len(files)} files under {root}") + + submitted = 0 + skipped = 0 + failed = 0 + for i, src in enumerate(files, 1): + if limit is not None and i > limit: + break + try: + result = submit_file( + src, sftp_block=sftp_block, actor=actor, validate=validate, + ) + except Exception as exc: # noqa: BLE001 + click.echo(f"UNEXPECTED {src.name}: {exc.__class__.__name__}: {exc}", err=True) + failed += 1 + continue + if result.outcome == SubmitOutcome.SUBMITTED: + submitted += 1 + click.echo(f"submitted {src.name} batch={result.batch_id}") + elif result.outcome == SubmitOutcome.SKIPPED: + skipped += 1 + click.echo(f"skipped {src.name} (already uploaded)") + else: + failed += 1 + click.echo( + f"{result.outcome.value:<10} {src.name} {result.error or ''}", + err=True, + ) + + click.echo(f"\nsummary: submitted={submitted} skipped={skipped} failed={failed}") + # Per-file failures don't bump exit code; details in stdout. + # (Per cyclone-cli convention.) + + @main.group() def backup() -> None: """Encrypted DB backup management (SP17).""" diff --git a/backend/src/cyclone/db.py b/backend/src/cyclone/db.py index 508b89a..b65135d 100644 --- a/backend/src/cyclone/db.py +++ b/backend/src/cyclone/db.py @@ -221,6 +221,13 @@ class Batch(Base): totals_json: Mapped[Optional[dict]] = mapped_column(JSONText, nullable=True) validation_json: Mapped[Optional[dict]] = mapped_column(JSONText, nullable=True) raw_result_json: Mapped[Optional[dict]] = mapped_column(JSONText, nullable=True) + # SP37 Task 2: source 837's ST02 (transaction set control number). + # Populated from ``Envelope.transaction_set_control_number`` by + # ``store.write.add_record`` for 837P batches; NULL for 835 batches + # (the column is an 837P-specific join key for 999 AK2 resolution). + # Migration 0020 adds the column additively; no backfill required for + # pre-existing rows that lack the value. + transaction_set_control_number: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) claims: Mapped[list["Claim"]] = relationship( back_populates="batch", cascade="all, delete-orphan" diff --git a/backend/src/cyclone/migrations/0020_add_batch_txn_set_control_number.sql b/backend/src/cyclone/migrations/0020_add_batch_txn_set_control_number.sql new file mode 100644 index 0000000..92dc36a --- /dev/null +++ b/backend/src/cyclone/migrations/0020_add_batch_txn_set_control_number.sql @@ -0,0 +1,27 @@ +-- version: 20 +-- SP37: Batch.transaction_set_control_number = parsed 837's ST02. +-- +-- Today's 999 ack join (claim_acks.batch_envelope_index, Pass 1) matches +-- on ``Batch.envelope.control_number == 999's set_control_number``. That +-- never resolves in production because 999's set_control_number (AK201) +-- echoes the source 837's ST02 (transaction set control number), not the +-- ISA13 (interchange control number) that Envelope.control_number stores. +-- Result: every AK2 set-response against a dzinesco-generated 837 turns +-- into an orphan. +-- +-- SP37 fixes this by adding a column populated from the parsed 837's +-- ST02 on every ``add_record`` write, then updating Pass 1 to match on +-- it (Task 2). This migration is the additive part: nullable, no +-- default, backfills from ``raw_result_json.envelope.transaction_set_control_number`` +-- for any pre-existing batch rows that already carry the value. +-- +-- No new index (column is a primary join key, not a range query; the +-- existing batches table is small enough for a full scan during the +-- 999 join — see SP37 §"Migration 0013"). + +ALTER TABLE batches ADD COLUMN transaction_set_control_number TEXT; + +UPDATE batches +SET transaction_set_control_number = json_extract(raw_result_json, '$.envelope.transaction_set_control_number') +WHERE raw_result_json IS NOT NULL + AND json_extract(raw_result_json, '$.envelope.transaction_set_control_number') IS NOT NULL; diff --git a/backend/src/cyclone/parsers/models.py b/backend/src/cyclone/parsers/models.py index 2f3090e..f4b8917 100644 --- a/backend/src/cyclone/parsers/models.py +++ b/backend/src/cyclone/parsers/models.py @@ -120,6 +120,13 @@ class Envelope(_Base): implementation_guide: str | None = None # SP3 P1 T2: BHT06 transaction type code (was: transaction_set_purpose_code, which is BHT02). transaction_type_code: str | None = None + # SP37 Task 2: X12 ST02 (transaction set control number). Distinct + # from ``control_number`` above, which is the ISA13 interchange + # control number. 999 acks echo ST02 back as AK201, so this is the + # join key that lets ``add_record``'s batch row round-trip back to + # its source 837. Populated only by the 837P parser today; other + # parsers share this class but leave the field None. + transaction_set_control_number: str | None = None class BatchSummary(_Base): diff --git a/backend/src/cyclone/parsers/parse_837.py b/backend/src/cyclone/parsers/parse_837.py index 1953d49..5f44d3a 100644 --- a/backend/src/cyclone/parsers/parse_837.py +++ b/backend/src/cyclone/parsers/parse_837.py @@ -83,6 +83,12 @@ def _build_envelope(segments: list[list[str]], input_file: str = "") -> tuple[En except (IndexError, ValueError) as exc: log.warning("Could not parse BHT date: %s", exc) elif seg[0] == "ST" and envelope is not None: + # SP37 Task 2: capture ST02 (transaction set control number). + # 999 acks echo this back as AK201, so this is what makes the + # batch row joinable once the 999 ingests. Distinct from ISA13 + # (which is already on ``control_number``). + if len(seg) > 2: + envelope = envelope.model_copy(update={"transaction_set_control_number": seg[2].strip()}) if len(seg) > 3: envelope = envelope.model_copy(update={"implementation_guide": seg[3]}) return envelope, summary diff --git a/backend/src/cyclone/store/__init__.py b/backend/src/cyclone/store/__init__.py index 32a1a39..5c6c772 100644 --- a/backend/src/cyclone/store/__init__.py +++ b/backend/src/cyclone/store/__init__.py @@ -325,7 +325,13 @@ class CycloneStore: return _remove_claim_ack(link_id, event_bus=event_bus) def batch_envelope_index(self): - """Return a {envelope.control_number: batch.id} map for D10 Pass 1.""" + """Return a {key: batch.id} map populated from two columns (D10 Pass 1). + + Each 837p batch contributes both ``Batch.raw_result_json.envelope.control_number`` + (ISA13) and ``Batch.transaction_set_control_number`` (ST02); 835 batches + are excluded. Single ``.get(set_control_number)`` lookup resolves + either key — SP37 closes the 999 AK201 → source-batch gap. + """ return _batch_envelope_index() # -- SP17: encrypted DB backups ------------------------------------- diff --git a/backend/src/cyclone/store/claim_acks.py b/backend/src/cyclone/store/claim_acks.py index dbbdc4e..2ba9107 100644 --- a/backend/src/cyclone/store/claim_acks.py +++ b/backend/src/cyclone/store/claim_acks.py @@ -11,8 +11,10 @@ Five methods on top of the new ``ClaimAck`` ORM table: ack-orphans lane). * ``remove_claim_ack`` — unlink. Publishes ``claim_ack_dropped``. * ``batch_envelope_index`` — D10 in-memory map of - ``Batch.envelope.control_number → batch.id`` (cheap to rebuild; - re-built once per ingest). + ``{Batch.raw_result_json.envelope.control_number (ISA13) OR + Batch.transaction_set_control_number (ST02)} → batch.id`` + (SP37: populated from two columns; cheap to rebuild, re-built + once per ingest). Each mutating method opens its own ``db.SessionLocal()()`` session so callers don't have to manage session lifecycles. Publishes are @@ -67,26 +69,43 @@ def _safe_publish(event_bus: "EventBus | None", kind: str, payload: dict) -> Non def batch_envelope_index() -> dict[str, str]: - """Build a ``{envelope.control_number: batch.id}`` map. + """Build a ``{key: batch.id}`` map populated from two columns. - D10 primary join key (spec §D10). Built once per ingest from - every Batch row whose ``raw_result_json`` carries an envelope — - cost is O(N batches), currently ~16, so trivial. Kept as a - plain dict so callers can ``index.get(scn)`` to resolve. + D10 primary join key (spec §D10). Each 837p batch row contributes + up to two entries: + + * ``Batch.raw_result_json.envelope.control_number`` (ISA13) + * ``Batch.transaction_set_control_number`` (ST02, new in SP37) + + Either key resolves to the same ``batch.id``; callers do a single + ``idx.get(set_control_number)`` lookup. Pass 2 (PCN) is unchanged. + + Built once per ingest from every Batch row whose kind is "837p"; + cost is O(N batches), currently small, so trivial. 835 batches + are excluded — the column is an 837P-specific join key (see + ``Batch.transaction_set_control_number`` docstring). """ - out: dict[str, str] = {} + idx: dict[str, str] = {} with db.SessionLocal()() as s: rows = ( - s.query(Batch.id, Batch.raw_result_json) + s.query( + Batch.id, + Batch.raw_result_json, + Batch.transaction_set_control_number, + ) .filter(Batch.kind == "837p") .all() ) - for bid, raw in rows: + for bid, raw, stcn in rows: env = (raw or {}).get("envelope") or {} - ctrl = env.get("control_number") - if isinstance(ctrl, str) and ctrl: - out[ctrl] = bid - return out + isa_cn = env.get("control_number") + if isinstance(isa_cn, str) and isa_cn: + # First write wins (setdefault) — if ISA13 and ST02 + # ever collide they map to the same batch anyway. + idx.setdefault(isa_cn, bid) + if isinstance(stcn, str) and stcn: + idx.setdefault(stcn, bid) + return idx # --------------------------------------------------------------------------- diff --git a/backend/src/cyclone/store/write.py b/backend/src/cyclone/store/write.py index e4b05a1..8da42d6 100644 --- a/backend/src/cyclone/store/write.py +++ b/backend/src/cyclone/store/write.py @@ -77,6 +77,17 @@ def add_record(record: BatchRecord, *, event_bus=None) -> None: totals_json=None, validation_json=None, raw_result_json=json.loads(record.result.model_dump_json()), + # SP37 Task 2: mirror the parsed 837's ST02 onto the batch + # row so 999 AK201 set_control_numbers can resolve back via + # Pass 1. The ``getattr`` chain handles the 835 path: the + # shared ``Envelope`` class is used by both 837P and 835 + # parsers, but only ``parse_837`` populates this field — for + # 835 records it stays ``None`` and the column is NULL. + transaction_set_control_number=getattr( + getattr(record.result, "envelope", None), + "transaction_set_control_number", + None, + ), ) s.add(batch_row) diff --git a/backend/src/cyclone/submission/__init__.py b/backend/src/cyclone/submission/__init__.py new file mode 100644 index 0000000..f0df3ed --- /dev/null +++ b/backend/src/cyclone/submission/__init__.py @@ -0,0 +1,17 @@ +"""SP37 Task 4: canonical 837P submission flow. + +The single public helper is ``submit_file`` — it owns parse → DB +write → SFTP upload per file. CLI (``cyclone submit-batch``) and HTTP +(``POST /api/submit-batch``) are thin wrappers; both call this helper +so the ordering, idempotency, and audit shape are identical. + +DB-first, upload-second (per spec decision §2): if the DB write fails, +no SFTP call is made. If the SFTP call fails after a successful DB +write, the row exists but the file never landed — re-running is safe +(idempotent DB write via add_record's s.get check; idempotent SFTP +upload via stat-then-put). +""" +from .core import submit_file +from .result import SubmitOutcome, SubmitResult + +__all__ = ["submit_file", "SubmitOutcome", "SubmitResult"] diff --git a/backend/src/cyclone/submission/core.py b/backend/src/cyclone/submission/core.py new file mode 100644 index 0000000..3b299a6 --- /dev/null +++ b/backend/src/cyclone/submission/core.py @@ -0,0 +1,225 @@ +"""SP37 Task 4: parse → DB write → SFTP upload, per file. + +Mirrors ``resubmit_rejected_claims`` but writes to the DB before +uploading. Same idempotency check (``stat().st_size == local_size``) +skips already-uploaded files without re-emitting audit events. + +DB-first, upload-second (per spec decision §2): if the DB write fails, +no SFTP call is made. If the SFTP call fails after a successful DB +write, the row exists but the file never landed — re-running is safe +(idempotent DB write via add_record's s.get check; idempotent SFTP +upload via stat-then-put). + +The default SFTP factory uses paramiko directly (not the +``SftpClient`` wrapper) because the wrapper exposes ``write_file``, +``list_inbound``, and ``read_file`` but no ``stat()`` — which makes +the SKIPPED outcome unreachable in production. +""" +from __future__ import annotations + +import logging +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +from cyclone import db as db_mod +from cyclone.audit_log import AuditEvent, append_event +from cyclone.parsers.parse_837 import parse as parse_837_text +from cyclone.parsers.payer import PayerConfig +from cyclone.providers import SftpBlock +from cyclone.store import store as cycl_store +from cyclone.store.records import BatchRecord837 + +from .result import SubmitOutcome, SubmitResult + +log = logging.getLogger(__name__) + +# The companion-guide payer id we enforce for CO Medicaid submits. +# Same value the legacy ``resubmit-rejected-claims`` CLI gates on. +EXPECTED_PAYER_ID = "CO_TXIX" + + +def _default_sftp_factory(sftp_block: SftpBlock) -> Any: + """Open a paramiko SFTP session to the real MFT. Mirrors + resubmit_rejected_claims._open_session (cli.py:620-640). + Caller is responsible for closing the session. + + The returned ``sftp`` is a ``paramiko.SFTPClient`` — it exposes + ``stat(remote_path)`` (used for the idempotency check) and + ``put(local_path, remote_path)`` (used for the upload). The + underlying SSH handle is stashed on ``sftp._cyclone_ssh`` so the + caller can close it cleanly via ``getattr(sftp, "_cyclone_ssh", + None).close()`` after the upload finishes. + + Raises: + RuntimeError: if the SFTP block is in stub mode (the CLI/HTTP + layer should already guard against this, but we re-check + here because paramiko will try to connect even when + ``stub=True``). + """ + if sftp_block.stub: + # Same posture as resubmit_rejected_claims in cli.py:592-595: + # the operator refuses to upload in stub mode, and the helper + # surfaces that as SFTP_FAILED with an explicit error. + raise RuntimeError("SFTP block is in stub mode") + + from paramiko import AutoAddPolicy, SSHClient + + from cyclone.secrets import get_secret + + pw = get_secret(sftp_block.auth.get("password_keychain_account", "")) + ssh = SSHClient() + ssh.set_missing_host_key_policy(AutoAddPolicy()) + ssh.connect( + sftp_block.host, port=sftp_block.port, + username=sftp_block.username, password=pw, + timeout=15, banner_timeout=15, auth_timeout=15, + ) + sftp = ssh.open_sftp() + # Attach ssh to sftp so the caller can close it cleanly. + sftp._cyclone_ssh = ssh + return sftp + + +def submit_file( + path: Path, + *, + sftp_block: SftpBlock, + actor: str, + validate: bool = True, + sftp_client_factory: Callable[[SftpBlock], Any] | None = None, +) -> SubmitResult: + """Submit one 837P file: parse → DB write → SFTP upload. + + Args: + path: local 837P file. + sftp_block: the clearhouse SftpBlock (for paths + auth). + actor: audit-log actor tag (e.g. "api-submit-batch"). + validate: parse the file before upload (default True). Catches + bad byte-fixes early. + sftp_client_factory: optional callable that returns an + SFTP-client-compatible object (must expose ``stat()`` and + ``write_file()``). Defaults to :func:`_default_sftp_factory` + which opens a real paramiko session. Tests inject a fake. + + Returns: + SubmitResult with file, outcome, batch_id (when written), + error (when failed). + """ + file_label = path.name + try: + content = path.read_bytes() + except OSError as exc: + return SubmitResult(file_label, SubmitOutcome.PARSE_FAILED, error=str(exc)) + + # 1. Validate via parse (optional but recommended). + parsed: Any = None + if validate: + try: + parsed = parse_837_text(content.decode(), PayerConfig.co_medicaid()) + except Exception as exc: # noqa: BLE001 + log.warning("submit_file %s: parse failed: %s", file_label, exc) + return SubmitResult(file_label, SubmitOutcome.PARSE_FAILED, error=str(exc)) + mismatch = next( + (c for c in parsed.claims if c.payer.id != EXPECTED_PAYER_ID), + None, + ) + if mismatch is not None: + return SubmitResult( + file_label, + SubmitOutcome.PAYER_MISMATCH, + error=f"payer.id={mismatch.payer.id!r} (expected {EXPECTED_PAYER_ID!r})", + ) + + # 2. DB write — DB-first, upload-second invariant. + # ``parsed`` is required to construct a BatchRecord837 (it embeds the + # full ParseResult). validate=False therefore isn't a real path — + # the CLI/HTTP always pass validate=True — so reject it loudly + # rather than silently building an empty row. + if parsed is None: + return SubmitResult( + file_label, + SubmitOutcome.PARSE_FAILED, + error="validate=False is not supported; must parse to construct a BatchRecord", + ) + + # Use the same uuid4().hex id the existing /api/parse-837 path uses. + batch_id = uuid.uuid4().hex + record = BatchRecord837( + id=batch_id, + input_filename=file_label, + parsed_at=datetime.now(timezone.utc), + result=parsed, + ) + + try: + # ``cycl_store.add`` is the public facade method (per the + # CycloneStore class in store/__init__.py:158). It delegates + # to ``store.write.add_record``, which is the underlying + # SQLAlchemy write path. + cycl_store.add(record) + except Exception as exc: # noqa: BLE001 + log.warning("submit_file %s: DB write failed: %s", file_label, exc) + return SubmitResult(file_label, SubmitOutcome.DB_FAILED, error=str(exc)) + + # 3. SFTP upload. ``_default_sftp_factory`` opens a paramiko + # session directly because the SftpClient wrapper has no ``stat()`` + # method — that omission made the SKIPPED outcome unreachable in + # production. Mirror cli.py:620-650. + remote_path = f"{sftp_block.paths['outbound']}/{file_label}" + factory = sftp_client_factory or _default_sftp_factory + sftp = factory(sftp_block) + local_size = len(content) + try: + try: + stat = sftp.stat(remote_path) + if stat.st_size == local_size: + # Already on remote at the right size — re-run is safe; + # do NOT emit a duplicate audit event. + return SubmitResult(file_label, SubmitOutcome.SKIPPED, batch_id=batch_id) + except (IOError, OSError): + pass # not on remote yet + sftp.write_file(remote_path, content) + except Exception as exc: # noqa: BLE001 + log.warning("submit_file %s: SFTP failed: %s", file_label, exc) + return SubmitResult( + file_label, SubmitOutcome.SFTP_FAILED, + batch_id=batch_id, error=str(exc), + ) + finally: + # Close the paramiko SSH handle the helper stashed on the + # SFTP client (paramiko does not auto-close on GC and a leak + # here costs a slot in MOVEit's per-IP session table). + ssh_handle = getattr(sftp, "_cyclone_ssh", None) + if ssh_handle is not None: + try: + ssh_handle.close() + except Exception: # noqa: BLE001 + pass + + # 4. Audit event — same shape the legacy resubmit_rejected_claims CLI + # uses (event_type="clearhouse.submitted", entity_type="claim_file", + # entity_id=filename, payload has remote_path + source + size). + # Best-effort: an audit failure must not roll back the upload. + try: + with db_mod.SessionLocal()() as session: + append_event(session, AuditEvent( + event_type="clearhouse.submitted", + entity_type="claim_file", + entity_id=file_label, + payload={ + "remote_path": remote_path, + "source": "submit-batch", + "size": local_size, + "batch_id": batch_id, + }, + actor=actor, + )) + session.commit() + except Exception as exc: # noqa: BLE001 + log.warning( + "submit_file %s: audit event failed: %s", file_label, exc, + ) + + return SubmitResult(file_label, SubmitOutcome.SUBMITTED, batch_id=batch_id) diff --git a/backend/src/cyclone/submission/result.py b/backend/src/cyclone/submission/result.py new file mode 100644 index 0000000..329c1cd --- /dev/null +++ b/backend/src/cyclone/submission/result.py @@ -0,0 +1,40 @@ +"""SP37 Task 4: typed result for submit_file. + +The CLI and HTTP layer both consume SubmitResult; keeping it in one +place means both surfaces agree on the response shape. +""" +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + + +class SubmitOutcome(str, Enum): + """Outcome of a single submit_file invocation. + + Audit-event invariant: ``SUBMITTED`` emits a ``clearhouse.submitted`` + audit event; ``SKIPPED`` does NOT (re-running on an already-uploaded + file must not flood the audit log with duplicate events). All + failure outcomes (``PARSE_FAILED``, ``PAYER_MISMATCH``, ``DB_FAILED``, + ``SFTP_FAILED``) emit no event either — failures should be observable + via the helper's return value, not the audit log. + """ + SUBMITTED = "submitted" + SKIPPED = "skipped" + PARSE_FAILED = "parse_failed" + PAYER_MISMATCH = "payer_mismatch" + DB_FAILED = "db_failed" + SFTP_FAILED = "sftp_failed" + + +@dataclass(frozen=True) +class SubmitResult: + """Return value of ``submit_file``: filename, outcome, and optional + ``batch_id`` (populated when the DB write succeeded) and ``error`` + (populated on any failure outcome; ``None`` on ``SUBMITTED`` / + ``SKIPPED``). + """ + file: str + outcome: SubmitOutcome + batch_id: str | None = None + error: str | None = None diff --git a/backend/tests/fixtures/submit-batch/single-claim.x12 b/backend/tests/fixtures/submit-batch/single-claim.x12 new file mode 100644 index 0000000..61ee4e6 --- /dev/null +++ b/backend/tests/fixtures/submit-batch/single-claim.x12 @@ -0,0 +1,30 @@ +ISA*00* *00* *ZZ*11525703 *ZZ*COMEDASSISTPROG*260611*0814*^*00501*991102977*1*P*:~ +GS*HC*11525703*COMEDASSISTPROG*20260611*081417*991102977*X*005010X222A1~ +ST*837*991102977*005010X222A1~ +BHT*0019*00*ref-001*20260611*081417*CH~ +NM1*41*2*Test Submitter*****46*11525703~ +PER*IC*Test Contact*EM*test@example.com~ +NM1*40*2*COLORADO MEDICAL ASSISTANCE PROGRAM*****46*COMEDASSISTPROG~ +HL*1**20*1~ +PRV*BI*PXC*251E00000X~ +NM1*85*2*Test Provider Inc*****XX*1993999998~ +N3*123 Test St~ +N4*Denver*CO*80202~ +REF*EI*123456789~ +HL*2*1*22*0~ +SBR*P*18*******MC~ +NM1*IL*1*Doe*John****MI*ABC123~ +N3*456 Member St~ +N4*Denver*CO*80203~ +DMG*D8*19800101*M~ +NM1*PR*2*CO_TXIX*****PI*CO_TXIX~ +CLM*CLM001*100.00***12:B:1*Y*A*Y*Y~ +REF*G1*PA123~ +HI*ABK:Z00~ +LX*1~ +SV1*HC:99213*100.00*UN*1***1~ +DTP*472*D8*20260611~ +REF*6R*REF001~ +SE*26*991102977~ +GE*1*991102977~ +IEA*1*991102977~ \ No newline at end of file diff --git a/backend/tests/test_acks.py b/backend/tests/test_acks.py index cb11a88..8870fcd 100644 --- a/backend/tests/test_acks.py +++ b/backend/tests/test_acks.py @@ -51,7 +51,7 @@ def test_migration_0002_creates_acks_table(): def test_migration_latest_idempotent_on_fresh_db(): """Re-running the migration on the same DB must be a no-op (PRAGMA - user_version already at the latest version — currently 19 after + user_version already at the latest version — currently 20 after 0004-0006 line_reconciliation, 0005 ta1_acks, SP9's 0007 providers/payers/clearhouse, SP10's 0008 payer_rejected, SP11's 0009 audit_log, SP14's 0010 payer_rejected_acknowledged, @@ -61,15 +61,16 @@ def test_migration_latest_idempotent_on_fresh_db(): claims.matched_remittance_id index, SP27-Task 17's 0017 claim.patient_control_number backfill UPDATE, SP28's 0018 claim_acks join table, SP32's 0019 - rendering_provider_npi + service_provider_npi).""" + rendering_provider_npi + service_provider_npi, and SP37's 0020 + transaction_set_control_number).""" with db.engine().begin() as c: v1 = c.exec_driver_sql("PRAGMA user_version").scalar() or 0 - assert v1 == 19 + assert v1 == 20 # A second run should not raise and should not bump the version. db_migrate.run(db.engine()) with db.engine().begin() as c: v2 = c.exec_driver_sql("PRAGMA user_version").scalar() or 0 - assert v2 == 19 + assert v2 == 20 def test_add_ack_persists_row(): diff --git a/backend/tests/test_api_submit_batch.py b/backend/tests/test_api_submit_batch.py new file mode 100644 index 0000000..f8e1c9c --- /dev/null +++ b/backend/tests/test_api_submit_batch.py @@ -0,0 +1,355 @@ +"""SP37 Task 6: POST /api/submit-batch endpoint. + +Thin HTTP wrapper around ``cyclone.submission.submit_file``. Mirrors the +``cyclone submit-batch`` CLI walker exactly: each ``batch-*-claims/*.x12`` +under ``ingest_dir`` is submitted in order, ``._*`` AppleDouble files are +skipped, ``limit`` truncates after collection. + +Tests monkey-patch ``cyclone.api_routers.submission.submit_file`` so we +never touch the real paramiko factory or a live MFT. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from cyclone.submission.result import SubmitOutcome, SubmitResult + +_FIXTURE = Path(__file__).parent / "fixtures" / "submit-batch" / "single-claim.x12" + + +@pytest.fixture +def client(tmp_path) -> TestClient: + """Standard TestClient; conftest.py autouse handles DB + auth gate. + + Does NOT touch env vars (per the plan-vs-reality correction in + Task 6's instructions). Seeds the clearhouse then flips + ``sftp_block.stub`` to ``False`` so the file-walking tests reach + the walker (otherwise the default-seeded stub=True trips the 409 + guard before any per-file work runs). The ``stub_mode`` test + re-seeds stub=True explicitly so its 409 still fires. + """ + from cyclone import db as db_mod + from cyclone.store import store as cycl_store + + db_mod._reset_for_tests() + db_mod.init_db() + cycl_store.ensure_clearhouse_seeded() + ch = cycl_store.get_clearhouse() + # Flip stub=False so the walker actually runs (SFTP calls are + # intercepted via the submit_file monkey-patch in each test). + cycl_store.update_clearhouse( + ch.model_copy(update={"sftp_block": ch.sftp_block.model_copy(update={"stub": False})}), + ) + + from cyclone.api import app + + return TestClient(app) + + +def _stage_batch(tmp_path: Path, n: int) -> Path: + """Copy N copies of the single-claim fixture into ``batch-test-claims/``.""" + batch_dir = tmp_path / "batch-test-claims" + batch_dir.mkdir() + payload = _FIXTURE.read_bytes() + for i in range(n): + (batch_dir / f"claim-{i}.x12").write_bytes(payload) + return batch_dir + + +# --------------------------------------------------------------------------- # +# Happy path + walker semantics +# --------------------------------------------------------------------------- # + + +def test_submit_batch_happy_path(client: TestClient, tmp_path, monkeypatch): + """3 valid 837 files → 200 + body shape with submitted=3 (or skipped=3 on re-run). + + Monkey-patches ``cyclone.api_routers.submission.submit_file`` to + return a synthetic SUBMITTED SubmitResult per file. Skipped stays + possible because SubmitResult's batch_id is what matters and the + fake returns it consistently — but a fresh DB should always land + in the SUBMITTED branch. + """ + _stage_batch(tmp_path, 3) + + def _fake_submit(path, **_kw): + return SubmitResult( + file=path.name, outcome=SubmitOutcome.SUBMITTED, batch_id="b1", + ) + + monkeypatch.setattr( + "cyclone.api_routers.submission.submit_file", _fake_submit, + ) + + resp = client.post( + "/api/submit-batch", + json={"ingest_dir": str(tmp_path), "validate": False}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + # Tolerate either counter — counter totals are deterministic with a + # fresh DB, but the assertion below focuses on the structural shape + # the operator cares about: every file is accounted for. + assert body["submitted"] + body["skipped"] + body["failed"] == 3 + assert isinstance(body["results"], list) + assert len(body["results"]) == 3 + for row in body["results"]: + assert "file" in row + assert "outcome" in row + assert "batch_id" in row + assert "error" in row + + +def test_submit_batch_skips_apple_double_files(client: TestClient, tmp_path, monkeypatch): + """``._foo.x12`` AppleDouble files must be skipped (mirrors the CLI walker).""" + batch_dir = tmp_path / "batch-test-claims" + batch_dir.mkdir() + (batch_dir / "real.x12").write_bytes(_FIXTURE.read_bytes()) + (batch_dir / "._real.x12").write_bytes(b"\x00\x01\x02") # macOS AppleDouble noise + + def _fake_submit(path, **_kw): + return SubmitResult( + file=path.name, outcome=SubmitOutcome.SUBMITTED, batch_id="b1", + ) + + monkeypatch.setattr( + "cyclone.api_routers.submission.submit_file", _fake_submit, + ) + + resp = client.post( + "/api/submit-batch", + json={"ingest_dir": str(tmp_path), "validate": False}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + # Only real.x12 was submitted (one call); AppleDouble was skipped. + assert len(body["results"]) == 1 + assert body["results"][0]["file"] == "real.x12" + + +def test_submit_batch_limit_truncates(client: TestClient, tmp_path, monkeypatch): + """``limit=1`` against 3 files → body has exactly 1 result.""" + _stage_batch(tmp_path, 3) + + submitted_files: list[str] = [] + + def _fake_submit(path, **_kw): + submitted_files.append(path.name) + return SubmitResult( + file=path.name, outcome=SubmitOutcome.SUBMITTED, batch_id="b1", + ) + + monkeypatch.setattr( + "cyclone.api_routers.submission.submit_file", _fake_submit, + ) + + resp = client.post( + "/api/submit-batch", + json={"ingest_dir": str(tmp_path), "validate": False, "limit": 1}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert len(body["results"]) == 1 + assert body["submitted"] + body["skipped"] + body["failed"] == 1 + assert len(submitted_files) == 1 + + +def test_submit_batch_empty_ingest_dir_returns_422(client: TestClient, tmp_path): + """No batch-*-claims dir under ingest_dir → 422 (matches the CLI's exit 2 path).""" + resp = client.post( + "/api/submit-batch", + json={"ingest_dir": str(tmp_path), "validate": False}, + ) + assert resp.status_code == 422, resp.text + + +def test_submit_batch_per_file_failure_keeps_status_200( + client: TestClient, tmp_path, monkeypatch, +): + """A SUBMITTED + SFTP_FAILED mix → 200 with failed=1, submitted=1. + + Per-file failures live in the JSON body, NOT in the status code. + The endpoint always returns 200 on a completed run. + """ + _stage_batch(tmp_path, 2) + + outcomes = iter([ + SubmitResult(file="claim-0.x12", outcome=SubmitOutcome.SUBMITTED, batch_id="b1"), + SubmitResult( + file="claim-1.x12", outcome=SubmitOutcome.SFTP_FAILED, + batch_id="b2", error="sftp down", + ), + ]) + + def _fake_submit(path, **_kw): + return next(outcomes) + + monkeypatch.setattr( + "cyclone.api_routers.submission.submit_file", _fake_submit, + ) + + resp = client.post( + "/api/submit-batch", + json={"ingest_dir": str(tmp_path), "validate": False}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["submitted"] == 1 + assert body["failed"] == 1 + assert body["skipped"] == 0 + assert len(body["results"]) == 2 + # The failure row carries the error string so the operator can see + # what went wrong without a separate API call. + assert body["results"][1]["outcome"] == "sftp_failed" + assert body["results"][1]["error"] == "sftp down" + + +def test_submit_batch_unexpected_exception_recorded_in_results( + client: TestClient, tmp_path, monkeypatch, +): + """If submit_file raises, the file lands in results[] as a failed row. + + The router catches the exception, builds a SubmitResult with + ``outcome=SubmitOutcome.SFTP_FAILED`` (per Task 6's instruction + that "any enum value works as a marker; choose whichever is least + misleading"), and counts it as ``failed``. The full error string + (including exception class name) is in ``error`` so the operator + can tell it was a real exception, not a typed failure path. + """ + _stage_batch(tmp_path, 1) + + def _explode(path, **_kw): + raise RuntimeError("kaboom") + + monkeypatch.setattr( + "cyclone.api_routers.submission.submit_file", _explode, + ) + + resp = client.post( + "/api/submit-batch", + json={"ingest_dir": str(tmp_path), "validate": False}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["failed"] == 1 + # The marker is the SFTP_FAILED enum value (per Task 6 design + # choice). The full exception class+message is in ``error`` so the + # operator can distinguish a real exception from a typed failure. + assert body["results"][0]["outcome"] == "sftp_failed" + assert "kaboom" in body["results"][0]["error"] + assert "RuntimeError" in body["results"][0]["error"] + + +# --------------------------------------------------------------------------- # +# Request validation +# --------------------------------------------------------------------------- # + + +def test_submit_batch_missing_ingest_dir_returns_422(client: TestClient): + """Body without ``ingest_dir`` → 422 from Pydantic (NOT your manual check).""" + resp = client.post("/api/submit-batch", json={"validate": False}) + assert resp.status_code == 422, resp.text + + +def test_submit_batch_ingest_dir_missing_on_disk_returns_422( + client: TestClient, tmp_path, +): + """Body has a path that doesn't exist on disk → 422.""" + resp = client.post( + "/api/submit-batch", + json={ + "ingest_dir": str(tmp_path / "does-not-exist"), + "validate": False, + }, + ) + assert resp.status_code == 422, resp.text + assert "does not exist" in resp.text or "does-not-exist" in resp.text + + +# --------------------------------------------------------------------------- # +# Clearhouse posture (404 / 409) +# --------------------------------------------------------------------------- # + + +def test_submit_batch_no_clearhouse_returns_404(client: TestClient, tmp_path, monkeypatch): + """Delete the clearhouse row → endpoint must refuse with 404, not 500. + + 404 per the Task 6 status code contract: a missing clearhouse is a + config-level "missing" (4xx), not an unexpected exception. + """ + from cyclone import db as db_mod + from cyclone.db import ClearhouseORM + + # Wipe the seeded clearhouse row. + with db_mod.SessionLocal()() as s: + row = s.get(ClearhouseORM, 1) + if row is not None: + s.delete(row) + s.commit() + + _stage_batch(tmp_path, 1) + + resp = client.post( + "/api/submit-batch", + json={"ingest_dir": str(tmp_path), "validate": False}, + ) + assert resp.status_code == 404, resp.text + assert "clearhouse" in resp.text.lower() + + +def test_submit_batch_stub_mode_returns_409(client: TestClient, tmp_path, monkeypatch): + """When the seeded clearhouse's ``sftp_block.stub`` is True → 409. + + The default client fixture flips stub=False so file-walking tests + reach the walker. This test flips it back to True so the + stub-mode 409 guard fires before any per-file work runs. + """ + from cyclone.store import store as cycl_store + + ch = cycl_store.get_clearhouse() + assert ch is not None + # Re-flip stub=True (the default client fixture set it to False). + cycl_store.update_clearhouse( + ch.model_copy(update={ + "sftp_block": ch.sftp_block.model_copy(update={"stub": True}), + }), + ) + + _stage_batch(tmp_path, 1) + + resp = client.post( + "/api/submit-batch", + json={"ingest_dir": str(tmp_path), "validate": False}, + ) + assert resp.status_code == 409, resp.text + assert "stub" in resp.text.lower() + + +# --------------------------------------------------------------------------- # +# Auth gate +# --------------------------------------------------------------------------- # + + +def test_submit_batch_auth_gate(client: TestClient, tmp_path, monkeypatch): + """No cookie + AUTH_DISABLED=False → 401 from matrix_gate. + + The conftest autouse fixture sets ``AUTH_DISABLED = True`` at + fixture setup and resets to False at teardown. Inside the test we + flip the module attribute so the gate fires — but to avoid leaking + the flip into a sibling test we use monkeypatch.setattr (which + restores at test teardown, before conftest's finally block runs). + """ + import cyclone.auth.deps as auth_deps + + monkeypatch.setattr(auth_deps, "AUTH_DISABLED", False) + + _stage_batch(tmp_path, 1) + + resp = client.post( + "/api/submit-batch", + json={"ingest_dir": str(tmp_path), "validate": False}, + ) + assert resp.status_code == 401, resp.text \ No newline at end of file diff --git a/backend/tests/test_batch_envelope_index_sp37.py b/backend/tests/test_batch_envelope_index_sp37.py new file mode 100644 index 0000000..d3ee54c --- /dev/null +++ b/backend/tests/test_batch_envelope_index_sp37.py @@ -0,0 +1,126 @@ +"""SP37 Task 3: batch_envelope_index populates from BOTH columns. + +The dict returned by ``batch_envelope_index`` should resolve lookups +by EITHER ``Batch.raw_result_json.envelope.control_number`` (ISA13, +preserved) OR ``Batch.transaction_set_control_number`` (ST02, new in +SP37 Task 2). One dict, both keys — a single ``.get(set_control_number)`` +call hits whichever matches, so the D10 Pass 1 join in +:func:`cyclone.claim_acks.lookup_claims_for_ack_set_response` resolves +999 AK201 (which echoes the source 837's ST02) back to the right batch +even when ST02 != ISA13. +""" +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from cyclone import db as db_mod +from cyclone.db import Batch +from cyclone.store.claim_acks import batch_envelope_index + + +@pytest.fixture(autouse=True) +def _db(tmp_path, monkeypatch): + """Per-test DB; conftest's autouse already wires one too, but we + pin the URL again so this module is self-contained if anyone ever + lifts it out of the conftest tree.""" + monkeypatch.setenv("CYCLONE_DB_URL", f"sqlite:///{tmp_path}/test.db") + db_mod._reset_for_tests() + db_mod.init_db() + yield + + +def _make_batch( + s, + *, + id: str, + icn: str, + stcn: str | None = None, + raw_json: dict | None = None, +) -> Batch: + """Insert one Batch row whose envelope + ST02 mirror what + ``store.write.add_record`` writes for an 837P batch.""" + row = Batch( + id=id, + kind="837p", + input_filename=id + ".x12", + parsed_at=datetime.now(timezone.utc), + raw_result_json=raw_json or {"envelope": {"control_number": icn}}, + transaction_set_control_number=stcn, + ) + s.add(row) + s.commit() + s.refresh(row) + return row + + +def test_index_resolves_by_envelope_control_number(): + """Pre-SP37 path: ISA13 (envelope.control_number) still resolves.""" + with db_mod.SessionLocal()() as s: + _make_batch(s, id="b1", icn="ISA000001") + idx = batch_envelope_index() + assert idx.get("ISA000001") == "b1" + + +def test_index_resolves_by_transaction_set_control_number(): + """SP37 path: ST02 (transaction_set_control_number) resolves too. + + The whole point of Task 3 — Gainwell batches have ST02 != ISA13, + so 999 AK201 echoes ST02 and must hit this branch. + """ + with db_mod.SessionLocal()() as s: + _make_batch( + s, id="b2", icn="ISA000002", stcn="ST000002", + raw_json={"envelope": {"control_number": "ISA000002"}}, + ) + idx = batch_envelope_index() + assert idx.get("ST000002") == "b2" + # Backward compat: ISA still resolves. + assert idx.get("ISA000002") == "b2" + + +def test_index_handles_row_with_no_transaction_set_control_number(): + """Pre-migration rows (stcn NULL) should still resolve by ISA. + + Spec says: 'Pre-migration rows (stcn NULL) should still resolve by ISA.' + Production row ``b1`` (the only one with claims) has ST02 NULL because + the migration's backfill only ran over rows whose raw_result_json + envelope had the ST02 key — and that row was written before the + parser populated it. + """ + with db_mod.SessionLocal()() as s: + _make_batch(s, id="b3", icn="ISA000003") # no stcn + idx = batch_envelope_index() + assert idx.get("ISA000003") == "b3" + assert idx.get("ST000003") is None + + +def test_index_ignores_non_837_batches(): + """835 batches don't contribute to the 999 join index. + + The D10 two-pass join is specifically for 837 → 999 linkage; 835 + remittances live on the response side of the loop and have no ST02 + join key. Filtering by ``kind == "837p"`` keeps the index scoped + correctly and avoids an 835 row shadowing an 837 ST02 by accident. + """ + with db_mod.SessionLocal()() as s: + # Add an 837 row first so we can prove the 835 doesn't leak in. + _make_batch(s, id="b-837", icn="ISA837", stcn="ST837") + # And an 835 row whose ISA13 / ST02 would otherwise pollute. + s.add(Batch( + id="b-835", + kind="835", + input_filename="b-835.x12", + parsed_at=datetime.now(timezone.utc), + raw_result_json={"envelope": {"control_number": "ISA999"}}, + transaction_set_control_number="ST999", + )) + s.commit() + idx = batch_envelope_index() + # 837 row resolves both ways. + assert idx.get("ISA837") == "b-837" + assert idx.get("ST837") == "b-837" + # 835 row contributes nothing. + assert "ISA999" not in idx + assert "ST999" not in idx diff --git a/backend/tests/test_batch_txn_set_cn.py b/backend/tests/test_batch_txn_set_cn.py new file mode 100644 index 0000000..ef68da5 --- /dev/null +++ b/backend/tests/test_batch_txn_set_cn.py @@ -0,0 +1,103 @@ +"""SP37 Task 2: add_record populates Batch.transaction_set_control_number. + +The column should mirror the source 837 envelope's +``transaction_set_control_number`` (ST02) so the join-key update in +Task 3 can resolve 999 ``set_control_number`` (AK201) values back to +the right batch. The 835 path leaves the column NULL because the +field is an 837P-specific join key (835 remittances don't need to +resolve back to a source 837 — they're the response side of the loop). +""" +from __future__ import annotations + +from datetime import date, datetime, timezone +from pathlib import Path + +from cyclone import db as db_mod +from cyclone.db import Batch +from cyclone.parsers.parse_837 import parse as parse_837_text +from cyclone.parsers.payer import PayerConfig +from cyclone.store import store as cycl_store +from cyclone.store.records import BatchRecord837, BatchRecord835 + +FIXTURE_837 = Path(__file__).parent / "fixtures" / "minimal_837p.txt" +FIXTURE_835 = Path(__file__).parent / "fixtures" / "minimal_835.txt" + + +def test_add_record_populates_transaction_set_control_number_for_837(): + """parse_837 → add_record writes the parsed ST02 onto the Batch row. + + The fixture's ST02 (``991102977``) must round-trip into the new + ``batches.transaction_set_control_number`` column. This is the + join key Task 3's 999-ingest Pass 1 update relies on. + """ + text = FIXTURE_837.read_text() + parsed = parse_837_text(text, PayerConfig.co_medicaid()) + + record = BatchRecord837( + id="b-txn-cn-1", + input_filename="minimal_837p.txt", + parsed_at=datetime.now(timezone.utc), + result=parsed, + ) + cycl_store.add(record) + + with db_mod.SessionLocal()() as s: + row = s.get(Batch, "b-txn-cn-1") + assert row is not None + assert row.transaction_set_control_number == parsed.envelope.transaction_set_control_number + # Sanity: the fixture's ST02 must be the parsed-and-stored value. + assert row.transaction_set_control_number == "991102977" + + +def test_envelope_model_exposes_transaction_set_control_number(): + """The Envelope Pydantic model carries the new field with a None default. + + Guards against a regression where someone removes the field from + the model — the rest of the chain (parser + ORM + write path) + silently degrades to None if the model loses the attribute. + """ + from cyclone.parsers.models import Envelope + + env = Envelope( + sender_id="S", + receiver_id="R", + control_number="000000001", + transaction_date=date(2026, 1, 1), + ) + assert env.transaction_set_control_number is None + + env2 = env.model_copy(update={"transaction_set_control_number": "0001"}) + assert env2.transaction_set_control_number == "0001" + + +def test_add_record_leaves_835_column_null(): + """835 batches don't carry an ST02 join key; column stays NULL. + + The shared ``Envelope`` class is used by both 837P and 835 parsers, + but only ``parse_837`` populates ``transaction_set_control_number``. + For 835 records the field is None and ``add_record`` writes NULL + to the column — verified end-to-end via a real 835 ingest. + """ + text = FIXTURE_835.read_text() + from cyclone.parsers.parse_835 import parse as parse_835_text + parsed835 = parse_835_text(text, payer_config=None) # type: ignore[arg-type] + + # Sanity: the 835 envelope also has the field (shared model class), + # but the parser doesn't populate it — so it must be None. + assert parsed835.envelope.transaction_set_control_number is None + + record = BatchRecord835( + id="b-txn-cn-835-1", + input_filename="minimal_835.txt", + parsed_at=datetime.now(timezone.utc), + result=parsed835, + ) + cycl_store.add(record) + + with db_mod.SessionLocal()() as s: + row = s.get(Batch, "b-txn-cn-835-1") + assert row is not None + assert row.transaction_set_control_number is None + # And the kind/kind round-trip is correct (sanity check that + # we did exercise the 835 path, not the 837 path). + assert row.kind == "835" \ No newline at end of file diff --git a/backend/tests/test_db_migrate.py b/backend/tests/test_db_migrate.py index add5174..9c2d8a1 100644 --- a/backend/tests/test_db_migrate.py +++ b/backend/tests/test_db_migrate.py @@ -126,14 +126,15 @@ def test_migration_latest_idempotent_on_fresh_db(tmp_path: Path) -> None: SP28 bumped it to 18 with the claim_acks join table. SP32 bumped it to 19 with rendering_provider_npi + service_provider_npi on claims and remittances. + SP37 bumped it to 20 with batch transaction_set_control_number. """ engine = _fresh_engine(tmp_path) db_migrate.run(engine) v_after_first = _user_version(engine) - assert v_after_first == 19, f"expected head=19, got {v_after_first}" + assert v_after_first == 20, f"expected head=20, got {v_after_first}" db_migrate.run(engine) - assert _user_version(engine) == 19, "second run should not bump version" + assert _user_version(engine) == 20, "second run should not bump version" def test_drop_claims_unique_constraint_migration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -159,7 +160,7 @@ def test_drop_claims_unique_constraint_migration(tmp_path: Path, monkeypatch: py engine = _fresh_engine(tmp_path) db_migrate.run(engine) - assert _user_version(engine) == 19, f"expected head=19, got {_user_version(engine)}" + assert _user_version(engine) == 20, f"expected head=20, got {_user_version(engine)}" # Two claims in one batch with the same patient_control_number # must be insertable. If 0015's table recreation re-introduced a diff --git a/backend/tests/test_migration_0020.py b/backend/tests/test_migration_0020.py new file mode 100644 index 0000000..11fa2ac --- /dev/null +++ b/backend/tests/test_migration_0020.py @@ -0,0 +1,173 @@ +"""Tests for migration 0020_add_batch_txn_set_control_number.sql. + +SP37 adds a nullable ``batches.transaction_set_control_number`` column +populated from the parsed 837's ST02 (transaction set control number) +on every write. The 999 ack join (Pass 1) needs to resolve by ST02, +not ISA13, so this column is the join key. This migration is purely +additive: nullable, no default, backfills from +``raw_result_json.envelope.transaction_set_control_number`` where the +source JSON already carries it. + +For the backfill-shape tests we point ``db_migrate.MIGRATIONS_DIR`` +at the real migrations directory, apply all migrations once to bring +the fresh DB up to v20, insert representative rows, then replay the +exact UPDATE statement the migration uses. Replaying the UPDATE +proves the SQL works as intended even though the migration itself +already ran over an empty table. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import sqlalchemy as sa + +from cyclone import db_migrate + + +# The backfill UPDATE the migration executes (extracted so the test can +# replay it against rows that didn't exist when init_db ran). +BACKFILL_SQL = ( + "UPDATE batches " + "SET transaction_set_control_number = " + "json_extract(raw_result_json, '$.envelope.transaction_set_control_number') " + "WHERE raw_result_json IS NOT NULL " + "AND json_extract(raw_result_json, '$.envelope.transaction_set_control_number') " + "IS NOT NULL" +) + + +def _fresh_engine(path: Path) -> sa.Engine: + return sa.create_engine(f"sqlite:///{path}", future=True) + + +def _table_info(engine: sa.Engine, table: str) -> list[tuple]: + """Return PRAGMA table_info rows for ``table`` as plain tuples.""" + with engine.connect() as conn: + return list(conn.exec_driver_sql(f"PRAGMA table_info({table});").tuples()) + + +def _real_migrations_dir() -> Path: + return Path(__file__).parent.parent / "src" / "cyclone" / "migrations" + + +@pytest.fixture +def migrated_engine(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Yield an engine at v20 against which every real migration has run. + + Points ``db_migrate.MIGRATIONS_DIR`` at the real migrations + directory so the test exercises the actual 0020 file, then runs + the migration runner on a per-test fresh DB. + """ + monkeypatch.setattr(db_migrate, "MIGRATIONS_DIR", _real_migrations_dir()) + engine = _fresh_engine(tmp_path / "mig0020.db") + db_migrate.run(engine) + + # Confirm head is 20 (every migration applied). + with engine.connect() as conn: + v = conn.exec_driver_sql("PRAGMA user_version").scalar() or 0 + assert v == 20, f"expected migration head=20, got {v}" + + yield engine + engine.dispose() + + +def test_migration_0020_creates_column(migrated_engine) -> None: + """Migration 0020 adds ``transaction_set_control_number`` to ``batches``.""" + cols = _table_info(migrated_engine, "batches") + col_names = {row[1] for row in cols} + assert "transaction_set_control_number" in col_names, ( + "batches.transaction_set_control_number missing — migration 0020 did not run. " + f"Existing columns: {sorted(col_names)}" + ) + + +def test_migration_0020_column_is_nullable(migrated_engine) -> None: + """The new column is nullable (no DEFAULT, no NOT NULL) so existing + batches that don't yet carry the ST02 stay valid.""" + cols = {row[1]: row for row in _table_info(migrated_engine, "batches")} + row = cols["transaction_set_control_number"] + # PRAGMA table_info tuples: (cid, name, type, notnull, dflt_value, pk) + assert row[3] == 0, f"notnull flag must be 0 (nullable), got {row[3]}" + assert row[4] is None, f"dflt_value must be NULL, got {row[4]!r}" + assert "TEXT" in (row[2] or "").upper(), f"expected TEXT column, got {row[2]!r}" + + +def test_migration_0020_backfills_when_key_present( + migrated_engine: sa.Engine, +) -> None: + """Replay the migration's backfill UPDATE against a row whose + ``raw_result_json`` carries the key — must populate the new column. + + Replay (rather than waiting for ``db_migrate.run()`` to do it) is + the only way to test the SQL against a row that didn't exist when + init_db() ran; the migration's UPDATE naturally runs only over + pre-existing rows. + """ + st02_value = "ST0001" + raw_json = {"envelope": {"transaction_set_control_number": st02_value}} + + with migrated_engine.begin() as conn: + conn.exec_driver_sql( + "INSERT INTO batches (id, kind, input_filename, parsed_at, raw_result_json) " + "VALUES ('B-ST02-1', '837p', 'mig0020-st02.txt', '2026-07-07 00:00:00', ?)", + (json.dumps(raw_json),), + ) + conn.exec_driver_sql(BACKFILL_SQL) + + with migrated_engine.connect() as conn: + row = conn.exec_driver_sql( + "SELECT transaction_set_control_number FROM batches WHERE id='B-ST02-1'" + ).first() + assert row is not None + assert row[0] == st02_value, ( + f"backfill failed: expected {st02_value!r}, got {row[0]!r}" + ) + + +def test_migration_0020_backfill_conditional_on_key_present( + migrated_engine: sa.Engine, +) -> None: + """Replay the UPDATE against a row whose ``raw_result_json`` does NOT + carry the key — must stay NULL. Proves the UPDATE is conditional + (the ``json_extract(...) IS NOT NULL`` guard) rather than + unconditionally overwriting with NULL.""" + raw_json = {"envelope": {"control_number": "ISA0001"}} # no txn-set key + with migrated_engine.begin() as conn: + conn.exec_driver_sql( + "INSERT INTO batches (id, kind, input_filename, parsed_at, raw_result_json) " + "VALUES ('B-NOST-1', '837p', 'mig0020-nost.txt', '2026-07-07 00:00:00', ?)", + (json.dumps(raw_json),), + ) + conn.exec_driver_sql(BACKFILL_SQL) + + with migrated_engine.connect() as conn: + row = conn.exec_driver_sql( + "SELECT transaction_set_control_number FROM batches WHERE id='B-NOST-1'" + ).first() + assert row is not None + assert row[0] is None, ( + f"backfill must not overwrite when key is absent; got {row[0]!r}" + ) + + +def test_migration_0020_backfill_handles_null_raw_result_json( + migrated_engine: sa.Engine, +) -> None: + """A Batch row with ``raw_result_json IS NULL`` (the prior SP's + unparsed state) must not crash the backfill and must stay NULL.""" + with migrated_engine.begin() as conn: + conn.exec_driver_sql( + "INSERT INTO batches (id, kind, input_filename, parsed_at) " + "VALUES ('B-NULL-1', '837p', 'mig0020-null.txt', '2026-07-07 00:00:00')" + ) + conn.exec_driver_sql(BACKFILL_SQL) + + with migrated_engine.connect() as conn: + row = conn.exec_driver_sql( + "SELECT transaction_set_control_number FROM batches WHERE id='B-NULL-1'" + ).first() + assert row is not None + assert row[0] is None diff --git a/backend/tests/test_submission.py b/backend/tests/test_submission.py new file mode 100644 index 0000000..f2a56ce --- /dev/null +++ b/backend/tests/test_submission.py @@ -0,0 +1,208 @@ +"""SP37 Task 4: submit_file owns parse → DB write → SFTP upload. + +Tests use a fake SFTP client to avoid hitting the real clearhouse. +The DB path is real (autouse ``_auto_init_db`` fixture in conftest.py) +so the join-key wiring is exercised end-to-end. +""" +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from cyclone import db as db_mod +from cyclone.submission.core import submit_file +from cyclone.submission.result import SubmitOutcome + + +_FIXTURE = Path(__file__).parent / "fixtures" / "submit-batch" / "single-claim.x12" + + +class _FakeSftp: + """Implements just enough of SftpClient for submit_file's idempotency check. + + After every successful ``write_file``, the file is added to + ``existing_files`` keyed by remote_path with the byte size — so a + subsequent ``stat`` returns the right size and the idempotency + check in ``submit_file`` short-circuits to ``SKIPPED``. Without + this bookkeeping, every re-run would look like the file isn't on + the remote and would re-upload. + """ + + def __init__(self, existing_files: dict[str, int] | None = None): + self.existing_files = existing_files or {} + self.put_calls: list[tuple[str, bytes]] = [] + + def stat(self, path: str): + if path not in self.existing_files: + raise IOError(f"no such file: {path}") + m = MagicMock() + m.st_size = self.existing_files[path] + return m + + def write_file(self, remote_path: str, content: bytes): + self.put_calls.append((remote_path, content)) + # Mirror the upload so a subsequent stat() sees it. + self.existing_files[remote_path] = len(content) + + +def test_submit_file_happy_path(): + """parse → DB write → SFTP upload, all steps succeed.""" + fake = _FakeSftp() + sftp_block = MagicMock() + sftp_block.stub = False + sftp_block.paths = {"outbound": "/ToHPE"} + + result = submit_file( + _FIXTURE, + sftp_block=sftp_block, + actor="test", + validate=True, + sftp_client_factory=lambda _block: fake, + ) + + assert result.outcome == SubmitOutcome.SUBMITTED + assert result.batch_id is not None + assert len(fake.put_calls) == 1 + # DB row landed. + with db_mod.SessionLocal()() as s: + assert s.get(db_mod.Batch, result.batch_id) is not None + + +def test_submit_file_skipped_when_remote_size_matches(): + """Re-running submit_file on an already-uploaded file is SKIPPED. + + The ``SKIPPED`` outcome is reached via the SFTP ``stat()`` check + (not via the DB) — same-size remote file ⇒ idempotency short-circuit. + Re-runs therefore don't emit a duplicate ``clearhouse.submitted`` + audit event. + """ + fake = _FakeSftp() + sftp_block = MagicMock(stub=False, paths={"outbound": "/ToHPE"}) + factory = lambda _b: fake + + r1 = submit_file(_FIXTURE, sftp_block=sftp_block, actor="t", sftp_client_factory=factory) + r2 = submit_file(_FIXTURE, sftp_block=sftp_block, actor="t", sftp_client_factory=factory) + + # First call SUBMITTED, second call SKIPPED (size match on stat). + assert r1.outcome == SubmitOutcome.SUBMITTED + assert r2.outcome == SubmitOutcome.SKIPPED + + +def test_submit_file_parse_fail(tmp_path): + """Bad 837 → PARSE_FAILED, no DB write, no SFTP call.""" + bad = tmp_path / "bad.x12" + bad.write_bytes(b"not x12") + sftp_block = MagicMock(stub=False, paths={"outbound": "/ToHPE"}) + fake = _FakeSftp() + factory = lambda _b: fake + + result = submit_file(bad, sftp_block=sftp_block, actor="t", sftp_client_factory=factory) + assert result.outcome == SubmitOutcome.PARSE_FAILED + assert len(fake.put_calls) == 0 + + +def test_submit_file_db_fail(monkeypatch): + """If add raises, no SFTP call is made (DB-first invariant).""" + from cyclone.store import store as cycl_store + monkeypatch.setattr(cycl_store, "add", MagicMock(side_effect=RuntimeError("db down"))) + + sftp_block = MagicMock(stub=False, paths={"outbound": "/ToHPE"}) + fake = _FakeSftp() + factory = lambda _b: fake + + result = submit_file(_FIXTURE, sftp_block=sftp_block, actor="t", sftp_client_factory=factory) + assert result.outcome == SubmitOutcome.DB_FAILED + assert len(fake.put_calls) == 0 + + +def test_submit_file_sftp_fail(): + """SFTP failure after DB write → SFTP_FAILED, DB row still present.""" + + class _Boom: + def stat(self, _p): + raise IOError("nope") + def write_file(self, _p, _c): + raise RuntimeError("sftp down") + + sftp_block = MagicMock(stub=False, paths={"outbound": "/ToHPE"}) + factory = lambda _b: _Boom() + + result = submit_file(_FIXTURE, sftp_block=sftp_block, actor="t", sftp_client_factory=factory) + assert result.outcome == SubmitOutcome.SFTP_FAILED + # DB row was written before the SFTP call. + assert result.batch_id is not None + with db_mod.SessionLocal()() as s: + assert s.get(db_mod.Batch, result.batch_id) is not None + + +def test_submit_file_payer_mismatch(): + """PAYER_MISMATCH → no DB write, no SFTP call. + + We monkeypatch ``parse_837_text`` to return a single claim whose + payer.id is ``OTHER_PAYER`` (the helper gates on the first + claim's payer — same posture as the legacy ``resubmit_rejected_claims`` + CLI at cli.py:657-668). + """ + from unittest.mock import patch + + bad_claim = MagicMock() + bad_claim.payer.id = "OTHER_PAYER" + parsed_with_bad_payer = MagicMock() + parsed_with_bad_payer.claims = [bad_claim] + parsed_with_bad_payer.parsed_at = None + parsed_with_bad_payer.envelope.transaction_set_control_number = "ST123" + + fake = _FakeSftp() + sftp_block = MagicMock(stub=False, paths={"outbound": "/ToHPE"}) + + with patch("cyclone.submission.core.parse_837_text", return_value=parsed_with_bad_payer): + result = submit_file( + _FIXTURE, + sftp_block=sftp_block, + actor="t", + validate=True, + sftp_client_factory=lambda _b: fake, + ) + assert result.outcome == SubmitOutcome.PAYER_MISMATCH + assert "OTHER_PAYER" in (result.error or "") + assert len(fake.put_calls) == 0 + + +def test_submit_file_uses_default_paramiko_factory(monkeypatch): + """When no factory is supplied, helper should use paramiko directly + (not SftpClient wrapper, which lacks stat()). + """ + fake_paramiko_sftp = _FakeSftp() + monkeypatch.setattr( + "cyclone.submission.core._default_sftp_factory", + lambda _block: fake_paramiko_sftp, + ) + + sftp_block = MagicMock() + sftp_block.stub = False + sftp_block.paths = {"outbound": "/ToHPE"} + + result = submit_file( + _FIXTURE, + sftp_block=sftp_block, + actor="t", + validate=True, + # NO sftp_client_factory — should use the paramiko default + ) + assert result.outcome == SubmitOutcome.SUBMITTED + assert len(fake_paramiko_sftp.put_calls) == 1 + + +def test_submit_batch_cli_help(): + """`cyclone submit-batch --help` exits 0 and shows the flags.""" + import subprocess + import sys + result = subprocess.run( + [sys.executable, "-m", "cyclone.cli", "submit-batch", "--help"], + capture_output=True, text=True, + cwd=str(Path(__file__).parent.parent), + ) + assert result.returncode == 0 + assert "--ingest-dir" in result.stdout diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 1271f43..cd241db 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -187,6 +187,52 @@ absent.) `processed_inbound_files` are skipped automatically — to re-process, delete the row first (`DELETE FROM processed_inbound_files WHERE name = ...`). +### Submitting claims (canonical — SP37) + +For 837P files generated upstream (dzinesco) that you want cyclone to +**track in the DB** before uploading, use the canonical submit path. +This is the preferred outbound path going forward — it captures the +batch's `transaction_set_control_number` (the ST02 control number that +999 acks reference in AK201) so future 999 ack links resolve instead +of becoming orphans. + +```bash +cd /home/tyler/dev/cyclone/backend + +# 1. Lay out files in batch-*-claims subdirs under your ingest dir: +# ingest/batch-2026-07-08-claims/claim-001.x12 +# ingest/batch-2026-07-08-claims/claim-002.x12 +# ... + +# 2. CLI — walks ingest/, parses, writes to DB, then SFTP-uploads. +.venv/bin/python -m cyclone submit-batch \ + --ingest-dir /home/tyler/dev/cyclone/ingest \ + --actor cli-submit-batch + +# Or via HTTP (auth-gated by matrix_gate): +curl -s -X POST http://localhost:8000/api/submit-batch \ + -H 'Content-Type: application/json' \ + -d '{"ingest_dir": "/home/tyler/dev/cyclone/ingest", "actor": "api-submit-batch"}' +``` + +Both surfaces share `cyclone.submission.submit_file` for the +parse → DB-write → SFTP-upload chain (DB-first, upload-second +invariant). The walker pattern is identical: `batch-*-claims/*.x12`, +sorted, with `._*` AppleDouble files skipped. + +**When to use `submit-batch` vs `resubmit-rejected-claims`:** +- `submit-batch` — canonical path for fresh 837s from dzinesco (or any + source) that should be tracked in the DB before upload. Default choice. +- `resubmit-rejected-claims` — one-off path for cases where you do NOT + want a DB row (e.g., dzinesco-generated fixes not ready for canonical + tracking). Legacy, retained for backward compat. + +**Status codes / exit codes:** +- HTTP 200 on completed runs (per-file failures live in the JSON body); + 401 unauthenticated; 404 no clearhouse; 409 stub mode; 422 validation. +- CLI exit 0 on completed runs (per-file failures counted, not bumped); + 2 on config-level failures (no clearhouse / stub mode / missing dir). + ### Note on per-file parse CLIs `parse-837` and `parse-835` exist as CLIs but only emit JSON files to diff --git a/docs/superpowers/plans/2026-07-07-cyclone-submit-batch-canonical-flow.md b/docs/superpowers/plans/2026-07-07-cyclone-submit-batch-canonical-flow.md new file mode 100644 index 0000000..00a44ac --- /dev/null +++ b/docs/superpowers/plans/2026-07-07-cyclone-submit-batch-canonical-flow.md @@ -0,0 +1,1391 @@ +# SP37 — Canonical submit-batch flow: Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a canonical `cyclone submit-batch` CLI + `POST /api/submit-batch` endpoint that does parse → DB write → SFTP upload per file, and add a `Batch.transaction_set_control_number` column so 999 acks can join back to source 837 batches via the ST02 control number. + +**Architecture:** New `cyclone.submission` package with one shared helper (`submit_file`) wrapping the parse-write-upload sequence. CLI + HTTP both thin-call the helper. Migration adds the new column additively + backfills from `raw_result_json`. `claim_acks.batch_envelope_index` returns a dict populated from BOTH `envelope.control_number` (preserved) AND `transaction_set_control_number` (new) — single `idx.get(set_control_number)` lookup matches either key. + +**Tech Stack:** Python 3.11, FastAPI, SQLAlchemy + Alembic, Click (CLI), paramiko (SFTP), Pydantic, pytest. + +**Spec:** `docs/superpowers/specs/2026-07-07-cyclone-submit-batch-canonical-flow-design.md` + +**Progress tracker:** `/tmp/refactor-cyclone.md` — update after every commit per the operator's standing directive (per-step log with pre/post pytest, live-test result, reviewer verdict). + +**Branch:** `sp37-submit-batch-canonical-flow` (already created, off `710104f`). + +**Conventions:** Per `cyclone-spec` skill — commit prefixes are `feat(sp37): …`, `merge: SP37 canonical submit-batch flow into main`. Per `cyclone-tests` skill — backend tests live under `backend/tests/test_*.py`, `test_api_*` for FastAPI integration, pure-unit otherwise; conftest points `CYCLONE_DB_URL` at `tmp_path/test.db`. Per `cyclone-store` skill — write through the `CycloneStore` facade, never directly into `db.py`. + +--- + +## Task 0: Progress-tracker bootstrap + baseline pytest + +**Files:** +- Modify: `/tmp/refactor-cyclone.md` +- (no code changes) + +- [ ] **Step 1: Capture baseline pytest** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest -x --tb=no -q 2>&1 | tail -3` +Expected: a count line like `X passed, Y skipped, Z failed`. Save to `/tmp/refactor-pre-baseline.txt`. + +- [ ] **Step 2: Update tracker baseline section** + +Edit `/tmp/refactor-cyclone.md` — replace the "Baseline" section's pytest line with the actual count from Step 1 (so future task diffs are honest). + +- [ ] **Step 3: Commit** + +```bash +git add /tmp/refactor-cyclone.md # the file lives outside the repo; this is a no-op marker +# No git commit needed — the tracker is out-of-repo. Just confirm the file is updated. +``` + +(End of Task 0 — no commit, no code change. The tracker is out-of-repo per the operator's standing directive.) + +--- + +## Task 1: Migration `0013_add_batch_txn_set_control_number.py` + +**Files:** +- Create: `backend/migrations/0013_add_batch_txn_set_control_number.py` +- Create: `backend/tests/test_migration_0013.py` + +- [ ] **Step 1: Write the failing test** + +Create `backend/tests/test_migration_0013.py`: + +```python +"""SP37 Task 1: migration 0013 adds Batch.transaction_set_control_number. + +The column is nullable, additive, and backfills from raw_result_json +when the parsed envelope carries a transaction_set_control_number key. +""" +from __future__ import annotations + +import json + +import pytest + +from cyclone import db as db_mod +from cyclone.db_migrate import apply_migrations + + +def _bootstrap_test_db(tmp_path, monkeypatch): + db_file = tmp_path / "mig.db" + monkeypatch.setenv("CYCLONE_DB_URL", f"sqlite:///{db_file}") + db_mod._reset_for_tests() + db_mod.init_db() + return db_file + + +def test_migration_0013_adds_column(tmp_path, monkeypatch): + _bootstrap_test_db(tmp_path, monkeypatch) + apply_migrations(target="0013") + + with db_mod.engine.connect() as conn: + cols = {row[1] for row in conn.exec_driver_sql("PRAGMA table_info(batches)").fetchall()} + assert "transaction_set_control_number" in cols + + +def test_migration_0013_backfills_from_raw_json(tmp_path, monkeypatch): + _bootstrap_test_db(tmp_path, monkeypatch) + + # Insert a batch row with an envelope that carries transaction_set_control_number. + with db_mod.SessionLocal()() as s: + s.add(db_mod.Batch( + id="b-backfill", + kind="837p", + input_filename="x.837p", + raw_result_json={ + "envelope": { + "sender_id": "CYCLONE", + "receiver_id": "HCPF", + "control_number": "000000001", + "transaction_set_control_number": "ST0001", + "transaction_date": "2026-07-07", + }, + "claims": [], + }, + )) + s.commit() + + apply_migrations(target="0013") + + with db_mod.SessionLocal()() as s: + row = s.get(db_mod.Batch, "b-backfill") + assert row.transaction_set_control_number == "ST0001" + + +def test_migration_0013_down(tmp_path, monkeypatch): + _bootstrap_test_db(tmp_path, monkeypatch) + apply_migrations(target="0013") + apply_migrations(target="zero") # walk back to nothing + + with db_mod.engine.connect() as conn: + cols = {row[1] for row in conn.exec_driver_sql("PRAGMA table_info(batches)").fetchall()} + assert "transaction_set_control_number" not in cols +``` + +- [ ] **Step 2: Run test, verify it fails** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_migration_0013.py -v` +Expected: all 3 tests FAIL (no migration yet). + +- [ ] **Step 3: Write the migration** + +Create `backend/migrations/0013_add_batch_txn_set_control_number.py`. Look at `backend/migrations/0012_backups.py` first to match the project's existing migration style (Alembic op vs raw SQL — match what's there). The body: + +```python +"""SP37 Task 1: add nullable batches.transaction_set_control_number. + +Backfills from raw_result_json.envelope.transaction_set_control_number +when present. Idempotent — re-running on a populated DB is a no-op +because the column is already populated and the WHERE clause skips NULLs. +""" +# (Replace this docstring with the project's standard migration header +# once you've matched the style of 0012_backups.py.) + +from alembic import op +import sqlalchemy as sa + + +revision = "0013_add_batch_txn_set_control_number" +down_revision = "0012_backups" +depends_on = None + + +def upgrade(): + op.add_column( + "batches", + sa.Column("transaction_set_control_number", sa.String(length=32), nullable=True), + ) + op.execute( + """ + UPDATE batches + SET transaction_set_control_number = json_extract( + raw_result_json, '$.envelope.transaction_set_control_number' + ) + WHERE raw_result_json IS NOT NULL + AND json_extract(raw_result_json, '$.envelope.transaction_set_control_number') IS NOT NULL + """ + ) + + +def downgrade(): + op.drop_column("batches", "transaction_set_control_number") +``` + +- [ ] **Step 4: Run test, verify it passes** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_migration_0013.py -v` +Expected: 3 passed. + +- [ ] **Step 5: Live test — apply on the real dev DB** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/python -m cyclone db migrate --to 0013` (use whatever command `db_migrate.py` exposes; check `backend/src/cyclone/db_migrate.py` for the actual CLI). +Expected: column added; existing 1 batch row (`b1`) shows NULL `transaction_set_control_number` (it has no envelope with that key). + +Verify: `sqlite3 ~/.local/share/cyclone/cyclone.db "PRAGMA table_info(batches);" | grep transaction` +Expected: one row matching `transaction_set_control_number`. + +- [ ] **Step 6: Commit** + +```bash +git add backend/migrations/0013_add_batch_txn_set_control_number.py backend/tests/test_migration_0013.py +git commit -m "feat(sp37): add batches.transaction_set_control_number column + +Migration 0013 adds the column additively (nullable, no default) and +backfills from raw_result_json.envelope.transaction_set_control_number +for any existing batch rows that already carry it. Required for the +SP37 join-key update so 999 acks can resolve by ST02 (the source 837's +transaction set control number) instead of just ISA13." +``` + +- [ ] **Step 7: Update tracker** + +Add a Task 1 section to `/tmp/refactor-cyclone.md` with: pre/post pytest (both should equal baseline from Task 0), migration applied successfully, no reviewer needed (migration only). + +--- + +## Task 2: Add `Batch.transaction_set_control_number` to ORM + populate from `_claim_837_row` flow + +**Files:** +- Modify: `backend/src/cyclone/db.py:Batch` (add mapped column) +- Modify: `backend/src/cyclone/store/orm_builders.py:_claim_837_row` (or wherever `add_record` populates the Batch row — see Step 1) + +- [ ] **Step 1: Locate the Batch row constructor** + +Search for the `Batch(...)` instantiation that `add_record` uses. It lives in `backend/src/cyclone/store/write.py:78-85` (per the spec exploration). The current constructor: + +```python +batch_row = Batch( + id=record.id, + kind=record.kind, + input_filename=record.input_filename, + parsed_at=record.parsed_at, + totals_json=None, + validation_json=None, + raw_result_json=json.loads(record.result.model_dump_json()), +) +``` + +The `record.result` is the parsed `ParseResult` (or `ParseResult835`). For 837P the envelope is at `result.envelope`. + +- [ ] **Step 2: Write the failing test** + +Create `backend/tests/test_batch_txn_set_cn.py`: + +```python +"""SP37 Task 2: add_record populates Batch.transaction_set_control_number. + +The column should mirror the source 837 envelope's +transaction_set_control_number (ST02) so the join-key index in +Task 3 can resolve 999 set_control_numbers back to the right batch. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from cyclone import db as db_mod +from cyclone.db import Batch +from cyclone.store import store as cycl_store +from cyclone.store.records import BatchRecord837 + +# Reuse a small 837 fixture if one exists in tests/fixtures/; otherwise +# copy the smallest real 837 from docs/prodfiles/ to fixtures/. +_FIXTURE = Path(__file__).parent / "fixtures" / "single-claim-837P.x12" + + +@pytest.fixture(autouse=True) +def _db(tmp_path, monkeypatch): + monkeypatch.setenv("CYCLONE_DB_URL", f"sqlite:///{tmp_path}/test.db") + db_mod._reset_for_tests() + db_mod.init_db() + yield + + +def test_add_record_populates_transaction_set_control_number(monkeypatch): + """parse_837 → add_record should write the parsed ST02 onto the Batch row.""" + from cyclone.parsers.parse_837 import parse as parse_837_text + from cyclone.parsers.payer import PayerConfig + + text = _FIXTURE.read_text() + result = parse_837_text(text, PayerConfig.co_medicaid()) + + record = BatchRecord837( + id="b-test-1", + input_filename="x.837p", + parsed_at=result.parsed_at, + result=result, + ) + cycl_store.add_record(record) + + with db_mod.SessionLocal()() as s: + row = s.get(Batch, "b-test-1") + assert row is not None + # Whichever ST02 the fixture has must round-trip into the column. + assert row.transaction_set_control_number == result.envelope.transaction_set_control_number +``` + +- [ ] **Step 3: Add the ORM column** + +Modify `backend/src/cyclone/db.py:Batch`. Add the new mapped column. Locate the existing `Batch` class (search for `class Batch(` in `db.py`) and add after the existing envelope-ish fields: + +```python +transaction_set_control_number: Mapped[Optional[str]] = mapped_column( + String(32), nullable=True +) +``` + +Confirm the import line `from sqlalchemy import ... String ...` already includes `String`; if not, add it. + +- [ ] **Step 4: Run test, verify it fails** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_batch_txn_set_cn.py -v` +Expected: FAIL with `AttributeError: type object 'Batch' has no attribute 'transaction_set_control_number'` (the ORM doesn't have the column yet, OR the migration hasn't been applied to the test DB). If the test DB is freshly created per-test (it is, via the autouse fixture), apply migration 0013 to it: + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/python -c "from cyclone.db_migrate import apply_migrations; from cyclone import db; db.init_db(); apply_migrations(target='0013')"` +(This step is for local sanity; the autouse test fixture should already trigger `init_db` which runs migrations.) + +If the test still fails because `add_record` doesn't populate the column, that's the next step. + +- [ ] **Step 5: Update `add_record` to populate the column** + +Modify `backend/src/cyclone/store/write.py:78-85`. Change the Batch constructor to include the new field. Read the `record.result` envelope and copy its `transaction_set_control_number` onto the row: + +```python +batch_row = Batch( + id=record.id, + kind=record.kind, + input_filename=record.input_filename, + parsed_at=record.parsed_at, + totals_json=None, + validation_json=None, + raw_result_json=json.loads(record.result.model_dump_json()), + transaction_set_control_number=getattr( + getattr(record.result, "envelope", None), "transaction_set_control_number", None + ), +) +``` + +The `getattr(..., None)` guards against the 835 path where the result has no `envelope` attribute (it's a `ParseResult835`, not `ParseResult`); `BatchRecord835` won't set the column. + +- [ ] **Step 6: Run test, verify it passes** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_batch_txn_set_cn.py -v` +Expected: PASS. + +- [ ] **Step 7: Run the existing `test_api_parse_837` to confirm no regression** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_api_parse_837.py -v` +Expected: same pass/fail as baseline (no new failures introduced). + +- [ ] **Step 8: Commit** + +```bash +git add backend/src/cyclone/db.py backend/src/cyclone/store/write.py backend/tests/test_batch_txn_set_cn.py +git commit -m "feat(sp37): populate batches.transaction_set_control_number on 837P ingest + +add_record now copies the parsed envelope's transaction_set_control_number +(the source 837's ST02) onto the Batch row. Unlocks the SP37 join-key +update so 999 set_control_number (AK201) values resolve back to the +right batch via Pass 1." +``` + +- [ ] **Step 9: Update tracker** + +Append Task 2 section with pre/post pytest, test result, live-test (no manual live test needed; the existing `clm-1` row can be re-ingested via `/api/parse-837` if you want to verify end-to-end — optional). + +--- + +## Task 3: Update `batch_envelope_index` to key by both envelope.control_number AND transaction_set_control_number + +**Files:** +- Modify: `backend/src/cyclone/store/claim_acks.py:batch_envelope_index` +- Create: `backend/tests/test_batch_envelope_index_sp37.py` + +- [ ] **Step 1: Read the existing function** + +Read `backend/src/cyclone/store/claim_acks.py` around `def batch_envelope_index`. Per spec, the current behavior is: read every Batch row whose `raw_result_json` has an envelope and return `{envelope.control_number: batch.id}`. We change it to ALSO populate from `Batch.transaction_set_control_number` (the new column from Task 2), so a single dict has both keys. + +- [ ] **Step 2: Write the failing test** + +Create `backend/tests/test_batch_envelope_index_sp37.py`: + +```python +"""SP37 Task 3: batch_envelope_index populates from BOTH columns. + +The dict returned by batch_envelope_index should resolve lookups by +either envelope.control_number (ISA13, preserved) OR +transaction_set_control_number (ST02, new). One dict, both keys — +a single .get(set_control_number) call hits whichever matches. +""" +from __future__ import annotations + +import json + +import pytest + +from cyclone import db as db_mod +from cyclone.db import Batch +from cyclone.store.claim_acks import batch_envelope_index + + +@pytest.fixture(autouse=True) +def _db(tmp_path, monkeypatch): + monkeypatch.setenv("CYCLONE_DB_URL", f"sqlite:///{tmp_path}/test.db") + db_mod._reset_for_tests() + db_mod.init_db() + yield + + +def _make_batch(s, **kw): + s.add(Batch( + id=kw["id"], + kind="837p", + input_filename=kw["id"] + ".x12", + raw_result_json=kw.get("raw_result_json", {"envelope": {"control_number": kw.get("icn", "000000001")}}), + transaction_set_control_number=kw.get("stcn"), + )) + s.commit() + + +def test_index_resolves_by_envelope_control_number(): + with db_mod.SessionLocal()() as s: + _make_batch(s, id="b1", icn="ISA000001") + idx = batch_envelope_index() + assert idx.get("ISA000001") == "b1" + + +def test_index_resolves_by_transaction_set_control_number(): + with db_mod.SessionLocal()() as s: + _make_batch(s, id="b2", icn="ISA000002", stcn="ST000002") + idx = batch_envelope_index() + assert idx.get("ST000002") == "b2" + # Backward compat: ISA still resolves. + assert idx.get("ISA000002") == "b2" + + +def test_index_handles_row_with_no_transaction_set_control_number(): + """Pre-migration rows (stcn NULL) should still resolve by ISA.""" + with db_mod.SessionLocal()() as s: + _make_batch(s, id="b3", icn="ISA000003") # no stcn + idx = batch_envelope_index() + assert idx.get("ISA000003") == "b3" + assert idx.get("ST000003") is None +``` + +- [ ] **Step 3: Run test, verify it fails** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_batch_envelope_index_sp37.py -v` +Expected: `test_index_resolves_by_transaction_set_control_number` and `test_index_handles_row_with_no_transaction_set_control_number` FAIL because the current function only reads `raw_result_json.envelope.control_number` and ignores the new column. + +- [ ] **Step 4: Update the function** + +Modify `backend/src/cyclone/store/claim_acks.py:batch_envelope_index`. Read both the raw_result_json envelope AND the new `transaction_set_control_number` column. Replace the function body so it returns a single dict populated from both sources. The new body (sketch — adapt to the existing function's style): + +```python +def batch_envelope_index() -> dict[str, str]: + """Build a ``{key: batch.id}`` map populated from two columns. + + D10 primary join key (spec §D10). Each batch row contributes up + to two entries: + + * ``Batch.raw_result_json.envelope.control_number`` (ISA13) + * ``Batch.transaction_set_control_number`` (ST02, new in SP37) + + Either key resolves to the same ``batch.id``; callers do a single + ``idx.get(set_control_number)`` lookup. Pass 2 (PCN) is unchanged. + + Built once per ingest from every Batch row whose kind is "837p"; + cost is O(N batches), currently small, so trivial. + """ + idx: dict[str, str] = {} + with db.SessionLocal()() as s: + rows = s.query(Batch.id, Batch.raw_result_json, Batch.transaction_set_control_number).filter( + Batch.kind == "837p" + ).all() + for bid, raw, stcn in rows: + env = (raw or {}).get("envelope") or {} + isa_cn = env.get("control_number") + if isa_cn: + idx.setdefault(str(isa_cn), bid) + if stcn: + idx.setdefault(str(stcn), bid) + return idx +``` + +(The exact rewrite depends on the current function's structure — read it first, then drop the new body in, preserving any existing docstring above it.) + +- [ ] **Step 5: Run test, verify it passes** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_batch_envelope_index_sp37.py -v` +Expected: 3 passed. + +- [ ] **Step 6: Run all ack-related tests to confirm no regression** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_claim_acks.py tests/test_api_ack_stream.py tests/test_apply_999_rejections.py -v` +Expected: same pass/fail as baseline. The new dict may have more entries, but Pass 1's `idx.get(...)` lookup semantics are unchanged. + +- [ ] **Step 7: Commit** + +```bash +git add backend/src/cyclone/store/claim_acks.py backend/tests/test_batch_envelope_index_sp37.py +git commit -m "feat(sp37): batch_envelope_index now resolves by ST02 or ISA13 + +Each batch row contributes up to two entries to the join-key index +(envelope.control_number + transaction_set_control_number). One +idx.get(set_control_number) call resolves either, so 999 acks whose +AK201 echoes the source 837's ST02 now hit Pass 1 where they +previously fell through to Pass 2 (and to the orphan log)." +``` + +- [ ] **Step 8: Update tracker** + +Append Task 3 section. + +--- + +## Task 4: `cyclone.submission` package — `submit_file` helper + unit tests + +**Files:** +- Create: `backend/src/cyclone/submission/__init__.py` +- Create: `backend/src/cyclone/submission/core.py` +- Create: `backend/src/cyclone/submission/result.py` (SubmitResult + SubmitOutcome) +- Create: `backend/tests/test_submission.py` +- Create: `backend/tests/fixtures/submit-batch/` directory + at least one `.x12` fixture + +- [ ] **Step 1: Pick a fixture** + +Copy a small valid 837P file from `docs/prodfiles/co_medicaid/` (or wherever the operator's recent files live — try `docs/prodfiles/2026-07/` or `ingest/july7/`) into `backend/tests/fixtures/submit-batch/`. Name it `single-claim.x12`. If no prodfile is available, synthesize one from the test fixtures used in `tests/test_api_parse_837.py` (find the smallest one and copy it). + +- [ ] **Step 2: Define the result types** + +Create `backend/src/cyclone/submission/result.py`: + +```python +"""SP37 Task 4: typed result for submit_file. + +The CLI and HTTP layer both consume SubmitResult; keeping it in one +place means both surfaces agree on the response shape. +""" +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + + +class SubmitOutcome(str, Enum): + SUBMITTED = "submitted" + SKIPPED = "skipped" + PARSE_FAILED = "parse_failed" + PAYER_MISMATCH = "payer_mismatch" + DB_FAILED = "db_failed" + SFTP_FAILED = "sftp_failed" + + +@dataclass(frozen=True) +class SubmitResult: + file: str + outcome: SubmitOutcome + batch_id: str | None = None + error: str | None = None +``` + +- [ ] **Step 3: Write the failing test (happy path)** + +Create `backend/tests/test_submission.py`: + +```python +"""SP37 Task 4: submit_file owns parse → DB write → SFTP upload. + +Tests use a fake SFTP client to avoid hitting the real clearhouse. +The DB path is real (autouse fixture) so the join-key wiring is +exercised end-to-end. +""" +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from cyclone import db as db_mod +from cyclone.clearhouse import SftpClient +from cyclone.submission.core import submit_file +from cyclone.submission.result import SubmitOutcome + + +_FIXTURE = Path(__file__).parent / "fixtures" / "submit-batch" / "single-claim.x12" + + +class _FakeSftp: + """Implements just enough of SftpClient for submit_file's idempotency check.""" + + def __init__(self, existing_files: dict[str, int] | None = None): + self.existing_files = existing_files or {} + self.put_calls: list[tuple[str, bytes]] = [] + + def stat(self, path: str): + if path not in self.existing_files: + raise IOError(f"no such file: {path}") + m = MagicMock() + m.st_size = self.existing_files[path] + return m + + def write_file(self, remote_path: str, content: bytes): + self.put_calls.append((remote_path, content)) + + +@pytest.fixture(autouse=True) +def _db(tmp_path, monkeypatch): + monkeypatch.setenv("CYCLONE_DB_URL", f"sqlite:///{tmp_path}/test.db") + db_mod._reset_for_tests() + db_mod.init_db() + yield + + +def test_submit_file_happy_path(monkeypatch): + """parse → DB write → SFTP upload, all steps succeed.""" + fake = _FakeSftp() + monkeypatch.setattr(SftpClient, "stat", fake.stat) + monkeypatch.setattr(SftpClient, "write_file", fake.write_file) + + sftp_block = MagicMock() + sftp_block.stub = False + sftp_block.paths = {"outbound": "/ToHPE"} + + result = submit_file( + _FIXTURE, + sftp_block=sftp_block, + actor="test", + validate=True, + sftp_client_factory=lambda _block: fake, + ) + + assert result.outcome == SubmitOutcome.SUBMITTED + assert result.batch_id is not None + assert len(fake.put_calls) == 1 + # DB row landed. + with db_mod.SessionLocal()() as s: + assert s.get(db_mod.Batch, result.batch_id) is not None +``` + +- [ ] **Step 4: Run test, verify it fails** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_submission.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'cyclone.submission'`. + +- [ ] **Step 5: Implement the package skeleton** + +Create `backend/src/cyclone/submission/__init__.py`: + +```python +"""SP37 Task 4: canonical 837P submission flow. + +The single public helper is ``submit_file`` — it owns parse → DB +write → SFTP upload per file. CLI (``cyclone submit-batch``) and HTTP +(``POST /api/submit-batch``) are thin wrappers; both call this helper +so the ordering, idempotency, and audit shape are identical. + +DB-first, upload-second (per spec decision §2): if the DB write fails, +no SFTP call is made. If the SFTP call fails after a successful DB +write, the row exists but the file never landed — re-running is safe +(idempotent DB write via add_record's s.get check; idempotent SFTP +upload via stat-then-put). +""" +from .core import submit_file +from .result import SubmitOutcome, SubmitResult + +__all__ = ["submit_file", "SubmitOutcome", "SubmitResult"] +``` + +- [ ] **Step 6: Implement `submit_file`** + +Create `backend/src/cyclone/submission/core.py`. Read the existing `resubmit_rejected_claims` CLI (around `backend/src/cyclone/cli.py:553-720`) and mirror its structure, but route the DB write through `cycl_store.add_record` first and the SFTP put second. + +```python +"""SP37 Task 4: parse → DB write → SFTP upload, per file. + +Mirrors ``resubmit_rejected_claims`` but writes to the DB before +uploading. Same idempotency check (``stat().st_size == local_size``) +skips already-uploaded files without re-emitting audit events. +""" +from __future__ import annotations + +import logging +from pathlib import Path + +from cyclone.clearhouse import SftpClient +from cyclone.parsers.parse_837 import parse as parse_837_text +from cyclone.parsers.payer import PayerConfig +from cyclone.store import store as cycl_store +from cyclone.store.records import BatchRecord837 + +from .result import SubmitOutcome, SubmitResult + +log = logging.getLogger(__name__) + + +def submit_file( + path: Path, + *, + sftp_block, + actor: str, + validate: bool = True, + sftp_client_factory=None, +) -> SubmitResult: + """Submit one 837P file: parse → DB write → SFTP upload. + + Args: + path: local 837P file. + sftp_block: the clearhouse SftpBlock (for paths + auth). + actor: audit-log actor tag (e.g. "api-submit-batch"). + validate: parse the file before upload (default True). Catches + bad byte-fixes early. + sftp_client_factory: optional callable that returns an + SftpClient-compatible object. Defaults to the real + SftpClient(block=sftp_block). Tests inject a fake. + + Returns: + SubmitResult with file, outcome, batch_id (when written), + error (when failed). + """ + file_label = path.name + try: + content = path.read_bytes() + except OSError as exc: + return SubmitResult(file_label, SubmitOutcome.PARSE_FAILED, error=str(exc)) + + # 1. Validate via parse. + if validate: + try: + parsed = parse_837_text(content.decode(), PayerConfig.co_medicaid()) + except Exception as exc: # noqa: BLE001 + log.warning("submit_file %s: parse failed: %s", file_label, exc) + return SubmitResult(file_label, SubmitOutcome.PARSE_FAILED, error=str(exc)) + mismatch = next( + (c for c in parsed.claims if c.payer.id != "CO_TXIX"), + None, + ) + if mismatch is not None: + return SubmitResult( + file_label, + SubmitOutcome.PAYER_MISMATCH, + error=f"payer.id={mismatch.payer.id!r} (expected 'CO_TXIX')", + ) + + # 2. DB write — DB-first, upload-second invariant. + record = BatchRecord837( + id=cycl_store._new_batch_id(), # or however existing code generates ids + input_filename=file_label, + parsed_at=parsed.parsed_at if validate else None, + result=parsed if validate else None, + ) + try: + cycl_store.add_record(record) + except Exception as exc: # noqa: BLE001 + log.warning("submit_file %s: DB write failed: %s", file_label, exc) + return SubmitResult(file_label, SubmitOutcome.DB_FAILED, error=str(exc)) + + # 3. SFTP upload. + remote_path = f"{sftp_block.paths['outbound']}/{file_label}" + factory = sftp_client_factory or (lambda b: SftpClient(block=b)) + sftp = factory(sftp_block) + local_size = len(content) + try: + try: + stat = sftp.stat(remote_path) + if stat.st_size == local_size: + return SubmitResult(file_label, SubmitOutcome.SKIPPED, batch_id=record.id) + except (IOError, OSError): + pass # not on remote yet + sftp.write_file(remote_path, content) + except Exception as exc: # noqa: BLE001 + log.warning("submit_file %s: SFTP failed: %s", file_label, exc) + return SubmitResult(file_label, SubmitOutcome.SFTP_FAILED, batch_id=record.id, error=str(exc)) + + # 4. Audit event. + cycl_store.append_clearhouse_submitted_event( + source_file=file_label, batch_id=record.id, actor=actor, + ) + return SubmitResult(file_label, SubmitOutcome.SUBMITTED, batch_id=record.id) +``` + +(Adapt the helpers you don't have — `cycl_store._new_batch_id` may not exist; check `cyclone/store/__init__.py` for the canonical ID generator, and `cycl_store.append_clearhouse_submitted_event` may need to be added if it doesn't exist. Mirror what `resubmit_rejected_claims` already does for these two operations.) + +- [ ] **Step 7: Run the happy-path test, verify it passes** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_submission.py::test_submit_file_happy_path -v` +Expected: PASS. + +- [ ] **Step 8: Add the remaining unit tests** + +Append to `backend/tests/test_submission.py`: + +```python +def test_submit_file_idempotent_db(monkeypatch): + """Re-running submit_file on the same file is a no-op on the DB side.""" + fake = _FakeSftp() + monkeypatch.setattr(SftpClient, "stat", fake.stat) + monkeypatch.setattr(SftpClient, "write_file", fake.write_file) + sftp_block = MagicMock(stub=False, paths={"outbound": "/ToHPE"}) + factory = lambda _b: fake + + r1 = submit_file(_FIXTURE, sftp_block=sftp_block, actor="t", sftp_client_factory=factory) + r2 = submit_file(_FIXTURE, sftp_block=sftp_block, actor="t", sftp_client_factory=factory) + + # First call SUBMITTED, second call SKIPPED (size match on stat). + assert r1.outcome == SubmitOutcome.SUBMITTED + assert r2.outcome == SubmitOutcome.SKIPPED + + +def test_submit_file_parse_fail(monkeypatch, tmp_path): + bad = tmp_path / "bad.x12" + bad.write_bytes(b"not x12") + sftp_block = MagicMock(stub=False, paths={"outbound": "/ToHPE"}) + fake = _FakeSftp() + factory = lambda _b: fake + + result = submit_file(bad, sftp_block=sftp_block, actor="t", sftp_client_factory=factory) + assert result.outcome == SubmitOutcome.PARSE_FAILED + assert len(fake.put_calls) == 0 + + +def test_submit_file_db_fail(monkeypatch): + """If add_record raises, no SFTP call is made (DB-first invariant).""" + from cyclone import store as cycl_store + + fake = _FakeSftp() + monkeypatch.setattr(cycl_store, "add_record", MagicMock(side_effect=RuntimeError("db down"))) + + sftp_block = MagicMock(stub=False, paths={"outbound": "/ToHPE"}) + factory = lambda _b: fake + + result = submit_file(_FIXTURE, sftp_block=sftp_block, actor="t", sftp_client_factory=factory) + assert result.outcome == SubmitOutcome.DB_FAILED + assert len(fake.put_calls) == 0 + + +def test_submit_file_sftp_fail(monkeypatch): + """SFTP failure after DB write → SFTP_FAILED outcome, DB row still present.""" + from cyclone import store as cycl_store + + class _Boom: + def stat(self, _p): raise IOError("nope") + def write_file(self, _p, _c): raise RuntimeError("sftp down") + + monkeypatch.setattr(cycl_store, "append_clearhouse_submitted_event", lambda **kw: None) + sftp_block = MagicMock(stub=False, paths={"outbound": "/ToHPE"}) + factory = lambda _b: _Boom() + + result = submit_file(_FIXTURE, sftp_block=sftp_block, actor="t", sftp_client_factory=factory) + assert result.outcome == SubmitOutcome.SFTP_FAILED + # DB row was written before the SFTP call. + assert result.batch_id is not None + with db_mod.SessionLocal()() as s: + assert s.get(db_mod.Batch, result.batch_id) is not None +``` + +- [ ] **Step 9: Run all submission tests** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_submission.py -v` +Expected: 5 passed. + +- [ ] **Step 10: Commit** + +```bash +git add backend/src/cyclone/submission/ backend/tests/test_submission.py backend/tests/fixtures/submit-batch/ +git commit -m "feat(sp37): cyclone.submission.submit_file helper + +Owns the parse → DB write → SFTP upload sequence in one place. CLI +and HTTP thin-call it. DB-first invariant: if add_record raises, no +SFTP call is made. Idempotency: add_record dedupes via s.get(Claim, +claim_id); SFTP layer dedupes via stat().st_size match." +``` + +- [ ] **Step 11: Autoreview (per the operator's standing directive)** + +Spawn a `pr-reviewer` subagent against the staged diff: + +``` +Subagent prompt: "Review the staged diff in +/home/tyler/dev/cyclone for SP37 Task 4 — the new cyclone.submission +package and its submit_file helper. Check (a) no orphan imports, +(b) no leaked business logic outside the helper, (c) the DB-first +ordering is preserved (no SFTP call when add_record raises), +(d) idempotency is correct on both layers, (e) audit event matches +the existing clearhouse.submitted shape. Return PASS or FAIL with +specific findings." +``` + +(Per `superpowers:pr-reviewer` — uses the `pr-reviewer` subagent type with the staged diff as input.) + +- [ ] **Step 12: Update tracker** + +Append Task 4 section with pre/post pytest, test result, reviewer verdict. + +--- + +## Task 5: `cyclone submit-batch` CLI + +**Files:** +- Modify: `backend/src/cyclone/cli.py` (add new `@main.command("submit-batch")` near `resubmit_rejected_claims`) + +- [ ] **Step 1: Write a smoke test** + +The CLI is hard to unit-test (Click + db init + walker), so the test is a subprocess invocation. Append to `backend/tests/test_submission.py`: + +```python +import subprocess +import sys + + +def test_submit_batch_cli_help(): + """`cyclone submit-batch --help` exits 0 and shows the flags.""" + result = subprocess.run( + [sys.executable, "-m", "cyclone.cli", "submit-batch", "--help"], + capture_output=True, text=True, cwd=str(Path(__file__).parent.parent), + ) + assert result.returncode == 0 + assert "--ingest-dir" in result.stdout +``` + +- [ ] **Step 2: Run test, verify it fails** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_submission.py::test_submit_batch_cli_help -v` +Expected: FAIL with `No such command 'submit-batch'` or non-zero exit. + +- [ ] **Step 3: Add the Click command** + +Modify `backend/src/cyclone/cli.py`. Insert near `resubmit_rejected_claims` (around line 539). Mirror its structure — same flags, same exit codes per `cyclone-cli`. Sketch: + +```python +@main.command("submit-batch") +@click.option("--ingest-dir", default="ingest", + show_default=True, + help="Root dir holding batch-*-claims/ subfolders of .x12 files.") +@click.option("--actor", default="cli-submit-batch", + show_default=True, + help="Audit-log actor tag for the clearhouse.submitted events.") +@click.option("--validate/--no-validate", default=True, + help="Parse each file via parse_837 before upload.") +@click.option("--limit", type=int, default=None, + help="Stop after N files (smoke-tests).") +def submit_batch(ingest_dir, actor, validate, limit): + """Parse → DB-write → SFTP-upload each batch-*-claims/*.x12 (SP37). + + Mirrors `resubmit-rejected-claims` but routes through the canonical + `cyclone.submission.submit_file` helper, which writes to the DB + before uploading. Use this as the preferred outbound path; the + legacy `resubmit-rejected-claims` remains for one-off cases where + you don't want a DB row (e.g., dzinesco-generated fixes that aren't + ready for canonical tracking). + + Exit codes: 0 = run completed (even with per-file failures), 1 = + unexpected exception, 2 = config-level failure (no clearhouse, + SFTP block in stub mode, ingest-dir missing). + """ + import sys + from pathlib import Path + + from cyclone import db as db_mod + from cyclone.clearhouse import SftpClient + from cyclone.submission.core import submit_file + from cyclone.submission.result import SubmitOutcome + from cyclone.store import store as cycl_store + + db_mod.init_db() + cycl_store.ensure_clearhouse_seeded() + clearhouse = cycl_store.get_clearhouse() + if clearhouse is None: + click.echo("No clearhouse seeded; cannot resolve SFTP block.", err=True) + sys.exit(2) + sftp_block = clearhouse.sftp_block + if sftp_block.stub: + click.echo("Clearhouse SFTP block is in stub mode; refusing to upload.", err=True) + sys.exit(2) + + root = Path(ingest_dir).resolve() + if not root.exists(): + click.echo(f"--ingest-dir does not exist: {root}", err=True) + sys.exit(2) + + files = [] + for batch_dir in sorted(root.glob("batch-*-claims")): + files.extend(sorted(p for p in batch_dir.glob("*.x12") if not p.name.startswith("._"))) + if not files: + click.echo(f"No .x12 files under {root}", err=True) + sys.exit(2) + + click.echo(f"found {len(files)} files under {root}") + + submitted = skipped = failed = 0 + for i, src in enumerate(files, 1): + if limit is not None and i > limit: + break + try: + result = submit_file( + src, sftp_block=sftp_block, actor=actor, validate=validate, + sftp_client_factory=lambda b: SftpClient(block=b), + ) + except Exception as exc: # noqa: BLE001 + click.echo(f"UNEXPECTED {src.name}: {exc.__class__.__name__}: {exc}", err=True) + failed += 1 + continue + if result.outcome == SubmitOutcome.SUBMITTED: + submitted += 1 + click.echo(f"submitted {src.name} batch={result.batch_id}") + elif result.outcome == SubmitOutcome.SKIPPED: + skipped += 1 + click.echo(f"skipped {src.name} (already uploaded)") + else: + failed += 1 + click.echo(f"{result.outcome.value:<10} {src.name} {result.error or ''}", err=True) + + click.echo(f"\nsummary: submitted={submitted} skipped={skipped} failed={failed}") + if failed: + sys.exit(0) # per-file failures don't bump exit code; check stdout +``` + +(Adapt the helper names — `cycl_store._new_batch_id` may not exist; check `store/__init__.py` for the canonical generator.) + +- [ ] **Step 4: Run the smoke test, verify it passes** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_submission.py::test_submit_batch_cli_help -v` +Expected: PASS. + +- [ ] **Step 5: Live-test against the stub** + +Run (with stub SFTP block, in this box's current posture): + +```bash +cd /home/tyler/dev/cyclone/backend +.venv/bin/python -m cyclone.cli submit-batch --ingest-dir /tmp/sp37-fixtures --limit 1 +``` + +Expected: `submitted single-claim.x12 batch=` (or `skipped` if re-run). + +Verify the DB row: `sqlite3 ~/.local/share/cyclone/cyclone.db "SELECT id, transaction_set_control_number FROM batches ORDER BY parsed_at DESC LIMIT 1;"` + +- [ ] **Step 6: Commit** + +```bash +git add backend/src/cyclone/cli.py backend/tests/test_submission.py +git commit -m "feat(sp37): `cyclone submit-batch` CLI + +Walks batch-*-claims/*.x12 under --ingest-dir, calls submit_file per +file, prints submitted/skipped/failed counts. Exits 0 even on +per-file failures (details in stdout); exits 2 on config-level +errors (no clearhouse, stub mode, missing dir)." +``` + +- [ ] **Step 7: Update tracker** + +Append Task 5 section. + +--- + +## Task 6: `POST /api/submit-batch` endpoint + +**Files:** +- Create: `backend/src/cyclone/api_routers/submission.py` +- Modify: `backend/src/cyclone/api_routers/__init__.py` (add to the routers list) +- Create: `backend/tests/test_api_submit_batch.py` + +- [ ] **Step 1: Read the existing router pattern** + +Read `backend/src/cyclone/api_routers/clearhouse.py` (smallest recent example of a router that talks to the clearhouse). Mirror its shape: Pydantic request/response models, auth-gated handler, `event_bus` access via `request.app.state.event_bus`. + +- [ ] **Step 2: Write the failing tests** + +Create `backend/tests/test_api_submit_batch.py`: + +```python +"""SP37 Task 6: POST /api/submit-batch endpoint.""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + + +_FIXTURE_DIR = Path(__file__).parent / "fixtures" / "submit-batch" + + +@pytest.fixture +def client(tmp_path, monkeypatch): + monkeypatch.setenv("CYCLONE_DB_URL", f"sqlite:///{tmp_path}/test.db") + monkeypatch.setenv("CYCLONE_AUTH_DISABLED", "1") # skip auth for unit test + + from cyclone import db as db_mod, store as cycl_store + db_mod._reset_for_tests() + db_mod.init_db() + cycl_store.ensure_clearhouse_seeded() + # Force real-SFTP flag off so the endpoint doesn't reject stub mode in tests. + cycl_store.update_clearhouse(sftp_block={"stub": True}) + + from cyclone.api import app + return TestClient(app) + + +def test_submit_batch_happy_path(client, tmp_path): + """3 valid 837 files → 200 + body shape with submitted=3.""" + # Copy 3 copies of the single-claim fixture into a batch-*-claims dir. + batch_dir = tmp_path / "batch-test-claims" + batch_dir.mkdir() + for i in range(3): + (batch_dir / f"claim-{i}.x12").write_bytes(_FIXTURE_DIR.joinpath("single-claim.x12").read_bytes()) + + resp = client.post("/api/submit-batch", json={"ingest_dir": str(tmp_path), "validate": False}) + assert resp.status_code == 200 + body = resp.json() + assert body["submitted"] == 3 or body["skipped"] == 3 # tolerate re-runs + assert isinstance(body["results"], list) + assert len(body["results"]) == 3 + + +def test_submit_batch_auth_gate(client): + """No cookie → 401 (matrix_gate).""" + # Reset CYCLONE_AUTH_DISABLED for this single test. + import os + os.environ.pop("CYCLONE_AUTH_DISABLED", None) + resp = client.post("/api/submit-batch", json={"ingest_dir": "/tmp"}) + assert resp.status_code == 401 + + +def test_submit_batch_missing_ingest_dir(client): + """Body without ingest_dir → 422.""" + resp = client.post("/api/submit-batch", json={"validate": False}) + assert resp.status_code == 422 +``` + +- [ ] **Step 3: Run test, verify it fails** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_api_submit_batch.py -v` +Expected: 3 FAIL (no router mounted yet). + +- [ ] **Step 4: Create the router** + +Create `backend/src/cyclone/api_routers/submission.py`. Mirror `clearhouse.py`'s pattern: + +```python +"""SP37 Task 6: HTTP endpoint for the canonical submit flow. + +Thin wrapper around ``cyclone.submission.submit_file`` — same logic +as the CLI, just different framing (JSON body in, JSON body out, +auth-gated by the existing matrix_gate). +""" +from __future__ import annotations + +import logging +from pathlib import Path + +from fastapi import APIRouter, HTTPException, Request +from pydantic import BaseModel + +from cyclone.submission.core import submit_file +from cyclone.submission.result import SubmitOutcome + +log = logging.getLogger(__name__) + +router = APIRouter(prefix="/api", tags=["submission"]) + + +class SubmitBatchRequest(BaseModel): + ingest_dir: str + validate: bool = True + actor: str = "api-submit-batch" + limit: int | None = None + + +@router.post("/submit-batch") +def post_submit_batch(body: SubmitBatchRequest, request: Request): + """Submit every batch-*-claims/*.x12 under ``ingest_dir``. + + Returns counts + per-file outcomes. Status code is always 200 on + a run that completes (per-file failures are in the body); 4xx + for auth/validation; 5xx for unexpected errors. + """ + from cyclone.clearhouse import SftpClient + from cyclone.store import store as cycl_store + + clearhouse = cycl_store.get_clearhouse() + if clearhouse is None: + raise HTTPException(status_code=409, detail="no clearhouse seeded") + sftp_block = clearhouse.sftp_block + if sftp_block.stub: + raise HTTPException(status_code=409, detail="clearhouse SFTP block is in stub mode") + + root = Path(body.ingest_dir).resolve() + if not root.exists(): + raise HTTPException(status_code=422, detail=f"ingest_dir does not exist: {root}") + + files = [] + for batch_dir in sorted(root.glob("batch-*-claims")): + files.extend(sorted(p for p in batch_dir.glob("*.x12") if not p.name.startswith("._"))) + + if body.limit is not None: + files = files[: body.limit] + + results = [] + submitted = skipped = failed = 0 + for src in files: + try: + r = submit_file( + src, + sftp_block=sftp_block, + actor=body.actor, + validate=body.validate, + sftp_client_factory=lambda b: SftpClient(block=b), + ) + except Exception as exc: # noqa: BLE001 + log.exception("submit-batch unexpected error on %s", src) + r = type("R", (), {"file": src.name, "outcome": "unexpected", "batch_id": None, "error": str(exc)})() + results.append({ + "file": r.file, + "outcome": r.outcome.value if hasattr(r.outcome, "value") else str(r.outcome), + "batch_id": r.batch_id, + "error": r.error, + }) + failed += 1 + continue + + if r.outcome == SubmitOutcome.SUBMITTED: + submitted += 1 + elif r.outcome == SubmitOutcome.SKIPPED: + skipped += 1 + else: + failed += 1 + + results.append({ + "file": r.file, + "outcome": r.outcome.value, + "batch_id": r.batch_id, + "error": r.error, + }) + + return { + "submitted": submitted, + "skipped": skipped, + "failed": failed, + "results": results, + } +``` + +- [ ] **Step 5: Mount the router** + +Modify `backend/src/cyclone/api_routers/__init__.py`. Add the import + entry to the `routers` list (exact shape depends on the file — read it first). + +```python +from .submission import router as submission_router +# ... add submission_router to the routers list +``` + +- [ ] **Step 6: Run tests, verify they pass** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest tests/test_api_submit_batch.py -v` +Expected: 3 passed (assuming you set up the test fixture correctly). + +- [ ] **Step 7: Live test** + +Restart the dev server (`python -m cyclone serve` in one terminal). With `CYCLONE_AUTH_DISABLED=1`, login is bypassed: + +```bash +curl -X POST http://localhost:8000/api/submit-batch \ + -H 'Content-Type: application/json' \ + -d '{"ingest_dir": "/tmp/sp37-fixtures", "validate": false}' +``` + +Expected: `{"submitted": 1, "skipped": 0, "failed": 0, "results": [{"file": "...", "outcome": "submitted", ...}]}` + +Verify the live-tail picks it up: open the Claims page in the UI, watch the new row appear. + +- [ ] **Step 8: Commit** + +```bash +git add backend/src/cyclone/api_routers/submission.py backend/src/cyclone/api_routers/__init__.py backend/tests/test_api_submit_batch.py +git commit -m "feat(sp37): POST /api/submit-batch endpoint + +Thin wrapper around cyclone.submission.submit_file with JSON +in/out. Auth-gated by matrix_gate. Same body shape as the CLI +summary (submitted/skipped/failed counts + per-file results). +Returns 200 on completed runs (per-file failures in body); 401 on +no auth; 422 on validation; 409 on clearhouse/stub-mode errors." +``` + +- [ ] **Step 9: Update tracker** + +Append Task 6 section. + +--- + +## Task 7: Integration + merge + +**Files:** +- (no code changes — verification + merge) + +- [ ] **Step 1: Run the full pytest** + +Run: `cd /home/tyler/dev/cyclone/backend && .venv/bin/pytest -x --tb=short -q 2>&1 | tail -5` +Expected: count line. Compare against `/tmp/refactor-pre-baseline.txt` from Task 0. Acceptable: same baseline ± rate-limit-noise (per Task 17 SP36 finding — failures pass in isolation). + +- [ ] **Step 2: Run the full vitest** + +Run: `cd /home/tyler/dev/cyclone && npm test 2>&1 | tail -5` +Expected: count line. Pre-existing baseline (no UI changes; should be unchanged). + +- [ ] **Step 3: Run the typecheck + lint** + +```bash +cd /home/tyler/dev/cyclone && npm run typecheck && npm run lint +``` +Expected: clean. + +- [ ] **Step 4: Live-test the join-key fix end-to-end** + +With the dev server running and `clm-1` already in the DB (or a fresh test DB), drop a synthetic 999 whose AK201 set_control_number matches the ST02 of an already-submitted batch. Run `pull-inbound`. Verify the ClaimAck link row is created (not an orphan). + +Use the existing helper for synthesizing 999s — `cyclone.submission.test_helpers.synth_999(set_control_number=...)` (add this helper to the test file if it doesn't exist). + +- [ ] **Step 5: pr-reviewer (per the operator's standing directive)** + +Spawn a `pr-reviewer` subagent against the full SP37 diff (all 7 commits on `sp37-submit-batch-canonical-flow`): + +``` +Subagent prompt: "Review the SP37 diff in +/home/tyler/dev/cyclone (branch sp37-submit-batch-canonical-flow, 7 +commits) for the canonical submit-batch flow. Check: (a) migration is +additive and reversible, (b) join-key update preserves backward compat, +(c) submit_file ordering (DB-first, upload-second) is invariant, (d) +idempotency is correct on both DB and SFTP layers, (e) audit events +match existing shape, (f) auth gate is wired, (g) no leaked business +logic in API/CLI wrappers, (h) tests cover happy path + idempotency + +each failure mode. Return PASS or FAIL with specific findings." +``` + +- [ ] **Step 6: Update RUNBOOK + CLAUDE.md** + +Document the canonical flow: + +- `docs/RUNBOOK.md`: add a "Submitting claims (canonical)" section showing `cyclone submit-batch --ingest-dir ingest/` and `POST /api/submit-batch`. Note that `resubmit-rejected-claims` remains for one-off cases. +- `CLAUDE.md`: under "Backend at a glance", add one bullet pointing at `cyclone/submission/` and the canonical flow. + +- [ ] **Step 7: Commit doc updates** + +```bash +git add docs/RUNBOOK.md CLAUDE.md +git commit -m "docs(sp37): document canonical submit-batch flow + +RUNBOOK gets a 'Submitting claims' section with both CLI and HTTP +examples. CLAUDE.md gets a pointer to cyclone/submission/ and the +preferred outbound path." +``` + +- [ ] **Step 8: Merge to main** + +Per `cyclone-spec` — single atomic merge commit, no squash, no rebase: + +```bash +git checkout main +git merge --no-ff sp37-submit-batch-canonical-flow \ + -m "merge: SP37 canonical submit-batch flow into main" +git push origin main +git branch -d sp37-submit-batch-canonical-flow +git push origin --delete sp37-submit-batch-canonical-flow +``` + +- [ ] **Step 9: Final tracker update** + +Append the "[SP37 final state]" section to `/tmp/refactor-cyclone.md` with: total commits on the branch (7 + doc update = 8), pytest delta (pre vs post), reviewer verdict, merge commit SHA, and the open follow-ups (backfill command, deprecation timeline, 277CA cross-link). + +--- + +## Self-Review + +**Spec coverage:** + +- Spec §1 Scope (CLI + HTTP + helper + new column + join-key update): Tasks 1, 2, 3, 4, 5, 6 cover it. +- Spec §2 Decisions (locked brainstorming choices): enforced throughout; CLI/API mirror the canonical submit shape, DB-first ordering is in Task 4 Step 6, new column is Tasks 1+2, additive deprecation is implicit (no tasks touch `parse-837` or `resubmit-rejected-claims`). +- Spec §3 Architecture (cyclone.submission package, submit_file algorithm, idempotency, failure modes): Task 4 owns the package; idempotency is Steps 3, 7, 8; failure modes are tests in Step 8. +- Spec §4 Data flow (request/response shapes, exit codes, audit events, migration shape, join-key update): Task 6 owns HTTP shapes; Task 5 owns exit codes; Task 1 owns the migration; Task 3 owns the join-key update. +- Spec §5 Testing (7 unit + 3 integration + 1 migration + 1 join-key + live test): spread across Tasks 1-7. +- Spec §6 Threat model (no new attack surface): implicit (no new auth bypass, no new SFTP path, no new SQL surface). +- Spec §7 Risks (3 named): each one addressed in the relevant task — Task 4 keeps `submit_file` as a pure helper (Risk 1), Task 4 docstring calls out layering (Risk 1), Task 1 Step 3 keeps the backfill to a single UPDATE (Risk 3), live-tail event volume is documented in Task 7 Step 6 (Risk 4). +- Spec §8 Rollout (6-step order, live-test cadence, merge shape): Tasks 1-6 follow it; Task 7 owns the merge. +- Spec §9 Open questions: NOT in this plan — they're explicitly follow-up SPs per the spec. + +**Placeholder scan:** No TBD/TODO/FIXME in any task. Every code block is complete. Every step has a concrete action. + +**Type consistency:** + +- `SubmitResult`, `SubmitOutcome` defined in Task 4 Step 2; used in Tasks 4, 5, 6 with identical attribute names (`file`, `outcome`, `batch_id`, `error`). +- `submit_file` signature in Task 4 Step 6; called with identical kwargs in Tasks 5 and 6. +- `Batch.transaction_set_control_number` introduced in Task 1 (migration) + Task 2 (ORM); used in Task 3 (index); not referenced elsewhere. No name drift. + +**Gaps found during review:** None. All spec requirements have a task. The follow-up items (backfill command, deprecation, 277CA) are explicitly out of scope per the spec. \ No newline at end of file diff --git a/docs/superpowers/specs/2026-07-07-cyclone-submit-batch-canonical-flow-design.md b/docs/superpowers/specs/2026-07-07-cyclone-submit-batch-canonical-flow-design.md new file mode 100644 index 0000000..7873a8b --- /dev/null +++ b/docs/superpowers/specs/2026-07-07-cyclone-submit-batch-canonical-flow-design.md @@ -0,0 +1,217 @@ +# Sub-project 37 — Canonical `submit-batch` flow: Design Spec + +**Date:** 2026-07-07 +**Status:** Draft, awaiting user sign-off +**Branch:** `sp37-submit-batch-canonical-flow` (off main, post-SP36 merge) +**Aesthetic direction:** No UI changes. Pure backend structural + data-model addition — the public HTTP surface gains one new endpoint; existing CLIs and endpoints are preserved unchanged. + +--- + +## 1. Scope + +Today, cyclone's `claims` table is essentially empty in production (`clm-1` is the only row as of 2026-07-07). The reason is structural: dzinesco generates the 837P files (in `ingest/corrected/batch-*-claims/*.x12`), cyclone's `resubmit-rejected-claims` CLI only validates and re-uploads them to Gainwell's MFT — it never writes the claims to the DB. When 999 acknowledgments come back referencing those batches, every AK2 set-response becomes an orphan because there is no `Batch` row to join against. + +The 999→claim join (the D10 two-pass join in `claim_acks.lookup_claims_for_ack_set_response`) has two paths: + +- **Pass 1 (primary):** `Batch.envelope.control_number == 999's set_control_number`. This currently fails because the 999's `set_control_number` (AK201) echoes the source 837's **ST02** (transaction set control number), not the **ISA13** (interchange control number). The parsed `Envelope` model stores ISA13 in `control_number`, so the keys never collide even when the DB row exists. +- **Pass 2 (fallback):** `Claim.patient_control_number == 999's set_control_number`. This also fails because the `_claim_837_row` ORM builder sets `patient_control_number = claim.claim_id` (CLM01), and CLM01 ≠ ST02 in production batches. + +The fix is twofold: (a) make cyclone write a `Batch` + `Claim` rows at submission time so there is something to join against, and (b) make the join key actually match — by storing the ST02 explicitly on `Batch` and using it in Pass 1. + +**In scope:** + +- A new top-level CLI `cyclone submit-batch` that does parse → DB write → SFTP upload per file, in that order. +- A new HTTP endpoint `POST /api/submit-batch` with the same semantics. +- A shared helper module `cyclone.submission.submit_file` so the CLI and HTTP path share one source of truth. +- A new nullable column `Batch.transaction_set_control_number`, populated from the parsed 837's ST02 on every `add_record` write. +- An Alembic migration `0013_add_batch_txn_set_control_number.py` (additive, nullable, backfills from `raw_result_json.envelope.transaction_set_control_number` where present). +- An update to `claim_acks.batch_envelope_index` so Pass 1 also matches on `transaction_set_control_number` (Pass 2 unchanged). +- Per-claim `claim_submitted` activity events and per-file `clearhouse.submitted` audit events (same shape `resubmit-rejected-claims` already emits). + +**Out of scope (explicit):** + +- Deprecating `parse-837` CLI or `/api/parse-837` endpoint. They remain for one-off testing and dry-run validation. +- Deprecating `resubmit-rejected-claims`. It remains the "validate-then-upload without DB write" path for the dzinesco-generated files that aren't ready for canonical tracking. +- UI changes. The Claims page already renders the fields a new tracked claim needs; the existing `claim_written` live-tail event fires automatically from `add_record`. +- Bulk backfill of the existing `ingest/corrected/batch-*-claims/*.x12` backlog. A follow-up SP will walk that directory once `submit-batch` is proven. +- Any change to the `claims` row schema beyond the new `Batch` column. +- SFTP auth work. This box stays in manual mode (post-2026-07-07 incident); `submit-batch` works in stub mode for local testing and against a whitelisted IP for production. + +## 2. Decisions (locked during brainstorming) + +These were each a multiple-choice question with operator sign-off on 2026-07-07. + +1. **Where does the DB write happen?** New canonical `submit` flow. `submit-batch` parses → writes to DB → uploads, replacing neither `parse-837` nor `resubmit-rejected-claims` but superseding them as the preferred path. +2. **DB vs upload ordering?** DB-first, upload-second. If DB write fails, the file never goes on the wire. If DB write succeeds but upload fails, re-running is safe (idempotent DB write, idempotent SFTP put). +3. **Join key for 999 acks?** Add `Batch.transaction_set_control_number` column. Update Pass 1 to match on it. Don't repurpose `envelope.control_number` (preserves backward compat with existing joins) and don't force CLM01 == ST02 (Gainwell's existing 999s use different values). +4. **Deprecation posture?** Add `submit-batch`; preserve `parse-837` and `resubmit-rejected-claims`. Document `submit-batch` as preferred in CLAUDE.md + RUNBOOK. Future SP can deprecate if usage shifts. + +## 3. Architecture + +The new flow is a single, small public surface — one CLI command, one HTTP endpoint, one shared helper. The helper owns the parse → write → upload sequence and is the only place that knows about the order. + +### Module layout + +A new package `cyclone.submission` (under `backend/src/cyclone/submission/`) holds the shared helper: + +- `__init__.py` — re-exports `submit_file`, `SubmitResult`, `SubmitOutcome` +- `core.py` — `submit_file(path: Path, *, sftp_block, actor: str, event_bus=None, validate: bool = True) -> SubmitResult`. Owns its own DB session for the parse-write step, returns a result dataclass the caller inspects. +- `cli.py` — Click handler for `cyclone submit-batch`. Walks `--ingest-dir`, calls `submit_file` per file, aggregates counts, exits 0/1/2 per cyclone-cli convention. +- `api.py` — FastAPI handler for `POST /api/submit-batch`. Mounted in `api_routers/submission.py`. Calls `submit_file` per file. Auth-gated by the existing `matrix_gate`. + +### `submit_file` algorithm + +The walker globs `/batch-*-claims/*.x12` (mirrors `resubmit-rejected-claims` exactly — same pattern, same `_`-prefix skip for AppleDouble residue). For each matched file: + +1. Read bytes; if `validate=True`, run `parse_837_text(bytes, payer_cfg)`. On parse failure, record `{path, "parse_failed", exc}` and return without writing or uploading. +2. Run a payer-id check (must be `CO_TXIX`); on mismatch, record and return. +3. Call `store.add_record(record, event_bus=...)`. On DB failure, record and return (no SFTP call). +4. Build the remote path `{outbound_root}/{filename}`. Open an `SftpClient` (the seeded singleton). On SFTP connect failure, record and return — the DB row is already written; the re-run will skip it via idempotency. +5. `sftp_client.stat(remote_path)` — if `st_size == local_size`, mark as `skipped` (already uploaded; do not re-emit audit). Otherwise `sftp_client.write_file(remote_path, bytes)`. +6. On successful upload, emit one `clearhouse.submitted` audit event with `source_file`, `batch_id`, `actor`. + +### Idempotency + +Two layers, both per-file: + +- **DB layer:** `store.add_record` already does a `s.get(Claim, claim.claim_id)` check before insert. A re-parse of the same file logs "claim already exists; skipping" and continues. Re-running `submit-batch` on an already-tracked file is a no-op on the DB. +- **SFTP layer:** the `sftp.stat().st_size == local_size` check before `write_file` short-circuits already-uploaded files. + +### Failure modes summary + +- Parse fails → no DB write, no SFTP put, file counted as `failed`. +- Payer-id mismatch → same as parse fail. +- DB write fails → no SFTP put, file counted as `failed`. The DB session rolls back; no orphan rows. +- SFTP connect fails → DB row already committed; file counted as `failed`. Re-run is safe. +- SFTP stat succeeds + size match → counted as `skipped`; no audit event. +- SFTP write fails → file counted as `failed`. DB row already committed. Re-run is safe. +- Crash between DB write and SFTP put → DB row exists, file never landed. Re-run will see the size mismatch (remote doesn't have it) and upload. + +## 4. Data flow & error handling + +### `POST /api/submit-batch` + +Request body (JSON): +- `ingest_dir: string` — root directory holding `batch-*-claims/*.x12` subfolders (mirrors the CLI flag). +- `validate: bool = true` — pass-through to `submit_file`. +- `actor: string = "api-submit-batch"` — audit actor tag. +- `limit: int | null = null` — stop after N files (smoke test). + +Response body (200): +- `submitted: int` — files uploaded this run. +- `skipped: int` — files already on remote (size match). +- `failed: int` — files that hit parse / payer / DB / SFTP errors. +- `results: list<{file: string, outcome: "submitted"|"skipped"|"parse_failed"|"payer_mismatch"|"db_failed"|"sftp_failed", batch_id: string|null, error: string|null}>` — per-file detail. + +Status codes: +- `200` — run completed (may have failures; details in the response body). +- `401` — no auth cookie (matrix_gate). +- `422` — body validation failure. +- `500` — unexpected exception (caught by the existing `@app.exception_handler(Exception)`). + +### CLI exit codes (per `cyclone-cli` convention) + +- `0` — run completed, even with some per-file failures (the per-file detail is in stdout). +- `1` — unexpected exception (uncaught error in the walker or the helper itself). +- `2` — config-level failure (no clearhouse seeded, SFTP block in stub mode for non-stubbed invocation, ingest-dir doesn't exist). + +### Audit + live-tail events + +- `claim_submitted` (one per Claim row) — emitted by `add_record` already; no new code path. +- `clearhouse.submitted` (one per successful upload) — emitted by `submit_file` after the SFTP put succeeds. Same actor tag and payload shape as the current `resubmit-rejected-claims` path uses. + +### Migration `0013_add_batch_txn_set_control_number.py` + +- `op.add_column('batches', sa.Column('transaction_set_control_number', sa.String(32), nullable=True))`. +- Backfill: `UPDATE batches SET transaction_set_control_number = json_extract(raw_result_json, '$.envelope.transaction_set_control_number') WHERE raw_result_json IS NOT NULL AND json_extract(raw_result_json, '$.envelope.transaction_set_control_number') IS NOT NULL`. +- Reversible: drop column. +- The down migration is `op.drop_column('batches', 'transaction_set_control_number')`. + +### Join-key update in `claim_acks.batch_envelope_index` + +The existing function reads every `Batch` row whose `raw_result_json` has an envelope and returns `dict[str, str]` keyed by `envelope.control_number`. The updated version returns the **same shape** — a single `dict[str, str]` — but **populated from two columns**: + +- Each row contributes one entry keyed by `envelope.control_number` (preserved). +- Each row also contributes one entry keyed by `transaction_set_control_number` (new), when that column is non-NULL. + +If both keys resolve to the same value (Gainwell batches where the ST02 happens to equal the ISA13 — unusual but legal), the second insert is a no-op. If they differ, both keys point to the same `batch.id` — a single `idx.get(set_control_number)` lookup will match either one. The dict may grow by up to `len(batches)` new entries; callers that previously assumed `len(index) == len(batches)` are unaffected because that invariant was already loose (multiple ST02s can share one ISA13). + +`lookup_claims_for_ack_set_response` Pass 1 is unchanged at the call site — one `idx.get(set_control_number)` lookup that succeeds whether the source 837's ST02 matches a `transaction_set_control_number` row or an `envelope.control_number` row. Pass 2 (PCN) fires as today if both miss. + +## 5. Testing approach + +Per `cyclone-tests` skill: pytest under `backend/tests/`, two flavors — `test_api_*` (FastAPI integration) and `test_*` (pure unit). Frontend tests out of scope (no UI changes). + +### Unit tests (`backend/tests/test_submission.py`) + +- `submit_file happy path` — mock SFTP, real DB; assert one Batch + N Claim rows + one audit event. +- `submit_file idempotency on DB` — call `submit_file` twice with the same file; assert second call returns `skipped` (DB no-op). +- `submit_file idempotency on SFTP` — mock SFTP `stat` to return matching size; assert `skipped`, no second `write_file` call, no second audit event. +- `submit_file parse fail` — pass a bad 837; assert no DB write, no SFTP call, `parse_failed` outcome. +- `submit_file payer mismatch` — pass a valid 837 with wrong payer_id; assert no DB write, no SFTP call, `payer_mismatch` outcome. +- `submit_file DB fail` — monkey-patch `add_record` to raise; assert no SFTP call, `db_failed` outcome. +- `submit_file SFTP fail` — mock SFTP `write_file` to raise; assert `sftp_failed` outcome, DB row still present. +- `submit_file re-run after partial failure` — submit a file that fails SFTP, then submit again with SFTP working; assert second run uploads and emits audit. + +### Integration tests (`backend/tests/test_api_submit_batch.py`) + +- `POST /api/submit-batch` happy path — 3 valid 837 fixtures in `ingest_dir`; assert 200 + body shape. +- `POST /api/submit-batch` auth gate — no cookie → 401 (matrix_gate). +- `POST /api/submit-batch` body validation — missing `ingest_dir` → 422. + +### Migration tests (`backend/tests/test_migration_0013.py`) + +- Up + down on a fresh DB; column exists + is nullable; backfill populates the column from raw JSON. + +### Join-key update tests (`backend/tests/test_claim_acks_index.py`) + +- `batch_envelope_index` returns both maps; Pass 1 finds a claim by ST02 even when ISA13 differs; Pass 1 finds by ISA13 (backward compat); Pass 2 fallback still works. + +### Live test (after merge) + +Walk an actual `ingest/batch-*-claims/` directory via the CLI; verify: +- `claims` table has new rows with the expected `patient_control_number` (CLM01). +- `batches` table has new rows with `transaction_set_control_number` matching the parsed ST02. +- Live-tail `claim_written` events fire on the API. + +Then drop a synthetic 999 (with the matching AK2 set_control_number) into `ingest/`, run `pull-inbound`, verify the `ClaimAck` link row is created (not an orphan). + +## 6. Threat model (post-SP24 alignment) + +No new attack surface. The new endpoint and CLI: + +- Are gated by the existing `matrix_gate` (HTTP) and `CYCLONE_AUTH_DISABLED` env var. +- Use the same SFTP client and credential paths as `resubmit-rejected-claims` today. +- Write to the same store facade; no new SQL surface. +- Emit the same audit events with the same payload shape. + +The `transaction_set_control_number` column is operator-visible (it's in the raw `batches` row), but no auth boundary depends on it being absent. PII exposure is unchanged (the column carries the same value the 999 ack would have echoed back — already public-to-Gainwell). + +## 7. Risks & mitigations + +- **Risk:** The new `submission` package is the third "command runner" in the codebase (alongside `cli.py`'s standalone commands and `scheduler.py`). **Mitigation:** Keep `submit_file` as a pure helper with no FastAPI/Click dependencies; the API and CLI handlers are thin wrappers. Document the layering in `submission/__init__.py`'s module docstring. +- **Risk:** `resubmit-rejected-claims` and `submit-batch` will diverge over time (one writes to DB, one doesn't). **Mitigation:** Both call the same `submit_file` helper if `submit-batch`'s `--write-to-db` flag is set to true (default). Document the divergence point in RUNBOOK. +- **Risk:** Migration backfill could be slow on a large DB. **Mitigation:** `UPDATE ... WHERE json_extract(...) IS NOT NULL` is one pass; on a million-row DB this is sub-second. Index on `transaction_set_control_number` not added (low cardinality per batch; existing `ix_claims_patient_control_number` covers the hot path). +- **Risk:** Live-tail event volume doubles when `submit-batch` runs (one `claim_submitted` per claim in addition to the historical `claim_written`). **Mitigation:** This is desired behavior — the operator wants the activity feed to show submissions. Document in the RUNBOOK that submitting a 1000-claim batch will publish ~1000 events. + +## 8. Rollout + +- **Branch:** `sp37-submit-batch-canonical-flow`, off main as of `710104f`. +- **Implementation order:** + 1. Migration `0013` + backfill (additive, no behavior change yet). + 2. Update `claim_acks.batch_envelope_index` + Pass 1 lookup (no behavior change yet — the new map is built but unused until `submit-batch` writes rows). + 3. `cyclone/submission/core.py` helper + unit tests. + 4. CLI handler `cyclone submit-batch` + smoke test. + 5. HTTP handler `POST /api/submit-batch` + integration tests. + 6. Live test with one of the existing `ingest/batch-*-claims/*.x12` directories (will be in stub mode on this host). +- **Live test cadence:** per the user's standing directive — live-test after each significant step, run autoreview, commit. +- **Merge shape:** single atomic merge commit per SP-N convention. No squash. No rebase. +- **Post-merge:** RUNBOOK update for the canonical flow; CLAUDE.md update to mark `submit-batch` as preferred. + +## 9. Open questions + +None blocking the spec. Tracked for follow-up SPs: + +- **Backfill command** — once `submit-batch` is proven, a `cyclone backfill-claims` CLI that walks the existing `ingest/corrected/batch-*-claims/*.x12` backlog (without uploading — they're already on Gainwell's side) and writes the DB rows. This is what would un-orphan the 1491+ orphan 999 acks currently in the DB. +- **Deprecation timeline for `resubmit-rejected-claims`** — once `submit-batch` is the default, consider folding `resubmit-rejected-claims` into a `--no-write-to-db` flag on `submit-batch` to consolidate the surface. +- **Cross-link to 277CA acks** — same `transaction_set_control_number` join key will resolve 277CA orphans if/when those arrive. The migration and index update cover both 999 and 277CA since both use Pass 1. \ No newline at end of file