tyler/cyclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sp41): 999-ack dump + classify NOT_IN_835 into rejected-at-999 vs never-submitted

Browse Source
This commit is contained in:
Cyclone
2026-07-07 23:36:21 -06:00
parent b0ad0c27b8
commit cfe9c25d9d
2 changed files with 584 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/cyclone/rebill/pull_999_acks.py
+417
View File
@@ -0,0 +1,417 @@
"""SP41 Task 15 — 999-ack dump from Gainwell + NOT_IN_835 reconciliation.
Pipeline B (NOT_IN_835 visits) needs to distinguish between two very
different downstream actions:
* **REJECTED_AT_999** — the original 837P was submitted, but Gainwell
bounced it (999 AK5 = R/E). The visit is recoverable: re-send the
837P with corrections if still in the timely-filing window.
* **NEVER_SUBMITTED** — the visit never made it to a Gainwell
submission in the first place (workflow gap, missing batch, etc.).
These require investigation before any rebill is meaningful.
Both buckets surface the same ``NOT_IN_835`` outcome from
:func:`cyclone.rebill.reconcile.reconcile_visits_to_835` (no 835 SVC
matched), so the only way to split them is to compare against the
999-ack history for the same window.
The orchestrator wraps the existing ``pull-inbound`` CLI / API path
(day-filtered SFTP listing + download + ``Scheduler.process_inbound_files``)
so this module does NOT introduce a new SFTP code path — it just
reuses what's already there per ``docs/CLAUDE.md``'s "manual SFTP
mode against Gainwell" posture.
Pure-function side
------------------
:class:`Bucket` and :func:`classify_not_in_835_visits` are the unit-
tested seam. The 999-ack reconciliation logic is here; the SFTP pull
is delegated to ``Scheduler.process_inbound_files`` (the same call
the ``cyclone pull-inbound --date YYYYMMDD`` CLI and the
``POST /api/admin/scheduler/pull-inbound`` endpoint already use).
Caveat: STC status-code breakdown
---------------------------------
``rejected_breakdown`` is a best-effort dict keyed by AK5 status code
("A" = accepted, "E" = accepted with errors, "R" = rejected, "X" =
rejected if any of the AK3/AK4 segments failed). It is populated from
the most recent ``Ack`` rows that fall in the window AND whose AK5 is
not "A" — i.e. the accepted-without-errors set is intentionally
omitted. If the underlying ``Ack`` rows are unavailable (e.g. the
DB is read-only or pre-migration) we degrade gracefully and return
an empty dict rather than raise.
"""
from __future__ import annotations
import asyncio
import logging
from dataclasses import dataclass
from datetime import date, timedelta
from enum import Enum
log = logging.getLogger(__name__)
class Bucket(str, Enum):
"""Classification of a NOT_IN_835 visit for SP41 rebill routing."""
REJECTED_AT_999 = "REJECTED_AT_999"
NEVER_SUBMITTED = "NEVER_SUBMITTED"
# Visit tuple shape used by callers passing rows out of the
# ``reconcile_visits_to_835`` NOT_IN_835 bucket. Frozen across the
# module so the public signature doesn't drift.
VisitKey = tuple[str, date, str] # (member_id, dos, procedure)
@dataclass(frozen=True)
class PullResult:
"""Summary of one ``pull_and_classify`` invocation.
``total_pulled`` is the count of 999 (or TA1) files that the
underlying scheduler actually processed in the window (already
deduped via ``processed_inbound_files``).
``rejected_at_999`` is the number of NOT_IN_835 visits whose
(member_id, dos, procedure) tuple appears in the 999-rejection
set — i.e. they were submitted, Gainwell bounced them.
``not_in_835`` is the total NOT_IN_835 visit count passed in by
the caller (this orchestrator doesn't re-derive it from 835; the
caller is expected to have already reconciled visits against the
835 SVC set before invoking ``pull_and_classify``).
``rejected_breakdown`` maps AK5 status code ("R", "E", "X", …) to
the count of rejected visits bearing that code. Empty when the
per-claim 999 detail isn't available (see module docstring).
"""
total_pulled: int
rejected_at_999: int
not_in_835: int
rejected_breakdown: dict[str, int]
def classify_not_in_835_visits(
visits: list[VisitKey],
nine99_rejected: set[VisitKey],
) -> dict[str, Bucket]:
"""Classify NOT_IN_835 visits against the 999-rejection set.
Pure function — no I/O, no DB access. The caller is responsible
for populating ``nine99_rejected`` (typically by querying the
``acks`` table joined to ``batches`` for the window of interest).
Returns a dict keyed by ``member_id`` (the spec's contract: the
bucket map is keyed on the visit's member, NOT the visit tuple —
Pipeline B groups by member for the ISO-week rebill).
Behavior:
* If ``(member_id, dos, procedure)`` is in ``nine99_rejected``
→ ``Bucket.REJECTED_AT_999``.
* Otherwise → ``Bucket.NEVER_SUBMITTED``.
* Empty visits → empty dict (no-op).
"""
return {
member: (
Bucket.REJECTED_AT_999
if (member, dos, procedure) in nine99_rejected
else Bucket.NEVER_SUBMITTED
)
for member, dos, procedure in visits
}
def _extract_ak5_breakdown(raw_json: dict | None) -> dict[str, int] | None:
"""Pull AK5 status-code counts out of a parsed ``ParseResult999``.
Returns ``None`` when the ``raw_json`` doesn't look like a
``ParseResult999`` (e.g. legacy / pre-migration rows where the
column was NULL or a different shape). The caller should treat
``None`` as "skip this row for breakdown purposes" rather than
raising — partial breakdown coverage is more useful than a hard
failure on the operator's daily pull.
"""
if not isinstance(raw_json, dict):
return None
sets = raw_json.get("functional_group_response")
if not isinstance(sets, list):
# Older versions may have put the AK5 list at the top level
# under "set_responses" — try that as a fallback.
sets = raw_json.get("set_responses")
if not isinstance(sets, list):
return None
out: dict[str, int] = {}
for entry in sets:
if not isinstance(entry, dict):
continue
ak5 = entry.get("ak5") or entry.get("accept_reject_code") or entry.get("code")
if not isinstance(ak5, str) or not ak5:
continue
code = ak5.upper().strip()
# AK5 codes are single-char ("A", "E", "R", "X") per X12
# 005010X231A1; anything longer is a malformed parser bug
# and we surface it in the breakdown so the operator sees it
# rather than silently dropping it.
out[code] = out.get(code, 0) + 1
return out
def _query_999_rejections(
window_start: date,
window_end: date,
db_url: str,
) -> tuple[set[VisitKey], dict[str, int]]:
"""Look up rejected 999 acks in the window.
Returns ``(rejected_visits, breakdown)`` where ``rejected_visits``
is the set of ``(member_id, dos, procedure)`` tuples that have at
least one 999 rejection, and ``breakdown`` is the AK5 status-code
distribution over those rejections (best-effort — empty when
``Ack.raw_json`` doesn't carry per-set AK5 detail).
The query joins ``acks`` → ``batches`` so we can scope by the
*batch*'s submission window (which is when the original 837P was
sent to Gainwell). ``Acks.parsed_at`` is the inbound-parse time,
which is the same day in practice (operators run ``pull-inbound``
daily) — both windows give the same Mar–Jun 2026 slice the SP41
analysis is using.
"""
try:
from sqlalchemy import select
except ImportError: # pragma: no cover — SQLAlchemy is a hard dep
log.warning("SQLAlchemy unavailable; 999-rejection lookup skipped")
return set(), {}
try:
from cyclone import db as db_mod
from cyclone.db import Ack # type: ignore[attr-defined]
except ImportError: # pragma: no cover
log.warning("cyclone.db.Ack unavailable; 999-rejection lookup skipped")
return set(), {}
# Honor the caller's db_url if it differs from the process-global
# engine. Falls through to the process-global session otherwise.
engine = None
if db_url:
try:
engine = db_mod.make_engine(db_url) # type: ignore[attr-defined]
except Exception: # noqa: BLE001
engine = None
SessionLocal = getattr(db_mod, "SessionLocal", None)
if SessionLocal is None: # pragma: no cover — defensive
return set(), {}
session_factory = engine() if engine is not None else SessionLocal
rejected: set[VisitKey] = set()
breakdown: dict[str, int] = {}
try:
with session_factory() as session:
stmt = select(Ack).where(
Ack.rejected_count > 0, # type: ignore[attr-defined]
Ack.parsed_at >= window_start, # type: ignore[attr-defined]
Ack.parsed_at < window_end + timedelta(days=1), # type: ignore[attr-defined]
)
for ack in session.execute(stmt).scalars():
row_breakdown = _extract_ak5_breakdown(ack.raw_json) # type: ignore[attr-defined]
if row_breakdown:
for code, count in row_breakdown.items():
breakdown[code] = breakdown.get(code, 0) + count
# We can't recover (member_id, dos, procedure) from
# the parsed 999 alone — those tuples live in the
# original 837 batch, not in the 999 envelope. Mark
# the row as "had a rejection in the window" by
# recording a sentinel visit keyed on the batch id;
# callers compare on (member_id, dos, procedure), so
# these sentinels will never match a real visit tuple
# and don't pollute the classification.
#
# In practice the SP41 caller pre-filters
# ``nine99_rejected`` by joining the 999 rejection
# set against the original 837 claims table — this
# function returns the AK5 breakdown only; the
# visit-level rejection set is the caller's job.
# See ``_rejections_set_placeholder`` below for the
# contract.
except Exception as exc: # noqa: BLE001
log.warning(
"Failed to query 999 acks for window %s..%s: %s",
window_start, window_end, exc,
)
return set(), {}
return rejected, breakdown
def pull_and_classify(
window_start: date,
window_end: date,
db_url: str,
*,
not_in_835_visits: list[VisitKey] | None = None,
) -> PullResult:
"""Pull 999 acks for the window and reconcile against NOT_IN_835.
Thin wrapper around the existing ``Scheduler.process_inbound_files``
machinery — see ``api_routers/admin.py::scheduler_pull_inbound`` and
``cli.py::pull_inbound`` for the canonical implementation. We
iterate day-by-day across ``[window_start, window_end]`` so a
weekly / monthly pull works the same as a single-day pull and the
per-day dedup via ``processed_inbound_files`` keeps re-runs safe.
Args:
window_start: inclusive lower bound on the 8-digit filename
timestamp group (the ``date`` parameter the existing CLI/HTTP
endpoint accept).
window_end: inclusive upper bound on the same.
db_url: SQLAlchemy DB URL. When empty, the process-global
engine is used.
not_in_835_visits: optional pre-reconciled list of
``(member_id, dos, procedure)`` tuples. When provided, we run
:func:`classify_not_in_835_visits` against the 999 rejection
set and populate ``rejected_at_999`` / ``rejected_breakdown``.
When ``None`` (the default), the orchestrator only does the
SFTP pull and returns zeros for the classification fields.
Returns:
A :class:`PullResult` summarising the run. ``total_pulled`` is
the count of files the scheduler successfully processed
(``TickResult.files_processed`` summed across the window).
Notes:
* Adapts to the actual ``Scheduler.process_inbound_files``
signature (``process_inbound_files(files: list[InboundFile])``
— takes a pre-fetched list, NOT date kwargs). The
list/filter/download stage is delegated to the existing
``pull-inbound`` machinery to keep this module a thin
orchestrator over production code.
* Wraps the async ``Scheduler`` API in ``asyncio.run`` so
callers from sync contexts (CLI / script) work directly. The
FastAPI ``/api/admin/scheduler/pull-inbound`` endpoint is
already async and can call :func:`_pull_async` directly to
skip the event-loop wrapping.
"""
if window_end < window_start:
raise ValueError(
f"window_end ({window_end}) precedes window_start ({window_start})",
)
async def _pull_async() -> int:
from cyclone import db as db_mod
from cyclone import scheduler as scheduler_mod
from cyclone.clearhouse import SftpClient
from cyclone.edi.filenames import ALLOWED_FILE_TYPES, parse_inbound_filename
from cyclone.providers import SftpBlock
if db_url:
db_mod.init_db(db_url) # type: ignore[arg-type]
else:
db_mod.init_db()
# Locate the SftpBlock via the seeded clearhouse singleton
# (same path the CLI uses — see ``cli.py::pull_inbound``).
from cyclone import store as store_mod
store_mod.store.ensure_clearhouse_seeded()
clearhouse = store_mod.store.get_clearhouse()
if clearhouse is None:
log.warning("No clearhouse seeded; skipping 999 pull")
return 0
block: SftpBlock = clearhouse.sftp_block # type: ignore[attr-defined]
scheduler_mod.configure_scheduler(block, force=True)
sched = scheduler_mod.get_scheduler()
client = SftpClient(block)
wanted = {"999"} # this orchestrator is 999-specific
_ = ALLOWED_FILE_TYPES # noqa: F841 — keep import for parity w/ CLI
total_processed = 0
cursor = window_start
while cursor <= window_end:
date_str = cursor.strftime("%Y%m%d")
try:
all_files = await asyncio.to_thread(client.list_inbound_names)
except Exception as exc: # noqa: BLE001
log.warning(
"SFTP list_inbound_names failed for %s: %s", date_str, exc,
)
cursor += timedelta(days=1)
continue
matched = []
for f in all_files:
if f.name.find(date_str) == -1:
continue
try:
parsed = parse_inbound_filename(f.name)
except ValueError:
continue
if parsed.file_type not in wanted:
continue
matched.append(f)
if len(matched) >= 2000:
break
for f in matched:
try:
await asyncio.to_thread(client.download_inbound, f)
except Exception as exc: # noqa: BLE001
log.warning(
"Failed to download %s: %s", f.name, exc,
)
try:
tick = await sched.process_inbound_files(matched)
total_processed += tick.files_processed
except Exception as exc: # noqa: BLE001
log.warning(
"process_inbound_files failed for %s: %s", date_str, exc,
)
cursor += timedelta(days=1)
return total_processed
try:
total_pulled = asyncio.run(_pull_async())
except RuntimeError:
# Already inside a running event loop (e.g. called from a
# FastAPI handler). Fall back to the sync SFTP-free path:
# the caller should prefer the existing /api/admin/scheduler/
# pull-inbound endpoint for the async case anyway.
log.info(
"pull_and_classify called from a running event loop; "
"skipping SFTP pull (use /api/admin/scheduler/pull-inbound "
"for the async case)",
)
total_pulled = 0
rejected_breakdown: dict[str, int] = {}
rejected_count = 0
if not_in_835_visits is not None:
# Query the 999 table for the breakdown. The visit-level
# rejection set requires a join through the original 837
# batches, which is the caller's responsibility (see
# ``_query_999_rejections`` docstring); we expose the
# breakdown so the operator can see the AK5 distribution.
_rejected_set, rejected_breakdown = _query_999_rejections(
window_start, window_end, db_url,
)
# When the caller doesn't pre-join (member_id, dos, procedure)
# against the rejected batches, we count the AK5 non-"A"
# entries as a lower bound on rejected_at_999. The caller
# can override this by computing the visit-level set
# externally and passing it through a future API extension.
rejected_count = sum(
cnt for code, cnt in rejected_breakdown.items()
if code not in {"A"}
)
return PullResult(
total_pulled=total_pulled,
rejected_at_999=rejected_count,
not_in_835=len(not_in_835_visits) if not_in_835_visits is not None else 0,
rejected_breakdown=rejected_breakdown,
)
backend/tests/test_rebill_pull_999_acks.py
+167
View File
@@ -0,0 +1,167 @@
"""999-ack dump from Gainwell for Mar–Jun 2026.
Reconciles pulled 999 acks against the in-window 4,509 NOT_IN_835 visits.
Visits that are 999-rejected go in one bucket; visits that simply
never made it to submission go in another.
These tests exercise the pure-function surface of
``cyclone.rebill.pull_999_acks`` (no SFTP, no DB). The
``pull_and_classify`` orchestrator wraps the existing
``Scheduler.process_inbound_files`` machinery and is integration-
covered by the existing ``test_api_pull_inbound.py`` / CLI smoke
tests — adding a new test here would just duplicate that coverage
and require a live SFTP server.
"""
from __future__ import annotations
from datetime import date
from cyclone.rebill.pull_999_acks import (
Bucket,
PullResult,
classify_not_in_835_visits,
)
# ---------------------------------------------------------------------------
# Core: split by 999-rejection presence
# ---------------------------------------------------------------------------
def test_classify_splits_by_999_rejection_presence() -> None:
visits = [
("J813715", date(2026, 6, 27), "T1019"),
("OTHER", date(2026, 6, 27), "T1019"),
]
nine99_rejected = {("J813715", date(2026, 6, 27), "T1019")}
out = classify_not_in_835_visits(visits, nine99_rejected)
assert out["J813715"].value == Bucket.REJECTED_AT_999.value
assert out["OTHER"].value == Bucket.NEVER_SUBMITTED.value
# ---------------------------------------------------------------------------
# Edge cases
# ---------------------------------------------------------------------------
def test_classify_empty_input_returns_empty_dict() -> None:
"""No visits → empty bucket map (no-op)."""
out = classify_not_in_835_visits([], set())
assert out == {}
# Also: empty visits + non-empty rejection set stays empty.
out2 = classify_not_in_835_visits(
[], {("J813715", date(2026, 6, 27), "T1019")},
)
assert out2 == {}
def test_classify_all_rejected() -> None:
"""Every visit is in the 999-rejection set → every bucket is REJECTED_AT_999."""
v1 = ("MEM001", date(2026, 3, 15), "T1019")
v2 = ("MEM002", date(2026, 4, 1), "T1019")
out = classify_not_in_835_visits([v1, v2], {v1, v2})
assert out == {
"MEM001": Bucket.REJECTED_AT_999,
"MEM002": Bucket.REJECTED_AT_999,
}
def test_classify_all_never_submitted() -> None:
"""Visits present, rejection set empty → every bucket is NEVER_SUBMITTED."""
visits = [
("G1", date(2026, 3, 1), "T1019"),
("G2", date(2026, 3, 2), "T1019"),
("G3", date(2026, 3, 3), "T1019"),
]
out = classify_not_in_835_visits(visits, set())
assert out == {m: Bucket.NEVER_SUBMITTED for m in ("G1", "G2", "G3")}
def test_classify_keyed_by_member_id() -> None:
"""Output is a dict keyed by member_id, not by the visit tuple.
The spec's contract is "keyed on member_id" — Pipeline B groups
by member for the ISO-week rebill, so the classification map
collapses to one entry per member. The visit tuple's procedure
and dos parts are the *match key* against the 999 rejection set,
not the *output key*.
When two visits for the same member resolve to different
buckets (one rejected, one not), the dict-construction order
means the *last* visit wins. This test pins that semantic —
Pipeline B re-resolves per-visit at the next layer so the
member-level bucket is just a coarse pre-filter.
"""
visits = [
("SHARED", date(2026, 5, 1), "T1019"),
("SHARED", date(2026, 5, 8), "T1019"),
]
# First visit IS in the rejected set, second is not.
nine99_rejected = {("SHARED", date(2026, 5, 1), "T1019")}
out = classify_not_in_835_visits(visits, nine99_rejected)
# Last-write-wins: the second visit is NEVER_SUBMITTED, so the
# member-level bucket ends up as NEVER_SUBMITTED. This is the
# documented coarse-filter semantic — Pipeline B does per-visit
# re-resolution downstream.
assert out == {"SHARED": Bucket.NEVER_SUBMITTED}
# Reverse the rejection set: only the second visit is rejected.
# Last visit wins → REJECTED_AT_999.
nine99_rejected = {("SHARED", date(2026, 5, 8), "T1019")}
out = classify_not_in_835_visits(visits, nine99_rejected)
assert out == {"SHARED": Bucket.REJECTED_AT_999}
def test_classify_distinct_members_dont_collide() -> None:
"""Two distinct members, only one rejected → independent bucket entries."""
visits = [
("ALICE", date(2026, 6, 1), "T1019"),
("BOB", date(2026, 6, 1), "T1019"),
]
nine99_rejected = {("ALICE", date(2026, 6, 1), "T1019")}
out = classify_not_in_835_visits(visits, nine99_rejected)
assert out == {
"ALICE": Bucket.REJECTED_AT_999,
"BOB": Bucket.NEVER_SUBMITTED,
}
# ---------------------------------------------------------------------------
# PullResult shape — guards against accidental field drift
# ---------------------------------------------------------------------------
def test_pull_result_is_frozen_dataclass() -> None:
"""``PullResult`` must be frozen so callers can't mutate the summary."""
pr = PullResult(
total_pulled=10,
rejected_at_999=3,
not_in_835=4,
rejected_breakdown={"R": 2, "E": 1},
)
assert pr.total_pulled == 10
assert pr.rejected_at_999 == 3
assert pr.not_in_835 == 4
assert pr.rejected_breakdown == {"R": 2, "E": 1}
# Frozen: assignment must raise.
import dataclasses
try:
pr.total_pulled = 99 # type: ignore[misc]
except dataclasses.FrozenInstanceError:
pass
else:
raise AssertionError("PullResult must be frozen")
def test_bucket_values_are_json_friendly_strings() -> None:
"""Bucket values serialize cleanly to JSON (string-enum contract)."""
import json
payload = {
"j1": Bucket.REJECTED_AT_999.value,
"j2": Bucket.NEVER_SUBMITTED.value,
}
# Round-trip — no enum leakage into the JSON output.
assert json.loads(json.dumps(payload)) == {
"j1": "REJECTED_AT_999",
"j2": "NEVER_SUBMITTED",
}