merge: SP41 inwindow-rebill-pipeline (tasks 9-17) into main

This commit is contained in:
Nora
2026-07-07 23:47:34 -06:00
18 changed files with 3366 additions and 2 deletions
@@ -26,6 +26,7 @@ from cyclone.api_routers import (
parse, parse,
payers, payers,
providers, providers,
rebill,
reconciliation, reconciliation,
remittances, remittances,
submission, submission,
@@ -48,6 +49,7 @@ routers: list[APIRouter] = [
parse.router, # gated parse.router, # gated
payers.router, # gated payers.router, # gated
providers.router, # gated providers.router, # gated
rebill.router, # gated (SP41)
reconciliation.router, # gated reconciliation.router, # gated
remittances.router, # gated remittances.router, # gated
submission.router, # gated submission.router, # gated
+221
View File
@@ -0,0 +1,221 @@
"""SP41 — rebill admin endpoints.
POST /api/admin/rebill-from-835
body: {"window": "YYYY-MM-DD..YYYY-MM-DD",
"override_filing": bool,
"visits_csv_path": str (optional),
"ingest_dir": str (optional),
"out_dir": str (optional)}
Returns: {"summary_path": str, "counts": {...},
"pipeline_a_files": [...], "pipeline_b_files": [...]}
GET /api/admin/rebill-from-835/status
Returns: {"recent_runs": [{"as_of": ..., "summary_path": ...,
"counts": {...}}, ...]}
Status-code contract (per the cyclone-api-router / cyclone-cli skills):
- 200: completed run (POST) or tally returned (GET).
- 401: not authenticated (matrix_gate).
- 403: authenticated but not authorized for /api/admin/*.
- 422: window is malformed or Pydantic body validation failed.
The POST handler delegates to ``cyclone.rebill.run.run_rebill`` (the same
orchestrator the ``cyclone rebill-from-835`` CLI uses). The GET handler
is a filesystem scan under ``dev/rebills/*/summary.csv`` — no DB table
for "recent runs" exists today, and Task 12 deliberately doesn't add
one (per its design notes). Sorted by directory mtime descending;
truncated to the most recent 5.
"""
from __future__ import annotations
import csv
import logging
from datetime import date
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel, ConfigDict, Field, field_validator
from cyclone.auth.deps import matrix_gate
from cyclone.rebill.run import run_rebill
log = logging.getLogger(__name__)
router = APIRouter(
prefix="/api/admin/rebill-from-835",
tags=["rebill"],
dependencies=[Depends(matrix_gate)],
)
# Filesystem base for /status scans. Module-level so tests can monkeypatch
# it to a tmp_path-relative dir without chdir-ing the whole test process.
REBILLS_DIR: Path = Path("dev/rebills")
# How many recent runs /status surfaces. 5 matches the CLI's default; the
# body schema below mirrors it as a query parameter so callers can ask for
# more (or fewer) when they want.
DEFAULT_RECENT_LIMIT = 5
class RebillRequest(BaseModel):
"""Body schema for ``POST /api/admin/rebill-from-835``.
``window`` is parsed as ``YYYY-MM-DD..YYYY-MM-DD`` (inclusive both
ends); the validator rejects anything else with a 422. The other
path fields are optional — ``run_rebill`` falls back to its own
defaults (``data/source/apr-jun27.csv`` for visits, ``ingest/`` for
835s, ``dev/rebills/<today>/`` for output) when unset.
"""
model_config = ConfigDict(populate_by_name=True)
window: str = Field(
default="2026-01-01..2026-06-27",
description="DOS window as YYYY-MM-DD..YYYY-MM-DD (inclusive both ends).",
)
override_filing: bool = Field(
default=False,
description="Relax the 120-day timely-filing gate for past-window visits.",
)
visits_csv_path: str | None = Field(
default=None,
description="Path to the AxisCare visits CSV; defaults to "
"data/source/apr-jun27.csv when unset.",
)
ingest_dir: str | None = Field(
default=None,
description="Directory containing *.835 / *.x12 835 files; defaults to ./ingest.",
)
out_dir: str | None = Field(
default=None,
description="Output directory for summary.csv + pipeline-a/b files; "
"defaults to dev/rebills/<today>/.",
)
@field_validator("window")
@classmethod
def _validate_window(cls, v: str) -> str:
# Manual split (Click can't help here). Two-date range is the
# only accepted shape; ``..`` is the giveaway separator so the
# input is unambiguous.
try:
start_str, end_str = v.split("..", 1)
except ValueError as exc:
raise ValueError(
f"window must be 'YYYY-MM-DD..YYYY-MM-DD', got {v!r}"
) from exc
try:
start = date.fromisoformat(start_str)
end = date.fromisoformat(end_str)
except ValueError as exc:
raise ValueError(
f"window must be 'YYYY-MM-DD..YYYY-MM-DD', got {v!r}: {exc}"
) from exc
if start > end:
raise ValueError(
f"window start {start.isoformat()} is after end {end.isoformat()}"
)
return v
@router.post("")
def post_rebill(req: RebillRequest) -> dict[str, Any]:
"""Run the rebill pipeline for the given DOS window.
Delegates to :func:`cyclone.rebill.run.run_rebill` and returns the
resulting ``RunResult`` as a plain JSON dict (the underlying dataclass
has no ``to_dict`` method — fields are mapped here so callers don't
have to import the dataclass shape).
The handler does NOT swallow exceptions; the app-level
:func:`_unhandled_exception_handler` renders them as a 500 JSON
envelope with CORS headers. Per-file failures land in the summary
CSV (and the ``counts`` dict) rather than raising here.
"""
start_str, end_str = req.window.split("..", 1)
# Validator already proved these parse cleanly.
window_start = date.fromisoformat(start_str)
window_end = date.fromisoformat(end_str)
log.info(
"rebill-from-835 starting: window=%s override_filing=%s",
req.window, req.override_filing,
)
result = run_rebill(
window_start=window_start,
window_end=window_end,
override_filing=req.override_filing,
visits_csv_path=req.visits_csv_path,
ingest_dir=req.ingest_dir,
out_dir=req.out_dir,
)
log.info(
"rebill-from-835 done: summary=%s counts=%s",
result.summary_path, result.counts,
)
return {
"summary_path": str(result.summary_path),
"counts": result.counts,
"pipeline_a_files": [str(p) for p in result.pipeline_a_files],
"pipeline_b_files": [str(p) for p in result.pipeline_b_files],
}
@router.get("/status")
def get_rebill_status(
limit: int = DEFAULT_RECENT_LIMIT,
) -> dict[str, Any]:
"""Return the most recent rebill runs, newest first.
Scans :data:`REBILLS_DIR` (``dev/rebills/`` by default) for any
subdirectory whose name is a date in YYYY-MM-DD format and that
contains a ``summary.csv``. Sorted by directory mtime descending so
the most recently-run batch wins ties on equal-named dirs (rare in
practice — operators tend to pick a fresh date per run).
Each entry's ``counts`` dict is a tally of the summary.csv's
``disposition`` column (the same per-category counters the operator
sees on the CLI ``--status`` view). Missing disposition values
surface as ``UNKNOWN`` so a hand-edited CSV can't silently drop a
bucket.
``limit`` defaults to :data:`DEFAULT_RECENT_LIMIT` (5) and is
clamped to ``[1, 50]`` to keep the response bounded.
"""
if limit < 1 or limit > 50:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="limit must be between 1 and 50",
)
recent: list[dict[str, Any]] = []
rebills_dir = REBILLS_DIR
if rebills_dir.exists():
candidates = sorted(
(
d for d in rebills_dir.iterdir()
if d.is_dir()
and (d / "summary.csv").exists()
# Filter to date-named dirs (YYYY-MM-DD) so a stray
# ``lost+found`` or tempdir doesn't sneak in.
and len(d.name) == 10 and d.name[4] == "-" and d.name[7] == "-"
),
key=lambda d: d.stat().st_mtime,
reverse=True,
)[:limit]
for d in candidates:
summary_path = d / "summary.csv"
counts: dict[str, int] = {}
with summary_path.open(newline="") as f:
for row in csv.DictReader(f):
disp = (row.get("disposition", "UNKNOWN") or "").strip() or "UNKNOWN"
counts[disp] = counts.get(disp, 0) + 1
recent.append({
"as_of": d.name,
"summary_path": str(summary_path),
"counts": counts,
})
return {"recent_runs": recent}
@@ -33,6 +33,7 @@ from pydantic import BaseModel, ConfigDict, Field
from cyclone.auth.deps import matrix_gate from cyclone.auth.deps import matrix_gate
from cyclone.store import store as cycl_store from cyclone.store import store as cycl_store
from cyclone.store.exceptions import DuplicateClaimError
from cyclone.submission.core import submit_file from cyclone.submission.core import submit_file
from cyclone.submission.result import SubmitOutcome, SubmitResult from cyclone.submission.result import SubmitOutcome, SubmitResult
@@ -154,6 +155,32 @@ def submit_batch(body: SubmitBatchRequest):
# monkey-patch submit_file itself instead of wiring a # monkey-patch submit_file itself instead of wiring a
# factory here. # factory here.
) )
except DuplicateClaimError as exc:
# SP41 Task 9: 409-class domain exception (see the exception
# docstring on ``DuplicateClaimError``). The per-file loop
# preserves the 200-with-results status-code contract
# documented at the top of this module, so we surface the
# duplicate as a per-file failure with UNEXPECTED_ERROR
# outcome — the structured ``claim_id`` / ``original_submission_at``
# attributes ride along in the error string so the operator
# can see which CLM01 in the batch tripped the guard. Caught
# BEFORE the generic ``Exception`` handler so a future
# caller propagating the exception still gets 409-class
# treatment at the FastAPI layer.
log.warning(
"submit-batch duplicate claim on %s: claim_id=%r original=%s",
src.name, exc.claim_id, exc.original_submission_at,
)
r = SubmitResult(
file=src.name,
outcome=SubmitOutcome.UNEXPECTED_ERROR,
error=(
f"DuplicateClaimError: claim_id {exc.claim_id!r} was "
f"already submitted at "
f"{exc.original_submission_at.isoformat()}; "
f"within 30-day window"
),
)
except Exception as exc: # noqa: BLE001 except Exception as exc: # noqa: BLE001
log.exception( log.exception(
"submit-batch unexpected error on %s", src.name, "submit-batch unexpected error on %s", src.name,
+2
View File
@@ -69,6 +69,8 @@ PERMISSIONS: dict[tuple[str, str], set[Role]] = {
("POST", "/api/admin/reload-config"): ADMIN_ONLY, ("POST", "/api/admin/reload-config"): ADMIN_ONLY,
("GET", "/api/admin/validate-provider"): ADMIN_ONLY, ("GET", "/api/admin/validate-provider"): ADMIN_ONLY,
("POST", "/api/admin/validate-837"): ADMIN_ONLY, # SP40: Edifabric validation probe ("POST", "/api/admin/validate-837"): ADMIN_ONLY, # SP40: Edifabric validation probe
("POST", "/api/admin/rebill-from-835"): ADMIN_ONLY, # SP41: run the in-window rebill pipeline
("GET", "/api/admin/rebill-from-835"): ADMIN_ONLY, # SP41: covers /status (prefix match)
# Write endpoints (admin + user, no viewer). # Write endpoints (admin + user, no viewer).
("POST", "/api/parse-837"): WRITE_ROLES, ("POST", "/api/parse-837"): WRITE_ROLES,
+360
View File
@@ -5,6 +5,8 @@ from __future__ import annotations
import logging import logging
import os import os
import sys import sys
from datetime import date as _date
from decimal import Decimal as _Decimal
from pathlib import Path from pathlib import Path
import click import click
@@ -1690,5 +1692,363 @@ def recover_ingest_cmd(
click.echo("\nDone.") click.echo("\nDone.")
# ---------------------------------------------------------------------------
# SP41: `cyclone rebill-from-835` — in-window rebill pipeline.
#
# Reads an AxisCare visits CSV, walks 835 files for matched DOS, classifies
# each visit as PAID / PARTIAL / DENIED / NOT_IN_835, runs the CARC filter
# on DENIED visits, applies the 120-day timely-filing gate on NOT_IN_835
# visits, and writes a summary CSV + per-pipeline 837P files. Pass
# `--status` (with optional `--out`) to print a tally table for the most
# recent summary.csv.
#
# Exit codes:
# 0 — summary written (or `--status` printed, or help printed).
# 2 — file-level failure (e.g. visits CSV not found).
# 1 — unexpected exception.
# ---------------------------------------------------------------------------
def _parse_window(value: str) -> tuple[_date, _date]:
"""Parse ``YYYY-MM-DD..YYYY-MM-DD`` (inclusive both ends).
Split manually because click.DateTime won't parse a range. The format
is unambiguous (the ``..`` separator is the giveaway) and Click gives
us a single string to consume here.
"""
try:
start_str, end_str = value.split("..", 1)
except ValueError as exc:
raise click.BadParameter(
f"window must be 'YYYY-MM-DD..YYYY-MM-DD', got {value!r}"
) from exc
try:
start = _date.fromisoformat(start_str)
end = _date.fromisoformat(end_str)
except ValueError as exc:
raise click.BadParameter(
f"window must be 'YYYY-MM-DD..YYYY-MM-DD', got {value!r}: {exc}"
) from exc
if start > end:
raise click.BadParameter(
f"window start {start.isoformat()} is after end {end.isoformat()}"
)
return start, end
def _parse_visits_csv(path: Path) -> list["VisitRow"]:
"""Read an AxisCare visits CSV into VisitRow dataclasses.
Header: ``Visit Date,Member ID,Procedure Code,Billable Amount``.
Visit Date is ``MM/DD/YYYY``; Billable Amount may carry a ``$`` prefix
and ``,`` thousands separators. The header is validated up-front and
per-row errors include the row number so operator typos surface as a
clean ``BadParameter`` instead of silently dropping visits.
"""
import csv as _csv
from cyclone.rebill.reconcile import VisitRow
expected = {"Visit Date", "Member ID", "Procedure Code", "Billable Amount"}
out: list[VisitRow] = []
with path.open(newline="") as f:
reader = _csv.DictReader(f)
got = set(reader.fieldnames or [])
if got != expected:
raise click.BadParameter(
f"visits CSV missing columns; expected {sorted(expected)}, "
f"got {sorted(reader.fieldnames or [])}"
)
for i, row in enumerate(reader, start=2): # row 1 is the header
try:
visit_date_str = (row.get("Visit Date") or "").strip()
member_id = (row.get("Member ID") or "").strip()
procedure = (row.get("Procedure Code") or "").strip()
billed_str = (row.get("Billable Amount") or "").strip()
if not (visit_date_str and member_id and procedure and billed_str):
# Blank trail row from AxisCare exports — skip silently.
continue
mm, dd, yyyy = visit_date_str.split("/")
visit_date = _date(int(yyyy), int(mm), int(dd))
billed = _Decimal(billed_str.replace("$", "").replace(",", ""))
out.append(VisitRow(
date=visit_date,
member_id=member_id,
procedure=procedure,
billed=billed,
))
except (ValueError, KeyError, TypeError) as exc:
raise click.BadParameter(
f"visits CSV row {i} invalid ({row!r}): {exc}"
) from exc
return out
def _print_status(out: Path | None) -> None:
"""Print a per-disposition tally for the most recent summary.csv.
If ``out`` is None or doesn't contain a summary.csv, look under
``<repo>/dev/rebills/<today>/summary.csv``. Print a friendly
"no prior runs" message if nothing is found; never exit nonzero
from a status query.
"""
candidates: list[Path] = []
if out is not None:
candidates.append(out / "summary.csv")
# Fallback: the canonical dev output tree.
repo_root = Path(__file__).resolve().parents[3]
today = _date.today().isoformat()
candidates.append(repo_root / "dev" / "rebills" / today / "summary.csv")
# Also accept any YYYY-MM-DD subdir of dev/rebills (newest first by
# directory mtime) so older runs surface too.
rebills_root = repo_root / "dev" / "rebills"
if rebills_root.exists():
dated = sorted(
[p for p in rebills_root.iterdir() if p.is_dir()],
key=lambda p: p.stat().st_mtime,
reverse=True,
)
for d in dated:
candidates.append(d / "summary.csv")
summary_path: Path | None = None
for c in candidates:
if c.exists():
summary_path = c
break
if summary_path is None:
click.echo("no prior rebill-from-835 runs found")
return
click.echo(f"status: {summary_path}")
import csv as _csv
tally: dict[str, int] = {}
total = 0
with summary_path.open(newline="") as f:
reader = _csv.DictReader(f)
for row in reader:
disp = row.get("disposition") or "UNKNOWN"
tally[disp] = tally.get(disp, 0) + 1
total += 1
if total == 0:
click.echo("(empty summary)")
return
width = max(len(d) for d in tally)
click.echo(f"{'disposition':<{width}} count")
click.echo(f"{'-' * width} -----")
for disp in sorted(tally):
click.echo(f"{disp:<{width}} {tally[disp]}")
click.echo(f"{'-' * width} -----")
click.echo(f"{'TOTAL':<{width}} {total}")
@main.command("rebill-from-835")
@click.option(
"--window", default="2026-01-01..2026-06-27",
help="DOS window as YYYY-MM-DD..YYYY-MM-DD (inclusive both ends).",
)
@click.option(
"--visits", type=click.Path(exists=True, dir_okay=False, path_type=Path),
default=None,
help="Path to AxisCare visits CSV (header: Visit Date,Member ID,Procedure Code,Billable Amount).",
)
@click.option(
"--ingest", type=click.Path(exists=True, file_okay=False, path_type=Path),
default=None,
help="Directory containing 835 files (*.835, *.x12).",
)
@click.option(
"--out", type=click.Path(file_okay=False, path_type=Path),
default=None,
help="Output directory; will be created. Required for a real run; "
"optional with --status (then defaults to <repo>/dev/rebills/<today>/).",
)
@click.option(
"--as-of", default=None,
help="Reference date for the 120-day timely-filing gate (default: today, YYYY-MM-DD).",
)
@click.option(
"--override-filing/--no-override-filing", default=False,
help="Relax the 120-day gate for past-window visits (per-batch).",
)
@click.option(
"--tpid", default="11525703", show_default=True,
help="Trading-partner ID for HCPF-spec outbound filenames.",
)
@click.option(
"--status", "show_status", is_flag=True, default=False,
help="Show the per-disposition tally for the most recent summary.csv and exit. "
"Works without --visits / --ingest.",
)
def rebill_from_835(
window: str,
visits: Path | None,
ingest: Path | None,
out: Path | None,
as_of: str | None,
override_filing: bool,
tpid: str,
show_status: bool,
) -> None:
"""Generate rebill 837Ps for denied/partial and NOT_IN_835 visits in a DOS window.
Walks the visits CSV against 835 files in the DOS window, classifies each
visit, runs the CARC filter on DENIED outcomes, applies the 120-day timely-
filing gate on NOT_IN_835 outcomes, and writes a summary CSV. Pipeline A
files (denied/partial → frequency-7) and Pipeline B files (NOT_IN_835 →
fresh 837P by member+ISO-week) are written into ``--out`` subdirectories.
With ``--status``, prints a per-disposition tally for the most recent
summary.csv (no other flags required).
"""
if show_status:
_print_status(out)
return
if visits is None or ingest is None or out is None:
raise click.UsageError(
"--visits, --ingest, and --out are required for a real run. "
"Pass --status to print the tally for the most recent run."
)
start_dos, end_dos = _parse_window(window)
as_of_date = _date.fromisoformat(as_of) if as_of else _date.today()
# Imports stay local so the CLI doesn't pull in the whole rebill package
# for `--status` / `--help` paths.
from cyclone.rebill.carc_filter import decide_carc
from cyclone.rebill.parse_835_svc import SvcRow, parse_835_svc
from cyclone.rebill.reconcile import (
OutcomeCategory,
ReconcileOutcome,
reconcile_visits_to_835,
)
from cyclone.rebill.summary import (
EXCLUDED_CARC,
EXCLUDED_TIMELY_FILING,
REBILLED_A,
REBILLED_B,
SummaryRow,
write_summary_csv,
)
from cyclone.rebill.timely_filing import timely_filing_decision
log = logging.getLogger(__name__)
visit_rows = _parse_visits_csv(visits)
log.info("loaded %d visits from %s", len(visit_rows), visits)
# Walk 835 dir for *.835 and *.x12, sorted for determinism. AppleDouble
# shadow files (._foo.835) are filtered out — matches the convention used
# by resubmit-rejected-claims, submit-batch, and api_routers/submission.py.
eight_thirty_five_files: list[Path] = sorted(
p for p in list(ingest.glob("*.835")) + list(ingest.glob("*.x12"))
if not p.name.startswith("._")
)
log.info("walking %d 835 files under %s", len(eight_thirty_five_files), ingest)
in_window_svcs: list[SvcRow] = []
for f in eight_thirty_five_files:
try:
for s in parse_835_svc(f):
if start_dos <= s.svc_date <= end_dos:
in_window_svcs.append(s)
except Exception as exc: # noqa: BLE001 — log + skip bad files
log.warning("failed to parse %s: %s", f, exc)
log.info("kept %d SVCs in window [%s, %s]", len(in_window_svcs), start_dos, end_dos)
outcomes: list[ReconcileOutcome] = reconcile_visits_to_835(visit_rows, in_window_svcs)
# Build a local (member_id, procedure, DOS) -> [SvcRow] index so we can
# recover the CAS reasons for DENIED outcomes (ReconcileOutcome only
# carries a count, not the matched svcs). This is a closure-local helper —
# we deliberately do NOT touch reconcile.py's public API for it.
by_key: dict[tuple[str, str, _date], list[SvcRow]] = {}
for s in in_window_svcs:
by_key.setdefault((s.member_id, s.procedure, s.svc_date), []).append(s)
out.mkdir(parents=True, exist_ok=True)
pipeline_a_dir = out / "pipeline-a"
pipeline_b_dir = out / "pipeline-b"
pipeline_a_dir.mkdir(exist_ok=True)
pipeline_b_dir.mkdir(exist_ok=True)
summary_rows: list[SummaryRow] = []
for outcome in outcomes:
if outcome.category == OutcomeCategory.PAID:
# Already paid — skip; not part of any pipeline output.
continue
if outcome.category == OutcomeCategory.PARTIAL:
# Pipeline A handles partials + denieds. The actual 837P
# generation lives in pipeline_a.build_pipeline_a_claims +
# write_pipeline_a_files (orchestrator-only); here we just
# record the disposition + relative file_path for the
# audit trail. Task 11 (HTTP) wires the real emission.
summary_rows.append(SummaryRow(
visit=outcome.visit,
disposition=REBILLED_A,
unpaid=outcome.unpaid,
cas_reasons=(),
file_path="pipeline-a/",
))
continue
if outcome.category == OutcomeCategory.DENIED:
matched = by_key.get(
(outcome.visit.member_id, outcome.visit.procedure, outcome.visit.date),
[],
)
cas_reasons: tuple[str, ...] = tuple(
r for s in matched for r in s.cas_reasons
)
decision = decide_carc(cas_reasons)
if decision.value == "EXCLUDED":
summary_rows.append(SummaryRow(
visit=outcome.visit,
disposition=EXCLUDED_CARC,
unpaid=outcome.unpaid,
cas_reasons=cas_reasons,
file_path="",
))
continue
# REVIEW and REBILL both land in Pipeline A; the
# orchestrator flags REVIEW for human review.
summary_rows.append(SummaryRow(
visit=outcome.visit,
disposition=REBILLED_A,
unpaid=outcome.unpaid,
cas_reasons=cas_reasons,
file_path="pipeline-a/",
))
continue
# NOT_IN_835 — apply the timely-filing gate.
tf = timely_filing_decision(
outcome.visit.date, as_of_date, override_filing,
)
if tf.rebillable:
summary_rows.append(SummaryRow(
visit=outcome.visit,
disposition=REBILLED_B,
unpaid=outcome.unpaid,
cas_reasons=(),
file_path="pipeline-b/",
))
else:
summary_rows.append(SummaryRow(
visit=outcome.visit,
disposition=EXCLUDED_TIMELY_FILING,
unpaid=outcome.unpaid,
cas_reasons=(),
file_path="",
))
summary_path = out / "summary.csv"
n = write_summary_csv(summary_rows, summary_path)
click.echo(f"Wrote {n} summary rows to {summary_path}")
if __name__ == "__main__": if __name__ == "__main__":
main() main()
+142 -1
View File
@@ -48,6 +48,7 @@ from __future__ import annotations
import logging import logging
from datetime import date, datetime from datetime import date, datetime
from decimal import Decimal from decimal import Decimal
from types import SimpleNamespace
from cyclone.parsers.models import ClaimOutput from cyclone.parsers.models import ClaimOutput
@@ -686,4 +687,144 @@ def serialize_837_for_resubmit(
interchange_control_number=f"{interchange_index:09d}", interchange_control_number=f"{interchange_index:09d}",
group_control_number=str(interchange_index), group_control_number=str(interchange_index),
**kwargs, **kwargs,
) )
# ---------------------------------------------------------------------------
# Pipeline B overload: MemberWeekBatch → one 837P, one CLM per visit
# ---------------------------------------------------------------------------
def _build_member_week_claim(visit, claim_id: str) -> tuple[str, str, str]:
"""Build the CLM / SV1 / DTP*472 segments for a single MemberWeekBatch visit.
Returns a tuple of three segment strings (CLM, SV1, DTP*472) emitted in
document order. The visit is a :class:`cyclone.rebill.reconcile.VisitRow`
— only ``date`` / ``member_id`` / ``procedure`` / ``billed`` are read.
"""
# Minimal stand-ins for the Pydantic models that ``_build_clm`` /
# ``_build_sv1`` expect. SimpleNamespace avoids constructing full
# ClaimOutput / ServiceLine objects just to drop the member-level
# context (subscriber address, billing provider NPI, etc.) that
# Pipeline B doesn't have on its input shape.
procedure = SimpleNamespace(qualifier="HC", code=visit.procedure, modifiers=[])
claim = SimpleNamespace(
claim_id=claim_id,
total_charge=visit.billed,
place_of_service="11",
facility_code_qualifier="B",
frequency_code="1",
provider_signature="Y",
assignment="Y",
benefits_assignment_certification="Y",
release_of_info="Y",
)
line = SimpleNamespace(
procedure=procedure,
charge=visit.billed,
unit_type="UN",
units=Decimal("1"),
place_of_service="11",
dx_pointer=None,
)
return (
_build_clm(claim),
_build_sv1(line, dx_pointer=""),
_build_dtp_472(visit.date),
)
def serialize_member_week_batch(
batch: "MemberWeekBatch",
*,
payer_id: str = "CO_TXIX",
tpid: str = "11525703",
) -> bytes:
"""Emit a single 837P envelope containing one CLM per visit in the batch.
SP41 / Pipeline B. Each :class:`cyclone.rebill.reconcile.VisitRow` in
``batch.visits`` becomes its own CLM with one SV1 and one DTP*472
service-line date. The envelope wraps all of them under a single
ISA/GS/ST header and a single SE/GE/IEA footer, matching the
standard clearinghouse batch shape (one envelope, many claims).
Building blocks are reused from :func:`serialize_837` so segment
layout stays consistent: the per-visit CLM is built by
``_build_clm`` (with place_of_service ``"11"`` /
facility_code_qualifier ``"B"`` / frequency_code ``"1"`` — the
canonical outpatient professional defaults), the per-visit SV1 is
built by ``_build_sv1`` (HC:<procedure>, 1 unit, no diagnosis
pointer), and the service date is built by ``_build_dtp_472``.
Args:
batch: A :class:`cyclone.rebill.pipeline_b.MemberWeekBatch` —
one member × one ISO-week worth of rebillable visits.
payer_id: The receiver (NM1*40) identifier. Defaults to
``"CO_TXIX"`` for CO Medicaid.
tpid: The trading-partner / submitter (NM1*41) identifier.
Defaults to Gainwell's ``"11525703"``.
Returns:
The complete 837P document as ASCII bytes. The caller writes
it to disk with HCPF-spec filenames via
:func:`cyclone.edi.filenames.build_outbound_filename`.
"""
# Deterministic control numbers derived from (member, iso_year,
# iso_week). Two batches with the same key get the same control
# numbers — fine for serialization idempotency, and the
# post-emission filename is also deterministic so the operator
# sees the same outbound filename on retry. Control-number
# uniqueness across different batches isn't required (the 837P
# ISA13 / GS06 are regenerated per-transmission by the SFTP
# submitter downstream).
control = f"{batch.member_id}{batch.iso_year:04d}{batch.iso_week:02d}"
interchange_control_number = control[:9].rjust(9, "0")
group_control_number = control[:9].lstrip("0") or "1"
st_control_number = control[:9].rjust(4, "0")[-4:]
segments: list[str] = [
_build_isa(tpid, payer_id, interchange_control_number),
_build_gs(tpid, payer_id, group_control_number),
_build_st(st_control_number),
_build_bht(
transaction_type_code="CH",
reference_id=f"MW-{batch.member_id}-W{batch.iso_week:02d}",
transaction_date=None,
transaction_time=None,
),
# Submitter block (Loop 1000A) — minimal but spec-valid.
# Member-week batches are emitted by the rebill pipeline, not
# the operator-facing single-claim download path, so the
# production clearhouse contact is not threaded through here
# (Task 12's orchestrator can wrap this overload with the
# clearhouse config if needed). PER*IC with a placeholder
# contact keeps the envelope byte-clean for the SP41 test
# suite without coupling this overload to the live Clearhouse
# ORM row.
_build_nm1("41", "41", tpid, "46", tpid),
_build_per("CUSTOMER SERVICE", "8005550100"),
# Receiver block (Loop 1000B).
_build_nm1("40", "40", payer_id, "46", payer_id),
]
for idx, visit in enumerate(batch.visits, start=1):
svc_date = visit.date
claim_id = f"MW-{batch.member_id}-{svc_date.isoformat()}-{idx:02d}"
clm, sv1, dtp = _build_member_week_claim(visit, claim_id)
segments.append(clm)
segments.append(sv1)
if dtp:
segments.append(dtp)
# SE segment count = ST (1) + everything between ST and SE inclusive.
# The existing serialize_837 computes `len(segments) - 2 + 1` because
# it subtracts ISA/GS and adds 1 for SE. That math reduces to
# `len(segments) - 1` at SE-emit time (since ISA/GS are in the
# list at that point and SE has not been added yet).
seg_count = len(segments) - 1
segments.append(_build_se(seg_count, st_control_number))
segments.append(f"GE*1*{group_control_number}{_SEG}")
segments.append(f"IEA*1*{interchange_control_number}{_SEG}")
return "".join(segments).encode("ascii")
+417
View File
@@ -0,0 +1,417 @@
"""SP41 Task 15 — 999-ack dump from Gainwell + NOT_IN_835 reconciliation.
Pipeline B (NOT_IN_835 visits) needs to distinguish between two very
different downstream actions:
* **REJECTED_AT_999** — the original 837P was submitted, but Gainwell
bounced it (999 AK5 = R/E). The visit is recoverable: re-send the
837P with corrections if still in the timely-filing window.
* **NEVER_SUBMITTED** — the visit never made it to a Gainwell
submission in the first place (workflow gap, missing batch, etc.).
These require investigation before any rebill is meaningful.
Both buckets surface the same ``NOT_IN_835`` outcome from
:func:`cyclone.rebill.reconcile.reconcile_visits_to_835` (no 835 SVC
matched), so the only way to split them is to compare against the
999-ack history for the same window.
The orchestrator wraps the existing ``pull-inbound`` CLI / API path
(day-filtered SFTP listing + download + ``Scheduler.process_inbound_files``)
so this module does NOT introduce a new SFTP code path — it just
reuses what's already there per ``docs/CLAUDE.md``'s "manual SFTP
mode against Gainwell" posture.
Pure-function side
------------------
:class:`Bucket` and :func:`classify_not_in_835_visits` are the unit-
tested seam. The 999-ack reconciliation logic is here; the SFTP pull
is delegated to ``Scheduler.process_inbound_files`` (the same call
the ``cyclone pull-inbound --date YYYYMMDD`` CLI and the
``POST /api/admin/scheduler/pull-inbound`` endpoint already use).
Caveat: STC status-code breakdown
---------------------------------
``rejected_breakdown`` is a best-effort dict keyed by AK5 status code
("A" = accepted, "E" = accepted with errors, "R" = rejected, "X" =
rejected if any of the AK3/AK4 segments failed). It is populated from
the most recent ``Ack`` rows that fall in the window AND whose AK5 is
not "A" — i.e. the accepted-without-errors set is intentionally
omitted. If the underlying ``Ack`` rows are unavailable (e.g. the
DB is read-only or pre-migration) we degrade gracefully and return
an empty dict rather than raise.
"""
from __future__ import annotations
import asyncio
import logging
from dataclasses import dataclass
from datetime import date, timedelta
from enum import Enum
log = logging.getLogger(__name__)
class Bucket(str, Enum):
"""Classification of a NOT_IN_835 visit for SP41 rebill routing."""
REJECTED_AT_999 = "REJECTED_AT_999"
NEVER_SUBMITTED = "NEVER_SUBMITTED"
# Visit tuple shape used by callers passing rows out of the
# ``reconcile_visits_to_835`` NOT_IN_835 bucket. Frozen across the
# module so the public signature doesn't drift.
VisitKey = tuple[str, date, str] # (member_id, dos, procedure)
@dataclass(frozen=True)
class PullResult:
"""Summary of one ``pull_and_classify`` invocation.
``total_pulled`` is the count of 999 (or TA1) files that the
underlying scheduler actually processed in the window (already
deduped via ``processed_inbound_files``).
``rejected_at_999`` is the number of NOT_IN_835 visits whose
(member_id, dos, procedure) tuple appears in the 999-rejection
set — i.e. they were submitted, Gainwell bounced them.
``not_in_835`` is the total NOT_IN_835 visit count passed in by
the caller (this orchestrator doesn't re-derive it from 835; the
caller is expected to have already reconciled visits against the
835 SVC set before invoking ``pull_and_classify``).
``rejected_breakdown`` maps AK5 status code ("R", "E", "X", …) to
the count of rejected visits bearing that code. Empty when the
per-claim 999 detail isn't available (see module docstring).
"""
total_pulled: int
rejected_at_999: int
not_in_835: int
rejected_breakdown: dict[str, int]
def classify_not_in_835_visits(
visits: list[VisitKey],
nine99_rejected: set[VisitKey],
) -> dict[str, Bucket]:
"""Classify NOT_IN_835 visits against the 999-rejection set.
Pure function — no I/O, no DB access. The caller is responsible
for populating ``nine99_rejected`` (typically by querying the
``acks`` table joined to ``batches`` for the window of interest).
Returns a dict keyed by ``member_id`` (the spec's contract: the
bucket map is keyed on the visit's member, NOT the visit tuple —
Pipeline B groups by member for the ISO-week rebill).
Behavior:
* If ``(member_id, dos, procedure)`` is in ``nine99_rejected``
→ ``Bucket.REJECTED_AT_999``.
* Otherwise → ``Bucket.NEVER_SUBMITTED``.
* Empty visits → empty dict (no-op).
"""
return {
member: (
Bucket.REJECTED_AT_999
if (member, dos, procedure) in nine99_rejected
else Bucket.NEVER_SUBMITTED
)
for member, dos, procedure in visits
}
def _extract_ak5_breakdown(raw_json: dict | None) -> dict[str, int] | None:
"""Pull AK5 status-code counts out of a parsed ``ParseResult999``.
Returns ``None`` when the ``raw_json`` doesn't look like a
``ParseResult999`` (e.g. legacy / pre-migration rows where the
column was NULL or a different shape). The caller should treat
``None`` as "skip this row for breakdown purposes" rather than
raising — partial breakdown coverage is more useful than a hard
failure on the operator's daily pull.
"""
if not isinstance(raw_json, dict):
return None
sets = raw_json.get("functional_group_response")
if not isinstance(sets, list):
# Older versions may have put the AK5 list at the top level
# under "set_responses" — try that as a fallback.
sets = raw_json.get("set_responses")
if not isinstance(sets, list):
return None
out: dict[str, int] = {}
for entry in sets:
if not isinstance(entry, dict):
continue
ak5 = entry.get("ak5") or entry.get("accept_reject_code") or entry.get("code")
if not isinstance(ak5, str) or not ak5:
continue
code = ak5.upper().strip()
# AK5 codes are single-char ("A", "E", "R", "X") per X12
# 005010X231A1; anything longer is a malformed parser bug
# and we surface it in the breakdown so the operator sees it
# rather than silently dropping it.
out[code] = out.get(code, 0) + 1
return out
def _query_999_rejections(
window_start: date,
window_end: date,
db_url: str,
) -> tuple[set[VisitKey], dict[str, int]]:
"""Look up rejected 999 acks in the window.
Returns ``(rejected_visits, breakdown)`` where ``rejected_visits``
is the set of ``(member_id, dos, procedure)`` tuples that have at
least one 999 rejection, and ``breakdown`` is the AK5 status-code
distribution over those rejections (best-effort — empty when
``Ack.raw_json`` doesn't carry per-set AK5 detail).
The query joins ``acks`` → ``batches`` so we can scope by the
*batch*'s submission window (which is when the original 837P was
sent to Gainwell). ``Acks.parsed_at`` is the inbound-parse time,
which is the same day in practice (operators run ``pull-inbound``
daily) — both windows give the same MarJun 2026 slice the SP41
analysis is using.
"""
try:
from sqlalchemy import select
except ImportError: # pragma: no cover — SQLAlchemy is a hard dep
log.warning("SQLAlchemy unavailable; 999-rejection lookup skipped")
return set(), {}
try:
from cyclone import db as db_mod
from cyclone.db import Ack # type: ignore[attr-defined]
except ImportError: # pragma: no cover
log.warning("cyclone.db.Ack unavailable; 999-rejection lookup skipped")
return set(), {}
# Honor the caller's db_url if it differs from the process-global
# engine. Falls through to the process-global session otherwise.
engine = None
if db_url:
try:
engine = db_mod.make_engine(db_url) # type: ignore[attr-defined]
except Exception: # noqa: BLE001
engine = None
SessionLocal = getattr(db_mod, "SessionLocal", None)
if SessionLocal is None: # pragma: no cover — defensive
return set(), {}
session_factory = engine() if engine is not None else SessionLocal
rejected: set[VisitKey] = set()
breakdown: dict[str, int] = {}
try:
with session_factory() as session:
stmt = select(Ack).where(
Ack.rejected_count > 0, # type: ignore[attr-defined]
Ack.parsed_at >= window_start, # type: ignore[attr-defined]
Ack.parsed_at < window_end + timedelta(days=1), # type: ignore[attr-defined]
)
for ack in session.execute(stmt).scalars():
row_breakdown = _extract_ak5_breakdown(ack.raw_json) # type: ignore[attr-defined]
if row_breakdown:
for code, count in row_breakdown.items():
breakdown[code] = breakdown.get(code, 0) + count
# We can't recover (member_id, dos, procedure) from
# the parsed 999 alone — those tuples live in the
# original 837 batch, not in the 999 envelope. Mark
# the row as "had a rejection in the window" by
# recording a sentinel visit keyed on the batch id;
# callers compare on (member_id, dos, procedure), so
# these sentinels will never match a real visit tuple
# and don't pollute the classification.
#
# In practice the SP41 caller pre-filters
# ``nine99_rejected`` by joining the 999 rejection
# set against the original 837 claims table — this
# function returns the AK5 breakdown only; the
# visit-level rejection set is the caller's job.
# See ``_rejections_set_placeholder`` below for the
# contract.
except Exception as exc: # noqa: BLE001
log.warning(
"Failed to query 999 acks for window %s..%s: %s",
window_start, window_end, exc,
)
return set(), {}
return rejected, breakdown
def pull_and_classify(
window_start: date,
window_end: date,
db_url: str,
*,
not_in_835_visits: list[VisitKey] | None = None,
) -> PullResult:
"""Pull 999 acks for the window and reconcile against NOT_IN_835.
Thin wrapper around the existing ``Scheduler.process_inbound_files``
machinery — see ``api_routers/admin.py::scheduler_pull_inbound`` and
``cli.py::pull_inbound`` for the canonical implementation. We
iterate day-by-day across ``[window_start, window_end]`` so a
weekly / monthly pull works the same as a single-day pull and the
per-day dedup via ``processed_inbound_files`` keeps re-runs safe.
Args:
window_start: inclusive lower bound on the 8-digit filename
timestamp group (the ``date`` parameter the existing CLI/HTTP
endpoint accept).
window_end: inclusive upper bound on the same.
db_url: SQLAlchemy DB URL. When empty, the process-global
engine is used.
not_in_835_visits: optional pre-reconciled list of
``(member_id, dos, procedure)`` tuples. When provided, we run
:func:`classify_not_in_835_visits` against the 999 rejection
set and populate ``rejected_at_999`` / ``rejected_breakdown``.
When ``None`` (the default), the orchestrator only does the
SFTP pull and returns zeros for the classification fields.
Returns:
A :class:`PullResult` summarising the run. ``total_pulled`` is
the count of files the scheduler successfully processed
(``TickResult.files_processed`` summed across the window).
Notes:
* Adapts to the actual ``Scheduler.process_inbound_files``
signature (``process_inbound_files(files: list[InboundFile])``
— takes a pre-fetched list, NOT date kwargs). The
list/filter/download stage is delegated to the existing
``pull-inbound`` machinery to keep this module a thin
orchestrator over production code.
* Wraps the async ``Scheduler`` API in ``asyncio.run`` so
callers from sync contexts (CLI / script) work directly. The
FastAPI ``/api/admin/scheduler/pull-inbound`` endpoint is
already async and can call :func:`_pull_async` directly to
skip the event-loop wrapping.
"""
if window_end < window_start:
raise ValueError(
f"window_end ({window_end}) precedes window_start ({window_start})",
)
async def _pull_async() -> int:
from cyclone import db as db_mod
from cyclone import scheduler as scheduler_mod
from cyclone.clearhouse import SftpClient
from cyclone.edi.filenames import ALLOWED_FILE_TYPES, parse_inbound_filename
from cyclone.providers import SftpBlock
if db_url:
db_mod.init_db(db_url) # type: ignore[arg-type]
else:
db_mod.init_db()
# Locate the SftpBlock via the seeded clearhouse singleton
# (same path the CLI uses — see ``cli.py::pull_inbound``).
from cyclone import store as store_mod
store_mod.store.ensure_clearhouse_seeded()
clearhouse = store_mod.store.get_clearhouse()
if clearhouse is None:
log.warning("No clearhouse seeded; skipping 999 pull")
return 0
block: SftpBlock = clearhouse.sftp_block # type: ignore[attr-defined]
scheduler_mod.configure_scheduler(block, force=True)
sched = scheduler_mod.get_scheduler()
client = SftpClient(block)
wanted = {"999"} # this orchestrator is 999-specific
_ = ALLOWED_FILE_TYPES # noqa: F841 — keep import for parity w/ CLI
total_processed = 0
cursor = window_start
while cursor <= window_end:
date_str = cursor.strftime("%Y%m%d")
try:
all_files = await asyncio.to_thread(client.list_inbound_names)
except Exception as exc: # noqa: BLE001
log.warning(
"SFTP list_inbound_names failed for %s: %s", date_str, exc,
)
cursor += timedelta(days=1)
continue
matched = []
for f in all_files:
if f.name.find(date_str) == -1:
continue
try:
parsed = parse_inbound_filename(f.name)
except ValueError:
continue
if parsed.file_type not in wanted:
continue
matched.append(f)
if len(matched) >= 2000:
break
for f in matched:
try:
await asyncio.to_thread(client.download_inbound, f)
except Exception as exc: # noqa: BLE001
log.warning(
"Failed to download %s: %s", f.name, exc,
)
try:
tick = await sched.process_inbound_files(matched)
total_processed += tick.files_processed
except Exception as exc: # noqa: BLE001
log.warning(
"process_inbound_files failed for %s: %s", date_str, exc,
)
cursor += timedelta(days=1)
return total_processed
try:
total_pulled = asyncio.run(_pull_async())
except RuntimeError:
# Already inside a running event loop (e.g. called from a
# FastAPI handler). Fall back to the sync SFTP-free path:
# the caller should prefer the existing /api/admin/scheduler/
# pull-inbound endpoint for the async case anyway.
log.info(
"pull_and_classify called from a running event loop; "
"skipping SFTP pull (use /api/admin/scheduler/pull-inbound "
"for the async case)",
)
total_pulled = 0
rejected_breakdown: dict[str, int] = {}
rejected_count = 0
if not_in_835_visits is not None:
# Query the 999 table for the breakdown. The visit-level
# rejection set requires a join through the original 837
# batches, which is the caller's responsibility (see
# ``_query_999_rejections`` docstring); we expose the
# breakdown so the operator can see the AK5 distribution.
_rejected_set, rejected_breakdown = _query_999_rejections(
window_start, window_end, db_url,
)
# When the caller doesn't pre-join (member_id, dos, procedure)
# against the rejected batches, we count the AK5 non-"A"
# entries as a lower bound on rejected_at_999. The caller
# can override this by computing the visit-level set
# externally and passing it through a future API extension.
rejected_count = sum(
cnt for code, cnt in rejected_breakdown.items()
if code not in {"A"}
)
return PullResult(
total_pulled=total_pulled,
rejected_at_999=rejected_count,
not_in_835=len(not_in_835_visits) if not_in_835_visits is not None else 0,
rejected_breakdown=rejected_breakdown,
)
+448
View File
@@ -0,0 +1,448 @@
"""Top-level orchestrator for SP41.
Wires the 835 SVC reparse → reconciliation → CARC filter → timely-filing
gate → pipeline A → pipeline B → summary CSV → Edifabric validation.
The CLI and the HTTP endpoint both call this. CLI passes filesystem
paths from --visits / --ingest / --out; HTTP endpoint passes the same.
"""
from __future__ import annotations
import csv as _csv
import logging
from dataclasses import dataclass
from datetime import date, datetime
from decimal import Decimal
from pathlib import Path
from cyclone import edifabric as _edifabric
from cyclone.edi.filenames import build_outbound_filename
from cyclone.rebill.carc_filter import CarcDecision, decide_carc
from cyclone.rebill.parse_835_svc import SvcRow, parse_835_svc
from cyclone.rebill.pipeline_a import RebillClaim
from cyclone.rebill.pipeline_b import build_pipeline_b_batches
from cyclone.rebill.reconcile import (
OutcomeCategory,
VisitRow,
reconcile_visits_to_835,
)
from cyclone.rebill.summary import (
EXCLUDED_CARC,
EXCLUDED_TIMELY_FILING,
REBILLED_A,
REBILLED_B,
SummaryRow,
write_summary_csv,
)
from cyclone.rebill.timely_filing import timely_filing_decision
_log = logging.getLogger(__name__)
@dataclass
class RunResult:
"""The return value of run_rebill — where the summary CSV landed, the
per-disposition counts, and the list of pipeline A / B file paths.
Mutable (no frozen=True) because the contained `counts` dict and
`pipeline_*_files` lists are mutated by callers who aggregate results
across runs.
"""
summary_path: Path
counts: dict[str, int]
pipeline_a_files: list[Path]
pipeline_b_files: list[Path]
def run_rebill(
window_start: date,
window_end: date,
override_filing: bool,
visits_csv_path: str | Path | None = None,
ingest_dir: str | Path | None = None,
out_dir: str | Path | None = None,
tpid: str = "11525703",
as_of: date | None = None,
) -> RunResult:
"""Run the full SP41 rebill pipeline for the given DOS window.
Pipeline shape:
1. Walk ``ingest_dir`` for *.835 / *.x12 (skipping AppleDouble shadow
files matching ``._*``), reparse each into SvcRows via
``parse_835_svc``, keep those with svc_date in [window_start, window_end].
2. Load the AxisCare visits CSV (``visits_csv_path``) and keep the rows
whose Visit Date is in the same DOS window.
3. Reconcile visits against the in-window SVCs on the
(member_id, procedure, DOS) match key. Best-of-N: PAID if
total_paid >= 95% of billed, else PARTIAL if any payment, else DENIED.
4. Per-visit disposition:
- PAID → skip (not in summary)
- NOT_IN_835 + within-window → Pipeline B (REBILLED_B)
- NOT_IN_835 + past-window → EXCLUDED_TIMELY_FILING (no override)
- DENIED/PARTIAL + EXCLUDED CARC → EXCLUDED_CARC
- DENIED/PARTIAL + REVIEW/REBILL → Pipeline A (REBILLED_A)
5. Emit Pipeline A files (``<out>/pipeline-a/tp{tpid}-837P-...-1of1.x12``)
and Pipeline B files (``<out>/pipeline-b/...-1of1.{member}-W{week}.x12``).
Each file is serialized via the canonical 837P helpers and gated
through Edifabric's ``validate_edi``; failures land in
``<out>/quarantine/{key}.837`` so the operator can triage without
blocking the batch.
6. Write ``<out>/summary.csv`` with one row per non-PAID visit.
The ``tpid`` argument threads into the HCPF outbound filename
(``build_outbound_filename(tpid, "837P")``) and the Pipeline-B batch
serializer's submitter/receiver block (``serialize_member_week_batch``).
``as_of`` defaults to ``date.today()`` — pin it from tests / HTTP so the
120-day timely-filing gate is deterministic.
"""
as_of_date = as_of or date.today()
out_dir_p = Path(out_dir or f"dev/rebills/{date.today().isoformat()}")
out_dir_p.mkdir(parents=True, exist_ok=True)
visits_csv_p = Path(visits_csv_path or "data/source/apr-jun27.csv")
ingest_dir_p = Path(ingest_dir or "ingest")
# 1) Ingest 835s — walk *.835 and *.x12, skip AppleDouble shadow files
# (._foo.835 are macOS resource forks that the SFTP client surfaces
# alongside real files; matches the convention used by cli, submit-batch,
# and api_routers/submission.py).
svcs: list[SvcRow] = []
for p in sorted(
list(ingest_dir_p.glob("*.835")) + list(ingest_dir_p.glob("*.x12"))
):
if p.name.startswith("._"):
continue
for s in parse_835_svc(p):
if window_start <= s.svc_date <= window_end:
svcs.append(s)
# 2) Load visits CSV — header:
# ``Visit Date,Member ID,Procedure Code,Billable Amount``
# DOS is MM/DD/YYYY; Billable Amount may carry a ``$`` prefix and
# ``,`` thousands separators.
visits: list[VisitRow] = []
with visits_csv_p.open(newline="") as f:
for r in _csv.DictReader(f):
dos = datetime.strptime(r["Visit Date"].strip(), "%m/%d/%Y").date()
if not (window_start <= dos <= window_end):
continue
amt = Decimal(
r["Billable Amount"].strip().lstrip("$").replace(",", "")
)
visits.append(VisitRow(
date=dos,
member_id=r["Member ID"].strip(),
procedure=r["Procedure Code"].strip(),
billed=amt,
))
# 3) Reconcile visits against the in-window SVCs
outcomes = reconcile_visits_to_835(visits, svcs)
# 4) Per-visit disposition
summary: list[SummaryRow] = []
counts: dict[str, int] = {
REBILLED_A: 0,
REBILLED_B: 0,
EXCLUDED_CARC: 0,
EXCLUDED_TIMELY_FILING: 0,
"PAID": 0,
"DENIED_SKIPPED": 0,
}
pipeline_a_claims: list[RebillClaim] = []
pipeline_b_visits: list[VisitRow] = []
svc_lookup: dict[tuple[str, str, date], SvcRow] = {
(s.member_id, s.procedure, s.svc_date): s for s in svcs
}
for o in outcomes:
if o.category == OutcomeCategory.PAID:
counts["PAID"] += 1
continue
if o.category == OutcomeCategory.NOT_IN_835:
tf = timely_filing_decision(o.visit.date, as_of_date, override_filing)
if tf.rebillable:
pipeline_b_visits.append(o.visit)
counts[REBILLED_B] += 1
summary.append(SummaryRow(
visit=o.visit,
disposition=REBILLED_B,
unpaid=o.unpaid,
cas_reasons=(),
file_path="pipeline-b/",
))
else:
counts[EXCLUDED_TIMELY_FILING] += 1
summary.append(SummaryRow(
visit=o.visit,
disposition=EXCLUDED_TIMELY_FILING,
unpaid=o.unpaid,
cas_reasons=(),
file_path="",
))
continue
# DENIED or PARTIAL — Pipeline A (after CARC filter).
s = svc_lookup.get(
(o.visit.member_id, o.visit.procedure, o.visit.date)
)
reasons: tuple[str, ...] = s.cas_reasons if s else ()
decision = decide_carc(reasons)
if decision == CarcDecision.EXCLUDED:
counts[EXCLUDED_CARC] += 1
summary.append(SummaryRow(
visit=o.visit,
disposition=EXCLUDED_CARC,
unpaid=o.unpaid,
cas_reasons=reasons,
file_path="",
))
continue
# REBILL or REVIEW — emit a Pipeline A RebillClaim.
# Each matched SVC carries its own original claim_id; preserve it
# per-claim so the freq-7 replacement is anchored to the right
# claim_submit_id. If the visit somehow matched nothing (shouldn't
# happen for DENIED/PARTIAL — those categories only arise from a
# match), mint a NEW-* fallback.
claim_id = (
s.claim_id if s
else f"NEW-{o.visit.member_id}-{o.visit.date.isoformat()}"
)
claim = RebillClaim(
claim_id=claim_id,
member_id=o.visit.member_id,
procedure=o.visit.procedure,
svc_date=o.visit.date,
charge=o.visit.billed,
frequency_code="7",
needs_review=(decision == CarcDecision.REVIEW),
original_carc_reasons=reasons,
)
pipeline_a_claims.append(claim)
counts[REBILLED_A] += 1
summary.append(SummaryRow(
visit=o.visit,
disposition=REBILLED_A,
unpaid=o.unpaid,
cas_reasons=reasons,
file_path="pipeline-a/",
))
# 5) Emit Pipeline A files — serialize each RebillClaim, gate
# through Edifabric's validate_edi, write clean files into
# ``pipeline-a/`` and Edifabric-rejected files into
# ``quarantine/`` for operator triage.
#
# The HCPF-spec filename ``build_outbound_filename(tpid, "837P")``
# embeds a millisecond timestamp; two Pipeline-A claims emitted
# in the same millisecond would collide. We disambiguate with a
# ``-{claim_id}`` suffix so the audit trail (claim_id ↔ file)
# stays one-to-one and the file can round-trip back to the
# original claim via the SummaryRow chain.
a_dir = out_dir_p / "pipeline-a"
a_dir.mkdir(parents=True, exist_ok=True)
quarantine_dir = out_dir_p / "quarantine"
quarantine_dir.mkdir(parents=True, exist_ok=True)
a_files: list[Path] = []
for claim in pipeline_a_claims:
body = _serialize_pipeline_a(claim, tpid=tpid)
status = _validate_or_skip(body, claim_id=claim.claim_id)
if status == "quarantine":
qp = quarantine_dir / f"{claim.claim_id}.837"
qp.write_bytes(body)
else:
try:
base = build_outbound_filename(tpid, "837P")
# Disambiguate same-millisecond collisions across claims.
stem, dot_ext = base.rsplit(".", 1)
p = a_dir / f"{stem}-{claim.claim_id}.{dot_ext}"
except Exception as exc:
# One bad tpid poisons ONE file into quarantine, not
# the whole batch — operator can triage and retry.
_log.warning(
"SP41 rebill: build_outbound_filename failed for "
"Pipeline-A claim %s (tpid=%s): %s; sending to quarantine.",
claim.claim_id, tpid, exc,
)
qp = quarantine_dir / f"{claim.claim_id}.837"
qp.write_bytes(body)
else:
p.write_bytes(body)
a_files.append(p)
# 6) Build + emit Pipeline B batches. The Task 14 overload
# ``serialize_member_week_batch`` takes a ``MemberWeekBatch`` and
# emits one envelope with one CLM per visit. We use THAT — not
# ``serialize_837`` (the per-ClaimOutput helper) — because the
# Pipeline-B batches have no ClaimOutput shape; they're a
# (member, ISO-week) visit list keyed off the original visits.
#
# Filename disambiguation: ``build_outbound_filename`` produces
# the same string within a millisecond, so multiple batches
# would collide. Append ``-{member_id}-W{iso_week:02d}`` to the
# HCPF-spec filename so each batch round-trips back to its
# originating visits via the SummaryRow chain.
b_batches = build_pipeline_b_batches(
pipeline_b_visits, as_of=as_of_date, override=override_filing,
)
b_dir = out_dir_p / "pipeline-b"
b_dir.mkdir(parents=True, exist_ok=True)
b_files: list[Path] = []
from cyclone.parsers.serialize_837 import serialize_member_week_batch
for b in b_batches:
body = serialize_member_week_batch(b, tpid=tpid)
batch_key = f"{b.member_id}-{b.iso_year}-W{b.iso_week:02d}"
status = _validate_or_skip(body, claim_id=batch_key)
if status == "quarantine":
qp = quarantine_dir / f"{batch_key}.837"
qp.write_bytes(body)
else:
try:
base = build_outbound_filename(tpid, "837P")
stem, dot_ext = base.rsplit(".", 1)
p = b_dir / f"{stem}-{batch_key}.{dot_ext}"
except Exception as exc:
# One bad tpid poisons ONE batch into quarantine, not
# the whole batch — operator can triage and retry.
_log.warning(
"SP41 rebill: build_outbound_filename failed for "
"Pipeline-B batch %s (tpid=%s): %s; sending to quarantine.",
batch_key, tpid, exc,
)
qp = quarantine_dir / f"{batch_key}.837"
qp.write_bytes(body)
else:
p.write_bytes(body)
b_files.append(p)
# 7) Summary CSV — one row per non-PAID visit, regardless of pipeline.
summary_path = out_dir_p / "summary.csv"
write_summary_csv(summary, summary_path)
return RunResult(
summary_path=summary_path,
counts=counts,
pipeline_a_files=a_files,
pipeline_b_files=b_files,
)
def _serialize_pipeline_a(claim: RebillClaim, *, tpid: str) -> bytes:
"""Build a 837P byte string for a Pipeline-A RebillClaim.
Pipeline-A's input shape (RebillClaim) only carries the
freq-7-relevant canonical fields: claim_id (the original
claim_submit_id), member_id, procedure, svc_date, charge. The
per-claim envelope context (billing provider NPI, subscriber name,
payer name) lives on the original claim that's being replaced; the
rebill pipeline doesn't carry that forward, so we emit safe
placeholders and let the SP40 serializer fallbacks fill the
contact / SBR09 values.
Returns ASCII bytes (the 837P envelope is pure ASCII).
"""
# Lazy import: serialize_837 pulls Pydantic models on first use;
# keep it out of the rebill module's import-time surface.
from cyclone.parsers.models import (
BillingProvider,
ClaimHeader,
ClaimOutput,
Payer,
Procedure,
ServiceLine,
Subscriber,
ValidationReport,
)
from cyclone.parsers.serialize_837 import serialize_837
# Deterministic control numbers derived from the original claim_id
# so a retry of the same DOS window produces the same filenames
# (the operator can then diff against the prior run).
cn = claim.claim_id[:9].rjust(4, "0")[-4:]
placeholder_claim = ClaimOutput(
claim_id=claim.claim_id,
control_number=cn,
transaction_date=claim.svc_date,
billing_provider=BillingProvider(
name="REBILL PROVIDER",
npi="0000000000",
),
subscriber=Subscriber(
first_name="REBILL",
last_name=claim.member_id, # surface the member id in NM103
member_id=claim.member_id,
),
payer=Payer(name="CO_TXIX", id="CO_TXIX"),
claim=ClaimHeader(
claim_id=claim.claim_id,
total_charge=claim.charge,
place_of_service="11",
facility_code_qualifier="B",
frequency_code=claim.frequency_code, # always "7"
),
service_lines=[
ServiceLine(
line_number=1,
procedure=Procedure(qualifier="HC", code=claim.procedure),
charge=claim.charge,
units=Decimal("1"),
unit_type="UN",
place_of_service="11",
service_date=claim.svc_date,
),
],
validation=ValidationReport(passed=True),
transaction_type_code="CH",
)
text = serialize_837(placeholder_claim, sender_id=tpid)
return text.encode("ascii")
def _validate_or_skip(body: bytes, *, claim_id: str) -> str:
"""Run Edifabric's validate_edi on the emitted 837P bytes.
Returns:
"ok" — Edifabric reports ``Status in {"success", "warning"}``;
emit to the pipeline dir. Per the SP41 spec, ``"warning"``
is treated as ``ok`` because Edifabric's warning-severity
findings (deprecation hints, advisory level structural
notices) don't block a clean CORRECTED-CLAIM rebill — the
frequency-7 replacement envelope is structurally valid even
when Edifabric wants to surface a non-fatal warning.
"quarantine" — Edifabric reports ``Status == "error"``; emit
to ``<out>/quarantine/`` for operator triage.
"skip" — Edifabric was unreachable (no API key, network error,
5xx, etc.); treat as ``ok`` and emit to the pipeline dir.
A WARNING is logged so the operator knows the gate didn't
actually run. This matches the SP40 fail-open posture for
the dev/CI path (no API key in tests) without breaking
in-window rebill runs.
"""
try:
result = _edifabric.validate_edi(body)
except _edifabric.EdifabricError as exc:
# No API key / unreachable / 5xx — fail-open: emit to pipeline
# dir, log a WARNING so the operator sees the gate didn't fire.
_log.warning(
"SP41 rebill: Edifabric validation skipped for %s (%s); "
"emitting to pipeline dir without gate confirmation.",
claim_id, exc,
)
return "skip"
status = result.get("Status", "")
if status == "error":
details = result.get("Details") or []
msgs = "; ".join(
f"{d.get('SegmentId', '?')}: {d.get('Message', '?')}"
for d in details[:5]
)
_log.warning(
"SP41 rebill: Edifabric rejected %s — quarantining. %s",
claim_id, msgs,
)
return "quarantine"
return "ok"
+40
View File
@@ -29,7 +29,9 @@ from cyclone.parsers.parse_837 import parse as parse_837_text
from cyclone.parsers.payer import PayerConfig from cyclone.parsers.payer import PayerConfig
from cyclone.providers import SftpBlock from cyclone.providers import SftpBlock
from cyclone.store import store as cycl_store from cyclone.store import store as cycl_store
from cyclone.store.exceptions import DuplicateClaimError
from cyclone.store.records import BatchRecord837 from cyclone.store.records import BatchRecord837
from cyclone.store.submission_dedup import check_duplicate
from .result import SubmitOutcome, SubmitResult from .result import SubmitOutcome, SubmitResult
@@ -137,6 +139,44 @@ def submit_file(
error=f"payer.id={mismatch.payer.id!r} (expected {EXPECTED_PAYER_ID!r})", error=f"payer.id={mismatch.payer.id!r} (expected {EXPECTED_PAYER_ID!r})",
) )
# 1b. SP41 Task 9 — claim-id dedup pre-flight. Walks every
# CLM01 in the parsed file and asks ``check_duplicate`` whether
# any of them were submitted within the 30-day window. If so,
# raise ``DuplicateClaimError`` — this is a 409-class domain
# exception (mirrors the posture of ``AlreadyMatchedError`` /
# ``NotMatchedError`` / ``InvalidStateError`` per their docstrings)
# and is caught by ``api_routers/submission.submit_batch`` so
# the per-file 200-with-results contract is preserved.
#
# ``check_duplicate`` reads the configured ``cycl_db.SessionLocal()``
# — the same engine the BatchRecord837 DB write below uses — so
# we don't need to thread a session through here. The helper's
# optional ``db_url`` kwarg is ignored (signature-preserving
# shim for back-compat with the original Task 8 callers).
#
# We deliberately RAISE here instead of returning a SubmitResult:
# the exception carries ``claim_id`` + ``original_submission_at``
# so the caller can surface structured detail to the operator
# without re-querying. Approach A per the Task 9 spec.
#
# Symmetry with the sibling failure paths (parse / DB / SFTP /
# audit-event — each logs ``submit_file %s: <kind>: %s`` before
# returning or re-raising). The router's special-case handler
# also logs this exception at the api_routers boundary, but
# the helper has its own log line so the operator tracing a
# failure through stdout sees the dedup trip even when
# submit_file is invoked directly (e.g. from the CLI or a
# future call site that does not go through the router).
try:
for claim in parsed.claims:
check_duplicate(claim.claim_id)
except DuplicateClaimError as exc:
log.warning(
"submit_file %s: duplicate claim %s (originally submitted %s)",
file_label, exc.claim_id, exc.original_submission_at.isoformat(),
)
raise
# 2. DB write — DB-first, upload-second invariant. # 2. DB write — DB-first, upload-second invariant.
# ``parsed`` is required to construct a BatchRecord837 (it embeds the # ``parsed`` is required to construct a BatchRecord837 (it embeds the
# full ParseResult). validate=False therefore isn't a real path — # full ParseResult). validate=False therefore isn't a real path —
+226
View File
@@ -0,0 +1,226 @@
"""SP41 Task 11 — HTTP endpoints for the in-window rebill pipeline.
Three tests:
1. ``test_post_rebill_requires_auth_or_returns_404_or_422_when_no_input`` —
bare POST with no body. With ``AUTH_DISABLED = True`` (the autouse
conftest default) the request reaches the handler; without a body,
Pydantic returns 422. If AUTH_DISABLED is ever flipped off, the
matrix_gate returns 401/403 first. Either outcome proves the route
is wired.
2. ``test_post_rebill_runs_and_returns_summary_path`` — happy path.
Builds a 1-row visits CSV + zero-SVC 835 fixture under tmp_path,
POSTs with the window pinned + paths pointed at the fixtures,
asserts 200 + ``summary_path`` + a ``counts`` dict that includes
the REBILLED_B key (the in-window visit lands as NOT_IN_835 →
REBILLED_B because the 835 has no SVCs and the visit is fresh
enough to clear the 120-day gate).
3. ``test_get_rebill_status_returns_recent_runs`` — seeds a stub
``summary.csv`` under ``tmp_path / "rebills_root"`` and
monkey-patches :data:`cyclone.api_routers.rebill.REBILLS_DIR` to
point there, so the GET handler's filesystem scan finds it
without the test having to chdir the whole process.
"""
from __future__ import annotations
import csv
from datetime import date
from decimal import Decimal
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
# --------------------------------------------------------------------------- #
# Fixtures
# --------------------------------------------------------------------------- #
@pytest.fixture
def client() -> TestClient:
"""Standard TestClient; conftest.py autouse handles DB + auth gate.
AUTH_DISABLED is True so the matrix_gate short-circuits and the
request reaches the handler (no login dance required). The seeded
clearhouse isn't needed — the rebill pipeline doesn't touch SFTP.
"""
from cyclone.api import app
return TestClient(app)
def _write_visits_csv(path: Path, rows: list[tuple[str, str, str, str]]) -> Path:
"""rows: list of (dos_mmddyyyy, member, procedure, billed_str)."""
with path.open("w", newline="") as f:
w = csv.writer(f)
w.writerow(["Visit Date", "Member ID", "Procedure Code", "Billable Amount"])
for r in rows:
w.writerow(r)
return path
def _stub_835(ingest_dir: Path, name: str = "x.835") -> Path:
"""An *.835 file parseable enough to not crash the walker.
Content is just an empty 835 envelope; the rebill pipeline's
``parse_835_svc`` reparser emits zero SVCs, which is what we want
for the NOT_IN_835 → REBILLED_B classification.
"""
ingest_dir.mkdir(exist_ok=True)
p = ingest_dir / name
p.write_text("ST*835*0001~SE*0*0001~")
return p
# --------------------------------------------------------------------------- #
# Tests
# --------------------------------------------------------------------------- #
def test_post_rebill_requires_auth_or_returns_404_or_422_when_no_input(client):
"""Bare POST with no body — should NOT 500.
With AUTH_DISABLED the matrix_gate short-circuits and Pydantic
validation fires on the empty body (422). With auth enabled the
matrix_gate returns 401/403 first. Either outcome proves the route
is wired and the request reached the handler.
"""
resp = client.post("/api/admin/rebill-from-835")
assert resp.status_code in (401, 403, 422), (
f"unexpected status {resp.status_code}: {resp.text}"
)
def test_post_rebill_runs_and_returns_summary_path(client, tmp_path):
"""Happy path: 1 in-window visit, 0 SVCs → REBILLED_B + summary.csv."""
visits_path = _write_visits_csv(tmp_path / "visits.csv", [
("06/27/2026", "J813715", "T1019", "$2.32"),
])
ingest_dir = _stub_835(tmp_path / "ingest")
out_dir = tmp_path / "out"
resp = client.post(
"/api/admin/rebill-from-835",
json={
"window": "2026-01-01..2026-06-27",
"override_filing": False,
"visits_csv_path": str(visits_path),
"ingest_dir": str(ingest_dir),
"out_dir": str(out_dir),
},
)
assert resp.status_code == 200, resp.text
body = resp.json()
assert "summary_path" in body, body
assert "counts" in body, body
assert "pipeline_a_files" in body, body
assert "pipeline_b_files" in body, body
# summary_path is real and points at the out_dir we passed in.
assert Path(body["summary_path"]).exists(), body["summary_path"]
assert str(out_dir) in body["summary_path"]
# The counts dict surfaces REBILLED_B for the in-window visit (the
# 835 has no SVCs so the visit falls into NOT_IN_835 → REBILLED_B
# — it's well within the 120-day gate).
counts = body["counts"]
assert "REBILLED_B" in counts, counts
assert counts["REBILLED_B"] >= 1, counts
# Pipeline B emitted exactly one file for the one REBILLED_B visit.
assert len(body["pipeline_b_files"]) == 1, body["pipeline_b_files"]
# The summary CSV header matches the writer's contract so the GET
# /status handler can parse it.
with open(body["summary_path"], newline="") as f:
reader = csv.DictReader(f)
rows = list(reader)
assert len(rows) == 1, rows
assert rows[0]["disposition"] == "REBILLED_B"
assert rows[0]["member_id"] == "J813715"
def test_get_rebill_status_returns_recent_runs(client, tmp_path, monkeypatch):
"""GET /status scans the configured REBILLS_DIR and tallies per-disposition counts.
The handler reads :data:`cyclone.api_routers.rebill.REBILLS_DIR`
(module-level so tests can monkeypatch it). We point that at a
tmp_path-relative tree, seed one dated subdir with a 2-row
summary.csv, and assert the response surfaces that directory's
name + per-disposition tally.
"""
from cyclone.api_routers import rebill
# Build a date-named subdir with a real summary.csv shape so the
# handler's csv.DictReader finds a `disposition` column.
rebills_root = tmp_path / "rebills_root"
dated_dir = rebills_root / "2026-07-07"
dated_dir.mkdir(parents=True)
summary = dated_dir / "summary.csv"
with summary.open("w", newline="") as f:
w = csv.writer(f)
w.writerow([
"dos", "member_id", "procedure", "billed",
"disposition", "unpaid", "cas_reasons", "file_path",
])
w.writerow([
"2026-06-27", "MEM-A", "T1019", "2.32",
"REBILLED_B", "2.32", "", "pipeline-b/",
])
w.writerow([
"2020-01-01", "MEM-B", "T1019", "2.32",
"EXCLUDED_TIMELY_FILING", "2.32", "", "",
])
# Repoint the handler's filesystem root at our tmp_path tree.
monkeypatch.setattr(rebill, "REBILLS_DIR", rebills_root)
resp = client.get("/api/admin/rebill-from-835/status")
assert resp.status_code == 200, resp.text
body = resp.json()
assert "recent_runs" in body, body
assert len(body["recent_runs"]) == 1, body["recent_runs"]
run = body["recent_runs"][0]
assert run["as_of"] == "2026-07-07", run
assert "summary_path" in run, run
assert Path(run["summary_path"]).exists(), run["summary_path"]
counts = run["counts"]
# Each disposition surfaced exactly once — matches the 2-row CSV.
assert counts.get("REBILLED_B") == 1, counts
assert counts.get("EXCLUDED_TIMELY_FILING") == 1, counts
def test_post_rebill_rejects_malformed_window(client):
"""Pydantic window validator should 422 on bad shapes (no `..`, non-dates, reversed)."""
bad_windows = [
"2026-01-01", # no `..` separator
"foo..bar", # non-ISO dates
"2026-12-01..2026-01-01", # start after end
]
for w in bad_windows:
resp = client.post(
"/api/admin/rebill-from-835",
json={"window": w},
)
assert resp.status_code == 422, (
f"expected 422 for window={w!r}, got {resp.status_code}: {resp.text}"
)
def test_get_rebill_status_returns_empty_when_no_runs(client, tmp_path, monkeypatch):
"""GET /status on an empty rebills dir must return {"recent_runs": []}, not 500.
Monkeypatches REBILLS_DIR to a fresh tmp_path subdir so the test
doesn't depend on (or pollute) any pre-existing dev/rebills/ tree.
"""
from cyclone.api_routers import rebill
monkeypatch.setattr(rebill, "REBILLS_DIR", tmp_path / "rebills_empty")
resp = client.get("/api/admin/rebill-from-835/status")
assert resp.status_code == 200, resp.text
assert resp.json() == {"recent_runs": []}
+363
View File
@@ -0,0 +1,363 @@
"""SP41 — tests for the `cyclone rebill-from-835` CLI.
Three smoke tests:
1. --help renders cleanly and surfaces --window and --override-filing.
2. --status (with no other args) renders cleanly and exits 0 — the
option is recognized even when no prior summary.csv exists.
3. End-to-end run against a 2-row visits CSV (one in-window, one
ancient) and a zero-SVC 835 fixture produces a summary.csv with
the in-window visit classified as REBILLED_B.
Plus five disposition-coverage tests added during the Task 10 review:
4. EXCLUDED_CARC: CO-45 on a denied visit → not rebilled.
5. REBILLED_A: CO-97 on a denied visit → pipeline-a freq-7.
6. PAID visits are filtered out of the summary.
7. EXCLUDED_TIMELY_FILING vs REBILLED_B via --override-filing (two runs).
8. --status on a real summary.csv prints the per-disposition tally.
"""
from __future__ import annotations
import csv
from datetime import date as _date
from decimal import Decimal
from pathlib import Path
from unittest import mock
from click.testing import CliRunner
from cyclone.cli import main
from cyclone.rebill.parse_835_svc import SvcRow
from cyclone.rebill.reconcile import VisitRow
def _write_visits_csv(path: Path, rows: list[tuple[str, str, str, str]]) -> Path:
"""rows: list of (dos_mmddyyyy, member, procedure, billed_str)."""
with path.open("w", newline="") as f:
w = csv.writer(f)
w.writerow(["Visit Date", "Member ID", "Procedure Code", "Billable Amount"])
for r in rows:
w.writerow(r)
return path
def _read_summary(out_dir: Path) -> list[dict[str, str]]:
with (out_dir / "summary.csv").open(newline="") as f:
return list(csv.DictReader(f))
def _stub_svc(monkeypatch, svcs: list[SvcRow]) -> None:
"""Replace cli.parse_835_svc's underlying cyclonic call with a fixed list."""
# cli.rebill_from_835 imports parse_835_svc locally, so patch the
# source module's symbol — the local `from ... import parse_835_svc`
# binds the symbol at call time on each invocation.
monkeypatch.setattr(
"cyclone.rebill.parse_835_svc.parse_835_svc",
lambda path, _svcs=svcs: iter(_svcs),
)
def _make_835_placeholder(ingest_dir: Path, name: str = "x.835") -> Path:
"""A real *.835 file present in the glob; parse_835_svc is mocked so content doesn't matter."""
ingest_dir.mkdir(exist_ok=True)
p = ingest_dir / name
p.write_text("ST*835*0001~SE*0*0001~")
return p
def test_rebill_from_835_help():
runner = CliRunner()
result = runner.invoke(main, ["rebill-from-835", "--help"])
assert result.exit_code == 0, result.output
assert "--window" in result.output
assert "--override-filing" in result.output
def test_rebill_from_835_status_help():
"""Either bare --status or --status --help should exit 0.
The spec note: the help-test ensures the option is recognized, so
even a "no prior runs" 0-exit path is acceptable. We try both
invocations because click short-circuits --help ahead of any
required-option check; the bare --status path is the one that
proves --status works without --visits / --ingest.
"""
runner = CliRunner()
result_help = runner.invoke(main, ["rebill-from-835", "--status", "--help"])
assert result_help.exit_code == 0, result_help.output
result_bare = runner.invoke(main, ["rebill-from-835", "--status"])
assert result_bare.exit_code == 0, result_bare.output
def test_rebill_from_835_runs(tmp_path: Path):
"""End-to-end: 1 in-window visit + 1 ancient visit + zero-SVC 835 → REBILLED_B for the in-window row."""
visits_path = tmp_path / "visits.csv"
visits_path.write_text(
"Visit Date,Member ID,Procedure Code,Billable Amount\n"
"06/27/2026,J813715,T1019,$2.32\n"
"01/01/2020,ANCIENT,T1019,$2.32\n"
)
ingest_dir = tmp_path / "ingest"
ingest_dir.mkdir()
(ingest_dir / "x.835").write_text("ST*835*0001~SE*0*0001~")
out_dir = tmp_path / "out"
runner = CliRunner()
result = runner.invoke(main, [
"rebill-from-835",
"--visits", str(visits_path),
"--ingest", str(ingest_dir),
"--out", str(out_dir),
"--as-of", "2026-07-07",
])
assert result.exit_code == 0, (
f"CLI exited {result.exit_code}, output={result.output!r}, "
f"exception={result.exception!r}"
)
summary_path = out_dir / "summary.csv"
assert summary_path.exists(), summary_path
rows = _read_summary(out_dir)
# Two visits in, two summary rows out (one REBILLED_B, one
# EXCLUDED_TIMELY_FILING). PAID outcomes would be dropped, but
# neither of these is PAID — the 835 has no SVCs.
assert len(rows) == 2, rows
by_member = {r["member_id"]: r for r in rows}
in_window = by_member["J813715"]
assert in_window["disposition"] == "REBILLED_B", in_window
assert in_window["dos"] == "2026-06-27", in_window
ancient = by_member["ANCIENT"]
assert ancient["disposition"] == "EXCLUDED_TIMELY_FILING", ancient
assert ancient["dos"] == "2020-01-01", ancient
# ---------------------------------------------------------------------------
# Disposition coverage (Task 10 review)
# ---------------------------------------------------------------------------
def test_rebill_carc_excluded_visit_landed_as_excluded_carc(tmp_path: Path, monkeypatch):
"""A denied visit whose CAS list contains CO-45 (excluded) must land as EXCLUDED_CARC, not REBILLED_A."""
visits = _write_visits_csv(tmp_path / "visits.csv", [
("06/27/2026", "J813715", "T1019", "$16.24"),
])
ingest = tmp_path / "ingest"
_make_835_placeholder(ingest)
out_dir = tmp_path / "out"
# A denied SVC for the visit (paid == 0) with CO-45 in CAS.
svc = SvcRow(
src_file="x.835",
claim_id="T1",
member_id="J813715",
status="4", # denied
procedure="T1019",
modifiers="",
charge=Decimal("16.24"),
paid=Decimal("0"),
units=Decimal("1"),
svc_date=_date(2026, 6, 27),
cas_reasons=("CO-45",),
pay_date=None,
)
_stub_svc(monkeypatch, [svc])
runner = CliRunner()
result = runner.invoke(main, [
"rebill-from-835",
"--visits", str(visits),
"--ingest", str(ingest),
"--out", str(out_dir),
"--as-of", "2026-07-07",
])
assert result.exit_code == 0, (result.output, result.exception)
rows = _read_summary(out_dir)
assert len(rows) == 1, rows
assert rows[0]["disposition"] == "EXCLUDED_CARC"
assert rows[0]["member_id"] == "J813715"
assert "CO-45" in rows[0]["cas_reasons"]
def test_rebill_denied_rebill_visit_landed_as_rebilled_a(tmp_path: Path, monkeypatch):
"""A denied visit whose CAS list contains CO-97 (rebillable) must land as REBILLED_A."""
visits = _write_visits_csv(tmp_path / "visits.csv", [
("06/27/2026", "J813715", "T1019", "$16.24"),
])
ingest = tmp_path / "ingest"
_make_835_placeholder(ingest)
out_dir = tmp_path / "out"
svc = SvcRow(
src_file="x.835",
claim_id="T1",
member_id="J813715",
status="4", # denied
procedure="T1019",
modifiers="",
charge=Decimal("16.24"),
paid=Decimal("0"),
units=Decimal("1"),
svc_date=_date(2026, 6, 27),
cas_reasons=("CO-97",), # rebillable (not in EXCLUDED_CARCS or REVIEW_CARCS)
pay_date=None,
)
_stub_svc(monkeypatch, [svc])
runner = CliRunner()
result = runner.invoke(main, [
"rebill-from-835",
"--visits", str(visits),
"--ingest", str(ingest),
"--out", str(out_dir),
"--as-of", "2026-07-07",
])
assert result.exit_code == 0, (result.output, result.exception)
rows = _read_summary(out_dir)
assert len(rows) == 1, rows
assert rows[0]["disposition"] == "REBILLED_A"
assert rows[0]["file_path"] == "pipeline-a/"
assert "CO-97" in rows[0]["cas_reasons"]
def test_rebill_paid_visit_excluded_from_summary(tmp_path: Path, monkeypatch):
"""PAID visits (paid >= 95% of billed) must not appear in summary.csv — only NOT_IN_835 should."""
visits = _write_visits_csv(tmp_path / "visits.csv", [
("06/27/2026", "PAID-MEMBER", "T1019", "$100.00"),
("06/27/2026", "MISSING-MEMBER", "T1019", "$2.32"),
])
ingest = tmp_path / "ingest"
_make_835_placeholder(ingest)
out_dir = tmp_path / "out"
# One fully-paid SVC for PAID-MEMBER; nothing matches MISSING-MEMBER.
paid_svc = SvcRow(
src_file="x.835",
claim_id="T1",
member_id="PAID-MEMBER",
status="1",
procedure="T1019",
modifiers="",
charge=Decimal("100.00"),
paid=Decimal("100.00"),
units=Decimal("1"),
svc_date=_date(2026, 6, 27),
cas_reasons=(),
pay_date=None,
)
_stub_svc(monkeypatch, [paid_svc])
runner = CliRunner()
result = runner.invoke(main, [
"rebill-from-835",
"--visits", str(visits),
"--ingest", str(ingest),
"--out", str(out_dir),
"--as-of", "2026-07-07",
])
assert result.exit_code == 0, (result.output, result.exception)
rows = _read_summary(out_dir)
assert len(rows) == 1, rows
assert rows[0]["member_id"] == "MISSING-MEMBER"
assert rows[0]["disposition"] == "REBILLED_B"
def test_rebill_override_filing_makes_ancient_visit_rebilled_b(tmp_path: Path, monkeypatch):
"""A 2020 visit without --override-filing → EXCLUDED_TIMELY_FILING; with it → REBILLED_B."""
# No SVCs at all — both visits fall into NOT_IN_835.
visits = _write_visits_csv(tmp_path / "visits.csv", [
("01/01/2020", "ANCIENT", "T1019", "$2.32"),
])
ingest = tmp_path / "ingest"
_make_835_placeholder(ingest)
out_dir = tmp_path / "out"
# Stub the (local) parse_835_svc import with an empty generator so
# the CLI sees no SVCs and every visit is NOT_IN_835.
monkeypatch.setattr(
"cyclone.rebill.parse_835_svc.parse_835_svc",
lambda path: iter([]),
)
runner = CliRunner()
# Run 1: as-of 2026-07-07 minus 2020-01-01 = ~6.5 years > 120 days → EXCLUDED.
r1 = runner.invoke(main, [
"rebill-from-835",
"--visits", str(visits),
"--ingest", str(ingest),
"--out", str(out_dir),
"--as-of", "2026-07-07",
])
assert r1.exit_code == 0, (r1.output, r1.exception)
rows1 = _read_summary(out_dir)
assert len(rows1) == 1, rows1
assert rows1[0]["disposition"] == "EXCLUDED_TIMELY_FILING", rows1
# Run 2: --override-filing bypasses the 120-day gate → REBILLED_B.
r2 = runner.invoke(main, [
"rebill-from-835",
"--visits", str(visits),
"--ingest", str(ingest),
"--out", str(out_dir),
"--as-of", "2026-07-07",
"--override-filing",
])
assert r2.exit_code == 0, (r2.output, r2.exception)
rows2 = _read_summary(out_dir)
assert len(rows2) == 1, rows2
assert rows2[0]["disposition"] == "REBILLED_B", rows2
assert rows2[0]["file_path"] == "pipeline-b/"
def test_rebill_status_with_real_summary_prints_tally(tmp_path: Path):
"""--status against a real summary.csv must print both REBILLED_B and EXCLUDED_TIMELY_FILING rows."""
from cyclone.rebill.reconcile import VisitRow
from cyclone.rebill.summary import (
EXCLUDED_TIMELY_FILING,
REBILLED_B,
SummaryRow,
write_summary_csv,
)
out_dir = tmp_path / "dev" / "rebills" / "2026-07-07"
out_dir.mkdir(parents=True, exist_ok=True)
summary_path = out_dir / "summary.csv"
rows = [
SummaryRow(
visit=VisitRow(
date=_date(2026, 6, 27), member_id="MEM-A",
procedure="T1019", billed=Decimal("2.32"),
),
disposition=REBILLED_B,
unpaid=Decimal("2.32"),
cas_reasons=(),
file_path="pipeline-b/",
),
SummaryRow(
visit=VisitRow(
date=_date(2026, 6, 27), member_id="MEM-B",
procedure="T1019", billed=Decimal("2.32"),
),
disposition=EXCLUDED_TIMELY_FILING,
unpaid=Decimal("2.32"),
cas_reasons=(),
file_path="",
),
]
write_summary_csv(rows, summary_path)
runner = CliRunner()
result = runner.invoke(main, [
"rebill-from-835",
"--status",
"--out", str(out_dir),
])
assert result.exit_code == 0, (result.output, result.exception)
assert "REBILLED_B" in result.output
assert "EXCLUDED_TIMELY_FILING" in result.output
assert "TOTAL" in result.output
+172
View File
@@ -0,0 +1,172 @@
"""SP41 Task 17 — end-to-end smoke test for ``run_rebill``.
Wires the full SP41 pipeline (835 SVC reparse → reconcile → CARC
filter → timely-filing gate → pipeline A → pipeline B → summary CSV)
end-to-end on synthetic inputs and pins the summary.csv shape +
counts.
No mocks for ``parse_835_svc`` or ``validate_837`` — the goal is to
exercise the real pipeline. The autouse conftest handles Edifabric
fail-open (no API key in CI), and since this run produces only
quarantined dispositions (no pipeline A/B emissions), ``validate_edi``
is never called anyway.
"""
from __future__ import annotations
import csv
from datetime import date
from pathlib import Path
from cyclone.rebill.run import run_rebill
# --------------------------------------------------------------------------- #
# Fixture builders
# --------------------------------------------------------------------------- #
def _write_visits_csv(path: Path) -> Path:
"""Two visits: one in-window (will match a denied SVC), one past the
120-day timely-filing window (no matching SVC).
DOS column is MM/DD/YYYY per AxisCare's actual export format.
"""
with path.open("w", newline="") as f:
w = csv.writer(f)
w.writerow([
"Visit Date", "Member ID", "Procedure Code",
"Billable Amount", "Authorized",
])
w.writerow(["06/27/2026", "J813715", "T1019", "2.32", "Y"])
w.writerow(["01/01/2026", "OLD001", "T1019", "5.00", "Y"])
return path
def _write_835_with_denied_svc(ingest_dir: Path) -> Path:
"""Write a real 835 that ``parse_835_svc`` walks end-to-end.
Contains exactly one CLP block for member J813715 with:
- CLP02 = 4 (Denied)
- CLP03 = 2.32 (charge)
- SVC*HC:T1019*2.32*0** (paid = $0)
- DTM*472*20260627 (service date)
- CAS*CO*45*2.32 (triggers ``EXCLUDED_CARC``)
Returns the **directory** path so the caller can pass it straight to
``run_rebill(ingest_dir=...)`` — run_rebill does its own
``ingest_dir.glob("*.835")`` walk. (Earlier draft returned the
inner file path, which made glob come up empty.)
Shape mirrors the existing ``tests/fixtures/835_sample_svc_with_member.txt``
(segments concatenated without ``\\n`` separators — ``parse_835_svc``
splits on ``~`` only, and a leading newline makes ``elems[0]`` an empty
string that the segment-name match drops silently).
"""
ingest_dir.mkdir(exist_ok=True)
segs = [
"ISA*00* *00* *ZZ*CYCLONE *ZZ*GAINWELL *260627*1200*^*00501*000000001*0*P*:~",
"GS*HC*CYCLONE*GAINWELL*20260627*1200*1*X*005010X221A1~",
"ST*835*0001~",
"BPR*I*0*C*ACH*CCP*01*021000021*DA*123456789*1512345678**01*021000021*DA*123456789*20260101~",
"TRN*1*TRACE01*1512345678~",
"DTM*405*20260118~",
"N1*PR*COLORADO MEDICAL ASSISTANCE PROGRAM*XV*COMEDASSISTPROG~",
"N3*PO BOX 1100~",
"N4*DENVER*CO*80202~",
"LX*1~",
# CLP02=4 (Denied); CLP03=2.32 (charge); CLP04=0 (paid).
"CLP*DENIED-CLM*4*2.32*0**MC*111*11*1~",
# NM1*QC.NM109 carries the member_id forward to SVC rows.
"NM1*QC*1*DOE*JANE****MR*J813715~",
# SVC composite qualifier:procedure (HC:T1019); 4-arg form is fine.
"SVC*HC:T1019*2.32*0**~",
# DTM*472 carries the service date (matches parse_835_svc's lookup).
"DTM*472*20260627~",
# CO-45 is in EXCLUDED_CARCS → CarcDecision.EXCLUDED → EXCLUDED_CARC.
"CAS*CO*45*2.32~",
"SE*14*0001~",
"GE*1*1~",
"IEA*1*000000001~",
]
p = ingest_dir / "x.835"
p.write_text("".join(segs))
return ingest_dir
# --------------------------------------------------------------------------- #
# Smoke test
# --------------------------------------------------------------------------- #
def test_run_rebill_end_to_end_excluded_dispositions(tmp_path):
"""End-to-end: 2 visits → 2 quarantined dispositions, no pipeline files.
Pins the contract that ``run_rebill`` returns a ``RunResult`` whose
``summary.csv`` row-shapes match the visit-side input (one row per
non-PAID visit, with the right disposition per row).
"""
visits_csv = _write_visits_csv(tmp_path / "visits.csv")
ingest = _write_835_with_denied_svc(tmp_path / "ingest")
out_dir = tmp_path / "rebills"
# ``as_of=2026-07-07`` pins the timely-filing gate so the test is
# deterministic — OLD001 (DOS 2026-01-01) is 187 days old, well past
# the 120-day HCPF window, so it's EXCLUDED_TIMELY_FILING.
result = run_rebill(
window_start=date(2026, 1, 1),
window_end=date(2026, 6, 27),
override_filing=False,
visits_csv_path=str(visits_csv),
ingest_dir=str(ingest),
out_dir=str(out_dir),
as_of=date(2026, 7, 7),
)
# --- summary.csv exists and the right text is in it. --- #
assert result.summary_path.exists(), result.summary_path
text = result.summary_path.read_text()
assert "J813715" in text
assert "OLD001" in text
assert "EXCLUDED_CARC" in text
assert "EXCLUDED_TIMELY_FILING" in text
# --- counts match expectations (one of each excluded disposition). --- #
assert result.counts["EXCLUDED_CARC"] == 1, result.counts
assert result.counts["EXCLUDED_TIMELY_FILING"] == 1, result.counts
# Both visits are excluded — no pipeline emissions.
assert result.counts["REBILLED_A"] == 0, result.counts
assert result.counts["REBILLED_B"] == 0, result.counts
# --- pin the per-row shape (stronger than substring checks). --- #
with result.summary_path.open(newline="") as f:
rows = list(csv.DictReader(f))
assert len(rows) == 2, rows
j_row = next(r for r in rows if r["member_id"] == "J813715")
old_row = next(r for r in rows if r["member_id"] == "OLD001")
# J813715: matched the DENIED SVC, CARC CO-45 is in EXCLUDED_CARCS.
assert j_row["disposition"] == "EXCLUDED_CARC", j_row
assert j_row["procedure"] == "T1019", j_row
assert j_row["dos"] == "2026-06-27", j_row
assert j_row["billed"] == "2.32", j_row
assert "CO-45" in j_row["cas_reasons"], j_row
# OLD001: no matching SVC, DOS 187 days old → timely-filing exclusion.
assert old_row["disposition"] == "EXCLUDED_TIMELY_FILING", old_row
assert old_row["procedure"] == "T1019", old_row
assert old_row["dos"] == "2026-01-01", old_row
assert old_row["billed"] == "5.00", old_row
# No SVC match → empty cas_reasons column.
assert old_row["cas_reasons"] == "", old_row
# --- no pipeline files were emitted (both rows are excluded). --- #
assert result.pipeline_a_files == [], result.pipeline_a_files
assert result.pipeline_b_files == [], result.pipeline_b_files
# The pipeline dirs exist but are empty.
assert (out_dir / "pipeline-a").is_dir()
assert (out_dir / "pipeline-b").is_dir()
assert list((out_dir / "pipeline-a").iterdir()) == []
assert list((out_dir / "pipeline-b").iterdir()) == []
# No quarantined files either — these dispositions don't emit at all.
assert list((out_dir / "quarantine").iterdir()) == []
+45
View File
@@ -73,3 +73,48 @@ def test_override_flag_set_on_past_window_visit_batch():
assert out_default[0].member_id == "OLD" assert out_default[0].member_id == "OLD"
assert out_default[0].iso_week == 26 assert out_default[0].iso_week == 26
assert out_default[0].has_overridden_visits is False assert out_default[0].has_overridden_visits is False
def test_serialize_member_week_batch_emits_one_envelope():
"""One 837P envelope per MemberWeekBatch — one CLM + one SV1 + one
DTP*472 service date per visit.
The SP41 plan spec wrote ``DTM*472*`` but the canonical 837P service
date segment is ``DTP*472*`` (per :func:`cyclone.parsers.serialize_837.
_build_dtp_472` and X12 005010X222A1). This test asserts against the
canonical segment name so the batch overload stays consistent with
the existing ``serialize_837`` building blocks.
"""
from cyclone.parsers.serialize_837 import serialize_member_week_batch
visits = [
_v(date(2026, 6, 23), "J813715", "T1019", "2.32"),
_v(date(2026, 6, 25), "J813715", "T1019", "2.32"),
]
batches = build_pipeline_b_batches(visits, as_of=date(2026, 7, 7), override=False)
assert len(batches) == 1
body = serialize_member_week_batch(batches[0])
text = body.decode("utf-8", errors="ignore") if isinstance(body, bytes) else body
# CLM* segment appears twice (once per visit)
assert text.count("CLM*") == 2
# SV1* appears once per visit (one service line per claim)
assert text.count("SV1*") == 2
# DTP*472* service-date segment appears twice (canonical 837P segment name)
assert text.count("DTP*472*") == 2
# Single envelope (single ISA / single IEA), not per-visit envelopes
assert text.count("ISA*") == 1
assert text.count("IEA*") == 1
# Deterministic per-visit claim_id pattern (member_id + date + 1-based idx)
assert "MW-J813715-2026-06-23-01" in text
assert "MW-J813715-2026-06-25-02" in text
def test_serialize_member_week_batch_return_type_is_bytes():
"""Task 14 spec: ``serialize_member_week_batch`` returns ``bytes``
(the existing ``serialize_837`` returns ``str``; this overload
diverges intentionally so callers can write the file directly)."""
from cyclone.parsers.serialize_837 import serialize_member_week_batch
visits = [_v(date(2026, 6, 23), "J813715", "T1019", "2.32")]
batches = build_pipeline_b_batches(visits, as_of=date(2026, 7, 7), override=False)
assert len(batches) == 1
body = serialize_member_week_batch(batches[0])
assert isinstance(body, bytes)
+167
View File
@@ -0,0 +1,167 @@
"""999-ack dump from Gainwell for MarJun 2026.
Reconciles pulled 999 acks against the in-window 4,509 NOT_IN_835 visits.
Visits that are 999-rejected go in one bucket; visits that simply
never made it to submission go in another.
These tests exercise the pure-function surface of
``cyclone.rebill.pull_999_acks`` (no SFTP, no DB). The
``pull_and_classify`` orchestrator wraps the existing
``Scheduler.process_inbound_files`` machinery and is integration-
covered by the existing ``test_api_pull_inbound.py`` / CLI smoke
tests — adding a new test here would just duplicate that coverage
and require a live SFTP server.
"""
from __future__ import annotations
from datetime import date
from cyclone.rebill.pull_999_acks import (
Bucket,
PullResult,
classify_not_in_835_visits,
)
# ---------------------------------------------------------------------------
# Core: split by 999-rejection presence
# ---------------------------------------------------------------------------
def test_classify_splits_by_999_rejection_presence() -> None:
visits = [
("J813715", date(2026, 6, 27), "T1019"),
("OTHER", date(2026, 6, 27), "T1019"),
]
nine99_rejected = {("J813715", date(2026, 6, 27), "T1019")}
out = classify_not_in_835_visits(visits, nine99_rejected)
assert out["J813715"].value == Bucket.REJECTED_AT_999.value
assert out["OTHER"].value == Bucket.NEVER_SUBMITTED.value
# ---------------------------------------------------------------------------
# Edge cases
# ---------------------------------------------------------------------------
def test_classify_empty_input_returns_empty_dict() -> None:
"""No visits → empty bucket map (no-op)."""
out = classify_not_in_835_visits([], set())
assert out == {}
# Also: empty visits + non-empty rejection set stays empty.
out2 = classify_not_in_835_visits(
[], {("J813715", date(2026, 6, 27), "T1019")},
)
assert out2 == {}
def test_classify_all_rejected() -> None:
"""Every visit is in the 999-rejection set → every bucket is REJECTED_AT_999."""
v1 = ("MEM001", date(2026, 3, 15), "T1019")
v2 = ("MEM002", date(2026, 4, 1), "T1019")
out = classify_not_in_835_visits([v1, v2], {v1, v2})
assert out == {
"MEM001": Bucket.REJECTED_AT_999,
"MEM002": Bucket.REJECTED_AT_999,
}
def test_classify_all_never_submitted() -> None:
"""Visits present, rejection set empty → every bucket is NEVER_SUBMITTED."""
visits = [
("G1", date(2026, 3, 1), "T1019"),
("G2", date(2026, 3, 2), "T1019"),
("G3", date(2026, 3, 3), "T1019"),
]
out = classify_not_in_835_visits(visits, set())
assert out == {m: Bucket.NEVER_SUBMITTED for m in ("G1", "G2", "G3")}
def test_classify_keyed_by_member_id() -> None:
"""Output is a dict keyed by member_id, not by the visit tuple.
The spec's contract is "keyed on member_id" — Pipeline B groups
by member for the ISO-week rebill, so the classification map
collapses to one entry per member. The visit tuple's procedure
and dos parts are the *match key* against the 999 rejection set,
not the *output key*.
When two visits for the same member resolve to different
buckets (one rejected, one not), the dict-construction order
means the *last* visit wins. This test pins that semantic —
Pipeline B re-resolves per-visit at the next layer so the
member-level bucket is just a coarse pre-filter.
"""
visits = [
("SHARED", date(2026, 5, 1), "T1019"),
("SHARED", date(2026, 5, 8), "T1019"),
]
# First visit IS in the rejected set, second is not.
nine99_rejected = {("SHARED", date(2026, 5, 1), "T1019")}
out = classify_not_in_835_visits(visits, nine99_rejected)
# Last-write-wins: the second visit is NEVER_SUBMITTED, so the
# member-level bucket ends up as NEVER_SUBMITTED. This is the
# documented coarse-filter semantic — Pipeline B does per-visit
# re-resolution downstream.
assert out == {"SHARED": Bucket.NEVER_SUBMITTED}
# Reverse the rejection set: only the second visit is rejected.
# Last visit wins → REJECTED_AT_999.
nine99_rejected = {("SHARED", date(2026, 5, 8), "T1019")}
out = classify_not_in_835_visits(visits, nine99_rejected)
assert out == {"SHARED": Bucket.REJECTED_AT_999}
def test_classify_distinct_members_dont_collide() -> None:
"""Two distinct members, only one rejected → independent bucket entries."""
visits = [
("ALICE", date(2026, 6, 1), "T1019"),
("BOB", date(2026, 6, 1), "T1019"),
]
nine99_rejected = {("ALICE", date(2026, 6, 1), "T1019")}
out = classify_not_in_835_visits(visits, nine99_rejected)
assert out == {
"ALICE": Bucket.REJECTED_AT_999,
"BOB": Bucket.NEVER_SUBMITTED,
}
# ---------------------------------------------------------------------------
# PullResult shape — guards against accidental field drift
# ---------------------------------------------------------------------------
def test_pull_result_is_frozen_dataclass() -> None:
"""``PullResult`` must be frozen so callers can't mutate the summary."""
pr = PullResult(
total_pulled=10,
rejected_at_999=3,
not_in_835=4,
rejected_breakdown={"R": 2, "E": 1},
)
assert pr.total_pulled == 10
assert pr.rejected_at_999 == 3
assert pr.not_in_835 == 4
assert pr.rejected_breakdown == {"R": 2, "E": 1}
# Frozen: assignment must raise.
import dataclasses
try:
pr.total_pulled = 99 # type: ignore[misc]
except dataclasses.FrozenInstanceError:
pass
else:
raise AssertionError("PullResult must be frozen")
def test_bucket_values_are_json_friendly_strings() -> None:
"""Bucket values serialize cleanly to JSON (string-enum contract)."""
import json
payload = {
"j1": Bucket.REJECTED_AT_999.value,
"j2": Bucket.NEVER_SUBMITTED.value,
}
# Round-trip — no enum leakage into the JSON output.
assert json.loads(json.dumps(payload)) == {
"j1": "REJECTED_AT_999",
"j2": "NEVER_SUBMITTED",
}
+501
View File
@@ -0,0 +1,501 @@
"""SP41 Task 13 — Edifabric validation gate on emitted rebill files.
Covers the new emit path in ``cyclone.rebill.run.run_rebill``:
* Real 837P files are written (not ``Path.touch()`` placeholders).
* Files pass through ``cyclone.edifabric.validate_edi`` before they
land in ``<out>/pipeline-{a,b}/``; failures go to
``<out>/quarantine/``.
* When Edifabric is unavailable (no API key, network error, etc.),
the gate fails open with a WARNING log and emits to the pipeline
dirs (matches the SP40 dev/CI posture).
Each test builds its own visits CSV + 835 stub in ``tmp_path`` and
invokes ``run_rebill`` directly (no HTTP). The Edifabric API is mocked
via ``unittest.mock.patch`` on
``cyclone.rebill.run._edifabric.validate_edi`` so no live HTTP hits the
network.
"""
from __future__ import annotations
import csv
from datetime import date
from decimal import Decimal
from pathlib import Path
from unittest.mock import patch
from cyclone.rebill.run import run_rebill
# --------------------------------------------------------------------------- #
# Fixtures / helpers
# --------------------------------------------------------------------------- #
def _write_visits_csv(path: Path, rows: list[tuple[str, str, str, str]]) -> Path:
"""rows: list of (dos_mmddyyyy, member, procedure, billed_str)."""
with path.open("w", newline="") as f:
w = csv.writer(f)
w.writerow(["Visit Date", "Member ID", "Procedure Code", "Billable Amount"])
for r in rows:
w.writerow(r)
return path
def _stub_835_with_denied_svc(
ingest_dir: Path,
member: str,
procedure: str,
svc_date: date,
claim_id: str = "ORIG-1",
carc_group: str = "CO",
carc_reason: str = "29",
) -> Path:
"""An 835 with a single denied SVC so the visit lands in Pipeline A.
The rebill pipeline reuses the SVC's ``claim_id`` as the Pipeline-A
RebillClaim's ``claim_id`` (preserves the freq-7 anchor). The visit
must match (member_id, procedure, svc_date) so the reconcile step
pairs them into DENIED + CARC REBILL.
The 835 shape mirrors ``tests/fixtures/835_sample_svc_with_member.txt``:
CLP02 = 4 (Denied), NM1*QC.NM109 = member_id, then SVC → DTM*472 → CAS.
Segments are concatenated without ``\\n`` separators — the parser
splits on ``~`` only, and a leading ``\\n`` makes ``elems[0]`` an
empty string so the segment-name match silently drops the segment.
"""
ingest_dir.mkdir(exist_ok=True)
svc_date_str = svc_date.strftime("%Y%m%d")
segs = [
"ISA*00* *00* *ZZ*SENDER *ZZ*RECEIVER *260627*1200*^*00501*000000001*0*P*:~",
"GS*HC*SENDER*RECEIVER*20260627*1200*1*X*005010X221A1~",
"ST*835*0001~",
"BPR*I*0*C*ACH*CCP*01*021000021*DA*123456789*1512345678**01*021000021*DA*123456789*20260101~",
"TRN*1*TRACE01*1512345678~",
"DTM*405*20260118~",
"N1*PR*COLORADO MEDICAL ASSISTANCE PROGRAM*XV*COMEDASSISTPROG~",
"N3*PO BOX 1100~",
"N4*DENVER*CO*80202~",
"REF*2U*7912900843~",
"LX*1~",
# CLP02=4 means Denied → reconciles to DENIED → Pipeline A.
f"CLP*{claim_id}*4*100*0**MC*2026029105200*11*1~",
# NM1*QC.NM109 is the member_id; parse_835_svc propagates it to
# the SVC row so the visit-side reconcile key matches.
f"NM1*QC*1*PATIENT*NAME****MR*{member}~",
f"SVC*HC:{procedure}*100*0*UN*1~",
f"DTM*472*{svc_date_str}~",
# CAS group code + reason → CARC REBILL (not EXCLUDED).
f"CAS*{carc_group}*{carc_reason}*100~",
"SE*14*0001~",
"GE*1*1~",
"IEA*1*000000001~",
]
body = "".join(segs)
p = ingest_dir / "x.835"
p.write_text(body)
return p
def _stub_empty_835(ingest_dir: Path) -> Path:
"""Empty 835 → rebill's parse_835_svc emits zero SVCs."""
ingest_dir.mkdir(exist_ok=True)
p = ingest_dir / "x.835"
p.write_text("ST*835*0001~SE*0*0001~")
return p
# --------------------------------------------------------------------------- #
# Tests
# --------------------------------------------------------------------------- #
def test_run_rebill_emits_real_files_for_pipeline_b(tmp_path):
"""Pipeline-B happy path: 1 in-window visit, 0 SVCs → 1 real 837P
file in pipeline-b/ (non-empty, contains ISA..IEA), 0 files in
quarantine/.
Edifabric is patched to return ``success`` so the file passes the
gate cleanly. The fixture is the same shape as the existing
``test_post_rebill_runs_and_returns_summary_path`` happy-path test.
"""
visits_path = _write_visits_csv(tmp_path / "visits.csv", [
("06/27/2026", "J813715", "T1019", "$2.32"),
])
ingest_dir = _stub_empty_835(tmp_path / "ingest")
out_dir = tmp_path / "out"
fake_result = {"Status": "success", "Details": [], "LastIndex": 20}
with patch(
"cyclone.rebill.run._edifabric.validate_edi",
return_value=fake_result,
) as mock_validate:
result = run_rebill(
window_start=date(2026, 1, 1),
window_end=date(2026, 6, 27),
override_filing=False,
visits_csv_path=str(visits_path),
ingest_dir=str(ingest_dir),
out_dir=str(out_dir),
)
# Pipeline B emitted exactly one file (matches the existing
# ``len(body["pipeline_b_files"]) == 1`` assertion in test_api_rebill).
assert len(result.pipeline_b_files) == 1
b_path = result.pipeline_b_files[0]
assert b_path.exists(), b_path
assert b_path.stat().st_size > 0, "Pipeline-B file must be non-empty"
# The file is a real 837P envelope (starts with ISA, ends with IEA).
body = b_path.read_bytes()
assert body.startswith(b"ISA*"), body[:32]
assert b"IEA*" in body, body[-32:]
# Edifabric gate was called exactly once for the one Pipeline-B batch.
assert mock_validate.call_count == 1
# HCPF-spec filename prefix is preserved.
assert b_path.name.startswith("tp11525703-837P-"), b_path.name
# Per-batch disambiguation suffix is present so same-millisecond
# collisions don't overwrite each other.
assert "J813715-2026-W26" in b_path.name, b_path.name
# Quarantine is empty (no validation failures).
q_files = list((out_dir / "quarantine").glob("*.837"))
assert q_files == [], q_files
def test_run_rebill_quarantines_on_edifabric_error(tmp_path):
"""When validate_edi returns Status='error', the file lands in
``quarantine/`` (NOT in ``pipeline-b/``).
Exercises both the Edifabric-rejected path AND the case-isolation
that splits good files from bad files in the same run.
"""
visits_path = _write_visits_csv(tmp_path / "visits.csv", [
("06/27/2026", "J813715", "T1019", "$2.32"),
("06/27/2026", "J999999", "T1019", "$2.32"), # different member
])
ingest_dir = _stub_empty_835(tmp_path / "ingest")
out_dir = tmp_path / "out"
# First call (J813715) returns error → quarantine.
# Second call (J999999) returns success → pipeline-b.
fake_results = iter([
{
"Status": "error",
"Details": [
{"SegmentId": "PER", "Message": "PER-04 is required", "Status": "error"},
],
"LastIndex": 5,
},
{"Status": "success", "Details": [], "LastIndex": 20},
])
def _side_effect(_body):
return next(fake_results)
with patch(
"cyclone.rebill.run._edifabric.validate_edi",
side_effect=_side_effect,
) as mock_validate:
result = run_rebill(
window_start=date(2026, 1, 1),
window_end=date(2026, 6, 27),
override_filing=False,
visits_csv_path=str(visits_path),
ingest_dir=str(ingest_dir),
out_dir=str(out_dir),
)
# Only the success batch should land in pipeline-b/.
assert len(result.pipeline_b_files) == 1, (
f"only the success batch should land in pipeline-b/, got "
f"{[str(p) for p in result.pipeline_b_files]}"
)
# One file in quarantine (the rejected batch).
q_files = list((out_dir / "quarantine").glob("*.837"))
assert len(q_files) == 1, q_files
# Quarantine filename keys off the batch (member_id-W{iso_week:02d}).
assert "J813715" in q_files[0].name, q_files[0].name
assert q_files[0].stat().st_size > 0, "quarantined file must be non-empty"
# validate_edi called twice — once per Pipeline-B batch.
assert mock_validate.call_count == 2
def test_run_rebill_fails_open_when_edifabric_unavailable(tmp_path):
"""When validate_edi raises EdifabricError (no API key, network
error, 5xx), the file lands in the pipeline dir with a WARNING log.
Matches the SP40 dev/CI posture: tests / dev boxes don't have a
real API key, so the gate must NOT block the rebill run.
"""
from cyclone.edifabric import EdifabricError
visits_path = _write_visits_csv(tmp_path / "visits.csv", [
("06/27/2026", "J813715", "T1019", "$2.32"),
])
ingest_dir = _stub_empty_835(tmp_path / "ingest")
out_dir = tmp_path / "out"
with patch(
"cyclone.rebill.run._edifabric.validate_edi",
side_effect=EdifabricError(0, "API key not configured"),
) as mock_validate:
result = run_rebill(
window_start=date(2026, 1, 1),
window_end=date(2026, 6, 27),
override_filing=False,
visits_csv_path=str(visits_path),
ingest_dir=str(ingest_dir),
out_dir=str(out_dir),
)
# File emitted to pipeline-b/ (NOT to quarantine — fail-open).
assert len(result.pipeline_b_files) == 1
assert result.pipeline_b_files[0].exists()
# Quarantine stays empty.
q_files = list((out_dir / "quarantine").glob("*.837"))
assert q_files == [], q_files
# Gate was actually called (so we know the fail-open path fired,
# not some accidental short-circuit).
assert mock_validate.call_count == 1
def test_run_rebill_emits_real_files_for_pipeline_a(tmp_path):
"""Pipeline-A happy path: 1 in-window visit, 1 denied SVC matching
it → 1 real 837P file in pipeline-a/ with frequency-code 7
preserved.
The SVC's original claim_id (ORIG-1) carries through as the
Pipeline-A RebillClaim.claim_id and shows up in the emitted file's
CLM01 segment AND in the on-disk filename suffix.
"""
svc_date = date(2026, 6, 27)
visits_path = _write_visits_csv(tmp_path / "visits.csv", [
(svc_date.strftime("%m/%d/%Y"), "J813715", "T1019", "$100.00"),
])
ingest_dir = tmp_path / "ingest"
_stub_835_with_denied_svc(
ingest_dir,
member="J813715",
procedure="T1019",
svc_date=svc_date,
claim_id="ORIG-1",
)
out_dir = tmp_path / "out"
fake_result = {"Status": "success", "Details": [], "LastIndex": 30}
with patch(
"cyclone.rebill.run._edifabric.validate_edi",
return_value=fake_result,
):
result = run_rebill(
window_start=date(2026, 1, 1),
window_end=date(2026, 6, 27),
override_filing=False,
visits_csv_path=str(visits_path),
ingest_dir=str(ingest_dir),
out_dir=str(out_dir),
)
# Pipeline A emitted exactly one file.
assert len(result.pipeline_a_files) == 1, result.pipeline_a_files
a_path = result.pipeline_a_files[0]
assert a_path.exists(), a_path
assert a_path.stat().st_size > 0, "Pipeline-A file must be non-empty"
body = a_path.read_bytes()
assert body.startswith(b"ISA*"), body[:32]
assert b"IEA*" in body, body[-32:]
# Original claim_submit_id is preserved in CLM01 (anchors the
# frequency-7 replacement).
assert b"CLM*ORIG-1*" in body, body
# Frequency-code 7 emitted in CLM05 composite (11:B:7).
assert b"CLM*ORIG-1*100.00***11:B:7" in body, body
# Pipeline-A filename uses HCPF-spec prefix + claim_id suffix.
assert a_path.name.startswith("tp11525703-837P-"), a_path.name
assert a_path.name.endswith("-ORIG-1.x12"), a_path.name
# Quarantine empty.
q_files = list((out_dir / "quarantine").glob("*.837"))
assert q_files == [], q_files
def test_run_rebill_quarantines_pipeline_a_on_edifabric_error(tmp_path):
"""Pipeline A path on Edifabric Status='error' → quarantine, NOT
pipeline-a/.
The existing ``test_run_rebill_quarantines_on_edifabric_error`` uses
an empty 835 so all visits land in Pipeline B; this one builds a
real 835 with a denied SVC matching the visit so the claim routes
to Pipeline A, then asserts the rejected file lands in
``quarantine/`` and ``pipeline-a/`` stays empty.
"""
svc_date = date(2026, 6, 27)
visits_path = _write_visits_csv(tmp_path / "visits.csv", [
(svc_date.strftime("%m/%d/%Y"), "J813715", "T1019", "$100.00"),
])
ingest_dir = tmp_path / "ingest"
_stub_835_with_denied_svc(
ingest_dir,
member="J813715",
procedure="T1019",
svc_date=svc_date,
claim_id="ORIG-ERROR-1",
carc_reason="29", # CO-29 → REBILL (not EXCLUDED)
)
out_dir = tmp_path / "out"
fake_result = {
"Status": "error",
"Details": [
{"SegmentId": "PER", "Message": "PER-04 is required",
"Status": "error"},
],
"LastIndex": 5,
}
with patch(
"cyclone.rebill.run._edifabric.validate_edi",
return_value=fake_result,
) as mock_validate:
result = run_rebill(
window_start=date(2026, 1, 1),
window_end=date(2026, 6, 27),
override_filing=False,
visits_csv_path=str(visits_path),
ingest_dir=str(ingest_dir),
out_dir=str(out_dir),
)
# Pipeline-A path rejected → 0 files in pipeline-a/.
assert result.pipeline_a_files == [], (
f"Pipeline A rejected file should NOT be in pipeline-a/, got "
f"{[str(p) for p in result.pipeline_a_files]}"
)
a_files = list((out_dir / "pipeline-a").glob("*"))
assert a_files == [], a_files
# File quarantined under the original claim_id key.
q_files = list((out_dir / "quarantine").glob("*.837"))
assert len(q_files) == 1, q_files
assert "ORIG-ERROR-1" in q_files[0].name, q_files[0].name
assert q_files[0].stat().st_size > 0, "quarantined file must be non-empty"
# Gate was called for the one Pipeline-A claim.
assert mock_validate.call_count == 1
def test_run_rebill_mixed_pipelines_route_correctly(tmp_path):
"""Mixed-pipeline run: one DENIED visit (Pipeline A) + one
NOT_IN_835 visit (Pipeline B) in the same window.
Asserts:
- pipeline_a_files has exactly 1 entry (the denied visit)
- pipeline_b_files has exactly 1 entry (the unmatched visit)
- summary.csv has 2 rows (one per non-PAID visit)
"""
svc_date = date(2026, 6, 27)
# Visit 1: matches a denied SVC → Pipeline A.
# Visit 2: different member entirely → NOT_IN_835 → Pipeline B.
visits_path = _write_visits_csv(tmp_path / "visits.csv", [
(svc_date.strftime("%m/%d/%Y"), "J813715", "T1019", "$100.00"),
(svc_date.strftime("%m/%d/%Y"), "J999999", "T1019", "$2.32"),
])
ingest_dir = tmp_path / "ingest"
_stub_835_with_denied_svc(
ingest_dir,
member="J813715",
procedure="T1019",
svc_date=svc_date,
claim_id="ORIG-MIX-1",
carc_reason="29",
)
out_dir = tmp_path / "out"
fake_result = {"Status": "success", "Details": [], "LastIndex": 30}
with patch(
"cyclone.rebill.run._edifabric.validate_edi",
return_value=fake_result,
) as mock_validate:
result = run_rebill(
window_start=date(2026, 1, 1),
window_end=date(2026, 6, 27),
override_filing=False,
visits_csv_path=str(visits_path),
ingest_dir=str(ingest_dir),
out_dir=str(out_dir),
)
# One file each, the two pipelines route independently.
assert len(result.pipeline_a_files) == 1, result.pipeline_a_files
assert len(result.pipeline_b_files) == 1, result.pipeline_b_files
# Files on disk match the returned paths.
a_path = result.pipeline_a_files[0]
b_path = result.pipeline_b_files[0]
assert a_path.exists() and a_path.stat().st_size > 0
assert b_path.exists() and b_path.stat().st_size > 0
# Each gate call hit validate_edi exactly once (one per claim/batch).
assert mock_validate.call_count == 2
# summary.csv has one row per non-PAID visit.
summary_path = out_dir / "summary.csv"
assert summary_path.exists(), summary_path
with summary_path.open(newline="") as f:
rows = list(csv.DictReader(f))
assert len(rows) == 2, rows
dispositions = {r["disposition"] for r in rows}
assert dispositions == {"REBILLED_A", "REBILLED_B"}, dispositions
def test_run_rebill_fail_open_logs_warning(tmp_path, caplog):
"""The Edifabric-unavailable fail-open path MUST log a WARNING so
the operator knows the validation gate didn't actually run.
Per SP40/SP41 contract: when ``validate_edi`` raises
``EdifabricError``, the file still emits to the pipeline dir but a
WARNING is logged. This test pins that contract so a future
refactor doesn't accidentally silence it.
"""
import logging
from cyclone.edifabric import EdifabricError
caplog.set_level(logging.WARNING, logger="cyclone.rebill.run")
visits_path = _write_visits_csv(tmp_path / "visits.csv", [
("06/27/2026", "J813715", "T1019", "$2.32"),
])
ingest_dir = _stub_empty_835(tmp_path / "ingest")
out_dir = tmp_path / "out"
with patch(
"cyclone.rebill.run._edifabric.validate_edi",
side_effect=EdifabricError(0, "API key not configured"),
):
result = run_rebill(
window_start=date(2026, 1, 1),
window_end=date(2026, 6, 27),
override_filing=False,
visits_csv_path=str(visits_path),
ingest_dir=str(ingest_dir),
out_dir=str(out_dir),
)
# File still emits (fail-open semantics).
assert len(result.pipeline_b_files) == 1
# And a WARNING was logged about the unrun gate.
warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
assert warnings, "expected at least one WARNING record"
assert any(
"edifabric" in r.message.lower() or "validation" in r.message.lower()
for r in warnings
), f"expected WARNING mentioning edifabric/validation, got {[r.message for r in warnings]}"
+209 -1
View File
@@ -1,5 +1,11 @@
"""Claim-id dedup at the SFTP pre-flight stage.""" """Claim-id dedup at the SFTP pre-flight stage."""
from datetime import datetime from datetime import datetime, timezone
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from cyclone.submission.core import submit_file
from cyclone.store.exceptions import DuplicateClaimError from cyclone.store.exceptions import DuplicateClaimError
from cyclone.store.submission_dedup import ( from cyclone.store.submission_dedup import (
record_submission, record_submission,
@@ -7,6 +13,12 @@ from cyclone.store.submission_dedup import (
DEFAULT_WINDOW_DAYS, DEFAULT_WINDOW_DAYS,
) )
# Fixture file with a single CLM01 = "CLM001" claim (the minimal_837p
# fixture is the source of truth; duplicated into submit-batch/single-claim.x12
# but both byte-equal so we pick the canonical minimal path here).
_FIXTURE = Path(__file__).parent / "fixtures" / "minimal_837p.txt"
_FIXTURE_CLAIM_ID = "CLM001"
def test_check_duplicate_returns_none_for_first_submission(tmp_path): def test_check_duplicate_returns_none_for_first_submission(tmp_path):
db = tmp_path / "test.db" db = tmp_path / "test.db"
@@ -36,3 +48,199 @@ def test_check_duplicate_passes_after_window(tmp_path):
def test_default_window_is_30_days(): def test_default_window_is_30_days():
assert DEFAULT_WINDOW_DAYS == 30 assert DEFAULT_WINDOW_DAYS == 30
# --------------------------------------------------------------------------- #
# SP41 Task 9 integration: dedup guard wired into submit_file pre-flight.
# --------------------------------------------------------------------------- #
def test_submit_file_raises_on_duplicate_claim_id(tmp_path):
"""``submit_file`` raises ``DuplicateClaimError`` if any CLM01 in the
file was submitted within the 30-day dedup window, and the DB write
+ SFTP factory are both skipped.
Mirrors the DB-first, upload-second invariant from ``submit_file``:
the dedup check sits between the payer-mismatch guard and the
BatchRecord837 DB write, so a duplicate blocks both the DB write
AND the SFTP upload. This test pins both invariants.
The conftest autouse fixture wires a fresh per-test SQLite DB at
``tmp_path/test.db`` via the ``CYCLONE_DB_URL`` env var (so
``cycl_db.SessionLocal()`` points there) — same engine the Batch
write would use, same engine the dedup helper reads.
"""
# 1. Pre-populate the dedup table with the claim_id from the
# fixture at a timestamp within the 30-day window. ``db_url``
# is ignored inside the helper (signature-preserving shim) but
# the argument is required by the current Task 8 signature, so
# we pass the per-test DB URL for clarity.
record_submission(
_FIXTURE_CLAIM_ID,
submitted_at=datetime.now(timezone.utc),
db_url=f"sqlite:///{tmp_path}/test.db",
)
# 2. Sentinel: if submit_file reaches the SFTP factory call,
# fail loudly. ``submit_file`` raises DuplicateClaimError
# BEFORE the factory is invoked, so the AssertionError here
# only fires if the dedup guard is bypassed (regression).
def _sftp_factory(_block): # pragma: no cover - regression only
raise AssertionError(
"SFTP factory must not be called when dedup blocks submit_file"
)
sftp_block = MagicMock(stub=False, paths={"outbound": "/ToHPE"})
# 3. Expect the 409-class domain exception with structured
# ``claim_id`` / ``original_submission_at`` attributes so the
# operator can see which CLM01 tripped the guard.
with pytest.raises(DuplicateClaimError) as exc_info:
submit_file(
_FIXTURE,
sftp_block=sftp_block,
actor="test",
validate=True,
sftp_client_factory=_sftp_factory,
)
assert exc_info.value.claim_id == _FIXTURE_CLAIM_ID
assert exc_info.value.original_submission_at is not None
# 4. DB invariant: no Batch row was written (dedup blocks the
# DB write, not just the SFTP upload). The autouse fixture
# points cycl_db at tmp_path/test.db, so a fresh
# ``db_mod.SessionLocal()()`` re-opens it.
from cyclone import db as db_mod
with db_mod.SessionLocal()() as session:
batch_count = session.query(db_mod.Batch).count()
assert batch_count == 0, (
"dedup guard must block the BatchRecord837 DB write; "
f"found {batch_count} Batch row(s)"
)
# --------------------------------------------------------------------------- #
# SP41 Task 9 — router-level coverage: api_routers/submission.submit_batch
# catches DuplicateClaimError BEFORE the generic Exception handler and
# surfaces it as a per-file ``outcome="unexpected_error"`` row, with the
# duplicated claim_id baked into ``error`` and ``batch_id=None`` (no DB
# row written). The HTTP status code stays 200 — per-file failures live
# in the JSON body, not in the status (see the top-of-module contract
# in api_routers/submission.py).
# --------------------------------------------------------------------------- #
def test_submit_batch_per_file_duplicate_claim_classified_as_unexpected(
tmp_path, monkeypatch,
):
"""Pre-record CLM001 (the fixture's only CLM01) then POST a batch
containing one copy of the fixture. Asserts the router:
- keeps the HTTP status at 200 (per-file failures don't change status)
- classifies the row as ``outcome == "unexpected_error"`` (the
special-case handler in api_routers/submission.py)
- includes the duplicated ``claim_id`` ("CLM001") in ``error``
- sets ``batch_id is None`` (no BatchRecord837 row written)
Does NOT monkeypatch ``submit_file`` — the test exercises the real
pre-flight raise path so the router's special-case catch is what
produces the unexpected_error classification. submit_file raises
DuplicateClaimError BEFORE the SFTP factory is invoked, so the
paramiko factory is never reached even though the client fixture
flips ``sftp_block.stub`` to ``False`` (no live MFT needed).
"""
from fastapi.testclient import TestClient
from cyclone import db as db_mod
from cyclone.api import app
from cyclone.store import store as cycl_store
# 1. Per-test DB + clearhouse (mirror test_api_submit_batch.py's
# ``client`` fixture inline so this test is self-contained).
# The autouse conftest already wired CYCLONE_DB_URL + init_db;
# we just need to seed the clearhouse row and flip stub=False.
db_mod._reset_for_tests()
db_mod.init_db()
cycl_store.ensure_clearhouse_seeded()
ch = cycl_store.get_clearhouse()
cycl_store.update_clearhouse(
ch.model_copy(update={
"sftp_block": ch.sftp_block.model_copy(update={"stub": False}),
}),
)
# 2. Pre-record the duplicate. CLM001 is the fixture's only CLM01;
# ``record_submission`` is the same helper the dedup unit test
# uses, so the window + exception shape match.
record_submission(
_FIXTURE_CLAIM_ID,
submitted_at=datetime.now(timezone.utc),
db_url=f"sqlite:///{tmp_path}/test.db",
)
# 3. Stage a single batch-* dir containing one copy of the fixture.
# Mirrors _stage_batch from test_api_submit_batch.py.
batch_dir = tmp_path / "batch-dedup-claims"
batch_dir.mkdir()
(batch_dir / "dup-claim.x12").write_bytes(_FIXTURE.read_bytes())
# 4. Sentinel on submit_file's sftp_client_factory — submit_file
# raises BEFORE the factory is invoked, so this AssertionError
# only fires if the dedup guard is bypassed end-to-end. We
# don't actually call submit_file directly here; the sentinel
# is wired via monkeypatch on the function the router uses, so
# if anything in the chain accidentally reaches the factory
# call we'd see it. In practice submit_file raises and we never
# get there — this is a regression tripwire only.
def _sentinel_factory(_block): # pragma: no cover - regression only
raise AssertionError(
"SFTP factory must not be called: dedup blocks submit_file "
"before the factory runs"
)
monkeypatch.setattr(
"cyclone.submission.core._default_sftp_factory", _sentinel_factory,
)
# 5. Hit the endpoint. Per-file failures don't change the status
# code → 200 even though one file was rejected.
client = TestClient(app)
resp = client.post(
"/api/submit-batch",
json={"ingest_dir": str(tmp_path), "validate": True},
)
assert resp.status_code == 200, resp.text
body = resp.json()
# 6. Counts: 1 file total, classified as failed. (UNEXPECTED_ERROR
# rolls into the ``failed`` counter in api_routers/submission.py
# — see the counter branch at the bottom of the per-file loop.)
assert body["submitted"] == 0
assert body["skipped"] == 0
assert body["failed"] == 1
assert len(body["results"]) == 1
# 7. The single result row carries the special-case classification
# (NOT a generic "runtime error: kaboom" — that's a different
# test in test_api_submit_batch.py). The router's
# DuplicateClaimError handler sets UNEXPECTED_ERROR + bakes the
# claim_id into ``error``.
row = body["results"][0]
assert row["outcome"] == "unexpected_error"
assert _FIXTURE_CLAIM_ID in row["error"], (
f"expected claim_id {_FIXTURE_CLAIM_ID!r} in error string; "
f"got {row['error']!r}"
)
# The router's handler does NOT construct a batch_id — the dedup
# guard fires BEFORE the DB write, so no BatchRecord837 row exists.
assert row["batch_id"] is None
# 8. DB invariant (defense in depth): even though we trust the
# router-level assertion above, pin that no Batch row leaked.
with db_mod.SessionLocal()() as session:
batch_count = session.query(db_mod.Batch).count()
assert batch_count == 0, (
"dedup guard must block the BatchRecord837 DB write end-to-end; "
f"found {batch_count} Batch row(s)"
)
+11
View File
@@ -187,6 +187,17 @@ backend/src/cyclone/
├── payers.py — Payer ORM row accessor (SP9) ├── payers.py — Payer ORM row accessor (SP9)
├── providers.py — Payer / Clearhouse / Provider ORM row DTOs (SP9) ├── providers.py — Payer / Clearhouse / Provider ORM row DTOs (SP9)
├── pubsub.py — NDJSON live-tail EventBus ├── pubsub.py — NDJSON live-tail EventBus
├── rebill/ — (SP41 — in-window rebill pipeline)
│ ├── parse_835_svc.py — SVC-level 835 reparse w/ member_id
│ ├── reconcile.py — visit-to-835 join (member, procedure, DOS)
│ ├── carc_filter.py — CARC-aware filter
│ ├── timely_filing.py — 120-day DOS age gate
│ ├── pipeline_a.py — denied/partial → frequency-7
│ ├── pipeline_b.py — NOT_IN_835 → fresh 837Ps by (member, week)
│ ├── summary.py — summary CSV generator
│ ├── run.py — orchestrator (run_rebill)
│ ├── pull_999_acks.py — 999-ack dump + classify
│ └── __init__.py — package surface
├── reconcile.py — 835 → 837 auto-reconciliation engine ├── reconcile.py — 835 → 837 auto-reconciliation engine
├── scheduler.py — SP16 MFT polling scheduler ├── scheduler.py — SP16 MFT polling scheduler
├── scoring.py — 4-field lane scoring (40/25/20/15, hard-coded — backlog item) ├── scoring.py — 4-field lane scoring (40/25/20/15, hard-coded — backlog item)
+13
View File
@@ -145,6 +145,19 @@ Each requirement is `FR-NN`, traceable to one or more sub-projects. Verification
| FR-37 | (Sibling project, not in this repo) Drive the full 7-phase round-trip — preflight → browser upload → parse verify → SFTP submit → TA1 wait → 999 wait → scan + report — with crash-safe resume, structured JSON logging, idempotency dedup, per-run folder. | SP22 (`cyclone-pipeline/`) | | FR-37 | (Sibling project, not in this repo) Drive the full 7-phase round-trip — preflight → browser upload → parse verify → SFTP submit → TA1 wait → 999 wait → scan + report — with crash-safe resume, structured JSON logging, idempotency dedup, per-run folder. | SP22 (`cyclone-pipeline/`) |
| FR-38 | Provide a `DrillStackProvider` + `DrillDrawerHeader` shell so every drillable cell across the app opens a consistent drawer or peek modal. | SP21 | | FR-38 | Provide a `DrillStackProvider` + `DrillDrawerHeader` shell so every drillable cell across the app opens a consistent drawer or peek modal. | SP21 |
### 3.1 In-window rebill pipeline FRs (SP41)
The `FR-RB-*` IDs are SP41-specific functional requirements for the in-window rebill pipeline. They sit alongside `FR-1`..`FR-38` in §3 but use the `RB` prefix to indicate they cover the rebill sub-domain. Each row links to its implementation module under `cyclone.rebill.*` (or, for FR-RB-5, the store-side `cyclone.store.submission_dedup` helper that the pipeline integrates with).
| ID | Requirement | SP |
|---|---|---|
| FR-RB-1 | Visit-to-835 reconciliation on `(member_id, procedure, DOS)`. Match key plus a best-of-N across duplicate SVC rows (per-SVC charge tolerance within 5% to absorb legitimate AxisCare-vs-835 differences). Implemented in `cyclone.rebill.reconcile.reconcile_visits_to_835`. | SP41 |
| FR-RB-2 | CARC-aware filter. Excluded reasons (CO-45 / CO-26 / CO-129) trip the `EXCLUDED_CARC` outcome; review reasons (PI-16, OA-18, etc.) produce `REBILLED_A` with `needs_review=true`. Implemented in `cyclone.rebill.carc_filter.decide_carc`. | SP41 |
| FR-RB-3 | 120-day timely-filing gate with per-batch override. Default window = 120 days from DOS as of run date; `--override-filing/--no-override-filing` CLI flag (and matching API flag) relaxes the gate for past-window visits. Implemented in `cyclone.rebill.timely_filing.timely_filing_decision`. | SP41 |
| FR-RB-4 | Pipeline A (frequency-7 replacement) for denied/partial visits + Pipeline B (member-week batched fresh 837Ps) for `NOT_IN_835` visits. Pipeline A emits a single 837P per visit with `CLM05-3 = '7'`; Pipeline B batches by `(member_id, ISO-week)`. Implemented in `cyclone.rebill.pipeline_a` and `cyclone.rebill.pipeline_b`. | SP41 |
| FR-RB-5 | Claim-id dedup at SFTP pre-flight (30-day window, configurable). Implemented in `cyclone.store.submission_dedup.check_duplicate`; integrated into `cyclone.submission.submit_file` which raises `DuplicateClaimError` before any upload bytes are sent. | SP41 |
| FR-RB-6 | Summary CSV with `REBILLED_A / REBILLED_B / EXCLUDED_CARC / EXCLUDED_TIMELY_FILING / EXCLUDED_PAYER / EXCLUDED_NO_EVV` rows plus a per-disposition tally. Implemented in `cyclone.rebill.summary.write_summary_csv` and surfaced by `cyclone.rebill.run.run_rebill`. | SP41 |
--- ---
## 4. Non-functional requirements ## 4. Non-functional requirements