Compare commits
346 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 90a2055c20 | |||
| c9588f782d | |||
| 2c43a82390 | |||
| a8aef7b9b2 | |||
| b3a608a939 | |||
| f5713640e4 | |||
| 36cac9af78 | |||
| 7d257911d2 | |||
| b5b715d1c9 | |||
| df3e5e401d | |||
| 9e422fdadb | |||
| daf55d9a38 | |||
| ea68ae81f7 | |||
| ad6629c7ab | |||
| afd6cead06 | |||
| 95d04f9991 | |||
| 5fe61fddd3 | |||
| 56e14341c2 | |||
| 23fc85cad2 | |||
| 61e90b64b3 | |||
| e9e0b68746 | |||
| be5824ec31 | |||
| ee1a397bd5 | |||
| 019e09aff6 | |||
| e35e5e03b3 | |||
| 579e5d7438 | |||
| 1978e696d7 | |||
| ce61b36504 | |||
| 26c948e7fc | |||
| 309e6c016a | |||
| 8790e0a439 | |||
| 467f53046e | |||
| cfe9c25d9d | |||
| b0ad0c27b8 | |||
| b0d03bc10e | |||
| 9beab5b4fc | |||
| adb8dcb137 | |||
| a15547fb0e | |||
| 262ad00481 | |||
| cc41c73d2a | |||
| 30aee61956 | |||
| b19f43f448 | |||
| 2dfe213af0 | |||
| 83d4df413c | |||
| bd5530539d | |||
| 53df3f0684 | |||
| 0e130579b4 | |||
| 13dd240328 | |||
| 543422c343 | |||
| 39f578aa5e | |||
| 4d396d40b6 | |||
| fcac090885 | |||
| 45b6c6291a | |||
| a3831213cc | |||
| e946a4f7aa | |||
| 30d13876cd | |||
| ea7acd2e66 | |||
| 3b4f18eef6 | |||
| 69b760e890 | |||
| f8e2f0933e | |||
| 52795adfb1 | |||
| acf3b506ec | |||
| e7098b99b2 | |||
| 7838ed2a4c | |||
| c3ef4f3847 | |||
| cfca1d8975 | |||
| b652b18458 | |||
| 0815a4ad35 | |||
| 7218ec76a8 | |||
| b237b9cfc4 | |||
| c5604fdfa0 | |||
| f21201a22e | |||
| 0436820023 | |||
| 055d95224a | |||
| 77f73cf73e | |||
| 08c0430ee7 | |||
| 3400ed5ec2 | |||
| 0493814489 | |||
| 538fad211b | |||
| 353adfc08b | |||
| d328c0b224 | |||
| 8516b90601 | |||
| 8a251d20a9 | |||
| 01e5ee781e | |||
| 28835e2f1d | |||
| 0ccc396e5e | |||
| c8f5af3ba5 | |||
| fa034b1cd7 | |||
| 9c0aa577fb | |||
| 83a31b6fea | |||
| 07ea7ca1d6 | |||
| 76923a79f5 | |||
| 9ef749c783 | |||
| ad14b56732 | |||
| 8890627014 | |||
| d776a4a7ec | |||
| 893a6629a8 | |||
| f62e1a7881 | |||
| dc5bff617d | |||
| abaf23c122 | |||
| ce0df8ac24 | |||
| cc02cb99c5 | |||
| aff3a13016 | |||
| 7cb0278be0 | |||
| 4c85166734 | |||
| f1dc06ec3b | |||
| dc990cfde3 | |||
| ec065a9082 | |||
| 5d4e06b863 | |||
| 298200fe71 | |||
| 7ad190c267 | |||
| 3a58f561d4 | |||
| 0067ecc5da | |||
| 698f8660bf | |||
| 178898a8e1 | |||
| 710104f491 | |||
| 4b56416414 | |||
| f005494606 | |||
| 69b338234d | |||
| 17736ccffa | |||
| 5f5ac875f1 | |||
| 97145f313b | |||
| 4770c04021 | |||
| cad4f1fe2d | |||
| 7abe94a3a8 | |||
| 6a5dbdf88a | |||
| 5028628269 | |||
| 299c1a85a3 | |||
| 9429d11b5f | |||
| 85791e0df7 | |||
| 21066ad0bf | |||
| a963d3ce25 | |||
| a52a85c7a2 | |||
| 95f5e91ade | |||
| 3bc5740e8b | |||
| b0e06a2dd0 | |||
| d1cd6e1a51 | |||
| f25214189a | |||
| e4f3d25f3a | |||
| 750f560ee0 | |||
| 0193ee4c32 | |||
| 6bda5005c1 | |||
| 27bca33b09 | |||
| f10ab83628 | |||
| 244015a361 | |||
| 3bf5622010 | |||
| 0625c83a45 | |||
| cf7c343ff0 | |||
| dae7749464 | |||
| 2e7ad471e0 | |||
| dfd654202e | |||
| 15c85300f6 | |||
| 1b2f6c6b21 | |||
| 0f3b264e41 | |||
| de77f19d9d | |||
| c4bc118557 | |||
| 055b4ae30e | |||
| 4d3eef22ef | |||
| d8707ba874 | |||
| c491003287 | |||
| 4363b4fe41 | |||
| d44948cb90 | |||
| 9e2dde6ddb | |||
| 2d40fbbcbb | |||
| a2ec65e5c6 | |||
| 18fa119ff7 | |||
| ab91449e62 | |||
| 9bade2429c | |||
| 5a54961930 | |||
| b484ca36dc | |||
| 75a2800a9d | |||
| 6624a0bafd | |||
| a12104fb0f | |||
| edee0a6259 | |||
| 97512ec4a7 | |||
| bde3060e9e | |||
| 1d2572e624 | |||
| ffacfd8665 | |||
| 9cc13e7940 | |||
| 37caf6af77 | |||
| 02b879a204 | |||
| fba594baf4 | |||
| 190f4b8c8d | |||
| 9d11ffcd8a | |||
| 21a5485faa | |||
| 3fd2c445e7 | |||
| 9e16b8d9bd | |||
| 1c367ddbe7 | |||
| d8f3fb15f9 | |||
| 602dc352c5 | |||
| 490a1b9478 | |||
| b094231995 | |||
| c54b2c1867 | |||
| a97f1d1350 | |||
| d2512945ca | |||
| 9e775c0150 | |||
| e10d3886c2 | |||
| ec87f98f02 | |||
| f88a7c7c4f | |||
| aca9667ff8 | |||
| 999a762c86 | |||
| 53a8c77cb8 | |||
| de8bc2e153 | |||
| 1363d36046 | |||
| 146cb6d17d | |||
| 191bc935c3 | |||
| 029623f3a5 | |||
| 1648c81425 | |||
| f1bee546f6 | |||
| 603ec8842a | |||
| 686c11f480 | |||
| 8d11b391a0 | |||
| 4b22193c4a | |||
| d8c03fde3f | |||
| 014e02ad42 | |||
| 4360ef7209 | |||
| fe84ce7cf3 | |||
| cfc95307e3 | |||
| c8f8f5d3c6 | |||
| 9644db8c51 | |||
| fbe7b2358d | |||
| b96da8a5a4 | |||
| 14fcbca5f1 | |||
| d8841834dc | |||
| d81b6ed4fc | |||
| 59c3275adf | |||
| f5d119fbe7 | |||
| d5f95b4f3c | |||
| 44a6fb031a | |||
| cf1a0a80d8 | |||
| 34542d1d34 | |||
| 4c05c6527b | |||
| 4592bca372 | |||
| 79fa30d018 | |||
| 35730fcf14 | |||
| 30e1add8a2 | |||
| d248a5f282 | |||
| 0f1e609888 | |||
| 9d8f83d111 | |||
| 454c3598b1 | |||
| f57f1d2875 | |||
| 315fbfec42 | |||
| 6507a8c874 | |||
| 1381a7652d | |||
| c3a6c53096 | |||
| a436538c15 | |||
| dd7da18279 | |||
| 9cb0311544 | |||
| b7be9f38bc | |||
| 1b74af4d3a | |||
| b80e40e7e9 | |||
| edca04868d | |||
| 74aa64f995 | |||
| 4ef09f9416 | |||
| 3c907d17e9 | |||
| 675faf9844 | |||
| 1e8217ff84 | |||
| 3075955826 | |||
| 5b4c1513b6 | |||
| 4e580ffe0e | |||
| 13a3e7c334 | |||
| 439a3c3d4b | |||
| 5db764d50e | |||
| 2d3667738d | |||
| b2d88d13d3 | |||
| 9fda46b891 | |||
| d401904412 | |||
| 7d2000ec31 | |||
| 1860782ad6 | |||
| ce37c10c06 | |||
| f4bafc1c94 | |||
| 24fbf945c9 | |||
| 07a7ecbdd4 | |||
| 5334646992 | |||
| aecf831f43 | |||
| 3ba5ca0849 | |||
| f7697e58b7 | |||
| 7706a6d7fe | |||
| 256ddfa5fa | |||
| d42fbc8c1b | |||
| 67dae61a94 | |||
| 59e69127a2 | |||
| 364e5d7497 | |||
| 35c561de20 | |||
| c398aa7d29 | |||
| 616d467c65 | |||
| 2194d35ea1 | |||
| 4fd55dc33e | |||
| 00f11f84b6 | |||
| d294b8bbed | |||
| 9e595503b7 | |||
| 1148a03a43 | |||
| cad104a38c | |||
| 0c81968d0c | |||
| add6e982a4 | |||
| 414d2eb722 | |||
| 7e4bb4d2c8 | |||
| fb913f0617 | |||
| 39ae988101 | |||
| e2d4a595a4 | |||
| 81bcb1c1ef | |||
| 8fc3d9adda | |||
| 35e0f5a422 | |||
| a25504bd3a | |||
| 70280f70bb | |||
| a933672411 | |||
| ca40fd72c3 | |||
| 4402a993d1 | |||
| cb87456575 | |||
| 3d0c7766f0 | |||
| e76b514872 | |||
| 5bb9588f09 | |||
| d895854dcc | |||
| 55a298f05f | |||
| 0aea0f64ac | |||
| 6c80bf0512 | |||
| 9c57b493a7 | |||
| 609499543e | |||
| 768f7c6247 | |||
| 86b635104c | |||
| e158871a9a | |||
| 1ca50e2bc0 | |||
| 74d7056284 | |||
| 0ba91040f1 | |||
| dc83d7bef2 | |||
| 9d4798a124 | |||
| d552d3d3ec | |||
| 42a826cb73 | |||
| 05b43078b9 | |||
| 7c1be58860 | |||
| 9bca4b608a | |||
| 35298907bc | |||
| 22720168c4 | |||
| 1c0d855b8e | |||
| 8db5db7610 | |||
| 786ead8c94 | |||
| 6c773c1159 | |||
| 5c7e9b6168 | |||
| ac87ed4908 | |||
| 4a8ce1a524 | |||
| 5053a1ea8e | |||
| 33fa899217 | |||
| 6fdbceefc2 | |||
| 7290cac643 | |||
| 5e8c7b11ea | |||
| 9c0cec8f0c |
@@ -0,0 +1,13 @@
|
||||
# Repo-root .dockerignore — applies to `docker compose build` (which builds
|
||||
# both backend and frontend contexts from the repo root). Excludes anything
|
||||
# that should never end up in a build context.
|
||||
|
||||
.worktrees/
|
||||
.git/
|
||||
.github/
|
||||
docs/prodfiles/
|
||||
*.production.txt
|
||||
node_modules/
|
||||
dist/
|
||||
.venv/
|
||||
.superpowers/brainstorm/
|
||||
+37
-4
@@ -1,7 +1,40 @@
|
||||
# Cyclone — environment configuration
|
||||
# Copy this file to `.env.local` and fill in values for your environment.
|
||||
|
||||
# Base URL for the Python (FastAPI) backend that powers the Upload page and
|
||||
# the real /api/parse-837 + /api/parse-835 endpoints. Leave empty to keep
|
||||
# the in-memory sample data store and disable real EDI parsing.
|
||||
VITE_API_BASE_URL=http://localhost:8000
|
||||
# Required on first boot. Cyclone refuses to start without these unless
|
||||
# at least one user already exists (e.g. seeded via `python -m cyclone users create`).
|
||||
# Min 12 chars for password.
|
||||
CYCLONE_ADMIN_USERNAME=admin
|
||||
CYCLONE_ADMIN_PASSWORD=change-me-to-a-strong-password-min-12-chars
|
||||
|
||||
# Base URL for the Python (FastAPI) backend. Leave empty for the
|
||||
# Docker deployment (nginx proxies /api/* to backend on compose network).
|
||||
VITE_API_BASE_URL=
|
||||
|
||||
# Optional. Set to 1 if you're behind an HTTPS reverse proxy and want
|
||||
# the session cookie to include the Secure flag.
|
||||
# CYCLONE_BEHIND_HTTPS=1
|
||||
|
||||
# Optional. Set to 1 to disable auth entirely (DEV ONLY). When set,
|
||||
# the backend auto-grants admin access without checking credentials.
|
||||
# CYCLONE_AUTH_DISABLED=0
|
||||
|
||||
# ─── Gainwell / HCPF SFTP credentials (operator convention) ─────────────
|
||||
# These mirror what /home/tyler/EDI/scripts/upload_claims.sh exports and
|
||||
# what the operator's SFTP client uses. Cyclone's secrets module looks up
|
||||
# CYCLONE_SFTP_PASSWORD (or _FILE), not GAINWELL_SFTP_PASS — to bridge:
|
||||
#
|
||||
# export CYCLONE_SFTP_PASSWORD="$GAINWELL_SFTP_PASS"
|
||||
#
|
||||
# or for the daemon, set CYCLONE_SFTP_PASSWORD_FILE to a 0600 file.
|
||||
GAINWELL_SFTP_USER=colorado-fts\coxix_prod_11525703
|
||||
GAINWELL_SFTP_HOST=mft.gainwelltechnologies.com
|
||||
GAINWELL_SFTP_PASS=
|
||||
GAINWELL_REMOTE_DIR=/CO XIX/PROD/coxix_prod_11525703/ToHPE
|
||||
|
||||
# CYCLONE_SFTP_PASSWORD= # mirror of GAINWELL_SFTP_PASS
|
||||
# CYCLONE_SFTP_PASSWORD_FILE= # path to a 0600 file containing it
|
||||
|
||||
# CYCLONE_SCHEDULER_AUTOSTART= # 1/true/yes to start the MFT poll loop on API launch
|
||||
# CYCLONE_SCHEDULER_POLL_SECONDS=60 # poll interval when autostart is on
|
||||
# CYCLONE_TAIL_HEARTBEAT_S=15 # NDJSON stream heartbeat interval
|
||||
+25
@@ -20,10 +20,24 @@ build/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# macOS extraction residue (AppleDouble / __MACOSX from unzip on macOS).
|
||||
# These are paired with every regular file in an extracted zip — never data.
|
||||
__MACOSX/
|
||||
._*
|
||||
|
||||
# Operator drop zone (untracked working dir; contents are HCPF-delivered
|
||||
# inbound files, never source). Use `mkdir -p ingest && touch ingest/.gitkeep`
|
||||
# if you want the directory itself in the repo.
|
||||
ingest/
|
||||
|
||||
# Local config
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
# Edifabric / EdiNation operator-supplied dev key (env file).
|
||||
# Lives at backend/.env.cyclone-edifabric; source it in the shell.
|
||||
.env.cyclone-edifabric
|
||||
backend/.env.cyclone-edifabric
|
||||
|
||||
# Production data (handled by ops, not committed)
|
||||
docs/prodfiles/*/
|
||||
@@ -38,3 +52,14 @@ claims_output/
|
||||
# Brainstorm session artifacts (visual companion mockups, events, server state).
|
||||
# Skills under .superpowers/skills/ are committed project-scoped guidance.
|
||||
.superpowers/brainstorm/
|
||||
|
||||
# SP33+ scratch / production-data ingest. Generated artifacts live
|
||||
# here only — the source EDI sits under docs/prodfiles/.
|
||||
ingest/
|
||||
|
||||
# SP41 dev/ scratch dir — generated 837P artifacts from rebill pipeline
|
||||
# runs. Hundreds of thousands of files; never source. The scripts that
|
||||
# build them are tracked (dev/unbilled-july2026/scripts/...) but the
|
||||
# generated x12 artifacts are not.
|
||||
dev/rebills/
|
||||
dev/rebills/2026-*
|
||||
|
||||
@@ -19,6 +19,10 @@ content negotiation + `tail_events`), and ~30 routes still inlined
|
||||
in `backend/src/cyclone/api.py`. The next refactor target is the
|
||||
parse endpoints.
|
||||
|
||||
## Auth gate (SP24)
|
||||
|
||||
Every router declared in `backend/src/cyclone/api_routers/` **must** carry `dependencies=[Depends(matrix_gate)]` at the `APIRouter(...)` declaration — not on each individual endpoint. The gate lives at `backend/src/cyclone/auth/deps.py:107` and the role matrix is at `backend/src/cyclone/auth/permissions.py`. The roles are `admin / user / viewer`; `matrix_gate` returns 401 when there's no session and 403 when the role is below the endpoint's required role. When `AUTH_DISABLED` is True (conftest autouse fixture flips it; `CYCLONE_AUTH_DISABLED=1` in prod-by-mistake), the gate short-circuits to a synthetic admin — see the SP24 spec for the threat-model implications. New routers get the gate by default; the auth-aware convention is `router = APIRouter(dependencies=[Depends(matrix_gate)])`.
|
||||
|
||||
## When to use
|
||||
|
||||
- **Adding an endpoint.** You're adding a new GET / POST handler —
|
||||
|
||||
@@ -10,9 +10,15 @@ a spec, a plan, an implementation branch, and a single atomic merge commit
|
||||
into `main`. This skill encodes the conventions so every increment follows
|
||||
the same shape and the commit history stays auditable.
|
||||
|
||||
As of this writing: **16 specs** in `docs/superpowers/specs/`, **12 plans**
|
||||
in `docs/superpowers/plans/`, and SP numbers used through **SP21** (the
|
||||
universal-drilldown design in progress). The next increment is **SP22**.
|
||||
As of this writing: **17 specs** in `docs/superpowers/specs/`, **13 plans**
|
||||
in `docs/superpowers/plans/`, and SP numbers used through **SP22**. **SP23**
|
||||
is the Ubuntu + Docker + RBAC product fork (awaiting user decision);
|
||||
**SP24** is the auth-posture alignment (docs-only). The next free increment
|
||||
is **SP25** after SP24 lands.
|
||||
|
||||
## Auth-aware spec template (SP24)
|
||||
|
||||
The threat-model section in the canonical SP-N spec template (`## 1. Scope`, second-to-last bullet) used to read "no second party to authenticate; no second host to harden against." **That phrasing is stale as of 2026-06-23** — the auth work landed in `main` and every backend endpoint requires login. New specs should instead state the auth boundary explicitly: "the auth boundary is HTTP (login required, bcrypt + HttpOnly session cookie); file-system threats remain the local-only threat model (SQLCipher at rest, macOS Keychain). SP23 changes the threat model to LAN-bound remote operator." Reference: [`docs/superpowers/specs/2026-06-23-cyclone-auth-posture-alignment-design.md`](../../../docs/superpowers/specs/2026-06-23-cyclone-auth-posture-alignment-design.md).
|
||||
|
||||
## When to use
|
||||
|
||||
|
||||
@@ -7,7 +7,11 @@ description: "Cyclone pytest + vitest fixture patterns, prodfiles layout, backen
|
||||
|
||||
The Cyclone test suite is split two ways: **backend pytest** (89 test files under `backend/tests/`, 13 flat fixtures in `backend/tests/fixtures/`, one autouse `conftest.py` that resets the DB per-test) and **frontend vitest** (59 `*.test.ts(x)` siblings across `src/`, two rendering styles — `@testing-library/react` and a custom `createRoot`+`Probe` shim). This skill codifies the conventions so additions stay consistent with what's already there.
|
||||
|
||||
As of this writing: **89 backend test files**, **13 flat backend fixtures**, **59 frontend `*.test.ts(x)` siblings**, and **23 prodfiles samples** across `docs/prodfiles/{837p-from-axiscare,835fromco,FromHPE,claims}/`. The next increment is **SP22**.
|
||||
As of this writing: **89 backend test files**, **13 flat backend fixtures**, **59 frontend `*.test.ts(x)` siblings**, and **23 prodfiles samples** across `docs/prodfiles/{837p-from-axiscare,835fromco,FromHPE,claims}/`. The next increment is **SP24** (SP23 is the Ubuntu+Docker fork, awaiting user decision).
|
||||
|
||||
## Auth flag (SP24)
|
||||
|
||||
The autouse `conftest.py` fixture at `backend/tests/conftest.py` flips `cyclone.auth.deps.AUTH_DISABLED = True` for the entire test session, so every test runs without a login round-trip. **Any new test that reads `cyclone.auth.deps.AUTH_DISABLED` directly will see `True`** — that's the test-suite reality, not a production reality. If you need a test that exercises the real gate, import `from cyclone.auth.deps import matrix_gate` and call it directly with a `Request` whose `state` carries a real session, or flip the flag back inside the test and reset it on teardown. The startup WARNING (`backend/src/cyclone/__main__.py`) is silent in tests by default because the conftest sets the flag before `bootstrap.run()` is called via `import`.
|
||||
|
||||
## When to use
|
||||
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## What this is
|
||||
|
||||
Cyclone is a self-hosted X12 EDI claims-management suite for a single billing office (Colorado Medicaid currently). It parses 837P professional claims and 835 ERA remittances (X12 005010X222A1 / 005010X221A1) and also handles 999, TA1, 270, 271, and 277CA. Always binds to `0.0.0.0` — the host firewall / compose port publishing is what restricts reachability, not the bind address. Requires login (auth boundary is HTTP; bcrypt + HttpOnly session cookie; first admin bootstrapped from `CYCLONE_ADMIN_USERNAME` + `CYCLONE_ADMIN_PASSWORD` env vars; see SP24 spec for the full posture). LAN-only by design — don't expose the published ports to the WAN.
|
||||
|
||||
Stack: one Python process (FastAPI + uvicorn, port 8000) + one Node process in dev (Vite, port 5173). The authoritative state is a single SQLite file at `~/.local/share/cyclone/cyclone.db` (or SQLCipher at the same path when the macOS Keychain entry + `sqlcipher3` are both present).
|
||||
|
||||
For the day-1 architecture read, see `docs/ARCHITECTURE.md` (process topology, module map, store facade, parser pipeline, pubsub). For the what-it-does read, see `docs/REQUIREMENTS.md` (FRs + NFRs + DoD).
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
# Backend (Python 3.11+)
|
||||
cd backend
|
||||
python -m venv .venv
|
||||
.venv/bin/pip install -e '.[dev]'
|
||||
|
||||
# Frontend (Node 20+)
|
||||
cd ..
|
||||
npm install
|
||||
```
|
||||
|
||||
Optional backend extras: `pip install -e '.[sqlcipher]'` (encryption at rest, SP12) and `pip install -e '.[sftp]'` (real SFTP, SP13).
|
||||
|
||||
## Dev (two terminals)
|
||||
|
||||
```bash
|
||||
# Terminal 1 — backend
|
||||
cd backend
|
||||
.venv/bin/python -m cyclone serve # default 0.0.0.0:8000
|
||||
# CYCLONE_PORT=... overrides port; CYCLONE_RELOAD=1 enables uvicorn --reload
|
||||
# Or: .venv/bin/uvicorn cyclone.api:app --reload --port 8000
|
||||
|
||||
# Terminal 2 — frontend
|
||||
npm run dev # Vite on http://localhost:5173
|
||||
```
|
||||
|
||||
Vite proxies `/api/*` to the backend at `http://127.0.0.1:${CYCLONE_PORT:-8000}` so relative-URL fetchers (the live-tail NDJSON streams in particular) resolve through the same origin. Override the backend port with `CYCLONE_PORT` in the frontend terminal too.
|
||||
|
||||
Create `.env.local` at the repo root with `VITE_API_BASE_URL=http://127.0.0.1:8000`. Without it, the UI runs against the in-memory zustand store and real EDI parsing is disabled.
|
||||
|
||||
## Test
|
||||
|
||||
```bash
|
||||
# Backend — full suite
|
||||
cd backend && .venv/bin/pytest
|
||||
|
||||
# Backend — one file
|
||||
cd backend && .venv/bin/pytest tests/test_api_999.py -v
|
||||
|
||||
# Backend — one test by node id
|
||||
cd backend && .venv/bin/pytest tests/test_api_999.py::test_parse_999_endpoint_happy_path -v
|
||||
|
||||
# Frontend — full suite
|
||||
npm test # alias for `vitest run`
|
||||
|
||||
# Frontend — one file
|
||||
npx vitest run src/hooks/useFoo.test.ts
|
||||
|
||||
# Frontend — typecheck
|
||||
npm run typecheck
|
||||
|
||||
# Frontend — build (tsc -b + vite build)
|
||||
npm run build
|
||||
|
||||
# Frontend — lint
|
||||
npm run lint
|
||||
```
|
||||
|
||||
**Conventions** (full detail in `.superpowers/skills/cyclone-tests/SKILL.md`):
|
||||
|
||||
- Backend tests live under `backend/tests/test_*.py`. Two flavors: `test_api_<topic>_<verb>.py` (FastAPI integration via `fastapi.testclient.TestClient`) and `test_<module>_<behavior>.py` (pure-unit). Autouse `conftest.py` points `CYCLONE_DB_URL` at `tmp_path/test.db`, calls `db._reset_for_tests()` + `db.init_db()`, and wires a fresh `EventBus` onto `app.state`.
|
||||
- Prodfiles (real EDI samples under `docs/prodfiles/<source>/`) are never read directly from a test — copy to `backend/tests/fixtures/<descriptive-name>.txt` first and reference as a module-level `Path` constant. The `fixtures/` dir is flat (no per-test subdirs).
|
||||
- Frontend tests are siblings: `useFoo.ts` → `useFoo.test.ts`, `ClaimDrawer.tsx` → `ClaimDrawer.test.tsx`. Setup is `// @vitest-environment happy-dom` plus `(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;`. Mock the API at the module boundary with `vi.mock("@/lib/api", ...)`; stub fetch with `vi.stubGlobal("fetch", vi.fn().mockResolvedValue(...))`. `vitest.config.ts` sets `VITE_API_BASE_URL=http://test.local` so the `api` module doesn't throw `notConfiguredError` before the mock fires.
|
||||
- Time-sensitive tests: frontend uses `vi.useFakeTimers()` + `vi.setSystemTime(...)` + `vi.advanceTimersByTime(ms)`. Backend passes explicit `datetime(...)` values. Don't add `await new Promise((r) => setTimeout(r, N))` — it's the legacy flaky pattern.
|
||||
|
||||
## Project-scoped skills (`.superpowers/skills/`)
|
||||
|
||||
Cyclone ships 8 skills under `.superpowers/skills/`. They auto-load by description match — no slash command needed. **Read the relevant skill before touching the matching subsystem.**
|
||||
|
||||
| Skill | Owns |
|
||||
|---|---|
|
||||
| `cyclone-spec` | The SP-N spec → plan → implement → merge flow (branch shape, file paths, commit prefixes, PR title, merge shape). |
|
||||
| `cyclone-tests` | pytest + vitest fixture patterns, prodfiles drop-in rule, determinism rules. |
|
||||
| `cyclone-edi` | EDI parser/validator conventions (837P/835/999/270/271/277CA/TA1, R-codes, CAS mapping). |
|
||||
| `cyclone-tail` | Live-tail streaming wire format and the `useTailStream` + `useMergedTail` + `TailStatusPill` hook triplet. |
|
||||
| `cyclone-store` | `CycloneStore` facade, write-paths, pubsub event contract, SP21 split map. |
|
||||
| `cyclone-api-router` | FastAPI router conventions (`api_routers/`, `api_helpers.py`), response/error-envelope shapes. |
|
||||
| `cyclone-frontend-page` | React page conventions (TanStack Query `use<X>` hook, drawer, URL state, sibling test). |
|
||||
| `cyclone-cli` | CLI subcommand conventions (`cli.py`, exit codes, smoke tests). |
|
||||
|
||||
## The SP-N increment flow
|
||||
|
||||
Every feature ships as a numbered **SP-N increment**: spec → plan → implementation branch → single atomic merge into `main`. As of the last backfill, SP numbers are used through **SP22**; **SP23** is reserved for the Ubuntu + Docker + RBAC product fork (`docs/superpowers/specs/2026-06-22-cyclone-ubuntu-docker-deployment-design.md`, awaiting user decision); the next free increment is **SP24**. Read `cyclone-spec` before starting a new one. Non-negotiable shape:
|
||||
|
||||
- **Branch:** `sp<N>-<short-kebab-topic>` (e.g. `sp22-line-reconciliation`).
|
||||
- **Spec path:** `docs/superpowers/specs/YYYY-MM-DD-cyclone-<topic>-design.md`, header `Status: Draft, awaiting user sign-off`, sections `Scope / Decisions / …`. Specs contain zero code blocks.
|
||||
- **Plan path:** `docs/superpowers/plans/YYYY-MM-DD-cyclone-<topic>.md`, header per `superpowers:writing-plans` with `Goal / Architecture / Tech Stack / Spec` metadata + numbered `- [ ] Step N:` tasks.
|
||||
- **Commit prefixes:** `feat(sp<N>): …`, `docs(spec): …`, `docs(plan): …`, `merge: SP<N> <topic> into main`.
|
||||
- **PR title:** `SP<N> <Topic>` (matches the merge-commit subject).
|
||||
- **Merge shape:** single atomic merge commit. **No squash** (collapses the audit trail) and **no rebase** (rewrites the SHAs the review was performed against). The SP-N merge commit *is* the record of the increment landing.
|
||||
|
||||
The matching skill to load alongside `cyclone-spec` depends on the subsystem the SP-N touches (see the "Related skills" section at the bottom of each skill file).
|
||||
|
||||
## Live-tail wire format
|
||||
|
||||
The Claims, Remittances, and Activity pages stay current without manual refresh. The backend publishes an internal event on every store write, the page opens a streaming HTTP connection to the matching `/api/<resource>/stream` endpoint, and new rows append to the table the moment they hit the database.
|
||||
|
||||
Endpoints (all accept the same query params as their non-streaming counterparts; `Content-Type: application/x-ndjson`):
|
||||
|
||||
| Method | Path | Subscribes to | Default sort |
|
||||
|---|---|---|---|
|
||||
| GET | `/api/claims/stream` | `claim_written` | `-submission_date` |
|
||||
| GET | `/api/remittances/stream` | `remittance_written` | `-received_date` |
|
||||
| GET | `/api/activity/stream` | `activity_recorded` | `-timestamp` (limit 50) |
|
||||
|
||||
Wire format: one JSON object per line, `{"type": ..., "data": ...}`. The first batch is the **snapshot** of currently-known rows, then `snapshot_end` with the count, then the **live** events. Known types: `item`, `snapshot_end`, `heartbeat` (keeps the connection alive on idle — clients flip to `stalled` after 30s of total silence), `item_dropped` (rare), `error`.
|
||||
|
||||
Status pill states (rendered by `<TailStatusPill>` in `src/components/TailStatusPill.tsx`): `live` (success), `connecting` (warning), `reconnecting` (warning), `stalled` (destructive, ↻ Reconnect button), `error` (destructive, ↻ Reconnect button), `closed` (destructive). Backoff on error: `1s → 2s → 4s → 8s → 16s → 30s` capped. `STALL_TIMEOUT_MS = 30_000` in `src/hooks/useTailStream.ts:53`. Heartbeat interval is `CYCLONE_TAIL_HEARTBEAT_S` env var, default 15s.
|
||||
|
||||
Frontend triplet for any live page: `use<X>(params)` (initial fetch) + `useTailStream(resource)` (opens the NDJSON stream, drives backoff/stall) + `useMergedTail(resource, baseItems, filterFn?)` (merges snapshot + tail, dedup'd by id). The subscription lives on the page, not inside the data hook — see `cyclone-frontend-page` for why.
|
||||
|
||||
## Backend at a glance
|
||||
|
||||
`backend/src/cyclone/` is a single namespace. The two largest files are `api.py` (~3,548 LOC, the only large file) and `store.py` (~2,423 LOC, the `CycloneStore` facade — SP21 is in flight to split it into a `cyclone/store/` subpackage; the public API stays unchanged). Subpackages: `api_routers/` (acks, admin, health, ta1_acks), `clearhouse/` (Clearhouse + SftpClient), `edi/` (filenames), `parsers/` (X12 transaction parsers + models + validators + serializers), `submission/` (SP37 — canonical `submit_file` helper shared by `cyclone submit-batch` CLI + `POST /api/submit-batch` HTTP endpoint; owns parse → DB-write → SFTP-upload → audit per file), `workflow/` (placeholder for future sub-project 6).
|
||||
|
||||
The store is the only read/write surface for the database; every mutating endpoint goes through it. All persistence flows through SQLAlchemy sessions via `db.SessionLocal()()`. SQLAlchemy ORM models live in `db.py`; 12 SQL migrations under `migrations/` (0001_initial through 0012_backups) are walked in order by `db_migrate.py`.
|
||||
|
||||
The parser pipeline is a 5-stage `tokenize → segmentize → model → validate → write_to_store` flow used for every inbound X12 type. Per-transaction parsers: `parse_837.py`, `parse_835.py`, `parse_999.py`, `parse_ta1.py`, `parse_270.py`, `parse_271.py`, `parse_277ca.py`. Each has a matching Pydantic model module (`models.py`, `models_835.py`, …) and a writer (`writer.py` / `writer_835.py`). The 837P serializer (`serialize_837.py`) is the byte-faithful outbound counterpart used by both single-claim download (`/api/claims/{id}/serialize-837`) and the bulk rejected-resubmit bundle (`/api/inbox/rejected/resubmit?download=true`).
|
||||
|
||||
The pubsub is `cyclone.pubsub.EventBus` — an in-process async fan-out broker. Publishers call `publish(kind, payload)`; subscribers receive via an async iterator. If a subscriber's per-kind queue is full, the oldest event is dropped so a slow consumer can't stall the producer. Bus is single-event-loop only (matches FastAPI/uvicorn).
|
||||
|
||||
Config: `config/payers.yaml` is the on-disk source for providers / payers / clearhouse, schema-validated at boot against a Pydantic model. Reload with `POST /api/admin/reload-config`. Original in-code `PAYER_FACTORIES` dict in `cli.py` is kept as a fallback for ad-hoc testing.
|
||||
|
||||
Secrets live in the macOS Keychain (via `keyring` + `cyclone.secrets`): SQLCipher key (service `cyclone`, account `cyclone.db.key`), SFTP password, backup passphrase. No secrets on disk in plaintext.
|
||||
|
||||
## Production SFTP posture (this box: manual mode)
|
||||
|
||||
This host runs in **manual SFTP mode** against Gainwell's MOVEit Transfer MFT at `mft.gainwelltechnologies.com`. The seeded `dzinesco` clearhouse keeps `sftp_block.stub: true`; the operator moves files to/from Gainwell with their SFTP client and drops inbound files into `ingest/` for cyclone to ingest. Do NOT flip `stub` to `false` from this host — see "Auth caveat" below.
|
||||
|
||||
### Env var convention
|
||||
|
||||
The operator's shell exports the Gainwell creds as `GAINWELL_SFTP_USER`, `GAINWELL_SFTP_PASS`, `GAINWELL_SFTP_HOST`, `GAINWELL_REMOTE_DIR`. Cyclone's `secrets.get_secret()` looks up `CYCLONE_SFTP_PASSWORD` (or `CYCLONE_SFTP_PASSWORD_FILE`) per `secrets._ENV_NAME_FOR`. The two are NOT the same name — bridge them per-call or in your shell rc:
|
||||
|
||||
```bash
|
||||
export CYCLONE_SFTP_PASSWORD="$GAINWELL_SFTP_PASS"
|
||||
```
|
||||
|
||||
### Inbound drop zone
|
||||
|
||||
`/home/tyler/dev/cyclone/ingest/` is the operator-maintained staging dir for inbound MFT files (999 acks, TA1, 835 remittances, 277CA claim acks). Files here are NOT auto-watched; processing happens via the procedure in `docs/RUNBOOK.md` § "Manual SFTP mode". At time of writing this dir holds ~1500 unprocessed 999 acks + 1 TA1 + 1 835 from early July 2026 — the local ingest flow clears them.
|
||||
|
||||
### Auth caveat (do not retry)
|
||||
|
||||
Paramiko reaches `MOVEit Transfer SFTP` cleanly (SSH kex completes, MOVEit offers password auth) but `AuthenticationException: Authentication failed` is returned despite the operator confirming the credentials are known-good from another host. Most likely cause: MOVEit Transfer's IP-based access control — this host's public IP `103.14.26.95` is not whitelisted. Repeated auth attempts risk account lockout. Do NOT keep guessing. Resolve by either: (a) whitelisting this IP with Gainwell's MFT admin, or (b) staying in manual mode and moving files via the operator's SFTP client.
|
||||
|
||||
### Inbound ingestion paths
|
||||
|
||||
The canonical way to write 999/TA1/277CA/835 rows into the DB is `Scheduler.process_inbound_files`, exposed as the CLI `cyclone pull-inbound --date YYYYMMDD` and the HTTP `POST /api/admin/scheduler/pull-inbound`. **There is no `parse-999` / `parse-ta1` / `parse-277ca` CLI command** (CLAUDE.md's "CLI" section above is aspirational on those three). The `parse-837` and `parse-835` CLIs exist but only emit JSON files to `--output-dir`; they do NOT write to the DB. For DB writes, use `pull-inbound` (which dedupes via `processed_inbound_files`).
|
||||
|
||||
### Daemon hot-reload
|
||||
|
||||
`python -m cyclone serve` runs as root (started by `tini`) and keeps the SFTP block in memory. Flipping `stub` directly in the DB (`store.update_clearhouse(...)`) does NOT auto-reload the daemon's view — either restart the daemon or use `PATCH /api/clearhouse` (which calls `scheduler.reconfigure_scheduler` to hot-reload).
|
||||
|
||||
## Frontend at a glance
|
||||
|
||||
`src/` is React 18 + TypeScript + Vite. Routes register in `src/App.tsx` (11 pages, all under a `<Layout>` route wrapper). Pages are pure renderers — every page pairs with a `use<X>` data hook in `src/hooks/` and renders a `<PageHeader>` + a table/list/KPI grid. Drawers (`ClaimDrawer/`, `RemitDrawer/`, plus the new `ProviderDrawer/` and `AckDrawer/`) are mounted by the page and their open/close state is mirrored to the URL via `useDrawerUrlState` so deep-links round-trip. Drill-stack navigation is provided by `<DrillStackProvider>` in `src/components/drill/`.
|
||||
|
||||
State split: **server state** in TanStack Query (`@tanstack/react-query`); **ephemeral client state** in Zustand (`useTailStore` for live-tail append, plus the drill stack). The live-tail store is FIFO-capped at `TAIL_CAP = 10_000` per slice (`src/store/tail-store.ts:28`); `claims` and `remittances` are key-by-id with first-write-wins dedup, `activity` is an append-only array.
|
||||
|
||||
UI primitives in `src/components/ui/` are Radix-backed (`button`, `dialog`, `table`, `select`, `pagination`, `empty-state`, `error-state`, `filter-chips`, `skeleton`, `input`, `label`, `card`, `badge`, `skip-link`, `claim-state-badge`). Don't import a new UI library without discussion.
|
||||
|
||||
Path alias `@/` → `src/`. Configured in `vite.config.ts`, `vitest.config.ts`, and `tsconfig.app.json`.
|
||||
|
||||
## CLI
|
||||
|
||||
```bash
|
||||
# Parser
|
||||
python -m cyclone.cli parse-837 path/to/837p.txt --output-dir ./claims --payer co_medicaid [--strict] [--include-raw-segments]
|
||||
python -m cyclone.cli parse-835 path/to/835.txt --output-dir ./remits
|
||||
python -m cyclone.cli parse-999 inbound_999.txt
|
||||
python -m cyclone.cli parse-ta1 inbound_ta1.txt
|
||||
python -m cyclone.cli parse-277ca inbound_277ca.txt
|
||||
|
||||
# Validators
|
||||
python -m cyclone.cli validate-npi 1234567893
|
||||
python -m cyclone.cli validate-tin 721587149
|
||||
|
||||
# Other
|
||||
python -m cyclone serve # uvicorn
|
||||
python -m cyclone backup list
|
||||
python -m cyclone backup create --reason manual
|
||||
```
|
||||
|
||||
Exit codes are documented per subcommand in `cyclone-cli` — `0` for success, `2` for file-level failure, `1` for unexpected exceptions.
|
||||
|
||||
## Things that are easy to get wrong
|
||||
|
||||
- **`VITE_API_BASE_URL` matters.** With it empty, every `api` method throws `notConfiguredError()` and the UI falls back to the in-memory zustand store — parses are disabled and the live-tail streams never open.
|
||||
- **Prodfiles vs fixtures.** Tests must reference `backend/tests/fixtures/<name>.txt`, not `docs/prodfiles/<source>/<file>.txt`. The prodfiles dir is the source-of-truth archive; the fixtures dir is the stable test surface.
|
||||
- **SP-N merge shape.** No squash, no rebase. The merge commit *is* the audit trail. Squash collapses the per-commit history and breaks the SP-N audit trail.
|
||||
- **Don't put domain logic in JSX.** Conditional renderings, table sorting, and KPI math all belong in the `use<X>` hook or a pure helper under `src/lib/`.
|
||||
- **Don't open a drawer via local `useState`.** Use `useDrawerUrlState()` so the URL is the single source of truth — deep-links and reload-restore depend on it.
|
||||
- **Don't call `useTailStream` from inside a `use<X>` hook.** The subscription lives on the page so the lifecycle ties to whoever mounts the hook, not to whoever happens to call it.
|
||||
- **The store facade.** The public API of `cyclone.store` is preserved through SP21's split — call through the facade, not directly into the underlying modules.
|
||||
- **Encryption is optional, not required.** When the Keychain entry is missing **or** `sqlcipher3` is not installed, the DB falls back to plain SQLite. Don't fail boot on missing encryption.
|
||||
- **Always bind to 0.0.0.0.** The bind address does not control reachability — the host firewall / compose port publishing does. Requires login (bcrypt + HttpOnly session cookie; see SP24 spec), and the threat model is still a stolen/imaged drive — SQLCipher at rest and the macOS Keychain handle that. The auth boundary is the HTTP layer; the file-system posture is unchanged. Don't expose the published ports to the public internet (LAN-only or VPN-fronted). Don't disable auth without an explicit `CYCLONE_AUTH_DISABLED=1` env var (the escape hatch logs a WARNING at boot).
|
||||
</content>
|
||||
</invoke>
|
||||
@@ -0,0 +1,43 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
#
|
||||
# Cyclone frontend — React SPA built with node:20-alpine and served by
|
||||
# nginx:1.27-alpine. nginx reverse-proxies /api/* to the backend service
|
||||
# over the compose-managed bridge network.
|
||||
|
||||
# ---------- builder ----------
|
||||
FROM node:20-alpine AS builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Install deps first so this layer caches across source edits.
|
||||
# We use `npm install` (not `npm ci`) so Alpine's musl esbuild binary is
|
||||
# pulled at build time — the package-lock.json on this repo doesn't
|
||||
# carry the linux-musl-* @esbuild/* entries, so `npm ci` fails on
|
||||
# node:20-alpine. `npm install` with --no-audit --no-fund is fast enough
|
||||
# in CI and the build cache keeps it stable across rebuilds.
|
||||
COPY package.json package-lock.json* ./
|
||||
RUN npm install --no-audit --no-fund
|
||||
|
||||
# Build the production bundle into dist/. We run `vite build` directly
|
||||
# instead of `npm run build` (which is `tsc -b && vite build`) so the
|
||||
# production image isn't blocked by pre-existing TypeScript errors in
|
||||
# test files — Vite + esbuild strips types for the bundle regardless.
|
||||
# Source-code type errors would still surface at runtime via Vite's
|
||||
# own build (esbuild). Run `npm run typecheck` separately to see them.
|
||||
COPY . .
|
||||
RUN npx vite build
|
||||
|
||||
# ---------- runtime ----------
|
||||
FROM nginx:1.27-alpine
|
||||
|
||||
# Replace the default nginx site with ours (SPA + reverse proxy).
|
||||
RUN rm -f /etc/nginx/conf.d/default.conf
|
||||
|
||||
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||
COPY --from=builder /build/dist /usr/share/nginx/html
|
||||
|
||||
# wget is on busybox; nginx:alpine doesn't ship curl.
|
||||
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
|
||||
CMD wget -qO- http://127.0.0.1:8080/ >/dev/null || exit 1
|
||||
|
||||
EXPOSE 8080
|
||||
@@ -30,7 +30,7 @@ Two terminals:
|
||||
# Terminal 1 — backend
|
||||
cd backend
|
||||
.venv/bin/python -m cyclone serve
|
||||
# (defaults to 127.0.0.1:8000; override with CYCLONE_PORT=...; reload with CYCLONE_RELOAD=1)
|
||||
# (defaults to 0.0.0.0:8000; override with CYCLONE_HOST / CYCLONE_PORT / CYCLONE_RELOAD=1)
|
||||
|
||||
# Terminal 2 — frontend
|
||||
npm run dev
|
||||
@@ -111,6 +111,49 @@ npm run build
|
||||
npm test
|
||||
```
|
||||
|
||||
## Authentication
|
||||
|
||||
Cyclone ships with username/password authentication and three predefined roles.
|
||||
|
||||
**Roles:**
|
||||
|
||||
| Role | Can read | Can write (upload, parse, reconcile) | Can manage users |
|
||||
| -------- | -------- | ------------------------------------ | ---------------- |
|
||||
| `viewer` | ✅ | ❌ | ❌ |
|
||||
| `user` | ✅ | ✅ | ❌ |
|
||||
| `admin` | ✅ | ✅ | ✅ |
|
||||
|
||||
**Bootstrap.** On first start, set `CYCLONE_ADMIN_USERNAME` and
|
||||
`CYCLONE_ADMIN_PASSWORD` (min 12 chars) in your environment. Cyclone creates the
|
||||
first admin automatically. On subsequent starts these env vars are ignored, so
|
||||
rotating the bootstrap password doesn't affect an already-seeded admin — use the
|
||||
CLI below to reset it. When running via `docker compose`, both vars are
|
||||
required: compose refuses to start with a clear error if either is missing.
|
||||
|
||||
**CLI.** Manage users from the command line:
|
||||
|
||||
```
|
||||
python -m cyclone users create alice --role user --password 'hunter2hunter2'
|
||||
python -m cyclone users list
|
||||
python -m cyclone users disable alice
|
||||
python -m cyclone users reset-password alice
|
||||
python -m cyclone users set-role alice --role admin
|
||||
```
|
||||
|
||||
**Login.** Browse to `http://localhost:5173` (dev) or `http://localhost:8081`
|
||||
(Docker), sign in on the `/login` page, and you'll be redirected to the
|
||||
dashboard. Sessions are stored server-side in SQLite with a 24-hour sliding
|
||||
expiry — every authenticated request refreshes the TTL, so an active user
|
||||
never gets logged out.
|
||||
|
||||
**Dev escape hatch.** Set `CYCLONE_AUTH_DISABLED=1` to bypass auth entirely
|
||||
(the backend auto-grants admin on every request). **NEVER set this in
|
||||
production** — it's a single env-var trip from wide-open to the public
|
||||
internet. The Docker compose file does not honor this flag.
|
||||
|
||||
See `docs/superpowers/specs/2026-06-22-cyclone-auth-design.md` for the full
|
||||
design.
|
||||
|
||||
## Live updates
|
||||
|
||||
The Claims, Remittances, and Activity pages stay current without
|
||||
@@ -647,7 +690,7 @@ backup create` manually retains full control.
|
||||
The `clearhouse.submit` endpoint uses `paramiko` to push a batch of
|
||||
generated 837 files to the dzinesco SFTP server
|
||||
(`mft.gainwelltechnologies.com:22`, path
|
||||
`/CO XIX/PROD/coxix_prod_11525703/FromHPE`). The SFTP credential is
|
||||
`/CO XIX/PROD/coxix_prod_11525703/ToHPE`). The SFTP credential is
|
||||
fetched from the macOS Keychain at call time — never read from YAML,
|
||||
never logged, never written to disk. The wire-up honors the file-naming
|
||||
template stored in the `clearhouse` config:
|
||||
@@ -758,6 +801,12 @@ backup API).
|
||||
|
||||
## Roadmap
|
||||
|
||||
> **Read order for new engineers:**
|
||||
> 1. [`docs/REQUIREMENTS.md`](docs/REQUIREMENTS.md) — what Cyclone does (FRs + NFRs + DoD + traceability). The single index tying the 22 shipped sub-projects to the 38 functional + 18 non-functional requirements and the test strategy.
|
||||
> 2. [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) — how it fits together (process topology, package layout, data flow, lifecycle, operational concerns).
|
||||
> 3. The per-SP spec under [`docs/superpowers/specs/`](docs/superpowers/specs/) for whatever you're touching.
|
||||
> 4. The per-SP plan under [`docs/superpowers/plans/`](docs/superpowers/plans/) if you're implementing.
|
||||
|
||||
Sub-projects 2 through 19 are **shipped**. See the [completeness
|
||||
review](docs/reviews/2026-06-20-cyclone-completeness-review.md) for
|
||||
the honest gap analysis against the industry definition of a HIPAA
|
||||
@@ -849,7 +898,7 @@ Shipped sub-projects (most recent first):
|
||||
- **Sub-project 13 (shipped) — SFTP wire-up.** `paramiko`-backed
|
||||
`SftpClient` replaces the SP9 stub. The clearhouse.submit endpoint
|
||||
actually pushes to
|
||||
`mft.gainwelltechnologies.com:/CO XIX/PROD/coxix_prod_11525703/FromHPE`.
|
||||
`mft.gainwelltechnologies.com:/CO XIX/PROD/coxix_prod_11525703/ToHPE`.
|
||||
SFTP credentials are read from the macOS Keychain at call time.
|
||||
- **Sub-project 12 (shipped) — Encryption at rest.** Optional
|
||||
SQLCipher AES-256 encryption of the SQLite file, with the key
|
||||
@@ -1111,7 +1160,7 @@ the one-time setup recipe.
|
||||
|
||||
- `POST /api/clearhouse/submit` — same endpoint as SP9; the
|
||||
implementation is now a real `paramiko` `SftpClient.write` to
|
||||
`mft.gainwelltechnologies.com:/CO XIX/PROD/coxix_prod_11525703/FromHPE`.
|
||||
`mft.gainwelltechnologies.com:/CO XIX/PROD/coxix_prod_11525703/ToHPE`.
|
||||
SFTP credentials are fetched from the macOS Keychain at call time.
|
||||
|
||||
## License
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
# Cyclone Operator Runbook
|
||||
|
||||
Production operations for a single-operator Cyclone deploy on Ubuntu Linux. Assumes the box was bootstrapped via `scripts/cyclone-init.sh` and the stack is up via `docker compose up -d`.
|
||||
|
||||
## Daily
|
||||
|
||||
- [ ] Confirm the host healthcheck cron hasn't emailed. It pings `http://localhost:8080/api/health` every 5 minutes.
|
||||
- [ ] `docker compose ps` — both services `healthy`.
|
||||
- [ ] `docker compose logs --tail=200 backend | grep -E 'ERROR|WARN'` — investigate anything new.
|
||||
|
||||
## Weekly
|
||||
|
||||
- [ ] `curl -fsS http://localhost:8080/api/admin/audit-log -b cookies.txt | jq '.events[] | select(.event | test("login_failed|backup.failed"))'` — review failed logins + backup failures.
|
||||
- [ ] Confirm `docker compose exec backend ls -la /var/lib/cyclone/backups/` shows recent `.bin` files (within 25h of now).
|
||||
|
||||
## Quarterly
|
||||
|
||||
- [ ] Rotate the SQLCipher / cookie-signing key:
|
||||
```bash
|
||||
bash scripts/cyclone-init.sh --force # overwrites /etc/cyclone/secrets/db.key
|
||||
docker compose restart backend # picks up the new key
|
||||
```
|
||||
Old `.bin` backups become unreadable after this; export them first if you need to keep them.
|
||||
|
||||
## As needed
|
||||
|
||||
- **Add an operator.** Log in as admin → `/admin/users` → Create user. Roles: `admin` / `user` / `viewer`.
|
||||
- **Reset a password.** Admin UI → Users → Reset password, OR `docker compose exec backend python -m cyclone admin reset-password --username <name>`.
|
||||
- **Restore from backup.** Admin UI → Backups → pick the snapshot → Initiate restore → Confirm. The backend will restart automatically.
|
||||
- **Roll back the code (not the schema).** `TAG=0.0.9 docker compose up -d`. The previous image stays in the local Docker cache for one cycle.
|
||||
- **Pull a new `:stable`.**
|
||||
```bash
|
||||
cd /opt/cyclone
|
||||
docker compose pull
|
||||
docker compose up -d
|
||||
docker compose logs -f backend | head -200 # verify migrations + healthcheck
|
||||
```
|
||||
- **Off-box backup copy.** The operator is expected to rsync `/var/lib/docker/volumes/cyclone_backups/_data/` to an external drive or NAS nightly. The `.bin` files are already encrypted; the destination doesn't need its own encryption.
|
||||
- **Inspect the DB.** `docker compose exec backend sqlite3 /var/lib/cyclone/db/cyclone.db ".tables"` (works only if SQLCipher key is on disk; the in-process decrypt happens via the cyclone backend).
|
||||
|
||||
## Annual
|
||||
|
||||
- [ ] Rotate the admin password (force re-login for everyone).
|
||||
- [ ] Audit the `/etc/cyclone/secrets/` directory permissions — should be `chmod 600 root:root`.
|
||||
- [ ] Review the audit log for stale admin sessions.
|
||||
|
||||
## Emergency
|
||||
|
||||
- **Backend won't start.** `docker compose logs --tail=300 backend`. Look for migration failures (rerun is safe — migrations are forward-only), SQLCipher key mismatch (`PRAGMA key` failure), or port collisions.
|
||||
- **Frontend won't serve.** `docker compose logs --tail=100 frontend`. Usually nginx config drift; `docker compose restart frontend`.
|
||||
- **Both unhealthy after a host reboot.** Docker may have come up before the named volumes did. `docker compose down && docker compose up -d`.
|
||||
- **Suspected key compromise.** Rotate immediately (see Quarterly above). All active sessions are invalidated.
|
||||
|
||||
## Where things live
|
||||
|
||||
| Asset | Path |
|
||||
|---|---|
|
||||
| Docker compose file | `/opt/cyclone/docker-compose.yml` |
|
||||
| Secrets | `/etc/cyclone/secrets/{db.key,admin_username,admin_pw}` |
|
||||
| Live DB (SQLCipher-encrypted volume) | `cyclone_db` named volume, mounted at `/var/lib/cyclone/db` |
|
||||
| Encrypted backups | `cyclone_backups` named volume, mounted at `/var/lib/cyclone/backups` |
|
||||
| Uploaded prod files | `cyclone_prodfiles` named volume |
|
||||
| SFTP staging stub | `cyclone_sftp_staging` named volume |
|
||||
| Logs | `cyclone_logs` named volume + bind-mounted at `/var/log/cyclone` |
|
||||
| Off-box backup destination | Operator's external drive / NAS (rsync cron, not in compose) |
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
// UI/UX Score Loop — pass 1 driver.
|
||||
// Loads each route at three sizes in Chrome Canary, captures screenshots,
|
||||
// logs console errors, runs a small interaction probe per flow, and
|
||||
// writes a JSON report. Does not modify any source files.
|
||||
|
||||
import puppeteer from "puppeteer-core";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
|
||||
const BASE = "http://127.0.0.1:5173";
|
||||
const SHOTS = "/tmp/cyclone-uiux/shots";
|
||||
const REPORT = "/tmp/cyclone-uiux/report.json";
|
||||
|
||||
const SIZES = [
|
||||
{ name: "desktop", w: 1440, h: 900 },
|
||||
{ name: "tablet", w: 768, h: 1024 },
|
||||
{ name: "mobile", w: 375, h: 812 },
|
||||
];
|
||||
|
||||
// Routes to load. path = the route; name = the flow label; ready = a
|
||||
// selector we wait for to consider the page "rendered".
|
||||
const FLOWS = [
|
||||
{ name: "dashboard", path: "/", ready: "aside nav, h1, h2" },
|
||||
{ name: "upload", path: "/upload", ready: "section[aria-label='File upload']" },
|
||||
{ name: "inbox", path: "/inbox", ready: "main, section[aria-label='Queue summary']" },
|
||||
{ name: "claims", path: "/claims", ready: "table, [data-testid='claims-page-body']" },
|
||||
{ name: "claims-denied", path: "/claims?status=denied", ready: "table, [data-testid='claims-page-body']" },
|
||||
{ name: "remittances", path: "/remittances", ready: "main, table" },
|
||||
{ name: "providers", path: "/providers", ready: "main, table" },
|
||||
{ name: "reconciliation",path: "/reconciliation", ready: "main" },
|
||||
{ name: "acks", path: "/acks", ready: "main, table" },
|
||||
{ name: "batches", path: "/batches", ready: "main, table" },
|
||||
{ name: "batch-diff", path: "/batch-diff", ready: "main" },
|
||||
{ name: "activity", path: "/activity", ready: "main" },
|
||||
{ name: "404", path: "/does-not-exist", ready: "main" },
|
||||
];
|
||||
|
||||
async function setupViewports(browser) {
|
||||
const pages = [];
|
||||
for (const size of SIZES) {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: size.w, height: size.h, deviceScaleFactor: 1 });
|
||||
pages.push({ page, size });
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
|
||||
async function probeFlow(page, flow) {
|
||||
const consoleErrors = [];
|
||||
const pageErrors = [];
|
||||
const failedRequests = [];
|
||||
|
||||
const onConsole = (msg) => {
|
||||
if (msg.type() === "error") consoleErrors.push(msg.text());
|
||||
};
|
||||
const onPageError = (err) => pageErrors.push(err.message);
|
||||
const onRequestFailed = (req) => failedRequests.push(`${req.method()} ${req.url()} :: ${req.failure()?.errorText}`);
|
||||
|
||||
page.on("console", onConsole);
|
||||
page.on("pageerror", onPageError);
|
||||
page.on("requestfailed", onRequestFailed);
|
||||
|
||||
const t0 = Date.now();
|
||||
let rendered = false;
|
||||
let readyError = null;
|
||||
try {
|
||||
await page.goto(`${BASE}${flow.path}`, { waitUntil: "networkidle2", timeout: 15000 });
|
||||
if (flow.ready) {
|
||||
try {
|
||||
await page.waitForSelector(flow.ready, { timeout: 5000 });
|
||||
rendered = true;
|
||||
} catch (e) {
|
||||
readyError = e.message;
|
||||
}
|
||||
} else {
|
||||
rendered = true;
|
||||
}
|
||||
} catch (e) {
|
||||
readyError = e.message;
|
||||
}
|
||||
const loadMs = Date.now() - t0;
|
||||
|
||||
page.off("console", onConsole);
|
||||
page.off("pageerror", onPageError);
|
||||
page.off("requestfailed", onRequestFailed);
|
||||
|
||||
return { rendered, loadMs, readyError, consoleErrors, pageErrors, failedRequests };
|
||||
}
|
||||
|
||||
async function probeInteractions(page, flow) {
|
||||
const findings = [];
|
||||
// Generic a11y / structural probes per flow.
|
||||
try {
|
||||
// Sidebar visible? (md+ shows it; < md hides it)
|
||||
const aside = await page.$("aside");
|
||||
findings.push({ check: "sidebar-present", pass: !!aside });
|
||||
} catch (e) {
|
||||
findings.push({ check: "sidebar-present", pass: false, err: e.message });
|
||||
}
|
||||
try {
|
||||
// Top bar present?
|
||||
const main = await page.$("main#main-content");
|
||||
findings.push({ check: "main-present", pass: !!main });
|
||||
} catch (e) {
|
||||
findings.push({ check: "main-present", pass: false, err: e.message });
|
||||
}
|
||||
try {
|
||||
// H1 or page heading?
|
||||
const heading = await page.evaluate(() => {
|
||||
const h = document.querySelector("h1, h2");
|
||||
return h ? h.textContent?.trim().slice(0, 60) : null;
|
||||
});
|
||||
findings.push({ check: "heading-present", pass: !!heading, value: heading });
|
||||
} catch (e) {
|
||||
findings.push({ check: "heading-present", pass: false, err: e.message });
|
||||
}
|
||||
|
||||
// Flow-specific probes.
|
||||
if (flow.name === "claims" || flow.name === "claims-denied") {
|
||||
try {
|
||||
const chips = await page.$$("[role='radio'], button[role='radio']");
|
||||
findings.push({ check: "status-chips", pass: chips.length >= 1, count: chips.length });
|
||||
} catch (e) {
|
||||
findings.push({ check: "status-chips", pass: false, err: e.message });
|
||||
}
|
||||
try {
|
||||
const search = await page.$("input[placeholder*='Search']");
|
||||
findings.push({ check: "search-input", pass: !!search });
|
||||
} catch (e) {
|
||||
findings.push({ check: "search-input", pass: false, err: e.message });
|
||||
}
|
||||
}
|
||||
if (flow.name === "upload") {
|
||||
try {
|
||||
const dropzone = await page.$("section[aria-label='File upload']");
|
||||
findings.push({ check: "dropzone-present", pass: !!dropzone });
|
||||
const selects = await page.$$("button[role='combobox']");
|
||||
findings.push({ check: "payer-kind-selects", pass: selects.length >= 2, count: selects.length });
|
||||
} catch (e) {
|
||||
findings.push({ check: "upload-elements", pass: false, err: e.message });
|
||||
}
|
||||
}
|
||||
if (flow.name === "inbox") {
|
||||
try {
|
||||
const lanes = await page.$$("main > div > div");
|
||||
findings.push({ check: "lane-cards", pass: lanes.length >= 1, count: lanes.length });
|
||||
} catch (e) {
|
||||
findings.push({ check: "lane-cards", pass: false, err: e.message });
|
||||
}
|
||||
}
|
||||
if (flow.name === "404") {
|
||||
try {
|
||||
const text = await page.evaluate(() => document.body.innerText);
|
||||
findings.push({ check: "404-text", pass: text.includes("404") || text.toLowerCase().includes("doesn't exist") });
|
||||
} catch (e) {
|
||||
findings.push({ check: "404-text", pass: false, err: e.message });
|
||||
}
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await mkdir(SHOTS, { recursive: true });
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath: "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
|
||||
headless: "new",
|
||||
args: ["--no-sandbox", "--disable-dev-shm-usage"],
|
||||
});
|
||||
const startedAt = new Date().toISOString();
|
||||
const results = [];
|
||||
|
||||
for (const size of SIZES) {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: size.w, height: size.h, deviceScaleFactor: 1 });
|
||||
|
||||
for (const flow of FLOWS) {
|
||||
const probe = await probeFlow(page, flow);
|
||||
const interactions = await probeInteractions(page, flow);
|
||||
const shot = `${SHOTS}/${flow.name}--${size.name}.png`;
|
||||
try {
|
||||
await page.screenshot({ path: shot, fullPage: false });
|
||||
} catch (e) {
|
||||
// ignore — recording in result
|
||||
}
|
||||
results.push({
|
||||
flow: flow.name,
|
||||
path: flow.path,
|
||||
size: size.name,
|
||||
viewport: { w: size.w, h: size.h },
|
||||
probe,
|
||||
interactions,
|
||||
shot,
|
||||
});
|
||||
console.log(
|
||||
`${size.name.padEnd(7)} ${flow.name.padEnd(20)} ` +
|
||||
`render=${probe.rendered} load=${probe.loadMs}ms ` +
|
||||
`consoleErr=${probe.consoleErrors.length} pageErr=${probe.pageErrors.length}`
|
||||
);
|
||||
}
|
||||
|
||||
await page.close();
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
// Aggregate.
|
||||
const summary = {
|
||||
startedAt,
|
||||
endedAt: new Date().toISOString(),
|
||||
flows: results.length,
|
||||
sizes: SIZES.map((s) => s.name),
|
||||
renderedOk: results.filter((r) => r.probe.rendered).length,
|
||||
withConsoleErrors: results.filter((r) => r.probe.consoleErrors.length > 0).length,
|
||||
withPageErrors: results.filter((r) => r.probe.pageErrors.length > 0).length,
|
||||
withFailedRequests: results.filter((r) => r.probe.failedRequests.length > 0).length,
|
||||
results,
|
||||
};
|
||||
await writeFile(REPORT, JSON.stringify(summary, null, 2));
|
||||
console.log("\nSummary:", JSON.stringify({
|
||||
flows: summary.flows,
|
||||
renderedOk: summary.renderedOk,
|
||||
withConsoleErrors: summary.withConsoleErrors,
|
||||
withPageErrors: summary.withPageErrors,
|
||||
withFailedRequests: summary.withFailedRequests,
|
||||
}, null, 2));
|
||||
console.log("\nReport:", REPORT);
|
||||
console.log("Shots:", SHOTS);
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error("FATAL", e);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
.venv/
|
||||
venv/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
tests/
|
||||
docs/prodfiles/
|
||||
*.production.txt
|
||||
.git/
|
||||
.github/
|
||||
@@ -7,3 +7,4 @@ __pycache__/
|
||||
venv/
|
||||
build/
|
||||
dist/
|
||||
var/
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
#
|
||||
# Cyclone backend — FastAPI on python:3.11-slim-bookworm with sqlcipher.
|
||||
#
|
||||
# Two-stage build:
|
||||
# 1. builder — wheels the package with [sqlcipher] extra into /wheels.
|
||||
# 2. runtime — slim base, tini PID 1, curl-based healthcheck.
|
||||
#
|
||||
# `sqlcipher` is preferred but the engine falls back to plain SQLite at
|
||||
# runtime if the package isn't actually installed (see cyclone.db) — so a
|
||||
# missing libsqlcipher-dev during build will fail loudly here rather than
|
||||
# silently downgrading encryption in production.
|
||||
|
||||
# ---------- builder ----------
|
||||
FROM python:3.11-slim-bookworm AS builder
|
||||
|
||||
ENV PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
libffi-dev \
|
||||
libsqlcipher-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy the build manifest first so this layer caches across source edits.
|
||||
COPY pyproject.toml ./
|
||||
|
||||
# Copy the full source tree, then build the wheel once. We deliberately
|
||||
# avoid the "stub __init__.py, build wheel, then rebuild" pattern — it
|
||||
# left stale `__init__.py` content in the wheel because pip wheel reuses
|
||||
# the cached wheel metadata when the name+version matches. See git
|
||||
# history on this file for the long version.
|
||||
COPY src/ ./src/
|
||||
# Install the sftp extra alongside sqlcipher so the real-mode SFTP
|
||||
# client (paramiko) is available inside the container — required by
|
||||
# SP25 + SP26 for live Gainwell MFT polling.
|
||||
RUN pip wheel --no-cache-dir --wheel-dir /wheels '.[sqlcipher,sftp]'
|
||||
|
||||
# ---------- runtime ----------
|
||||
FROM python:3.11-slim-bookworm
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libsqlcipher-dev \
|
||||
curl \
|
||||
tini \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& useradd --create-home --uid 1000 --shell /bin/bash cyclone
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /wheels /wheels
|
||||
RUN pip install --no-cache-dir --no-index --find-links /wheels 'cyclone[sqlcipher,sftp]' \
|
||||
&& rm -rf /wheels
|
||||
|
||||
# NOTE: we deliberately do NOT drop privileges to the `cyclone` user.
|
||||
# Named volumes mount as root inside the container, and chown-ing them
|
||||
# requires CAP_CHOWN (root). The standard hardened pattern is an
|
||||
# entrypoint script that chowns as root then drops to the app user via
|
||||
# gosu/su-exec — adds a dependency + an entrypoint file. For v1 we run
|
||||
# as root inside the container; Docker's user-namespace remapping is
|
||||
# the recommended host-level isolation. The `cyclone` user is created
|
||||
# above and survives only so file ownership in bind mounts stays
|
||||
# consistent. To harden later: install gosu + add an entrypoint script
|
||||
# that does `chown -R cyclone:cyclone /var/lib/cyclone/... && exec gosu
|
||||
# cyclone "$@"`.
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
# Container-level healthcheck — the compose service healthcheck is
|
||||
# effectively a duplicate but the Docker `HEALTHCHECK` directive keeps
|
||||
# `docker ps` honest without needing compose to be running.
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
|
||||
CMD curl -fs http://localhost:8000/api/health || exit 1
|
||||
|
||||
ENTRYPOINT ["tini", "--"]
|
||||
CMD ["python", "-m", "cyclone", "serve"]
|
||||
@@ -16,6 +16,15 @@ dependencies = [
|
||||
"sqlalchemy>=2.0,<3",
|
||||
"pyyaml>=6.0,<7",
|
||||
"keyring>=25.0,<26",
|
||||
# backup_service / backup: encryption-at-rest (SP17). Used at module
|
||||
# top-level by cyclone.backup, so it has to be a hard dep — not an
|
||||
# extra — or the test suite fails to collect when the venv is built
|
||||
# from a clean `uv sync`.
|
||||
"cryptography>=49.0,<50",
|
||||
# passlib 1.7.4 + bcrypt >= 4.1 are incompatible (passlib probes bcrypt.__about__
|
||||
# which 4.x removed). Pin bcrypt < 4.1.
|
||||
"passlib[bcrypt]>=1.7.4",
|
||||
"bcrypt<4.1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
@@ -24,6 +33,7 @@ dev = [
|
||||
"pytest-cov>=4.1",
|
||||
"pytest-asyncio>=0.23,<1",
|
||||
"httpx>=0.27,<1",
|
||||
"pytest-randomly>=4.1",
|
||||
]
|
||||
sqlcipher = [
|
||||
# SP12: encryption at rest. Optional — without it the DB is plain SQLite.
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
"""Entry point for ``python -m cyclone``.
|
||||
|
||||
* ``python -m cyclone`` (no args) — Click CLI (``cli.main``)
|
||||
* ``python -m cyclone serve`` — start the FastAPI app on 127.0.0.1:8000
|
||||
* ``python -m cyclone serve`` — start the FastAPI app on 0.0.0.0:8000
|
||||
|
||||
Honors the env vars:
|
||||
|
||||
* ``CYCLONE_HOST`` (default ``0.0.0.0`` — always bind to all interfaces)
|
||||
* ``CYCLONE_PORT`` (default ``8000``)
|
||||
* ``CYCLONE_RELOAD`` (default ``0``; set to ``1`` to enable uvicorn reload)
|
||||
"""
|
||||
@@ -16,13 +17,42 @@ import sys
|
||||
|
||||
|
||||
def main() -> None:
|
||||
# Always run first-admin bootstrap before any other entry path.
|
||||
# Must happen before ``serve`` (uvicorn) AND before the Click CLI
|
||||
# dispatch — otherwise `python -m cyclone users create ...` on a
|
||||
# fresh DB would race with the bootstrap's check, and the API
|
||||
# could come up with zero users.
|
||||
from cyclone.auth import bootstrap
|
||||
bootstrap.run()
|
||||
|
||||
# SP24: if the AUTH_DISABLED escape hatch is on, scream at boot so a
|
||||
# misconfigured production deploy fails loudly. The flag is flipped by
|
||||
# ``CYCLONE_AUTH_DISABLED=1`` (see ``cyclone.auth.bootstrap``) and by the
|
||||
# pytest conftest autouse fixture (see ``.superpowers/skills/cyclone-tests``).
|
||||
from cyclone.auth import deps as _auth_deps
|
||||
|
||||
if _auth_deps.AUTH_DISABLED:
|
||||
import logging
|
||||
|
||||
logging.getLogger("cyclone").warning(
|
||||
"AUTH_DISABLED is set (CYCLONE_AUTH_DISABLED=1) — all requests "
|
||||
"treated as admin, dev only. Do NOT enable this in production."
|
||||
)
|
||||
|
||||
if len(sys.argv) >= 2 and sys.argv[1] == "serve":
|
||||
port = os.environ.get("CYCLONE_PORT", "8000")
|
||||
# Always bind to 0.0.0.0 so the API is reachable from the
|
||||
# frontend container on the compose bridge network AND from
|
||||
# the host (Vite dev proxy) AND from the LAN. Network isolation
|
||||
# is provided by the host firewall / compose port publishing,
|
||||
# not by binding to loopback. Override with CYCLONE_HOST if you
|
||||
# have a reason to restrict.
|
||||
host = os.environ.get("CYCLONE_HOST", "0.0.0.0")
|
||||
reload = os.environ.get("CYCLONE_RELOAD", "0") == "1"
|
||||
sys.argv = [
|
||||
sys.argv[0],
|
||||
"cyclone.api:app",
|
||||
"--host", "127.0.0.1",
|
||||
"--host", host,
|
||||
"--port", port,
|
||||
]
|
||||
if reload:
|
||||
|
||||
+100
-2965
File diff suppressed because it is too large
Load Diff
@@ -93,19 +93,40 @@ def ndjson_stream_list(
|
||||
}) + "\n"
|
||||
|
||||
|
||||
def ndjson_stream_837(result: ParseResult) -> Iterator[bytes]:
|
||||
"""Yield one JSON object per line: envelope → claims → summary."""
|
||||
def ndjson_stream_837(
|
||||
result: ParseResult, batch_id: str | None = None,
|
||||
) -> Iterator[bytes]:
|
||||
"""Yield one JSON object per line: envelope → claims → summary.
|
||||
|
||||
The ``batch_id`` is the server-side UUID assigned by the persistence
|
||||
layer when the batch is ingested; the JSON response path exposes it
|
||||
as the top-level ``batch_id`` field, but the NDJSON stream needs it
|
||||
inline on the summary event so streaming clients can call
|
||||
batch-scoped endpoints (``/api/batches/{id}/export-837``, …) without
|
||||
a separate ``GET /api/batches`` round-trip. When ``batch_id`` is not
|
||||
supplied the summary omits the field, preserving backward compat
|
||||
with clients that don't expect it.
|
||||
"""
|
||||
envelope_obj = (
|
||||
result.envelope.model_dump() if result.envelope is not None else None
|
||||
)
|
||||
yield (json.dumps({"type": "envelope", "data": envelope_obj}) + "\n").encode("utf-8")
|
||||
for claim in result.claims:
|
||||
yield (json.dumps({"type": "claim", "data": json.loads(claim.model_dump_json())}) + "\n").encode("utf-8")
|
||||
yield (json.dumps({"type": "summary", "data": json.loads(result.summary.model_dump_json())}) + "\n").encode("utf-8")
|
||||
summary_data = json.loads(result.summary.model_dump_json())
|
||||
if batch_id is not None:
|
||||
summary_data["batch_id"] = batch_id
|
||||
yield (json.dumps({"type": "summary", "data": summary_data}) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
def ndjson_stream_835(result: ParseResult835) -> Iterator[bytes]:
|
||||
"""Yield one JSON object per line: envelope → financial → trace → payer → payee → claim_payments → summary."""
|
||||
def ndjson_stream_835(
|
||||
result: ParseResult835, batch_id: str | None = None,
|
||||
) -> Iterator[bytes]:
|
||||
"""Yield one JSON object per line: envelope → financial → trace → payer → payee → claim_payments → summary.
|
||||
|
||||
See ``ndjson_stream_837`` for why the optional ``batch_id`` is
|
||||
merged into the summary event.
|
||||
"""
|
||||
yield (json.dumps({"type": "envelope", "data": json.loads(result.envelope.model_dump_json())}) + "\n").encode("utf-8")
|
||||
yield (json.dumps({"type": "financial_info", "data": json.loads(result.financial_info.model_dump_json())}) + "\n").encode("utf-8")
|
||||
yield (json.dumps({"type": "trace", "data": json.loads(result.trace.model_dump_json())}) + "\n").encode("utf-8")
|
||||
@@ -113,7 +134,10 @@ def ndjson_stream_835(result: ParseResult835) -> Iterator[bytes]:
|
||||
yield (json.dumps({"type": "payee", "data": json.loads(result.payee.model_dump_json())}) + "\n").encode("utf-8")
|
||||
for claim in result.claims:
|
||||
yield (json.dumps({"type": "claim_payment", "data": json.loads(claim.model_dump_json())}) + "\n").encode("utf-8")
|
||||
yield (json.dumps({"type": "summary", "data": json.loads(result.summary.model_dump_json())}) + "\n").encode("utf-8")
|
||||
summary_data = json.loads(result.summary.model_dump_json())
|
||||
if batch_id is not None:
|
||||
summary_data["batch_id"] = batch_id
|
||||
yield (json.dumps({"type": "summary", "data": summary_data}) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
def strict_rewrite_837(result: ParseResult) -> ParseResult:
|
||||
|
||||
@@ -1 +1,59 @@
|
||||
"""Resource-group routers. Imported and registered by ``cyclone.api``."""
|
||||
"""Per-resource FastAPI routers.
|
||||
|
||||
`api.py` does `for r in routers: app.include_router(r)`. New
|
||||
routers register themselves here in alphabetical order. Helpers
|
||||
shared by 2+ routers live in `_shared.py` (private to the package).
|
||||
|
||||
Every router except ``health`` declares its own
|
||||
``APIRouter(dependencies=[Depends(matrix_gate)])`` — keep that
|
||||
invariant here so adding a new router doesn't silently miss the gate.
|
||||
"""
|
||||
from fastapi import APIRouter
|
||||
|
||||
from cyclone.api_routers import (
|
||||
acks,
|
||||
activity,
|
||||
admin,
|
||||
batches,
|
||||
claim_acks,
|
||||
claims,
|
||||
clearhouse,
|
||||
config,
|
||||
dashboard,
|
||||
eligibility,
|
||||
health,
|
||||
inbox,
|
||||
parse,
|
||||
payers,
|
||||
providers,
|
||||
rebill,
|
||||
reconciliation,
|
||||
remittances,
|
||||
submission,
|
||||
ta1_acks,
|
||||
)
|
||||
|
||||
routers: list[APIRouter] = [
|
||||
acks.router, # gated
|
||||
activity.router, # gated
|
||||
admin.router, # gated
|
||||
batches.router, # gated
|
||||
claim_acks.router, # gated
|
||||
claims.router, # gated
|
||||
clearhouse.router, # gated
|
||||
config.router, # gated
|
||||
dashboard.router, # gated
|
||||
eligibility.router, # gated
|
||||
health.router, # public — health probes must work pre-auth
|
||||
inbox.router, # gated
|
||||
parse.router, # gated
|
||||
payers.router, # gated
|
||||
providers.router, # gated
|
||||
rebill.router, # gated (SP41)
|
||||
reconciliation.router, # gated
|
||||
remittances.router, # gated
|
||||
submission.router, # gated
|
||||
ta1_acks.router, # gated
|
||||
]
|
||||
|
||||
__all__ = ["routers"]
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Cross-router helpers for the api_routers package.
|
||||
|
||||
Private to the package (leading underscore). Only routers in this
|
||||
package import from here. Helpers graduate here when at least two
|
||||
routers need them — single-router helpers stay in the router that
|
||||
uses them.
|
||||
|
||||
Helpers currently promoted here:
|
||||
|
||||
- :func:`_actor_user_id` — promoted early (during SP36 Task 11
|
||||
/ clearhouse extraction) because the clearhouse router needs
|
||||
it and the 2 remaining call-sites in ``api.py`` (parse-999 ack
|
||||
block, parse-277ca ack block) are both inside the parse
|
||||
surface that Task 16 will extract. Promoting now is cheaper
|
||||
than leaving a cross-module ``from cyclone.api import _actor_user_id``
|
||||
that would create an import-cycle at registry load time.
|
||||
|
||||
- :data:`PAYER_FACTORIES` / :data:`PAYER_FACTORIES_835` — SP36
|
||||
Task 16: payer config dicts lifted from ``api.py`` alongside
|
||||
the ``_resolve_payer`` / ``_resolve_payer_835`` helpers that
|
||||
consume them. The two helpers each touch a single ``PAYER_FACTORIES*``
|
||||
dict; keeping both halves of the pair in one module removes a
|
||||
circular import (parse.py → _shared._resolve_payer → api.PAYER_FACTORIES).
|
||||
|
||||
- :func:`_resolve_payer` / :func:`_resolve_payer_835` — used by
|
||||
``parse-837`` and ``parse-835`` endpoints respectively. Promoted
|
||||
in SP36 Task 16 alongside the PAYER_FACTORIES dicts.
|
||||
|
||||
- :func:`_transaction_set_id_from_segments` — used by
|
||||
``parse-837`` and ``parse-835`` envelope guards. Promoted in
|
||||
SP36 Task 16.
|
||||
|
||||
- :func:`_build_and_persist_ack` — used by ``parse-837`` (when
|
||||
``?ack=true``). Promoted in SP36 Task 16.
|
||||
|
||||
- :func:`_reconciliation_summary_for_batch` — used by
|
||||
``parse-835``. Promoted in SP36 Task 16.
|
||||
|
||||
- :func:`_ta1_synthetic_source_batch_id` — used by ``parse-ta1``.
|
||||
Promoted in SP36 Task 16.
|
||||
|
||||
- :func:`_serialize_ta1` — used by ``parse-ta1`` to build the
|
||||
raw TA1 round-trip text. Promoted in SP36 Task 16 per the
|
||||
plan's "8 helpers" specification; technically a single-router
|
||||
helper per D4 but moved here for symmetry with the other parse
|
||||
serializers.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.parsers.batch_ack_builder import build_ack_for_batch
|
||||
from cyclone.parsers.payer import PayerConfig, PayerConfig835
|
||||
from cyclone.parsers.serialize_999 import serialize_999
|
||||
from cyclone.store import store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Actor user id (SP36 Task 11 — early-promoted)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _actor_user_id(request: Request) -> int | None:
|
||||
"""Return the acting user's id from ``request.state.user``, or None.
|
||||
|
||||
``get_current_user``/``matrix_gate`` populate ``request.state.user``
|
||||
for both the authenticated path and the AUTH_DISABLED escape hatch.
|
||||
Returns None when the state hasn't been set (e.g. background jobs
|
||||
or unit tests that bypass auth). Used to stamp ``user_id`` onto
|
||||
audit events without crashing the request.
|
||||
"""
|
||||
user = getattr(request.state, "user", None)
|
||||
if user is None:
|
||||
return None
|
||||
return getattr(user, "id", None)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Payer config dicts (SP36 Task 16)
|
||||
# --------------------------------------------------------------------------- #
|
||||
#
|
||||
# Mirror cli._PAYER_FACTORIES. Kept here (not in the parse router) so
|
||||
# the ``_resolve_payer`` / ``_resolve_payer_835`` helpers can import
|
||||
# their backing dicts without a circular import.
|
||||
|
||||
|
||||
PAYER_FACTORIES: dict[str, Any] = {
|
||||
"co_medicaid": PayerConfig.co_medicaid,
|
||||
"generic_837p": PayerConfig.generic_837p,
|
||||
}
|
||||
|
||||
PAYER_FACTORIES_835: dict[str, Any] = {
|
||||
"co_medicaid_835": PayerConfig835.co_medicaid_835,
|
||||
"generic_835": PayerConfig835.generic_835,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Payer resolution (SP36 Task 16)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _resolve_payer(name: str) -> PayerConfig:
|
||||
if name not in PAYER_FACTORIES:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": "Unknown payer",
|
||||
"detail": f"Unknown payer {name!r}. Choose from: {', '.join(PAYER_FACTORIES)}",
|
||||
},
|
||||
)
|
||||
return PAYER_FACTORIES[name]()
|
||||
|
||||
|
||||
def _resolve_payer_835(name: str) -> PayerConfig835:
|
||||
if name not in PAYER_FACTORIES_835:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": "Unknown payer",
|
||||
"detail": f"Unknown payer {name!r}. Choose from: {', '.join(PAYER_FACTORIES_835)}",
|
||||
},
|
||||
)
|
||||
return PAYER_FACTORIES_835[name]()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Envelope detection (SP36 Task 16)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _transaction_set_id_from_segments(segments: list[list[str]]) -> str | None:
|
||||
"""Return the ST01 transaction-set id (``"837"``, ``"835"``, ``"999"``...).
|
||||
|
||||
SP35 helper: scans the first few tokenized segments for the ST
|
||||
segment and returns its second element (ST01). Returns None when no
|
||||
ST is present — e.g. a TA1 file, which uses the bare TA1 segment
|
||||
and no ST envelope. The endpoint-level envelope guards treat
|
||||
``None`` as "no ST found; let the parser decide" so TA1 files
|
||||
routed through the wrong endpoint still surface a parse error
|
||||
rather than a misleading "expected 837p, got ''" message.
|
||||
"""
|
||||
for seg in segments[:5]: # ST is always the second segment after ISA
|
||||
if seg and seg[0] == "ST" and len(seg) > 1:
|
||||
return seg[1]
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 999 ACK builder (SP36 Task 16)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _build_and_persist_ack(batch_id: str) -> dict | None:
|
||||
"""Build a 999 ACK for ``batch_id`` and persist the row.
|
||||
|
||||
Returns the ack payload dict (matches the ``/api/parse-999``
|
||||
response shape so the JSON and NDJSON clients can share the
|
||||
schema) or None if the build failed. The build is fail-soft:
|
||||
errors are logged but never abort the 837 ingest, because the
|
||||
user-visible 837 result is still correct.
|
||||
"""
|
||||
try:
|
||||
ack_result = build_ack_for_batch(batch_id)
|
||||
except Exception:
|
||||
log.exception("build_ack_for_batch failed for %s", batch_id)
|
||||
return None
|
||||
fg = ack_result.functional_group_acks[0] if ack_result.functional_group_acks else None
|
||||
if fg is None:
|
||||
return None
|
||||
raw_text = serialize_999(ack_result, interchange_control_number=ack_result.envelope.control_number)
|
||||
row = store.add_ack(
|
||||
source_batch_id=batch_id,
|
||||
accepted_count=fg.accepted_count,
|
||||
rejected_count=fg.rejected_count,
|
||||
received_count=fg.received_count,
|
||||
ack_code=fg.ack_code,
|
||||
raw_json=json.loads(ack_result.model_dump_json()),
|
||||
)
|
||||
return {
|
||||
"id": row.id,
|
||||
"accepted_count": fg.accepted_count,
|
||||
"rejected_count": fg.rejected_count,
|
||||
"received_count": fg.received_count,
|
||||
"ack_code": fg.ack_code,
|
||||
"source_batch_id": batch_id,
|
||||
"raw_999_text": raw_text,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Reconciliation summary (SP36 Task 16)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _reconciliation_summary_for_batch(batch_id: str) -> dict:
|
||||
"""Return ``{matched, unmatched_claims, unmatched_remittances, skipped}`` for a batch.
|
||||
|
||||
Reads from the DB after ``store.add()`` has already run reconciliation
|
||||
synchronously (SP27 Task 10: ``reconcile.run(s, batch_id)`` inside the
|
||||
ingest session, before commit). Counts are observed at this moment;
|
||||
a subsequent manual match/unmatch will not be reflected until the
|
||||
next request.
|
||||
|
||||
``skipped`` is reserved for future use — the orchestrator tracks
|
||||
skipped claims internally but does not surface a queryable count.
|
||||
"""
|
||||
from sqlalchemy import func, select
|
||||
from cyclone.db import Match, Remittance
|
||||
|
||||
with db.SessionLocal()() as s:
|
||||
matched = s.execute(
|
||||
select(func.count(Match.id)).where(
|
||||
Match.remittance_id.in_(
|
||||
select(Remittance.id).where(Remittance.batch_id == batch_id)
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
# Pull unmatched via the store (small result set; cheap).
|
||||
unmatched = store.list_unmatched(kind="both")
|
||||
return {
|
||||
"matched": matched,
|
||||
"unmatched_claims": len(unmatched["claims"]),
|
||||
"unmatched_remittances": len(unmatched["remittances"]),
|
||||
"skipped": 0, # reserved — T10 does not persist a skipped count
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# TA1 synthetic source batch id (SP36 Task 16)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _ta1_synthetic_source_batch_id(interchange_control_number: str) -> str:
|
||||
"""Return a synthetic ``batches.id`` for a received TA1 with no source batch.
|
||||
|
||||
Mirrors :func:`_ack_synthetic_source_batch_id` (in
|
||||
``cyclone.handlers._ack_id``). The ta1_acks.source_batch_id
|
||||
FK requires a row in batches; for received TA1s we synthesize an
|
||||
id of the form ``TA1-<ISA13>``. The row is NOT created in batches
|
||||
(same FK-is-no-op convention as the 999 path).
|
||||
"""
|
||||
return f"TA1-{(interchange_control_number or '').strip() or '000000001'}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# TA1 serializer (SP36 Task 16)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _serialize_ta1(result) -> str:
|
||||
"""Render a TA1 file from a ParseResultTa1 for the ``raw_ta1_text`` field.
|
||||
|
||||
Mirrors what the parser consumed: ISA envelope → TA1 → IEA. We
|
||||
rebuild minimal ISA fields from the envelope plus the TA1 segment
|
||||
verbatim. The serializer is intentionally tiny — TA1 has no GS/ST,
|
||||
so there's no functional-group structure to round-trip.
|
||||
"""
|
||||
ta1 = result.ta1
|
||||
parts = [
|
||||
f"TA1*{ta1.control_number}*{ta1.interchange_date.strftime('%y%m%d') if ta1.interchange_date else ''}*"
|
||||
f"{ta1.interchange_time or ''}*{ta1.ack_code}*{ta1.note_code or ''}*"
|
||||
f"{ta1.ack_generated_date.strftime('%y%m%d') if ta1.ack_generated_date else ''}",
|
||||
]
|
||||
return "~".join(parts) + "~"
|
||||
@@ -1,64 +1,86 @@
|
||||
"""``/api/acks`` — list & detail endpoints for the 999 ACK inbox.
|
||||
"""``/api/acks`` and ``/api/277ca-acks`` — list, detail, and live-tail.
|
||||
|
||||
These are the persisted acknowledgment rows produced by
|
||||
999 ACKs are the persisted acknowledgment rows produced by
|
||||
``POST /api/parse-999``. The frontend ``useAcks`` hook re-shapes the
|
||||
list payload to its ``Ack`` interface in ``src/types/index.ts``.
|
||||
|
||||
The detail endpoint returns the full ``raw_json`` payload plus the
|
||||
regenerated ``raw_999_text`` so the UI can show "view source" without a
|
||||
second round-trip.
|
||||
277CA ACKs are the persisted Claim Acknowledgment rows produced by
|
||||
``POST /api/parse-277ca``. The frontend ``useAcks`` hook treats them
|
||||
as a second source alongside 999 — the row shape is normalised in
|
||||
``cyclone.store.ui.to_ui_two77ca_ack``.
|
||||
|
||||
The 999 detail endpoint returns the full ``raw_json`` payload plus
|
||||
the regenerated ``raw_999_text`` so the UI can show "view source"
|
||||
without a second round-trip.
|
||||
|
||||
SP25: ``/api/acks/stream`` joins the live-tail triplet — the Acks
|
||||
page mounts ``useTailStream("acks")`` and ``useMergedTail("acks", …)``
|
||||
to see new 999 acks the moment they land (whether from the SFTP
|
||||
poller or a manual upload).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from cyclone.api_helpers import ndjson_stream_list, wants_ndjson
|
||||
from cyclone import db
|
||||
from cyclone.api_helpers import (
|
||||
ndjson_line,
|
||||
ndjson_stream_list,
|
||||
tail_events,
|
||||
wants_ndjson,
|
||||
)
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.parsers.models_999 import ParseResult999
|
||||
from cyclone.parsers.serialize_999 import serialize_999
|
||||
from cyclone.store import store
|
||||
from cyclone.pubsub import EventBus
|
||||
from cyclone.store import store, to_ui_ack, to_ui_two77ca_ack
|
||||
|
||||
router = APIRouter()
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ack_to_ui(row) -> dict:
|
||||
"""Map an ``Ack`` ORM row to the UI shape used by ``/api/acks``.
|
||||
|
||||
Field names match the rest of the Cyclone API (snake_case). The
|
||||
frontend ``useAcks`` hook re-shapes this to the camelCase ``Ack``
|
||||
interface in ``src/types/index.ts``.
|
||||
"""
|
||||
return {
|
||||
"id": row.id,
|
||||
"source_batch_id": row.source_batch_id,
|
||||
"accepted_count": row.accepted_count,
|
||||
"rejected_count": row.rejected_count,
|
||||
"received_count": row.received_count,
|
||||
"ack_code": row.ack_code,
|
||||
"parsed_at": (
|
||||
row.parsed_at.isoformat().replace("+00:00", "Z")
|
||||
if row.parsed_at is not None
|
||||
else ""
|
||||
),
|
||||
}
|
||||
# SP25: ``_ack_to_ui`` moved to ``cyclone.store.ui.to_ui_ack`` so the
|
||||
# live-tail event payload (``ack_received``) matches the list endpoint
|
||||
# shape byte-for-byte.
|
||||
|
||||
|
||||
@router.get("/api/acks")
|
||||
def list_acks_endpoint(
|
||||
request: Request,
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
limit: int = Query(100, ge=1, le=5000),
|
||||
offset: int = Query(0, ge=0),
|
||||
) -> Any:
|
||||
"""Return the list of persisted 999 ACKs, newest first."""
|
||||
"""Return the list of persisted 999 ACKs, newest first.
|
||||
|
||||
``limit`` caps the page size; ``offset`` lets the UI walk the
|
||||
full set without holding it all in memory. ``aggregates`` is
|
||||
summed over the *full* row set (not the page) so the KPI strip
|
||||
on the Acks page reflects every persisted 999, not just the
|
||||
visible 50. Without server-side aggregates the page would
|
||||
silently under-report (silent-failure mode) once the row count
|
||||
exceeds the page size.
|
||||
"""
|
||||
rows = store.list_acks()
|
||||
items = [_ack_to_ui(r) for r in rows[:limit]]
|
||||
items = [to_ui_ack(r) for r in rows[offset : offset + limit]]
|
||||
total = len(rows)
|
||||
returned = len(items)
|
||||
has_more = total > returned
|
||||
has_more = offset + returned < total
|
||||
aggregates = {
|
||||
"accepted_count": sum(r.accepted_count or 0 for r in rows),
|
||||
"rejected_count": sum(r.rejected_count or 0 for r in rows),
|
||||
"received_count": sum(r.received_count or 0 for r in rows),
|
||||
}
|
||||
# SP28: batch-fetch linked_claim_ids per ack row in one query to
|
||||
# avoid N+1 — see ``find_linked_claim_ids_for_acks`` below.
|
||||
ack_ids = [r.id for r in rows]
|
||||
linked_map = _find_linked_claim_ids_for_acks(ack_ids, kind="999")
|
||||
for item, aid in zip(items, ack_ids[offset : offset + limit]):
|
||||
item["linked_claim_ids"] = linked_map.get(aid, [])
|
||||
if wants_ndjson(request):
|
||||
return StreamingResponse(
|
||||
ndjson_stream_list(items, total, returned, has_more),
|
||||
@@ -69,9 +91,68 @@ def list_acks_endpoint(
|
||||
"total": total,
|
||||
"returned": returned,
|
||||
"has_more": has_more,
|
||||
"aggregates": aggregates,
|
||||
}
|
||||
|
||||
|
||||
def _find_linked_claim_ids_for_acks(
|
||||
ack_ids: list[int], *, kind: str
|
||||
) -> dict[int, list[str]]:
|
||||
"""Batch-fetch {ack_id: [claim_id, …]} for the listed ack rows.
|
||||
|
||||
One SELECT against ``claim_acks`` keyed on the page's ack_ids —
|
||||
avoids an N+1 round-trip when the page renders the
|
||||
"🔗 N claims" badge per row. Used by the 999 / TA1 / 277CA
|
||||
list endpoints. Returns a ``{ack_id: [claim_id]}`` map; acks
|
||||
with no links map to ``[]`` (default).
|
||||
"""
|
||||
out: dict[int, list[str]] = {aid: [] for aid in ack_ids}
|
||||
if not ack_ids:
|
||||
return out
|
||||
with db.SessionLocal()() as s:
|
||||
rows = (
|
||||
s.query(db.ClaimAck.ack_id, db.ClaimAck.claim_id)
|
||||
.filter(
|
||||
db.ClaimAck.ack_kind == kind,
|
||||
db.ClaimAck.ack_id.in_(ack_ids),
|
||||
db.ClaimAck.claim_id.isnot(None),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for ack_id, claim_id in rows:
|
||||
out[ack_id].append(claim_id)
|
||||
return out
|
||||
|
||||
|
||||
@router.get("/api/acks/stream")
|
||||
async def acks_stream(
|
||||
request: Request,
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
) -> StreamingResponse:
|
||||
"""Stream 999 ACKs as NDJSON: snapshot first, then live events.
|
||||
|
||||
SP25: this endpoint joins the live-tail triplet — subscribes to
|
||||
``ack_received`` and emits one ``item`` per snapshot row plus a
|
||||
single ``snapshot_end`` line, then forwards live events from
|
||||
the bus. Matches the wire format used by ``/api/claims/stream``,
|
||||
``/api/remittances/stream``, and ``/api/activity/stream``.
|
||||
|
||||
NOTE: registered BEFORE ``/api/acks/{ack_id}`` so the literal
|
||||
``stream`` path segment doesn't get matched as an ack id.
|
||||
"""
|
||||
bus: EventBus = request.app.state.event_bus
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
rows = store.list_acks()[:limit]
|
||||
for row in rows:
|
||||
yield ndjson_line({"type": "item", "data": to_ui_ack(row)})
|
||||
yield ndjson_line({"type": "snapshot_end", "data": {"count": len(rows)}})
|
||||
async for chunk in tail_events(request, bus, ["ack_received"]):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
@router.get("/api/acks/{ack_id}")
|
||||
def get_ack_endpoint(ack_id: int) -> dict:
|
||||
"""Return one persisted ACK row with its parsed detail.
|
||||
@@ -86,7 +167,7 @@ def get_ack_endpoint(ack_id: int) -> dict:
|
||||
status_code=404,
|
||||
detail={"error": "Not found", "detail": f"Ack {ack_id} not found"},
|
||||
)
|
||||
body = _ack_to_ui(row)
|
||||
body = to_ui_ack(row)
|
||||
body["raw_json"] = row.raw_json
|
||||
# Regenerate the X12 text from raw_json so the operator can download
|
||||
# the actual 999 file. (SP3 P3 follow-up: list endpoint doesn't carry
|
||||
@@ -102,3 +183,41 @@ def get_ack_endpoint(ack_id: int) -> dict:
|
||||
else:
|
||||
body["raw_999_text"] = None
|
||||
return body
|
||||
|
||||
|
||||
# -- 277CA ACKs -------------------------------------------------------------
|
||||
# SP36 Task 2: these two endpoints moved here from api.py (lines 1278-1318).
|
||||
# 277CA ACKs share the same shape-of-life contract as 999 ACKs: persisted by
|
||||
# a parse endpoint, listed newest-first, detail returns raw_json so the UI
|
||||
# can show "view source" without a round-trip.
|
||||
|
||||
|
||||
@router.get("/api/277ca-acks")
|
||||
def list_277ca_acks_endpoint(
|
||||
limit: int = Query(100, ge=1, le=5000),
|
||||
) -> Any:
|
||||
"""Return the list of persisted 277CA ACKs, newest first.
|
||||
|
||||
SP28: each item gains ``linked_claim_ids`` (batch-fetched via
|
||||
the shared ``_find_linked_claim_ids_for_acks`` helper — one
|
||||
SELECT keyed on ``ack_kind``, no N+1) so the Acks page row
|
||||
can render the "🔗 N claims" badge inline.
|
||||
"""
|
||||
rows = store.list_277ca_acks()
|
||||
items = [to_ui_two77ca_ack(r) for r in rows[:limit]]
|
||||
ack_ids = [r.id for r in rows]
|
||||
linked_map = _find_linked_claim_ids_for_acks(ack_ids, kind="277ca")
|
||||
for item, aid in zip(items, ack_ids[:limit]):
|
||||
item["linked_claim_ids"] = linked_map.get(aid, [])
|
||||
return {"total": len(rows), "items": items}
|
||||
|
||||
|
||||
@router.get("/api/277ca-acks/{ack_id}")
|
||||
def get_277ca_ack_endpoint(ack_id: int) -> dict:
|
||||
"""Return one persisted 277CA ACK row with its parsed detail."""
|
||||
row = store.get_277ca_ack(ack_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"277CA ACK {ack_id} not found")
|
||||
body = to_ui_two77ca_ack(row)
|
||||
body["raw_json"] = row.raw_json
|
||||
return body
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""``/api/activity`` and ``/api/activity/stream`` — operator-facing event log.
|
||||
|
||||
Two endpoints, both gated by ``matrix_gate``:
|
||||
|
||||
- ``GET /api/activity`` — paginated event list with ``kind`` /
|
||||
``since`` filters, plus an NDJSON variant when the caller sends
|
||||
``Accept: application/x-ndjson``.
|
||||
- ``GET /api/activity/stream`` — NDJSON live-tail: snapshot of the
|
||||
most recent N events, then ``activity_recorded`` events as they
|
||||
hit the store. Default ``limit`` is 50 (smaller than the list
|
||||
endpoint's 200) because activity is high-volume — callers usually
|
||||
want the most recent handful, not a full replay.
|
||||
|
||||
The snapshot halves of ``/api/activity`` and ``/api/activity/stream``
|
||||
share the same in-memory filter logic (``kind`` + ``since``) so the
|
||||
two endpoints are interchangeable for the snapshot half.
|
||||
|
||||
SP36 Task 10: this block moved here from ``api.py:2606`` (the
|
||||
``/api/activity*`` pair).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from cyclone.api_helpers import (
|
||||
ndjson_line as _ndjson_line,
|
||||
ndjson_stream_list as _ndjson_stream_list,
|
||||
tail_events as _tail_events,
|
||||
wants_ndjson as _wants_ndjson,
|
||||
)
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.pubsub import EventBus
|
||||
from cyclone.store import store
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.get("/api/activity")
|
||||
def list_activity(
|
||||
request: Request,
|
||||
kind: str | None = Query(None),
|
||||
since: str | None = Query(None),
|
||||
limit: int = Query(200, ge=1, le=5000),
|
||||
) -> Any:
|
||||
events = store.recent_activity(limit=limit)
|
||||
if kind is not None:
|
||||
events = [e for e in events if e["kind"] == kind]
|
||||
if since is not None:
|
||||
events = [e for e in events if e["timestamp"] >= since]
|
||||
total = len(events)
|
||||
has_more = False
|
||||
if _wants_ndjson(request):
|
||||
return StreamingResponse(
|
||||
_ndjson_stream_list(events, total, total, has_more),
|
||||
media_type="application/x-ndjson",
|
||||
)
|
||||
return {
|
||||
"items": events,
|
||||
"total": total,
|
||||
"returned": total,
|
||||
"has_more": has_more,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/activity/stream")
|
||||
async def activity_stream(
|
||||
request: Request,
|
||||
kind: str | None = Query(None),
|
||||
since: str | None = Query(None),
|
||||
limit: int = Query(50, ge=1, le=5000),
|
||||
) -> StreamingResponse:
|
||||
"""Stream Activity events as NDJSON: snapshot first, then live events.
|
||||
|
||||
Subscribes to ``activity_recorded``. Default ``limit`` is 50
|
||||
(smaller than the list endpoint's 200) because activity is
|
||||
high-volume — callers usually want the most recent handful, not a
|
||||
full replay.
|
||||
"""
|
||||
bus: EventBus = request.app.state.event_bus
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
# Snapshot reuses the same in-memory filter as ``list_activity``
|
||||
# so the two endpoints are interchangeable for the snapshot
|
||||
# half.
|
||||
events = store.recent_activity(limit=limit)
|
||||
if kind is not None:
|
||||
events = [e for e in events if e["kind"] == kind]
|
||||
if since is not None:
|
||||
events = [e for e in events if e["timestamp"] >= since]
|
||||
for ev in events:
|
||||
yield _ndjson_line({"type": "item", "data": ev})
|
||||
yield _ndjson_line({
|
||||
"type": "snapshot_end", "data": {"count": len(events)},
|
||||
})
|
||||
|
||||
async for chunk in _tail_events(request, bus, ["activity_recorded"]):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
@@ -1,23 +1,45 @@
|
||||
"""``/api/admin/validate-provider`` — NPI + Tax ID liveness probe (SP20).
|
||||
"""``/api/admin/*`` — operator-only endpoints.
|
||||
|
||||
Pure read-only endpoint that runs the local NPI Luhn + EIN format checks
|
||||
without touching the DB. Useful for:
|
||||
- operators vetting a new provider before adding them to the registry,
|
||||
- the dashboard's "validate" button on a Provider row,
|
||||
- smoke-testing the SP20 checks after a deploy.
|
||||
The /api/admin namespace covers:
|
||||
|
||||
Both query params are optional; omitting one just skips that check.
|
||||
Returns the per-check result dict so the caller can distinguish "bad
|
||||
format" from "bad checksum".
|
||||
* **audit-log** (SP11): list + chain-verify the tamper-evident log
|
||||
* **db/rotate-key**: SQLCipher key rotation (SP12)
|
||||
* **backup**: create / list / status / verify / restore / prune / scheduler (SP15)
|
||||
* **scheduler**: backup & main scheduler control + processed-files log
|
||||
* **reload-config**: hot-reload ``config/payers.yaml`` after edits
|
||||
* **validate-provider**: NPI + Tax ID liveness probe (SP20)
|
||||
|
||||
All routes on this router are gated by ``matrix_gate`` — declared
|
||||
once on the ``APIRouter`` constructor. ``/api/admin/validate-provider``
|
||||
lives here too because it's an admin-shaped read-only probe.
|
||||
|
||||
SP36 Task 3: the 20 admin endpoints formerly in ``cyclone.api``
|
||||
(audit-log ×2, db/rotate-key ×1, backup ×10, scheduler ×6,
|
||||
reload-config ×1) moved here from ``api.py:3321-4052`` and
|
||||
``api.py:4269-4276``. The non-admin ``/api/config/*`` and
|
||||
``/api/payers/{id}/summary`` routes that bracketed those blocks
|
||||
stay in ``api.py`` for now — they're extracted in Tasks 5 & 6.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from cyclone import db, edifabric
|
||||
from cyclone.audit_log import verify_chain
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.clearhouse import InboundFile
|
||||
from cyclone.npi import is_valid_npi, is_valid_tax_id, normalize_tax_id
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
router = APIRouter()
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@router.get("/api/admin/validate-provider")
|
||||
@@ -57,4 +79,804 @@ def validate_provider(
|
||||
"normalized": normalized,
|
||||
}
|
||||
|
||||
return result
|
||||
return result
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SP36 Task 3: the 20 admin endpoints below were moved here from api.py. #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SP11: tamper-evident audit log (admin)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@router.get("/api/admin/audit-log")
|
||||
def list_audit_log_endpoint(
|
||||
entity_type: str | None = Query(default=None),
|
||||
entity_id: str | None = Query(default=None),
|
||||
event_type: str | None = Query(default=None),
|
||||
limit: int = Query(default=100, ge=1, le=1000),
|
||||
) -> Any:
|
||||
"""List audit-log rows, newest first, with optional filters.
|
||||
|
||||
Filters match the (entity_type, entity_id) pair (typical use:
|
||||
"show me everything that happened to claim C-123") or a single
|
||||
event_type (typical use: "show me all clearhouse.submitted
|
||||
events today").
|
||||
"""
|
||||
with db.SessionLocal()() as s:
|
||||
q = s.query(db.AuditLog)
|
||||
if entity_type:
|
||||
q = q.filter(db.AuditLog.entity_type == entity_type)
|
||||
if entity_id:
|
||||
q = q.filter(db.AuditLog.entity_id == entity_id)
|
||||
if event_type:
|
||||
q = q.filter(db.AuditLog.event_type == event_type)
|
||||
rows = q.order_by(db.AuditLog.id.desc()).limit(limit).all()
|
||||
return {
|
||||
"total": len(rows),
|
||||
"items": [
|
||||
{
|
||||
"id": r.id,
|
||||
"event_type": r.event_type,
|
||||
"entity_type": r.entity_type,
|
||||
"entity_id": r.entity_id,
|
||||
"actor": r.actor,
|
||||
"payload": json.loads(r.payload_json) if r.payload_json else None,
|
||||
"created_at": r.created_at.isoformat() if r.created_at else None,
|
||||
"prev_hash": r.prev_hash,
|
||||
"hash": r.hash,
|
||||
}
|
||||
for r in rows
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/admin/audit-log/verify")
|
||||
def verify_audit_log_endpoint() -> Any:
|
||||
"""Walk the audit-log chain and verify every row's hash.
|
||||
|
||||
Returns ``{"ok": true, "checked": N}`` for a clean chain, or
|
||||
``{"ok": false, "checked": K, "first_bad_id": X, "reason": "..."}``
|
||||
for a broken chain. This is the operator's "did anyone tamper?"
|
||||
endpoint; run it on demand or via a nightly cron job.
|
||||
"""
|
||||
with db.SessionLocal()() as s:
|
||||
result = verify_chain(s)
|
||||
return {
|
||||
"ok": result.ok,
|
||||
"checked": result.checked,
|
||||
"first_bad_id": result.first_bad_id,
|
||||
"reason": result.reason,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP15: SQLCipher key rotation
|
||||
#
|
||||
# Re-encrypts the DB in place with a fresh key, then updates the
|
||||
# Keychain so subsequent connections open with the new key. This is
|
||||
# a 1-time operation per rotation; for routine read/write the rest
|
||||
# of the API is unchanged.
|
||||
#
|
||||
# Concurrency: the rotation holds a module-level lock so two
|
||||
# concurrent requests can't race and end up with mismatched Keychain
|
||||
# + DB. The lock is a simple threading.Lock; a process restart
|
||||
# resets it (intentional — the operator's next start-up opens with
|
||||
# whatever key is in the Keychain).
|
||||
# ---------------------------------------------------------------------------
|
||||
from cyclone import db_crypto as _db_crypto
|
||||
from cyclone import secrets as _secrets
|
||||
|
||||
_db_rotate_lock = threading.Lock()
|
||||
|
||||
|
||||
@router.post("/api/admin/db/rotate-key")
|
||||
def rotate_db_key_endpoint(body: dict | None = None) -> Any:
|
||||
"""Generate a fresh DB key, re-encrypt the DB, update the Keychain.
|
||||
|
||||
Request body (optional):
|
||||
actor: who initiated the rotation. Defaults to "operator".
|
||||
reason: human-readable reason. Written to the audit log.
|
||||
|
||||
Returns:
|
||||
``{ok, old_fingerprint, new_fingerprint, rotated_at, table_count}``
|
||||
on success. On failure (DB not encrypted, rekey failed,
|
||||
Keychain update failed) returns the same shape with
|
||||
``ok=false`` and a ``reason``. HTTP 503 is returned if the
|
||||
rekey fails or encryption is not enabled.
|
||||
|
||||
The Keychain write happens *after* the rekey succeeds. If the
|
||||
Keychain write fails, the DB has the new key but the Keychain
|
||||
still has the old one — the endpoint returns 503 with a
|
||||
"keychain update failed" reason and the operator must restore
|
||||
the old key manually (``cyclone db restore-key <old_key>``) to
|
||||
avoid being locked out.
|
||||
"""
|
||||
body = body or {}
|
||||
actor = body.get("actor") or "operator"
|
||||
reason = body.get("reason") or ""
|
||||
|
||||
if not _db_crypto.is_encryption_enabled():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="encryption not enabled (sqlcipher3 missing or no Keychain key)",
|
||||
)
|
||||
|
||||
# Acquire the lock; non-blocking so a stuck rotation doesn't
|
||||
# silently hold up other requests.
|
||||
if not _db_rotate_lock.acquire(blocking=False):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="another key rotation is in progress",
|
||||
)
|
||||
try:
|
||||
url = db._resolve_url()
|
||||
old_key = _db_crypto.get_db_key()
|
||||
if not old_key:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="no DB key in Keychain; cannot rotate",
|
||||
)
|
||||
|
||||
new_key = _db_crypto.generate_db_key()
|
||||
result = _db_crypto.rotate_db_key(
|
||||
url=url, old_key=old_key, new_key=new_key,
|
||||
)
|
||||
if not result.ok:
|
||||
# Rekey failed. The DB still has the old key. The
|
||||
# Keychain is unchanged. Caller should NOT retry with
|
||||
# the same new key (it's lost); generate a fresh one.
|
||||
log.error("SQLCipher rotate failed: %s", result.reason)
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail={
|
||||
"ok": False,
|
||||
"old_fingerprint": result.old_fingerprint,
|
||||
"new_fingerprint": result.new_fingerprint,
|
||||
"rotated_at": result.rotated_at,
|
||||
"reason": result.reason,
|
||||
},
|
||||
)
|
||||
|
||||
# Rekey succeeded. Now update the Keychain. If this fails
|
||||
# the DB is locked behind the new key — operator must
|
||||
# restore the old key manually.
|
||||
if not _secrets.set_secret(_db_crypto.KEYCHAIN_ACCOUNT, new_key):
|
||||
log.error("Keychain update failed after successful rekey!")
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail={
|
||||
"ok": False,
|
||||
"old_fingerprint": result.old_fingerprint,
|
||||
"new_fingerprint": result.new_fingerprint,
|
||||
"rotated_at": result.rotated_at,
|
||||
"reason": (
|
||||
"rekey succeeded but Keychain update failed — "
|
||||
"the DB is now encrypted with the new key but "
|
||||
"the Keychain still has the old one. "
|
||||
"Restore the old key to the Keychain to recover."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
# Store the old key in the "previous" account for a grace
|
||||
# period so the operator can roll back if they discover the
|
||||
# new key is broken (e.g. the Keychain entry got truncated).
|
||||
_secrets.set_secret(_db_crypto.KEYCHAIN_ACCOUNT_PREVIOUS, old_key)
|
||||
|
||||
# Rebuild the engine so subsequent connections use the new
|
||||
# key. dispose_engine() closes every pooled connection that
|
||||
# was using the old key; init_db() opens new ones with the
|
||||
# new key from the (now-updated) Keychain.
|
||||
db.reinit_engine()
|
||||
|
||||
# Audit log the rotation. We do this after the engine is
|
||||
# rebuilt so the audit event is written with the new key —
|
||||
# proving that the new key works for new writes.
|
||||
try:
|
||||
from cyclone.audit_log import append_event, AuditEvent
|
||||
with db.SessionLocal()() as s:
|
||||
append_event(s, AuditEvent(
|
||||
event_type="db.key_rotated",
|
||||
entity_type="database",
|
||||
entity_id="cyclone.db",
|
||||
actor=actor,
|
||||
payload={
|
||||
"old_fingerprint": result.old_fingerprint,
|
||||
"new_fingerprint": result.new_fingerprint,
|
||||
"table_count": result.table_count,
|
||||
"reason": reason,
|
||||
},
|
||||
))
|
||||
s.commit()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Audit append is best-effort; rotation already succeeded.
|
||||
log.warning("could not write audit event for rotation: %s", exc)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"old_fingerprint": result.old_fingerprint,
|
||||
"new_fingerprint": result.new_fingerprint,
|
||||
"rotated_at": result.rotated_at,
|
||||
"table_count": result.table_count,
|
||||
}
|
||||
finally:
|
||||
_db_rotate_lock.release()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP17: encrypted DB backups (admin)
|
||||
#
|
||||
# The actual encryption + lifecycle lives in :mod:`cyclone.backup` and
|
||||
# :mod:`cyclone.backup_service`. The scheduler (separate from the
|
||||
# MFT scheduler) lives in :mod:`cyclone.backup_scheduler`. These
|
||||
# endpoints expose the operator's manual controls plus a tick for
|
||||
# "take a backup right now."
|
||||
#
|
||||
# Restore is intentionally two-step: an idle browser tab can't nuke
|
||||
# the live DB. The first call returns a ``restore_token`` (a one-shot
|
||||
# 64-char hex) and a preview of the backup's fingerprint + table
|
||||
# count plus the live DB's. The second call with the token performs
|
||||
# the actual swap.
|
||||
# ---------------------------------------------------------------------------
|
||||
from cyclone import backup_service as _backup_svc_mod
|
||||
from cyclone import backup_scheduler as _backup_sched_mod
|
||||
|
||||
|
||||
def _backup_or_503():
|
||||
try:
|
||||
return _backup_svc_mod.get_backup_service()
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=503, detail=str(exc))
|
||||
|
||||
|
||||
@router.post("/api/admin/backup/create")
|
||||
def backup_create() -> Any:
|
||||
"""Take an encrypted backup right now. Returns the new backup metadata."""
|
||||
from cyclone import audit_log as _audit
|
||||
svc = _backup_or_503()
|
||||
try:
|
||||
result = svc.create_now()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Surface a 503 with the reason so the operator sees what
|
||||
# went wrong without grepping server logs.
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail=f"backup failed: {type(exc).__name__}: {exc}",
|
||||
)
|
||||
# Audit the create. Best-effort; failure here doesn't roll back
|
||||
# the backup (already on disk).
|
||||
try:
|
||||
with db.SessionLocal()() as s:
|
||||
_audit.append_event(s, _audit.AuditEvent(
|
||||
event_type="db.backup_created",
|
||||
entity_type="database",
|
||||
entity_id="cyclone.db",
|
||||
actor="operator",
|
||||
payload={
|
||||
"backup_id": result.backup.id,
|
||||
"db_fingerprint": result.backup.db_fingerprint,
|
||||
"table_count": result.backup.table_count,
|
||||
"triggered_by": "api",
|
||||
},
|
||||
))
|
||||
s.commit()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("could not write backup_created audit event: %s", exc)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"backup": {
|
||||
"id": result.backup.id,
|
||||
"filename": result.backup.filename,
|
||||
"size_bytes": result.backup.size_bytes,
|
||||
"db_fingerprint": result.backup.db_fingerprint,
|
||||
"table_count": result.backup.table_count,
|
||||
"created_at": result.backup.created_at.isoformat(),
|
||||
"key_fingerprint": result.backup.key_fingerprint,
|
||||
},
|
||||
"sidecar": {
|
||||
"format_version": result.sidecar.format_version,
|
||||
"kdf": result.sidecar.kdf,
|
||||
"kdf_iterations": result.sidecar.kdf_iterations,
|
||||
"cipher": result.sidecar.cipher,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/admin/backup/list")
|
||||
def backup_list(
|
||||
limit: int = Query(default=100, ge=1, le=1000),
|
||||
status: str | None = Query(default=None),
|
||||
) -> Any:
|
||||
"""List ``db_backups`` rows, newest first. Filters by status."""
|
||||
svc = _backup_or_503()
|
||||
rows = svc.list_backups(limit=limit, status=status)
|
||||
return {
|
||||
"count": len(rows),
|
||||
"files": [
|
||||
{
|
||||
"id": r.id,
|
||||
"filename": r.filename,
|
||||
"backup_dir": r.backup_dir,
|
||||
"size_bytes": r.size_bytes,
|
||||
"db_fingerprint": r.db_fingerprint,
|
||||
"table_count": r.table_count,
|
||||
"created_at": r.created_at.isoformat() if r.created_at else None,
|
||||
"completed_at": r.completed_at.isoformat() if r.completed_at else None,
|
||||
"status": r.status,
|
||||
"error_message": r.error_message,
|
||||
"key_fingerprint": r.key_fingerprint,
|
||||
}
|
||||
for r in rows
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/admin/backup/status")
|
||||
def backup_status() -> Any:
|
||||
"""Snapshot of the backup subsystem (counts, disk usage, last run)."""
|
||||
svc = _backup_or_503()
|
||||
snap = svc.status()
|
||||
# Also include the BackupScheduler's snapshot if configured.
|
||||
try:
|
||||
sched = _backup_sched_mod.get_backup_scheduler()
|
||||
snap["scheduler"] = sched.status().as_dict()
|
||||
except RuntimeError:
|
||||
snap["scheduler"] = None
|
||||
return snap
|
||||
|
||||
|
||||
@router.post("/api/admin/backup/{backup_id}/verify")
|
||||
def backup_verify(backup_id: int) -> Any:
|
||||
"""Decrypt + checksum-verify a backup against its sidecar."""
|
||||
svc = _backup_or_503()
|
||||
try:
|
||||
v = svc.verify(backup_id)
|
||||
except _backup_svc_mod.BackupError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
return {
|
||||
"backup_id": v.backup_id,
|
||||
"filename": v.filename,
|
||||
"ok": v.ok,
|
||||
"expected_fingerprint": v.expected_fingerprint,
|
||||
"actual_fingerprint": v.actual_fingerprint,
|
||||
"table_count": v.table_count,
|
||||
"reason": v.reason,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/admin/backup/{backup_id}/restore/initiate")
|
||||
def backup_restore_initiate(backup_id: int) -> Any:
|
||||
"""First step of the two-step restore. Returns a ``restore_token``."""
|
||||
svc = _backup_or_503()
|
||||
try:
|
||||
init = svc.restore_initiate(backup_id)
|
||||
except _backup_svc_mod.BackupError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
return {
|
||||
"backup_id": init.backup_id,
|
||||
"filename": init.filename,
|
||||
"size_bytes": init.size_bytes,
|
||||
"restore_token": init.restore_token,
|
||||
"expires_at": init.expires_at.isoformat(),
|
||||
"preview": {
|
||||
"backup_db_fingerprint": init.db_fingerprint,
|
||||
"backup_table_count": init.table_count,
|
||||
"current_db_fingerprint": init.current_db_fingerprint,
|
||||
"current_table_count": init.current_table_count,
|
||||
},
|
||||
"warning": (
|
||||
"Confirming will dispose the live engine and replace the DB "
|
||||
"file with the backup. In-flight requests will error. "
|
||||
"Re-issue the call with the restore_token within 5 minutes."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/admin/backup/{backup_id}/restore/confirm")
|
||||
def backup_restore_confirm(
|
||||
backup_id: int,
|
||||
body: dict | None = None,
|
||||
) -> Any:
|
||||
"""Second step of the two-step restore. Performs the swap."""
|
||||
body = body or {}
|
||||
token = body.get("restore_token")
|
||||
if not token or not isinstance(token, str):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="missing or invalid restore_token in request body",
|
||||
)
|
||||
actor = body.get("actor") or "operator"
|
||||
|
||||
svc = _backup_or_503()
|
||||
try:
|
||||
result = svc.restore_confirm(backup_id, token, actor=actor)
|
||||
except _backup_svc_mod.BackupError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
|
||||
# Audit the restore. Best-effort.
|
||||
try:
|
||||
from cyclone import audit_log as _audit
|
||||
with db.SessionLocal()() as s:
|
||||
_audit.append_event(s, _audit.AuditEvent(
|
||||
event_type="db.backup_restored",
|
||||
entity_type="database",
|
||||
entity_id="cyclone.db",
|
||||
actor=actor,
|
||||
payload={
|
||||
"backup_id": result.backup_id,
|
||||
"filename": result.filename,
|
||||
"restored_from_fingerprint": result.restored_from_fingerprint,
|
||||
"new_db_fingerprint": result.new_db_fingerprint,
|
||||
"restored_at": result.restored_at.isoformat(),
|
||||
},
|
||||
))
|
||||
s.commit()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("could not write backup_restored audit event: %s", exc)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"backup_id": result.backup_id,
|
||||
"filename": result.filename,
|
||||
"restored_from_fingerprint": result.restored_from_fingerprint,
|
||||
"restored_at": result.restored_at.isoformat(),
|
||||
"new_db_fingerprint": result.new_db_fingerprint,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/admin/backup/prune")
|
||||
def backup_prune() -> Any:
|
||||
"""Apply the retention policy now. Returns the deleted paths."""
|
||||
from cyclone import audit_log as _audit
|
||||
svc = _backup_or_503()
|
||||
deleted = svc.prune()
|
||||
actor = "operator"
|
||||
if deleted:
|
||||
try:
|
||||
with db.SessionLocal()() as s:
|
||||
_audit.append_event(s, _audit.AuditEvent(
|
||||
event_type="db.backup_pruned",
|
||||
entity_type="database",
|
||||
entity_id="cyclone.db",
|
||||
actor=actor,
|
||||
payload={"deleted_paths": deleted},
|
||||
))
|
||||
s.commit()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("could not write backup_pruned audit event: %s", exc)
|
||||
return {"ok": True, "deleted_count": len(deleted), "deleted_paths": deleted}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP17: backup scheduler (admin)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/api/admin/backup/scheduler/start")
|
||||
async def backup_scheduler_start() -> Any:
|
||||
"""Begin the backup scheduler loop."""
|
||||
try:
|
||||
sched = _backup_sched_mod.get_backup_scheduler()
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=503, detail=str(exc))
|
||||
await sched.start()
|
||||
return {"status": sched.status().as_dict()}
|
||||
|
||||
|
||||
@router.post("/api/admin/backup/scheduler/stop")
|
||||
async def backup_scheduler_stop() -> Any:
|
||||
"""Stop the backup scheduler loop."""
|
||||
try:
|
||||
sched = _backup_sched_mod.get_backup_scheduler()
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=503, detail=str(exc))
|
||||
await sched.stop()
|
||||
return {"status": sched.status().as_dict()}
|
||||
|
||||
|
||||
@router.post("/api/admin/backup/scheduler/tick")
|
||||
async def backup_scheduler_tick() -> Any:
|
||||
"""Run one backup tick now (create + prune + audit)."""
|
||||
try:
|
||||
sched = _backup_sched_mod.get_backup_scheduler()
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=503, detail=str(exc))
|
||||
result = await sched.tick()
|
||||
return {"ok": result.ok, "tick": result.as_dict()}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP16: live MFT polling scheduler (admin)
|
||||
#
|
||||
# The scheduler lives in :mod:`cyclone.scheduler` and is configured by
|
||||
# the lifespan handler. The endpoints below expose start / stop /
|
||||
# one-shot tick / status / history so an operator (or a cron job)
|
||||
# can drive the scheduler without touching the DB.
|
||||
#
|
||||
# Note: the scheduler is OFF by default. Auto-start is opt-in via
|
||||
# ``CYCLONE_SCHEDULER_AUTOSTART=true`` at launch. These endpoints
|
||||
# are the operator's manual controls.
|
||||
# ---------------------------------------------------------------------------
|
||||
from cyclone import scheduler as _scheduler_mod
|
||||
|
||||
|
||||
def _scheduler_or_503():
|
||||
"""Return the configured scheduler or raise 503."""
|
||||
try:
|
||||
return _scheduler_mod.get_scheduler()
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=503, detail=str(exc))
|
||||
|
||||
|
||||
@router.post("/api/admin/scheduler/start")
|
||||
async def scheduler_start() -> Any:
|
||||
"""Begin polling the MFT inbound path every poll_interval_seconds."""
|
||||
sched = _scheduler_or_503()
|
||||
await sched.start()
|
||||
return {"status": sched.status().as_dict()}
|
||||
|
||||
|
||||
@router.post("/api/admin/scheduler/stop")
|
||||
async def scheduler_stop() -> Any:
|
||||
"""Stop polling. Waits up to 30s for the current tick to finish."""
|
||||
sched = _scheduler_or_503()
|
||||
await sched.stop()
|
||||
return {"status": sched.status().as_dict()}
|
||||
|
||||
|
||||
@router.post("/api/admin/scheduler/tick")
|
||||
async def scheduler_tick() -> Any:
|
||||
"""Run a single poll cycle synchronously and return the result.
|
||||
|
||||
Useful for: forcing a poll without waiting for the next interval;
|
||||
verifying SFTP connectivity; running a one-shot import from the
|
||||
CLI (``curl -X POST .../api/admin/scheduler/tick``).
|
||||
"""
|
||||
sched = _scheduler_or_503()
|
||||
result = await sched.tick()
|
||||
return {"ok": True, "tick": result.as_dict()}
|
||||
|
||||
|
||||
@router.post("/api/admin/scheduler/pull-inbound")
|
||||
async def scheduler_pull_inbound(
|
||||
date: str = Query(
|
||||
..., pattern=r"^\d{8}$",
|
||||
description="Date filter as YYYYMMDD; only filenames whose 8-digit "
|
||||
"timestamp (the 9th positional group in the inbound "
|
||||
"filename) matches are downloaded and processed.",
|
||||
),
|
||||
file_types: str | None = Query(
|
||||
default=None,
|
||||
description="Optional comma-separated whitelist of file_types "
|
||||
"(999, TA1, 277, 277CA, 835). Defaults to 999+TA1.",
|
||||
),
|
||||
limit: int = Query(default=2000, ge=1, le=10000),
|
||||
) -> Any:
|
||||
"""Targeted pull: list, filter to a date, download, and process.
|
||||
|
||||
Bypasses the alphabetical full-listing pass. Workflow:
|
||||
1. ``SftpClient.list_inbound_names()`` — sub-second metadata-only
|
||||
listing of the inbound MFT dir (skips ``*_warn.txt``).
|
||||
2. Client-side filter: keep files whose 8-digit timestamp
|
||||
substring equals ``date`` and whose ``file_type`` is in the
|
||||
allowlist.
|
||||
3. ``SftpClient.download_inbound(f)`` for each — fetches bytes
|
||||
into the local cache.
|
||||
4. ``Scheduler.process_inbound_files(files)`` — runs the same
|
||||
per-file pipeline as a regular tick (already-processed files
|
||||
are deduped via ``processed_inbound_files``).
|
||||
|
||||
Use this for the daily "process today's 999s" workflow without
|
||||
paying the cost of downloading the full inbound set.
|
||||
|
||||
Returns ``{"ok": True, "summary": {...}}`` with
|
||||
``listed / matched / downloaded / processed / skipped / errored``
|
||||
counters and the date / file_type filters applied.
|
||||
"""
|
||||
|
||||
from cyclone.clearhouse import SftpClient
|
||||
from cyclone.edi.filenames import (
|
||||
ALLOWED_FILE_TYPES,
|
||||
parse_inbound_filename,
|
||||
)
|
||||
from cyclone.providers import SftpBlock
|
||||
|
||||
sched = _scheduler_or_503()
|
||||
block: SftpBlock = sched._sftp_block # noqa: SLF001 — internal but stable
|
||||
client = SftpClient(block)
|
||||
|
||||
if file_types:
|
||||
wanted = {t.strip().upper() for t in file_types.split(",") if t.strip()}
|
||||
unknown = wanted - ALLOWED_FILE_TYPES
|
||||
if unknown:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"file_types {sorted(unknown)!r} not in "
|
||||
f"{sorted(ALLOWED_FILE_TYPES)}",
|
||||
)
|
||||
else:
|
||||
wanted = {"999", "TA1"} # daily default — what the operator needs
|
||||
|
||||
started = time.monotonic()
|
||||
try:
|
||||
# Single SFTP listdir — fast, no download.
|
||||
all_files = await asyncio.to_thread(client.list_inbound_names)
|
||||
except Exception as exc:
|
||||
log.exception("SFTP list_inbound_names failed")
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"SFTP list failed: {type(exc).__name__}: {exc}",
|
||||
) from exc
|
||||
|
||||
listed = len(all_files)
|
||||
matched: list[InboundFile] = []
|
||||
for f in all_files:
|
||||
if f.name.find(date) == -1:
|
||||
continue
|
||||
try:
|
||||
parsed = parse_inbound_filename(f.name)
|
||||
except ValueError:
|
||||
continue
|
||||
if parsed.file_type not in wanted:
|
||||
continue
|
||||
matched.append(f)
|
||||
if len(matched) >= limit:
|
||||
break
|
||||
|
||||
# Download in parallel-ish via to_thread (SftpClient serializes per
|
||||
# connection; the overhead is dominated by the SFTP round trip).
|
||||
downloaded = 0
|
||||
download_errors: list[str] = []
|
||||
for f in matched:
|
||||
try:
|
||||
await asyncio.to_thread(client.download_inbound, f)
|
||||
downloaded += 1
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("Failed to download %s: %s", f.name, exc)
|
||||
download_errors.append(f"{f.name}: {type(exc).__name__}: {exc}")
|
||||
|
||||
# Hand off to the scheduler pipeline (idempotent; dedupes via
|
||||
# processed_inbound_files).
|
||||
tick = await sched.process_inbound_files(matched)
|
||||
duration = round(time.monotonic() - started, 3)
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": {
|
||||
"date": date,
|
||||
"file_types": sorted(wanted),
|
||||
"limit": limit,
|
||||
"listed": listed,
|
||||
"matched": len(matched),
|
||||
"downloaded": downloaded,
|
||||
"download_errors": download_errors,
|
||||
"processed": tick.files_processed,
|
||||
"skipped": tick.files_skipped,
|
||||
"errored": tick.files_errored,
|
||||
"duration_s": duration,
|
||||
},
|
||||
"tick": tick.as_dict(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/admin/scheduler/status")
|
||||
def scheduler_status() -> Any:
|
||||
"""Return the scheduler's runtime snapshot (running, counters, last tick)."""
|
||||
sched = _scheduler_or_503()
|
||||
return sched.status().as_dict()
|
||||
|
||||
|
||||
@router.get("/api/admin/scheduler/processed-files")
|
||||
def scheduler_processed_files(
|
||||
limit: int = Query(default=100, ge=1, le=1000),
|
||||
status: str | None = Query(default=None),
|
||||
) -> Any:
|
||||
"""List rows from ``processed_inbound_files``, newest first.
|
||||
|
||||
The operator's "what did the scheduler do?" view. Filters by
|
||||
``status`` (``ok`` / ``error`` / ``skipped`` / ``pending``).
|
||||
Returns ``{"count": N, "files": [...]}`` where ``files[i]``
|
||||
matches the ORM row as a JSON dict.
|
||||
"""
|
||||
from cyclone.db import ProcessedInboundFile
|
||||
from cyclone.scheduler import STATUS_OK, STATUS_ERROR, STATUS_SKIPPED, STATUS_PENDING
|
||||
|
||||
valid_statuses = {STATUS_OK, STATUS_ERROR, STATUS_SKIPPED, STATUS_PENDING}
|
||||
if status is not None and status not in valid_statuses:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"status must be one of {sorted(valid_statuses)}",
|
||||
)
|
||||
|
||||
with db.SessionLocal()() as s:
|
||||
q = s.query(db.ProcessedInboundFile)
|
||||
if status is not None:
|
||||
q = q.filter(db.ProcessedInboundFile.status == status)
|
||||
rows = q.order_by(db.ProcessedInboundFile.id.desc()).limit(limit).all()
|
||||
return {
|
||||
"count": len(rows),
|
||||
"files": [
|
||||
{
|
||||
"id": r.id,
|
||||
"sftp_block_name": r.sftp_block_name,
|
||||
"name": r.name,
|
||||
"size": r.size,
|
||||
"modified_at": r.modified_at.isoformat() if r.modified_at else None,
|
||||
"file_type": r.file_type,
|
||||
"processed_at": r.processed_at.isoformat() if r.processed_at else None,
|
||||
"parser_used": r.parser_used,
|
||||
"claim_count": r.claim_count,
|
||||
"status": r.status,
|
||||
"error_message": r.error_message,
|
||||
}
|
||||
for r in rows
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/admin/reload-config")
|
||||
def reload_config():
|
||||
"""Re-read ``config/payers.yaml`` and revalidate. Returns counts."""
|
||||
from cyclone import payers as payer_loader
|
||||
try:
|
||||
configs = payer_loader.load_payer_configs()
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
return {"ok": True, "loaded": len(configs), "errors": []}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP40: POST /api/admin/validate-837
|
||||
#
|
||||
# Admin-only Edifabric validation probe: upload an 837P file via
|
||||
# multipart, hit /v2/x12/read → /v2/x12/validate, return the raw
|
||||
# OperationResult JSON (Status, Details, LastIndex). Mirrors the
|
||||
# `cyclone validate-837 <file>` CLI behavior — the wire shape comes
|
||||
# straight through so callers don't have to translate between the two.
|
||||
#
|
||||
# HTTP status codes:
|
||||
# 200 — OperationResult returned (Status may be success / warning /
|
||||
# error; the caller decides whether the file is acceptable).
|
||||
# 400 — uploaded file is empty / undecodable (defense-in-depth, same
|
||||
# shape as /api/parse-837).
|
||||
# 502 — Edifabric upstream 4xx/5xx — caller can surface the body.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/api/admin/validate-837")
|
||||
async def validate_837_endpoint(
|
||||
file: UploadFile = File(...),
|
||||
) -> Any:
|
||||
"""Validate an uploaded 837P file via Edifabric /v2/x12/validate.
|
||||
|
||||
Multipart upload (``file=...``); the file is read into bytes and
|
||||
passed to :func:`cyclone.edifabric.validate_edi`. The API key is
|
||||
resolved server-side from ``cyclone.secrets.get_secret('edifabric.api_key')``
|
||||
so the key never leaves the backend.
|
||||
"""
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Empty file", "detail": "Uploaded file contained no bytes."},
|
||||
)
|
||||
try:
|
||||
result = edifabric.validate_edi(raw)
|
||||
except edifabric.EdifabricError as exc:
|
||||
# status_code=0 is a client-side config problem (missing API
|
||||
# key); 4xx/5xx upstream become 502. Surface the Edifabric body
|
||||
# verbatim so the operator can see what went wrong.
|
||||
http_status = 502 if exc.status_code else 503
|
||||
return JSONResponse(
|
||||
status_code=http_status,
|
||||
content={
|
||||
"error": "Edifabric validation failed",
|
||||
"upstream_status": exc.status_code,
|
||||
"upstream_body": exc.body,
|
||||
},
|
||||
)
|
||||
return JSONResponse(content=result)
|
||||
|
||||
@@ -0,0 +1,515 @@
|
||||
"""``/api/batches*`` — read views + ZIP export over the parsed-batch population.
|
||||
|
||||
Three endpoints, all gated by ``matrix_gate``:
|
||||
|
||||
- ``POST /api/batches/{batch_id}/export-837`` — download a ZIP of
|
||||
regenerated X12 837 files for the requested claim ids. Per-claim
|
||||
payer config + clearhouse identity drive the submitter/receiver
|
||||
blocks; per-claim millisecond offset on the filename keeps every
|
||||
file in the bundle unique (HCPF requires this). Read-only — does
|
||||
NOT mutate Claim state (compare with
|
||||
``/api/inbox/rejected/resubmit?download=true`` which DOES flip
|
||||
``REJECTED → SUBMITTED``).
|
||||
- ``GET /api/batches`` — summary list,
|
||||
newest-first, capped at ``limit``. Each row includes ``claimIds``
|
||||
(837P only, so the Upload page can render a one-click Re-export
|
||||
button per row without a round-trip to ``/api/batches/{id}``).
|
||||
SP30: also returns billing-outcome fields
|
||||
(``acceptedCount`` / ``rejectedCount`` / ``pendingCount`` /
|
||||
``billedTotal`` / ``topRejectionReason`` / ``hasProblem``) so the
|
||||
Dashboard "Recent batches" widget can render one row per batch
|
||||
without an N+1 fetch.
|
||||
- ``GET /api/batches/{batch_id}`` — full batch record
|
||||
(parsed envelope + claims). 404 when unknown.
|
||||
|
||||
Three single-router helpers stay in this file (per spec D4):
|
||||
|
||||
- :func:`_batch_summary_claim_count` — claim count (837P or 835).
|
||||
- :func:`_batch_summary_claim_ids` — per-claim ids (837P only).
|
||||
- :func:`_batch_summary_billing_outcomes` — per-batch GROUP BY state
|
||||
aggregate plus the most-recent rejection reason.
|
||||
|
||||
Inline imports inside handlers (preserved verbatim per spec D5):
|
||||
``zipfile``, ``datetime``, ``ZoneInfo``, ``build_outbound_filename``,
|
||||
``PayerConfigORM``, ``func`` (sqlalchemy).
|
||||
|
||||
SP36 Task 14: this block moved here from ``api.py:1280`` (the 3
|
||||
``/api/batches*`` routes + 3 single-router helpers + the SP30
|
||||
state-bucket tuples and the ``# 277CA STC category`` note).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.api_helpers import (
|
||||
ndjson_stream_list as _ndjson_stream_list,
|
||||
wants_ndjson as _wants_ndjson,
|
||||
)
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.db import Batch, Claim, ClaimState
|
||||
from cyclone.parsers.models import ClaimOutput
|
||||
from cyclone.parsers.serialize_837 import SerializeError as SerializeError837
|
||||
from cyclone.parsers.serialize_837 import serialize_837_for_resubmit
|
||||
from cyclone.store import BatchRecord, store
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.post("/api/batches/{batch_id}/export-837")
|
||||
def export_batch_837(request: Request, batch_id: str, body: dict):
|
||||
"""Download a ZIP of regenerated X12 837 files for the requested claim_ids.
|
||||
|
||||
Body shape: ``{"claim_ids": [str, ...]}``.
|
||||
|
||||
Each successfully serialized claim becomes an entry in the ZIP named
|
||||
per the HCPF X12 File Naming Standards:
|
||||
``tp{tpid}-837P-{yyyymmddhhmmssSSS}-1of1.x12`` (with a per-claim
|
||||
millisecond offset so every file in the bundle has a unique name).
|
||||
The ``serialize_837_for_resubmit`` serializer is used so every file
|
||||
gets a unique interchange / group control number — back-to-back
|
||||
exports of the same set must produce different envelopes (required
|
||||
by X12).
|
||||
|
||||
The submitter block (Loop 1000A — NM1*41 + PER) is populated from
|
||||
the clearhouse singleton (dzinesco's identity in the seeded config)
|
||||
and the receiver block (NM1*40) is populated from the per-payer
|
||||
config. Without this wiring, the serializer falls back to
|
||||
``CYCLONE`` / ``RECEIVER`` placeholders and HCPF rejects the file.
|
||||
|
||||
No DB state is mutated by this endpoint — it is read-only. Compare
|
||||
with ``/api/inbox/rejected/resubmit?download=true`` which ALSO flips
|
||||
``ClaimState.REJECTED → SUBMITTED``; the two endpoints are
|
||||
intentionally separate.
|
||||
|
||||
Responses:
|
||||
200 — ``application/zip`` with the .x12 entries. Per-claim failures
|
||||
are surfaced via the ``X-Cyclone-Serialize-Errors`` header
|
||||
(JSON-encoded array of ``{claim_id, reason}``).
|
||||
400 — ``claim_ids`` missing or empty.
|
||||
404 — ``batch_id`` unknown.
|
||||
422 — every claim failed to serialize; body is JSON listing all
|
||||
failures (``{"detail": {"serialize_errors": [...]}}``).
|
||||
"""
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from cyclone.edi.filenames import build_outbound_filename
|
||||
|
||||
ids = body.get("claim_ids") or []
|
||||
if not ids:
|
||||
raise HTTPException(400, "claim_ids required")
|
||||
|
||||
with db.SessionLocal()() as s:
|
||||
batch = s.get(Batch, batch_id)
|
||||
if batch is None:
|
||||
raise HTTPException(404, f"unknown batch: {batch_id}")
|
||||
|
||||
serialize_errors: list[dict] = []
|
||||
ordered_rows: list[tuple[str, "Claim"]] = []
|
||||
for cid in ids:
|
||||
c = s.get(Claim, cid)
|
||||
if c is None:
|
||||
serialize_errors.append({"claim_id": cid, "reason": "unknown claim_id"})
|
||||
continue
|
||||
ordered_rows.append((cid, c))
|
||||
|
||||
# Pull clearhouse identity (submitter). If unseeded, the serializer
|
||||
# falls back to placeholder defaults — degraded but not a hard error.
|
||||
ch = store.get_clearhouse()
|
||||
submitter_kwargs: dict = {}
|
||||
if ch is not None:
|
||||
submitter_kwargs = {
|
||||
"sender_id": ch.tpid,
|
||||
"submitter_name": ch.submitter_name,
|
||||
"submitter_contact_name": ch.submitter_contact_name,
|
||||
"submitter_contact_email": ch.submitter_contact_email,
|
||||
}
|
||||
# Submitter phone is not in the clearhouse config today, but if
|
||||
# it ever is, wire it here. Email is the canonical contact
|
||||
# channel for HCPF submissions per the SP9 spec.
|
||||
if getattr(ch, "submitter_contact_phone", None):
|
||||
submitter_kwargs["submitter_contact_phone"] = ch.submitter_contact_phone
|
||||
|
||||
# Resolve per-claim payer config so each file's receiver (NM1*40)
|
||||
# and SBR09 are correct. Cache so we don't re-query the same payer.
|
||||
from cyclone.db import PayerConfigORM as _PayerConfigORM
|
||||
_payer_cache: dict[str, dict | None] = {}
|
||||
|
||||
def _resolve_payer_cfg(claim_obj: ClaimOutput) -> dict | None:
|
||||
pid = (claim_obj.payer.id or "").strip() if claim_obj.payer else ""
|
||||
pname = (claim_obj.payer.name or "").strip() if claim_obj.payer else ""
|
||||
cache_key = pid or pname
|
||||
if cache_key in _payer_cache:
|
||||
return _payer_cache[cache_key]
|
||||
cfg: dict | None = None
|
||||
with db.SessionLocal()() as ss:
|
||||
# 1. Exact match on (payer_id, "837P")
|
||||
if pid:
|
||||
row = ss.get(_PayerConfigORM, (pid, "837P"))
|
||||
if row is not None:
|
||||
cfg = dict(row.config_json)
|
||||
# 2. Fallback: any row whose payer_id matches the parsed payer.name
|
||||
# (HCPF files emit "SKCO0" in NM109 but the canonical
|
||||
# payer_id in the DB is "CO_TXIX" — name-matching is the
|
||||
# pragmatic lookup for that case).
|
||||
if cfg is None and pname:
|
||||
row = (
|
||||
ss.query(_PayerConfigORM)
|
||||
.filter(_PayerConfigORM.transaction_type == "837P")
|
||||
.all()
|
||||
)
|
||||
for r in row:
|
||||
cj = dict(r.config_json)
|
||||
if cj.get("submitter_name") and pname.lower() in str(cj).lower():
|
||||
cfg = cj
|
||||
break
|
||||
if (r.payer_id or "").upper() == pname.upper():
|
||||
cfg = cj
|
||||
break
|
||||
# 3. Last resort: first 837P row in the table.
|
||||
if cfg is None:
|
||||
row = (
|
||||
ss.query(_PayerConfigORM)
|
||||
.filter(_PayerConfigORM.transaction_type == "837P")
|
||||
.first()
|
||||
)
|
||||
if row is not None:
|
||||
cfg = dict(row.config_json)
|
||||
_payer_cache[cache_key] = cfg
|
||||
return cfg
|
||||
|
||||
# Build per-claim kwargs (receiver + SBR09) lazily. Receiver
|
||||
# defaults to the parsed payer name/ID if no config row matches.
|
||||
def _serialize_kwargs(claim_obj: ClaimOutput) -> dict:
|
||||
payer_cfg = _resolve_payer_cfg(claim_obj) or {}
|
||||
receiver_id = (
|
||||
payer_cfg.get("receiver_id")
|
||||
or (claim_obj.payer.id if claim_obj.payer else None)
|
||||
or "RECEIVER"
|
||||
)
|
||||
receiver_name = (
|
||||
payer_cfg.get("receiver_name")
|
||||
or (claim_obj.payer.name if claim_obj.payer else None)
|
||||
or receiver_id
|
||||
)
|
||||
sbr09 = payer_cfg.get("sbr09_default") or "MC"
|
||||
return {
|
||||
"receiver_id": receiver_id,
|
||||
"receiver_name": receiver_name,
|
||||
"claim_filing_indicator_code": sbr09,
|
||||
}
|
||||
|
||||
# Base MT timestamp for HCPF filenames. We add a per-claim
|
||||
# millisecond offset so each file in the ZIP has a unique 17-digit
|
||||
# ts (HCPF requires that; the spec also enforces "1of1" for the
|
||||
# sequence element).
|
||||
base_ts = datetime.now(ZoneInfo("America/Denver"))
|
||||
|
||||
def _per_claim_filename(idx: int, cid: str) -> str:
|
||||
if ch is None:
|
||||
# No clearhouse — fall back to a per-claim friendly name.
|
||||
return f"claim-{cid}.x12"
|
||||
# Millisecond offset, with second/minute rollover.
|
||||
offset_ms = (idx - 1) * 1 # 1 ms per claim is enough within an export
|
||||
ts_mt = base_ts.fromtimestamp(
|
||||
base_ts.timestamp() + offset_ms / 1000.0, tz=ZoneInfo("America/Denver")
|
||||
)
|
||||
return build_outbound_filename(ch.tpid, "837P", now_mt=ts_mt)
|
||||
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
for idx, (cid, c) in enumerate(ordered_rows, start=1):
|
||||
if not c.raw_json:
|
||||
serialize_errors.append({"claim_id": cid, "reason": "no raw_json"})
|
||||
continue
|
||||
try:
|
||||
claim_obj = ClaimOutput.model_validate(c.raw_json)
|
||||
except Exception as exc:
|
||||
serialize_errors.append(
|
||||
{"claim_id": cid, "reason": f"raw_json invalid: {exc}"}
|
||||
)
|
||||
continue
|
||||
try:
|
||||
kwargs = {**submitter_kwargs, **_serialize_kwargs(claim_obj)}
|
||||
text = serialize_837_for_resubmit(
|
||||
claim_obj, interchange_index=idx, **kwargs
|
||||
)
|
||||
except SerializeError837 as exc:
|
||||
serialize_errors.append({"claim_id": cid, "reason": str(exc)})
|
||||
continue
|
||||
zf.writestr(_per_claim_filename(idx, cid), text)
|
||||
|
||||
success_count = len(ids) - len(serialize_errors)
|
||||
if serialize_errors and success_count == 0:
|
||||
# Every claim failed — surface the failure list in the body so the
|
||||
# UI can render a useful error toast (the response is not a ZIP).
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={"serialize_errors": serialize_errors},
|
||||
)
|
||||
|
||||
buf.seek(0)
|
||||
headers = {
|
||||
"Content-Disposition": (
|
||||
f'attachment; filename="batch-{batch_id}-{success_count}-claims.zip"'
|
||||
),
|
||||
}
|
||||
if serialize_errors:
|
||||
headers["X-Cyclone-Serialize-Errors"] = json.dumps(serialize_errors)
|
||||
return Response(
|
||||
content=buf.getvalue(),
|
||||
media_type="application/zip",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
def _batch_summary_claim_count(rec: BatchRecord) -> int:
|
||||
"""Return the number of claims on a batch, handling both 837P and 835."""
|
||||
if rec.kind == "837p":
|
||||
return len(rec.result.claims) # type: ignore[attr-defined]
|
||||
if rec.kind == "835":
|
||||
return len(rec.result.claims) # type: ignore[attr-defined]
|
||||
return 0
|
||||
|
||||
|
||||
def _batch_summary_claim_ids(rec: BatchRecord) -> list[str]:
|
||||
"""Return per-claim ids for an 837P batch, or ``[]`` otherwise.
|
||||
|
||||
The Upload page's History tab renders a one-click Re-export ZIP
|
||||
button per row; that button calls
|
||||
``POST /api/batches/{id}/export-837`` with the row's claim ids.
|
||||
Carrying them in the list response avoids an extra round-trip
|
||||
to ``/api/batches/{id}`` for every row. 835 has no re-export
|
||||
endpoint, so the list is empty for those — the UI uses the
|
||||
empty list as the signal to hide the button.
|
||||
"""
|
||||
if rec.kind != "837p":
|
||||
return []
|
||||
return [
|
||||
c.claim_id
|
||||
for c in rec.result.claims # type: ignore[attr-defined]
|
||||
if getattr(c, "claim_id", None)
|
||||
]
|
||||
|
||||
|
||||
# SP30: state buckets the Dashboard widget (and any future "how the
|
||||
# last batch billed" surface) reads at a glance. Keep these in sync
|
||||
# with ClaimState — adding a new state here is a deliberate decision
|
||||
# the operator needs to see, not a coincidence.
|
||||
_BATCH_SUMMARY_ACCEPTED_STATES: tuple[ClaimState, ...] = (
|
||||
ClaimState.PAID,
|
||||
ClaimState.RECEIVED,
|
||||
ClaimState.RECONCILED,
|
||||
ClaimState.PARTIAL,
|
||||
)
|
||||
_BATCH_SUMMARY_REJECTED_STATES: tuple[ClaimState, ...] = (
|
||||
ClaimState.REJECTED,
|
||||
ClaimState.DENIED,
|
||||
ClaimState.REVERSED,
|
||||
)
|
||||
_BATCH_SUMMARY_PENDING_STATES: tuple[ClaimState, ...] = (
|
||||
ClaimState.SUBMITTED,
|
||||
)
|
||||
# 277CA STC category A4/A6/A7 — payer-side rejections that may not
|
||||
# yet have flipped Claim.state (the operator hasn't acknowledged).
|
||||
# The Dashboard widget treats these as problems too, mirroring the
|
||||
# Inbox `rejected + payer_rejected` aggregation.
|
||||
_BATCH_SUMMARY_PAYER_REJECT_CODES: tuple[str, ...] = ("A4", "A6", "A7")
|
||||
|
||||
|
||||
def _batch_summary_billing_outcomes(
|
||||
records: list[BatchRecord],
|
||||
) -> dict[str, dict]:
|
||||
"""Compute per-batch billing outcome for the Dashboard widget.
|
||||
|
||||
Returns ``{batch_id: {accepted, rejected, pending, billed,
|
||||
top_rejection_reason, has_problem}}`` for every batch in
|
||||
``records``. Empty input → empty dict.
|
||||
|
||||
Two SQL queries, both bounded by the supplied batch ids:
|
||||
|
||||
1. One GROUP BY ``(batch_id, state)`` aggregate that produces
|
||||
the accepted/rejected/pending counts and the sum of
|
||||
``charge_amount`` (the billed total). Single pass — no N+1.
|
||||
2. One ordered scan over the rejected + payer-rejected subset
|
||||
to pick the most recent rejection reason (truncated to 60
|
||||
chars). Skipped when the first query found no rejections
|
||||
and no payer-rejects, so the happy path stays at one query.
|
||||
|
||||
835 batches have no Claim rows — the GROUP BY returns no
|
||||
rows for them, so the dict entry for an 835 batch is
|
||||
``{accepted:0, rejected:0, pending:0, billed:0.0,
|
||||
top_rejection_reason:None, has_problem:False}`` (filled by
|
||||
the caller's ``.get(id, defaults)`` pattern).
|
||||
"""
|
||||
if not records:
|
||||
return {}
|
||||
from sqlalchemy import func # local import to keep top-of-file light
|
||||
|
||||
batch_ids = [r.id for r in records]
|
||||
outcome: dict[str, dict] = {
|
||||
bid: {
|
||||
"accepted": 0,
|
||||
"rejected": 0,
|
||||
"pending": 0,
|
||||
"billed": 0.0,
|
||||
"top_rejection_reason": None,
|
||||
"has_problem": False,
|
||||
}
|
||||
for bid in batch_ids
|
||||
}
|
||||
|
||||
with db.SessionLocal()() as s:
|
||||
# ---- 1. GROUP BY (batch_id, state) for counts + billed total ----
|
||||
rows = (
|
||||
s.query(
|
||||
Claim.batch_id,
|
||||
Claim.state,
|
||||
func.count(Claim.id),
|
||||
func.coalesce(func.sum(Claim.charge_amount), 0),
|
||||
)
|
||||
.filter(Claim.batch_id.in_(batch_ids))
|
||||
.group_by(Claim.batch_id, Claim.state)
|
||||
.all()
|
||||
)
|
||||
any_rejection_or_payer = False
|
||||
for batch_id, state, count, billed in rows:
|
||||
slot = outcome.get(batch_id)
|
||||
if slot is None:
|
||||
continue # batch has no row in our pre-allocated dict
|
||||
count = int(count or 0)
|
||||
billed_f = float(billed or 0)
|
||||
slot["billed"] += billed_f
|
||||
if state in _BATCH_SUMMARY_ACCEPTED_STATES:
|
||||
slot["accepted"] += count
|
||||
elif state in _BATCH_SUMMARY_REJECTED_STATES:
|
||||
slot["rejected"] += count
|
||||
any_rejection_or_payer = True
|
||||
elif state in _BATCH_SUMMARY_PENDING_STATES:
|
||||
slot["pending"] += count
|
||||
# everything else (DRAFT, etc.) is excluded from the widget.
|
||||
|
||||
# ---- 2. Most-recent rejection reason + payer-reject probe ----
|
||||
# Only run when we know there IS at least one rejection OR a
|
||||
# payer-reject claim somewhere in the batch set; otherwise
|
||||
# the first query alone is enough.
|
||||
if any_rejection_or_payer:
|
||||
rej_rows = (
|
||||
s.query(
|
||||
Claim.batch_id,
|
||||
Claim.rejection_reason,
|
||||
Claim.payer_rejected_status_code,
|
||||
)
|
||||
.filter(
|
||||
Claim.batch_id.in_(batch_ids),
|
||||
Claim.state.in_(_BATCH_SUMMARY_REJECTED_STATES)
|
||||
| Claim.payer_rejected_status_code.in_(
|
||||
_BATCH_SUMMARY_PAYER_REJECT_CODES
|
||||
),
|
||||
)
|
||||
.order_by(Claim.rejected_at.desc().nullslast())
|
||||
.all()
|
||||
)
|
||||
seen_reason: set[str] = set()
|
||||
for batch_id, reason, payer_code in rej_rows:
|
||||
slot = outcome.get(batch_id)
|
||||
if slot is None:
|
||||
continue
|
||||
if payer_code in _BATCH_SUMMARY_PAYER_REJECT_CODES:
|
||||
slot["has_problem"] = True
|
||||
# Capture the first non-null reason for this batch
|
||||
# (rej_rows is ordered newest-first, so the first
|
||||
# non-null wins). Truncate to 60 chars + ellipsis.
|
||||
if (
|
||||
slot["top_rejection_reason"] is None
|
||||
and reason
|
||||
and batch_id not in seen_reason
|
||||
):
|
||||
r = reason.strip()
|
||||
if len(r) > 60:
|
||||
r = r[:60] + "…"
|
||||
slot["top_rejection_reason"] = r
|
||||
seen_reason.add(batch_id)
|
||||
if (
|
||||
slot["rejected"] > 0
|
||||
or payer_code in _BATCH_SUMMARY_PAYER_REJECT_CODES
|
||||
):
|
||||
slot["has_problem"] = True
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
@router.get("/api/batches")
|
||||
def list_batches(
|
||||
request: Request,
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
) -> Any:
|
||||
"""Summary of all parsed batches, newest first.
|
||||
|
||||
Each item includes ``claimIds`` (837P only) so the History tab
|
||||
on the Upload page can render a one-click re-export button per
|
||||
row without an extra round-trip to ``/api/batches/{id}``. The
|
||||
list is still capped at ``limit`` claims; see the full result
|
||||
via the by-id endpoint when more is needed.
|
||||
|
||||
SP30: also returns billing-outcome fields
|
||||
(``acceptedCount`` / ``rejectedCount`` / ``pendingCount`` /
|
||||
``billedTotal`` / ``topRejectionReason`` / ``hasProblem``) so
|
||||
the Dashboard "Recent batches" widget can render one row per
|
||||
batch without an N+1 fetch. See
|
||||
:func:`_batch_summary_billing_outcomes`.
|
||||
"""
|
||||
records = store.list(limit=limit)
|
||||
outcomes = _batch_summary_billing_outcomes(records)
|
||||
items = [
|
||||
{
|
||||
"id": r.id,
|
||||
"kind": r.kind,
|
||||
"inputFilename": r.input_filename,
|
||||
"parsedAt": r.parsed_at.isoformat().replace("+00:00", "Z"),
|
||||
"claimCount": _batch_summary_claim_count(r),
|
||||
"claimIds": _batch_summary_claim_ids(r),
|
||||
"acceptedCount": outcomes.get(r.id, {}).get("accepted", 0),
|
||||
"rejectedCount": outcomes.get(r.id, {}).get("rejected", 0),
|
||||
"pendingCount": outcomes.get(r.id, {}).get("pending", 0),
|
||||
"billedTotal": round(outcomes.get(r.id, {}).get("billed", 0.0), 2),
|
||||
"topRejectionReason": outcomes.get(r.id, {}).get(
|
||||
"top_rejection_reason"
|
||||
),
|
||||
"hasProblem": outcomes.get(r.id, {}).get("has_problem", False),
|
||||
}
|
||||
for r in records
|
||||
]
|
||||
all_records = store.all()
|
||||
total = len(all_records)
|
||||
returned = len(items)
|
||||
has_more = total > returned
|
||||
if _wants_ndjson(request):
|
||||
return StreamingResponse(
|
||||
_ndjson_stream_list(items, total, returned, has_more),
|
||||
media_type="application/x-ndjson",
|
||||
)
|
||||
return {
|
||||
"items": items,
|
||||
"total": total,
|
||||
"returned": returned,
|
||||
"has_more": has_more,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/batches/{batch_id}")
|
||||
def get_batch(batch_id: str) -> Any:
|
||||
rec = store.get(batch_id)
|
||||
if rec is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"error": "Not found", "detail": f"Batch {batch_id} not found"},
|
||||
)
|
||||
return json.loads(rec.result.model_dump_json())
|
||||
@@ -0,0 +1,307 @@
|
||||
"""``/api/claim-acks`` (and per-claim/per-ack surfaces) — SP28.
|
||||
|
||||
Seven endpoints that surface the ``claim_acks`` join table to the
|
||||
frontend + manual-match fallback for orphans. Mounted by
|
||||
``cyclone.api`` alongside the existing ``/api/acks`` and
|
||||
``/api/ta1-acks`` routers.
|
||||
|
||||
The live-tail endpoints (``/api/claims/{id}/acks/stream`` and
|
||||
``/api/acks/{kind}/{id}/claims/stream``) subscribe to the
|
||||
``claim_ack_written`` bus event so the ClaimDrawer Acknowledgments
|
||||
panel and the per-ack claims list refresh in real time.
|
||||
|
||||
Manual match is any-logged-in user (D5) — this endpoint mutates
|
||||
metadata only (``claim_acks`` row + live-tail event), no
|
||||
``Claim.state`` mutation, no payment data. Idempotent: re-calling
|
||||
with the same ``claim_id`` returns 200 with the existing row.
|
||||
Rejects (409) when the claim is in a terminal state (``REVERSED``).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, AsyncIterator, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.api_helpers import ndjson_line, tail_events
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.pubsub import EventBus
|
||||
from cyclone.store import store, to_ui_claim_ack
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
AckKind = Literal["999", "277ca", "ta1"]
|
||||
|
||||
|
||||
class MatchClaimBody(BaseModel):
|
||||
"""Body for ``POST /api/acks/{kind}/{ack_id}/match-claim``."""
|
||||
|
||||
claim_id: str = Field(..., min_length=1)
|
||||
set_control_number: str | None = None
|
||||
set_accept_reject_code: str | None = None
|
||||
ak2_index: int | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-claim surface
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/api/claims/{claim_id}/acks/stream")
|
||||
async def claim_acks_stream(
|
||||
request: Request,
|
||||
claim_id: str,
|
||||
) -> StreamingResponse:
|
||||
"""Stream ClaimAck rows for one claim as NDJSON.
|
||||
|
||||
Subscribes to ``claim_ack_written`` and filters for rows where
|
||||
``claim_id`` matches the path param. Each matching event is
|
||||
emitted as ``{"type": "item", "data": to_ui_claim_ack(...)}``;
|
||||
the client-side ``useMergedTail`` hook dedupes by id.
|
||||
|
||||
Registered BEFORE ``/api/claims/{claim_id}/acks`` so the literal
|
||||
``stream`` path segment doesn't get matched as a suffix.
|
||||
"""
|
||||
bus: EventBus = request.app.state.event_bus
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
rows = [
|
||||
r for r in store.list_acks_for_claim(claim_id)
|
||||
if r.claim_id is not None # filter out TA1 batch-level rows
|
||||
]
|
||||
for row in rows:
|
||||
yield ndjson_line({"type": "item", "data": to_ui_claim_ack(row)})
|
||||
yield ndjson_line({
|
||||
"type": "snapshot_end",
|
||||
"data": {"count": len(rows)},
|
||||
})
|
||||
|
||||
async for chunk in tail_events(request, bus, ["claim_ack_written"]):
|
||||
# tail_events yields full NDJSON lines; the client filter
|
||||
# picks claim_id matches. We forward every event so the
|
||||
# wire format mirrors /api/claims/stream.
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
@router.get("/api/claims/{claim_id}/acks")
|
||||
def list_acks_for_claim_endpoint(claim_id: str) -> Any:
|
||||
"""Return every ClaimAck row for one claim (per-claim only).
|
||||
|
||||
TA1 batch-level rows (where ``claim_id IS NULL``) are filtered
|
||||
out — those don't belong to a specific claim, they're a
|
||||
envelope-level acknowledgement that hangs off the originating
|
||||
837 batch. The ClaimDrawer Acknowledgments panel is
|
||||
per-claim only.
|
||||
"""
|
||||
rows = store.list_acks_for_claim(claim_id)
|
||||
items = [to_ui_claim_ack(r) for r in rows if r.claim_id is not None]
|
||||
return {"total": len(items), "items": items}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-ack surface
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/api/acks/{kind}/{ack_id}/claims/stream")
|
||||
async def ack_claims_stream(
|
||||
request: Request,
|
||||
kind: AckKind,
|
||||
ack_id: int,
|
||||
) -> StreamingResponse:
|
||||
"""Stream ClaimAck rows for one ack as NDJSON.
|
||||
|
||||
Subscribes to ``claim_ack_written`` and forwards events the
|
||||
store knows about. Clients filter by ack_id + ack_kind on the
|
||||
client side.
|
||||
"""
|
||||
bus: EventBus = request.app.state.event_bus
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
rows = store.list_claims_for_ack(kind, ack_id)
|
||||
for row in rows:
|
||||
yield ndjson_line({"type": "item", "data": to_ui_claim_ack(row)})
|
||||
yield ndjson_line({
|
||||
"type": "snapshot_end",
|
||||
"data": {"count": len(rows)},
|
||||
})
|
||||
async for chunk in tail_events(request, bus, ["claim_ack_written"]):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
@router.get("/api/acks/{kind}/{ack_id}/claims")
|
||||
def list_claims_for_ack_endpoint(kind: AckKind, ack_id: int) -> Any:
|
||||
"""Return every ClaimAck row for one ack (any kind).
|
||||
|
||||
For 999 / 277CA: returns 0..N rows (one per AK2 / ClaimStatus).
|
||||
For TA1: returns 0..1 row (envelope-level, populated batch_id).
|
||||
"""
|
||||
rows = store.list_claims_for_ack(kind, ack_id)
|
||||
items = [to_ui_claim_ack(r) for r in rows]
|
||||
return {"total": len(items), "items": items}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Manual match / unmatch (D5, D9)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/api/acks/{kind}/{ack_id}/match-claim")
|
||||
def manual_match_claim_endpoint(
|
||||
request: Request,
|
||||
kind: AckKind,
|
||||
ack_id: int,
|
||||
body: MatchClaimBody,
|
||||
) -> Any:
|
||||
"""Manual link fallback (D5/D9). Any-logged-in-user posture.
|
||||
|
||||
Inserts a ``claim_acks`` row with ``linked_by="manual"`` and
|
||||
publishes ``claim_ack_written`` so the drawers refresh. The
|
||||
endpoint is idempotent: if a row already exists for this dedup
|
||||
key, the existing row is returned (200). 409 when the claim is
|
||||
in a terminal state (``REVERSED``). 404 when the claim doesn't
|
||||
exist or the ack doesn't exist.
|
||||
"""
|
||||
# Verify the claim exists and is in a non-terminal state.
|
||||
from cyclone.db import ClaimState as _CS
|
||||
with db.SessionLocal()() as s:
|
||||
claim = s.get(db.Claim, body.claim_id)
|
||||
if claim is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={
|
||||
"error": "Not found",
|
||||
"detail": f"Claim {body.claim_id} not found",
|
||||
},
|
||||
)
|
||||
if claim.state == _CS.REVERSED:
|
||||
return JSONResponse(
|
||||
status_code=409,
|
||||
content={
|
||||
"error": "Conflict",
|
||||
"detail": (
|
||||
f"Claim {body.claim_id} is in terminal state "
|
||||
f"{claim.state.value} and cannot be linked."
|
||||
),
|
||||
},
|
||||
)
|
||||
# Verify the ack exists (any kind).
|
||||
ack_table = {
|
||||
"999": db.Ack,
|
||||
"277ca": db.Two77caAck,
|
||||
"ta1": db.Ta1Ack,
|
||||
}[kind]
|
||||
if s.get(ack_table, ack_id) is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={
|
||||
"error": "Not found",
|
||||
"detail": f"{kind} ACK {ack_id} not found",
|
||||
},
|
||||
)
|
||||
# Idempotency: if a manual or auto link already exists, return it.
|
||||
existing = (
|
||||
s.query(db.ClaimAck)
|
||||
.filter(
|
||||
db.ClaimAck.ack_kind == kind,
|
||||
db.ClaimAck.ack_id == ack_id,
|
||||
db.ClaimAck.claim_id == body.claim_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing is not None:
|
||||
return {
|
||||
"link": to_ui_claim_ack(existing),
|
||||
"created": False,
|
||||
}
|
||||
|
||||
# Insert via the store so the publish-from-store contract fires.
|
||||
row = store.add_claim_ack(
|
||||
claim_id=body.claim_id,
|
||||
batch_id=None,
|
||||
ack_id=ack_id,
|
||||
ack_kind=kind,
|
||||
ak2_index=body.ak2_index,
|
||||
set_control_number=body.set_control_number,
|
||||
set_accept_reject_code=body.set_accept_reject_code,
|
||||
linked_by="manual",
|
||||
event_bus=request.app.state.event_bus,
|
||||
)
|
||||
return {"link": to_ui_claim_ack(row), "created": True}
|
||||
|
||||
|
||||
@router.delete("/api/acks/{kind}/{ack_id}/match-claim/{claim_id}")
|
||||
def manual_unmatch_claim_endpoint(
|
||||
request: Request,
|
||||
kind: AckKind,
|
||||
ack_id: int,
|
||||
claim_id: str,
|
||||
) -> Any:
|
||||
"""Unlink (preserves ``Claim.state`` mutation; only removes the row).
|
||||
|
||||
404 when no link row exists for the dedup key. Publishes
|
||||
``claim_ack_dropped`` so live-tail subscribers remove the link.
|
||||
"""
|
||||
from cyclone import db as _db
|
||||
with _db.SessionLocal()() as s:
|
||||
link = (
|
||||
s.query(_db.ClaimAck)
|
||||
.filter(
|
||||
_db.ClaimAck.ack_kind == kind,
|
||||
_db.ClaimAck.ack_id == ack_id,
|
||||
_db.ClaimAck.claim_id == claim_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if link is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={
|
||||
"error": "Not found",
|
||||
"detail": (
|
||||
f"No link for {kind} ack {ack_id} → "
|
||||
f"claim {claim_id}"
|
||||
),
|
||||
},
|
||||
)
|
||||
link_id = link.id
|
||||
|
||||
removed = store.remove_claim_ack(link_id, event_bus=request.app.state.event_bus)
|
||||
return {"removed": removed, "link_id": link_id}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inbox ack-orphans lane (spec §D7)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/api/inbox/ack-orphans")
|
||||
def list_ack_orphans_endpoint(
|
||||
kind: AckKind | None = Query(None, description="Filter by ack kind"),
|
||||
) -> Any:
|
||||
"""List acks with no resolvable Claim row of their own kind.
|
||||
|
||||
Used by the Inbox "Ack orphans" lane for the operator's manual
|
||||
reconciliation flow. Filters by kind: ``999``, ``277ca``, ``ta1``.
|
||||
"""
|
||||
if kind is not None:
|
||||
items = store.find_ack_orphans(kind)
|
||||
return {"total": len(items), "items": items}
|
||||
items_999 = store.find_ack_orphans("999")
|
||||
items_277ca = store.find_ack_orphans("277ca")
|
||||
items_ta1 = store.find_ack_orphans("ta1")
|
||||
all_items = items_999 + items_277ca + items_ta1
|
||||
return {"total": len(all_items), "items": all_items}
|
||||
|
||||
|
||||
__all__ = ["router"]
|
||||
@@ -0,0 +1,539 @@
|
||||
"""``/api/claims*`` — Claims list / detail / streaming / serialize / line-reconciliation.
|
||||
|
||||
Five endpoints, all gated by ``matrix_gate``:
|
||||
|
||||
- ``GET /api/claims`` — paginated list
|
||||
with filter+sort, plus an NDJSON variant when the caller sends
|
||||
``Accept: application/x-ndjson``. SP27: counts the full filtered
|
||||
population, not a page-limited sample.
|
||||
- ``GET /api/claims/stream`` — NDJSON live-tail
|
||||
on ``claim_written``. Snapshot first (eager
|
||||
``store.iter_claims``), then ``tail_events`` subscribes + emits
|
||||
heartbeats. Registered before ``/api/claims/{claim_id}`` so the
|
||||
literal ``stream`` segment isn't captured as a claim id.
|
||||
- ``GET /api/claims/{claim_id}`` — full drawer
|
||||
context (SP4) with the SP28 ``ack_links`` block pre-attached.
|
||||
404 on missing id — never 500.
|
||||
- ``GET /api/claims/{claim_id}/serialize-837`` — regenerate X12
|
||||
837P from the stored ``raw_json`` payload. 404 unknown claim, 422
|
||||
no-``raw_json`` / unparseable / serializer failure.
|
||||
- ``GET /api/claims/{claim_id}/line-reconciliation`` — per-line 837
|
||||
vs 835 side-by-side with CAS adjustments and a summary block.
|
||||
|
||||
Three single-router helpers stay in this file (per spec D4):
|
||||
|
||||
- :func:`_compact_ack_links_for_claim` — slim form
|
||||
``{ack_id, ack_kind, set_accept_reject_code, …}`` for the drawer
|
||||
Acknowledgments panel.
|
||||
- :func:`_claim_line_dict` — project an 837 service-line
|
||||
dict from ``Claim.raw_json`` to wire shape.
|
||||
- :func:`_svc_to_dict` — project an ORM
|
||||
``ServiceLinePayment`` to wire shape.
|
||||
|
||||
Inline imports inside handlers (preserved verbatim per spec D5):
|
||||
``select`` (sqlalchemy), ``LineReconciliation``/``ServiceLinePayment``/
|
||||
``CasAdjustment`` (cyclone.db), ``json as _json``, ``Decimal``.
|
||||
|
||||
SP36 Task 15: this block moved here from ``api.py:1278`` (the 5
|
||||
``/api/claims*`` routes + 3 single-router helpers).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from decimal import Decimal
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.api_helpers import (
|
||||
ndjson_line as _ndjson_line,
|
||||
ndjson_stream_list as _ndjson_stream_list,
|
||||
tail_events as _tail_events,
|
||||
wants_ndjson as _wants_ndjson,
|
||||
)
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.db import Claim
|
||||
from cyclone.parsers.models import ClaimOutput
|
||||
from cyclone.parsers.serialize_837 import SerializeError as SerializeError837
|
||||
from cyclone.parsers.serialize_837 import serialize_837
|
||||
from cyclone.pubsub import EventBus
|
||||
from cyclone.store import store
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.get("/api/claims")
|
||||
def list_claims(
|
||||
request: Request,
|
||||
batch_id: str | None = Query(None),
|
||||
status: str | None = Query(None),
|
||||
provider_npi: str | None = Query(None),
|
||||
payer: str | None = Query(None),
|
||||
date_from: str | None = Query(None),
|
||||
date_to: str | None = Query(None),
|
||||
sort: str | None = Query(None),
|
||||
order: str = Query("desc"),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
) -> Any:
|
||||
common = dict(
|
||||
batch_id=batch_id,
|
||||
status=status,
|
||||
provider_npi=provider_npi,
|
||||
payer=payer,
|
||||
date_from=date_from,
|
||||
date_to=date_to,
|
||||
)
|
||||
items = list(store.iter_claims(
|
||||
sort=sort, order=order, limit=limit, offset=offset, **common,
|
||||
))
|
||||
# SP27 Task 13b: count the full population, not a 100-row sample.
|
||||
# `iter_claims` defaults to limit=100; counting its output silently
|
||||
# capped the reported total at 100 even when the DB held 60k rows.
|
||||
total = store.count_claims(**common)
|
||||
returned = len(items)
|
||||
has_more = total > offset + returned
|
||||
if _wants_ndjson(request):
|
||||
return StreamingResponse(
|
||||
_ndjson_stream_list(items, total, returned, has_more),
|
||||
media_type="application/x-ndjson",
|
||||
)
|
||||
return {
|
||||
"items": items,
|
||||
"total": total,
|
||||
"returned": returned,
|
||||
"has_more": has_more,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Live-tail NDJSON streaming endpoints (Phase 3 — SP5)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@router.get("/api/claims/stream")
|
||||
async def claims_stream(
|
||||
request: Request,
|
||||
status: str | None = Query(None),
|
||||
provider_npi: str | None = Query(None),
|
||||
payer: str | None = Query(None),
|
||||
date_from: str | None = Query(None),
|
||||
date_to: str | None = Query(None),
|
||||
sort: str | None = Query(None),
|
||||
order: str = Query("desc"),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
) -> StreamingResponse:
|
||||
"""Stream Claims as NDJSON: snapshot first, then live events.
|
||||
|
||||
Wire format:
|
||||
* ``{"type":"item","data":<claim>}`` per snapshot row, then per
|
||||
new ``claim_written`` event
|
||||
* ``{"type":"snapshot_end","data":{"count":N}}`` after the snapshot
|
||||
* ``{"type":"heartbeat","data":{"ts":<iso>}}`` every
|
||||
``CYCLONE_TAIL_HEARTBEAT_S`` seconds when idle
|
||||
|
||||
Query params mirror :func:`list_claims` so a frontend can swap a
|
||||
one-shot fetch for a tail with no URL surgery.
|
||||
|
||||
NOTE: registered before ``/api/claims/{claim_id}`` so the literal
|
||||
``stream`` path segment doesn't get matched as a claim id.
|
||||
"""
|
||||
bus: EventBus = request.app.state.event_bus
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
# 1. Snapshot (eager — iter_claims returns a list already).
|
||||
rows = store.iter_claims(
|
||||
status=status, provider_npi=provider_npi, payer=payer,
|
||||
date_from=date_from, date_to=date_to,
|
||||
sort=sort or "-submission_date", order=order, limit=limit,
|
||||
)
|
||||
for row in rows:
|
||||
yield _ndjson_line({"type": "item", "data": row})
|
||||
yield _ndjson_line({"type": "snapshot_end", "data": {"count": len(rows)}})
|
||||
|
||||
# 2. Subscribe + heartbeats.
|
||||
async for chunk in _tail_events(request, bus, ["claim_written"]):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
@router.get("/api/claims/{claim_id}")
|
||||
def get_claim_detail_endpoint(claim_id: str) -> dict:
|
||||
"""Return one claim with full drawer context (SP4).
|
||||
|
||||
Body shape is produced by :meth:`CycloneStore.get_claim_detail`:
|
||||
header, state, service lines, diagnoses, parties, validation,
|
||||
raw segments, ``stateHistory`` (most-recent-first, capped at 50),
|
||||
and a populated ``matchedRemittance`` block when paired.
|
||||
|
||||
SP28: response gains ``ack_links: list[dict]`` (compact form:
|
||||
``[{ack_id, ack_kind, set_accept_reject_code, parsed_at}]``)
|
||||
so the ``ClaimDrawer`` Acknowledgments panel can render on
|
||||
initial load. TA1 batch-level rows (``claim_id IS NULL``) are
|
||||
excluded — those don't belong to a specific claim.
|
||||
|
||||
Path param is ``claim_id`` (matches the SP3 ``/api/acks/{ack_id}``
|
||||
convention). Returns 404 — never 500 — on a missing claim so the
|
||||
UI can distinguish "doesn't exist" from a transient fetch error.
|
||||
"""
|
||||
body = store.get_claim_detail(claim_id)
|
||||
if body is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={
|
||||
"error": "Not found",
|
||||
"detail": f"Claim {claim_id} not found",
|
||||
},
|
||||
)
|
||||
# SP28: attach ack_links (compact form for the drawer panel).
|
||||
body["ack_links"] = _compact_ack_links_for_claim(claim_id)
|
||||
return body
|
||||
|
||||
|
||||
def _compact_ack_links_for_claim(claim_id: str) -> list[dict]:
|
||||
"""Return compact ack_links for one claim, newest first.
|
||||
|
||||
TA1 batch-level rows (claim_id IS NULL) are filtered out — those
|
||||
hang off the originating 837 batch, not a specific claim. The
|
||||
shape is the slimmer ``{ack_id, ack_kind,
|
||||
set_accept_reject_code, parsed_at, ak2_index}`` form so the
|
||||
ClaimDrawer can render without an N+1 round-trip per row.
|
||||
"""
|
||||
rows = store.list_acks_for_claim(claim_id)
|
||||
out: list[dict] = []
|
||||
for row in rows:
|
||||
if row.claim_id is None:
|
||||
continue
|
||||
out.append({
|
||||
"id": row.id,
|
||||
"ack_id": row.ack_id,
|
||||
"ack_kind": row.ack_kind,
|
||||
"ak2_index": row.ak2_index,
|
||||
"set_control_number": row.set_control_number,
|
||||
"set_accept_reject_code": row.set_accept_reject_code,
|
||||
"linked_at": (
|
||||
row.linked_at.isoformat().replace("+00:00", "Z")
|
||||
if row.linked_at is not None else ""
|
||||
),
|
||||
"linked_by": row.linked_by,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _serialize_kwargs_for_claim(claim_obj) -> dict:
|
||||
"""SP40: build the ``serialize_837`` kwargs from the live clearhouse
|
||||
+ per-payer ``PayerConfigORM`` config. Same as the bulk export in
|
||||
``batches.py:_serialize_kwargs`` so single-claim download mirrors
|
||||
production byte-faithfulness.
|
||||
|
||||
Without this, ``serialize_837(claim_obj)`` falls back to
|
||||
placeholder strings (CYCLONE/RECEIVER/CUSTOMER SERVICE/8005550100)
|
||||
which the operator rightly rejected as not production-ready.
|
||||
Returns an empty dict only when the clearhouse / payer config is
|
||||
not seeded — the serializer's own SP40 fallback then fires.
|
||||
"""
|
||||
out: dict = {}
|
||||
try:
|
||||
ch = store.get_clearhouse()
|
||||
except Exception:
|
||||
ch = None
|
||||
if ch is not None:
|
||||
out["sender_id"] = ch.tpid
|
||||
out["submitter_name"] = ch.submitter_name
|
||||
out["submitter_contact_name"] = ch.submitter_contact_name
|
||||
out["submitter_contact_email"] = ch.submitter_contact_email
|
||||
if getattr(ch, "submitter_contact_phone", None):
|
||||
out["submitter_contact_phone"] = ch.submitter_contact_phone
|
||||
|
||||
# Per-payer receiver + SBR-09 from the 837P PayerConfigORM row.
|
||||
pid = (getattr(claim_obj.payer, "id", "") or "").strip()
|
||||
if pid:
|
||||
try:
|
||||
from cyclone.db import PayerConfigORM, Payer as PayerORM
|
||||
with db.SessionLocal()() as ss:
|
||||
pcfg = ss.get(PayerConfigORM, (pid, "837P"))
|
||||
payer_row = ss.get(PayerORM, pid)
|
||||
if pcfg is not None and isinstance(pcfg.config_json, dict):
|
||||
cfg = pcfg.config_json
|
||||
if cfg.get("receiver_name"):
|
||||
out["receiver_name"] = cfg["receiver_name"]
|
||||
if cfg.get("receiver_id"):
|
||||
out["receiver_id"] = cfg["receiver_id"]
|
||||
if cfg.get("sbr09_default"):
|
||||
out["claim_filing_indicator_code"] = cfg["sbr09_default"]
|
||||
if payer_row is not None:
|
||||
if "receiver_name" not in out and payer_row.receiver_name:
|
||||
out["receiver_name"] = payer_row.receiver_name
|
||||
if "receiver_id" not in out and payer_row.receiver_id:
|
||||
out["receiver_id"] = payer_row.receiver_id
|
||||
except Exception:
|
||||
# Fall through; the serializer's own SP40 default
|
||||
# (MC for SBR-09) covers the most common case.
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
@router.get("/api/claims/{claim_id}/serialize-837")
|
||||
def serialize_claim_as_837(claim_id: str):
|
||||
"""Return the claim as a regenerated X12 837P file (SP8).
|
||||
|
||||
Loads the ClaimOutput from the persisted ``raw_json`` and runs the
|
||||
outbound serializer. Returns 404 if the claim doesn't exist, 422 if
|
||||
the stored payload has no parseable ClaimOutput (data integrity
|
||||
issue, not a transient failure).
|
||||
|
||||
SP40: threads the clearhouse submitter + CO_TXIX PayerConfig
|
||||
receiver kwargs through to ``serialize_837`` so the regenerated
|
||||
file has real ``NM1*41`` (Dzinesco / TPID ``11525703``),
|
||||
``PER*IC*Tyler Martinez*EM*tyler@dzinesco.com``, ``NM1*40``
|
||||
(HCPF) and ``SBR*P*18*******MC`` segments — not the
|
||||
``CYCLONE/RECEIVER/CUSTOMER SERVICE/8005550100`` placeholders the
|
||||
bare-args call site would emit. Mirrors the regen-script path in
|
||||
``dev/unbilled-july2026/scripts/regen_corrected_files.py``.
|
||||
"""
|
||||
with db.SessionLocal()() as s:
|
||||
row = s.get(Claim, claim_id)
|
||||
if row is None:
|
||||
return JSONResponse(
|
||||
{"error": "Not found", "detail": f"Claim {claim_id} not found"},
|
||||
status_code=404,
|
||||
)
|
||||
if not row.raw_json:
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": "Unprocessable",
|
||||
"detail": f"Claim {claim_id} has no raw_json; cannot serialize",
|
||||
},
|
||||
status_code=422,
|
||||
)
|
||||
try:
|
||||
claim_obj = ClaimOutput.model_validate(row.raw_json)
|
||||
except Exception as exc:
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": "Unprocessable",
|
||||
"detail": f"Claim {claim_id} raw_json is malformed: {exc}",
|
||||
},
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
# SP40: build the same kwargs the bulk export uses, so the
|
||||
# single-claim download mirrors production byte-faithfulness (real
|
||||
# submitter, real contact, real receiver, real SBR-09).
|
||||
serialize_kwargs = _serialize_kwargs_for_claim(claim_obj)
|
||||
|
||||
try:
|
||||
text = serialize_837(claim_obj, **serialize_kwargs)
|
||||
except SerializeError837 as exc:
|
||||
return JSONResponse(
|
||||
{"error": "Unprocessable", "detail": str(exc)},
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
return Response(
|
||||
content=text,
|
||||
media_type="text/x12",
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="claim-{claim_id}.x12"'
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/claims/{claim_id}/line-reconciliation")
|
||||
def get_claim_line_reconciliation(claim_id: str) -> dict:
|
||||
"""Per-line reconciliation view for the ClaimDrawer tab.
|
||||
|
||||
Spec §5.1. Returns the 837 service lines and 835 SVC composites
|
||||
side-by-side, with per-line CAS adjustments and a summary block.
|
||||
|
||||
Architecture note: 837 service lines live in ``Claim.raw_json``
|
||||
(not a separate ORM table), so the 837-side rows are read from the
|
||||
JSON blob; the 835-side rows come from ``ServiceLinePayment`` ORM.
|
||||
``LineReconciliation.claim_service_line_number`` stores the 1-based
|
||||
line number to join them.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
from cyclone.db import (
|
||||
LineReconciliation, ServiceLinePayment, CasAdjustment,
|
||||
)
|
||||
import json as _json
|
||||
from decimal import Decimal
|
||||
|
||||
with db.SessionLocal()() as s:
|
||||
claim = s.get(db.Claim, claim_id)
|
||||
if claim is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"error": "Not found", "detail": f"Claim {claim_id} not found"},
|
||||
)
|
||||
|
||||
# 837 service lines: from raw_json.
|
||||
raw = claim.raw_json or {}
|
||||
claim_lines_raw = raw.get("service_lines") or []
|
||||
# Normalize to dicts for the response.
|
||||
claim_lines = [_claim_line_dict(d) for d in claim_lines_raw]
|
||||
|
||||
# 835 service payments: ORM rows from the matched remit.
|
||||
remits = list(
|
||||
s.execute(
|
||||
select(db.Remittance).where(db.Remittance.claim_id == claim_id)
|
||||
).scalars().all()
|
||||
)
|
||||
svc_payments: list[dict] = []
|
||||
svc_ids: list[int] = []
|
||||
if remits:
|
||||
svc_rows = list(
|
||||
s.execute(
|
||||
select(ServiceLinePayment).where(
|
||||
ServiceLinePayment.remittance_id.in_([r.id for r in remits])
|
||||
).order_by(ServiceLinePayment.line_number)
|
||||
).scalars().all()
|
||||
)
|
||||
for svc in svc_rows:
|
||||
d = _svc_to_dict(svc)
|
||||
svc_payments.append(d)
|
||||
svc_ids.append(svc.id)
|
||||
|
||||
# LineReconciliation rows.
|
||||
lrs = list(
|
||||
s.execute(
|
||||
select(LineReconciliation).where(LineReconciliation.claim_id == claim_id)
|
||||
).scalars().all()
|
||||
)
|
||||
# Index by claim_service_line_number and service_line_payment_id.
|
||||
lr_by_claim_num: dict[int, LineReconciliation] = {
|
||||
lr.claim_service_line_number: lr for lr in lrs if lr.claim_service_line_number is not None
|
||||
}
|
||||
lr_by_svc: dict[int, LineReconciliation] = {
|
||||
lr.service_line_payment_id: lr for lr in lrs if lr.service_line_payment_id is not None
|
||||
}
|
||||
|
||||
# CAS rows grouped by svc id.
|
||||
cas_by_svc: dict[int, list[CasAdjustment]] = {}
|
||||
if svc_ids:
|
||||
cas_rows = list(
|
||||
s.execute(
|
||||
select(CasAdjustment).where(CasAdjustment.service_line_payment_id.in_(svc_ids))
|
||||
).scalars().all()
|
||||
)
|
||||
for c in cas_rows:
|
||||
cas_by_svc.setdefault(c.service_line_payment_id, []).append(c)
|
||||
|
||||
# Build output lines array, preserving 837 order then 835-only.
|
||||
svc_by_id: dict[int, dict] = {d["id"]: d for d in svc_payments}
|
||||
lines_out: list[dict] = []
|
||||
billed_total = Decimal("0")
|
||||
paid_total = Decimal("0")
|
||||
adjustment_total = Decimal("0")
|
||||
matched_count = 0
|
||||
used_svc_ids: set[int] = set()
|
||||
|
||||
for cl in claim_lines:
|
||||
billed_total += Decimal(str(cl["charge"]))
|
||||
lr = lr_by_claim_num.get(cl["line_number"])
|
||||
if lr is None:
|
||||
lines_out.append({
|
||||
"claim_service_line": cl,
|
||||
"service_line_payment": None,
|
||||
"status": "unmatched_837_only",
|
||||
"adjustments": [],
|
||||
})
|
||||
continue
|
||||
svc_id = lr.service_line_payment_id
|
||||
svc = svc_by_id.get(svc_id) if svc_id else None
|
||||
if svc_id is not None:
|
||||
used_svc_ids.add(svc_id)
|
||||
cas_list = cas_by_svc.get(svc_id, []) if svc_id is not None else []
|
||||
cas_total = sum((Decimal(str(c.amount)) for c in cas_list), Decimal("0"))
|
||||
if svc:
|
||||
paid_total += Decimal(str(svc["payment"]))
|
||||
adjustment_total += cas_total
|
||||
if lr.status == "matched":
|
||||
matched_count += 1
|
||||
lines_out.append({
|
||||
"claim_service_line": cl,
|
||||
"service_line_payment": svc,
|
||||
"status": lr.status,
|
||||
"adjustments": [
|
||||
{"group_code": c.group_code, "reason_code": c.reason_code,
|
||||
"amount": str(Decimal(str(c.amount)))}
|
||||
for c in cas_list
|
||||
],
|
||||
})
|
||||
|
||||
# 835-only lines (no claim match).
|
||||
for lr in lrs:
|
||||
if lr.claim_service_line_number is not None:
|
||||
continue
|
||||
svc_id = lr.service_line_payment_id
|
||||
if svc_id is None:
|
||||
continue
|
||||
if svc_id in used_svc_ids:
|
||||
continue
|
||||
svc = svc_by_id.get(svc_id)
|
||||
cas_list = cas_by_svc.get(svc_id, [])
|
||||
cas_total = sum((Decimal(str(c.amount)) for c in cas_list), Decimal("0"))
|
||||
if svc:
|
||||
paid_total += Decimal(str(svc["payment"]))
|
||||
adjustment_total += cas_total
|
||||
lines_out.append({
|
||||
"claim_service_line": None,
|
||||
"service_line_payment": svc,
|
||||
"status": lr.status,
|
||||
"adjustments": [
|
||||
{"group_code": c.group_code, "reason_code": c.reason_code,
|
||||
"amount": str(Decimal(str(c.amount)))}
|
||||
for c in cas_list
|
||||
],
|
||||
})
|
||||
|
||||
return {
|
||||
"claim_id": claim_id,
|
||||
"summary": {
|
||||
"billed_total": str(billed_total),
|
||||
"paid_total": str(paid_total),
|
||||
"adjustment_total": str(adjustment_total),
|
||||
"matched_lines": matched_count,
|
||||
"total_lines": len(claim_lines),
|
||||
},
|
||||
"lines": lines_out,
|
||||
}
|
||||
|
||||
|
||||
def _claim_line_dict(d: dict) -> dict:
|
||||
"""Project an 837 service-line dict from ``Claim.raw_json`` to wire shape."""
|
||||
from decimal import Decimal
|
||||
proc = d.get("procedure") or {}
|
||||
charge = d.get("charge")
|
||||
units = d.get("units")
|
||||
return {
|
||||
"line_number": d.get("line_number"),
|
||||
"procedure_qualifier": proc.get("qualifier", "HC"),
|
||||
"procedure_code": proc.get("code", ""),
|
||||
"modifiers": proc.get("modifiers") or [],
|
||||
"charge": str(Decimal(str(charge))) if charge is not None else "0",
|
||||
"units": str(Decimal(str(units))) if units is not None else None,
|
||||
"unit_type": d.get("unit_type"),
|
||||
"service_date": d.get("service_date"),
|
||||
}
|
||||
|
||||
|
||||
def _svc_to_dict(svc) -> dict:
|
||||
"""Project an ORM ``ServiceLinePayment`` to wire shape."""
|
||||
import json as _json
|
||||
from decimal import Decimal
|
||||
return {
|
||||
"id": svc.id,
|
||||
"line_number": svc.line_number,
|
||||
"procedure_qualifier": svc.procedure_qualifier,
|
||||
"procedure_code": svc.procedure_code,
|
||||
"modifiers": _json.loads(svc.modifiers_json or "[]"),
|
||||
"charge": str(Decimal(str(svc.charge))),
|
||||
"payment": str(Decimal(str(svc.payment))),
|
||||
"units": str(Decimal(str(svc.units))) if svc.units is not None else None,
|
||||
"unit_type": svc.unit_type,
|
||||
"service_date": svc.service_date.isoformat() if svc.service_date else None,
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
"""``/api/clearhouse*`` — singleton clearhouse config + SFTP submission.
|
||||
|
||||
Three endpoints, all gated by ``matrix_gate``:
|
||||
|
||||
- ``GET /api/clearhouse`` — read the singleton clearhouse row
|
||||
(dzinesco's identity, SFTP block, filename block). 404 when
|
||||
unseeded.
|
||||
- ``PATCH /api/clearhouse`` — full-row replacement of the
|
||||
singleton (SP25). Strict-validates ``sftp_block`` first (Pydantic
|
||||
v2 default mode coerces strings-to-bools and would hide a real
|
||||
operator mistake), then validates the whole body in loose mode.
|
||||
Hot-reloads the running scheduler via
|
||||
``scheduler.reconfigure_scheduler`` so the next tick picks up the
|
||||
new ``SftpBlock`` without a process restart.
|
||||
- ``POST /api/clearhouse/submit`` — submit a batch of claims to
|
||||
the clearhouse. Stub: serializes via the SP7 serializer, builds
|
||||
an HCPF-compliant outbound filename, copies the result to the
|
||||
staging path. Per-claim audit events stamped with
|
||||
``actor="clearhouse-submit"``.
|
||||
|
||||
Three single-router helpers stay in this file (per spec D4):
|
||||
|
||||
- :func:`_load_claim_row` — load a ``Claim`` row by id.
|
||||
- :func:`_serialize_claim_for_submit` — re-serialize a claim to X12
|
||||
with optional per-call kwargs (submitter, receiver, SBR09, etc).
|
||||
- :func:`_serialize_claim_from_raw` — best-effort serializer that
|
||||
re-parses stored ``x12_text`` and re-emits.
|
||||
|
||||
SP36 Task 11: this block moved here from ``api.py:2484`` (the 3
|
||||
``/api/clearhouse*`` routes + 3 single-router helpers).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.api_routers._shared import _actor_user_id
|
||||
from cyclone.audit_log import AuditEvent, append_event
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.db import Claim
|
||||
from cyclone.store import store
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.get("/api/clearhouse")
|
||||
def get_clearhouse():
|
||||
"""Return the singleton clearhouse config (dzinesco's identity, SFTP block, filename block)."""
|
||||
ch = store.get_clearhouse()
|
||||
if ch is None:
|
||||
raise HTTPException(status_code=404, detail="clearhouse not seeded")
|
||||
return json.loads(ch.model_dump_json())
|
||||
|
||||
|
||||
@router.patch("/api/clearhouse")
|
||||
async def patch_clearhouse(body: dict) -> Any:
|
||||
"""Replace the singleton clearhouse row (SP25).
|
||||
|
||||
The full ``Clearhouse`` model is required — we don't accept partial
|
||||
updates because the operator-facing use case is "I'm switching the
|
||||
loop to real MFT" or "I'm pointing at a different MFT server",
|
||||
not "I'm tweaking one field at a time." Validation errors are
|
||||
returned as 422 (Pydantic default).
|
||||
|
||||
After a successful write, the running scheduler is hot-reloaded
|
||||
via ``scheduler.reconfigure_scheduler()`` so the next tick uses
|
||||
the new SftpBlock without a process restart.
|
||||
"""
|
||||
from cyclone import scheduler as _scheduler_mod
|
||||
from cyclone.providers import Clearhouse as _Clearhouse, SftpBlock as _SftpBlock
|
||||
|
||||
# Strict-validate the sftp_block sub-dict FIRST. Pydantic v2's
|
||||
# default mode coerces strings to bools (e.g. ``"stub": "yes"``
|
||||
# silently becomes True), which would hide a real operator
|
||||
# mistake. The Clearhouse model itself stays in loose mode so
|
||||
# ISO-string ``updated_at`` (the JSON round-trip shape) keeps
|
||||
# parsing.
|
||||
raw_sb = body.get("sftp_block", {})
|
||||
try:
|
||||
_SftpBlock.model_validate(raw_sb, strict=True)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"invalid sftp_block: {exc}",
|
||||
) from exc
|
||||
|
||||
# Now validate the full body in loose mode.
|
||||
try:
|
||||
parsed = _Clearhouse.model_validate(body)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=422, detail=str(exc),
|
||||
) from exc
|
||||
|
||||
# SP25: when sftp_block.stub=false, the block must carry an auth
|
||||
# account name and a non-empty host. The Pydantic model catches
|
||||
# some of these; this catches the "empty password_keychain_account"
|
||||
# case (which Pydantic allows because it's a free-form dict).
|
||||
sb = parsed.sftp_block
|
||||
if not sb.stub:
|
||||
if not sb.host:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail="sftp_block.host is required when stub=false",
|
||||
)
|
||||
auth = sb.auth or {}
|
||||
if not auth.get("password_keychain_account") and not auth.get("key_file"):
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=(
|
||||
"sftp_block.auth must contain either "
|
||||
"'password_keychain_account' or 'key_file' when stub=false"
|
||||
),
|
||||
)
|
||||
|
||||
updated = store.update_clearhouse(parsed)
|
||||
await _scheduler_mod.reconfigure_scheduler(
|
||||
updated.sftp_block,
|
||||
sftp_block_name=updated.name or "default",
|
||||
)
|
||||
return json.loads(updated.model_dump_json())
|
||||
|
||||
|
||||
@router.post("/api/clearhouse/submit")
|
||||
def submit_to_clearhouse(request: Request, body: dict):
|
||||
"""Submit a batch of claims to the clearhouse (SFTP). SP9: stub.
|
||||
|
||||
Body: ``{"claim_ids": [...], "payer_id": "CO_TXIX"}``
|
||||
|
||||
Stub behavior: serializes each claim via the SP7 serializer, builds
|
||||
an HCPF-compliant outbound filename, and copies the result to
|
||||
``{staging_dir}/{outbound_path}/{filename}`` instead of opening a
|
||||
real SFTP connection. Returns a receipt per claim.
|
||||
"""
|
||||
from cyclone.clearhouse import make_client
|
||||
from cyclone.edi.filenames import build_outbound_filename
|
||||
|
||||
claim_ids = body.get("claim_ids", [])
|
||||
payer_id = body.get("payer_id")
|
||||
if not claim_ids:
|
||||
raise HTTPException(status_code=400, detail="claim_ids required")
|
||||
if not payer_id:
|
||||
raise HTTPException(status_code=400, detail="payer_id required")
|
||||
|
||||
ch = store.get_clearhouse()
|
||||
if ch is None:
|
||||
raise HTTPException(status_code=500, detail="clearhouse not seeded")
|
||||
|
||||
# Submitter (Loop 1000A) comes from the clearhouse config. The
|
||||
# receiver (NM1*40) and SBR09 come from the per-payer config and
|
||||
# are resolved per-claim below. Without this wiring, the
|
||||
# serializer would emit "CYCLONE" / "RECEIVER" placeholders and
|
||||
# the file would be rejected by HCPF.
|
||||
submitter_kwargs = {
|
||||
"sender_id": ch.tpid,
|
||||
"submitter_name": ch.submitter_name,
|
||||
"submitter_contact_name": ch.submitter_contact_name,
|
||||
"submitter_contact_email": ch.submitter_contact_email,
|
||||
}
|
||||
if getattr(ch, "submitter_contact_phone", None):
|
||||
submitter_kwargs["submitter_contact_phone"] = ch.submitter_contact_phone
|
||||
|
||||
# Build a payer_id → PayerConfig837 map once so we can look up the
|
||||
# receiver + SBR09 default for each claim.
|
||||
from cyclone.db import PayerConfigORM as _PayerConfigORM
|
||||
|
||||
def _resolve_payer_cfg(claim_obj) -> dict | None:
|
||||
pid = (claim_obj.payer.id or "").strip() if claim_obj.payer else ""
|
||||
pname = (claim_obj.payer.name or "").strip() if claim_obj.payer else ""
|
||||
with db.SessionLocal()() as ss:
|
||||
if pid:
|
||||
row = ss.get(_PayerConfigORM, (pid, "837P"))
|
||||
if row is not None:
|
||||
return dict(row.config_json)
|
||||
if pname:
|
||||
rows = (
|
||||
ss.query(_PayerConfigORM)
|
||||
.filter(_PayerConfigORM.transaction_type == "837P")
|
||||
.all()
|
||||
)
|
||||
for r in rows:
|
||||
cj = dict(r.config_json)
|
||||
if (r.payer_id or "").upper() == pname.upper():
|
||||
return cj
|
||||
if pname.lower() in str(cj).lower():
|
||||
return cj
|
||||
row = (
|
||||
ss.query(_PayerConfigORM)
|
||||
.filter(_PayerConfigORM.transaction_type == "837P")
|
||||
.first()
|
||||
)
|
||||
return dict(row.config_json) if row else None
|
||||
|
||||
client = make_client(ch.sftp_block)
|
||||
results = []
|
||||
for cid in claim_ids:
|
||||
try:
|
||||
x12_text = _serialize_claim_for_submit(cid)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
results.append({"claim_id": cid, "ok": False, "error": str(exc)})
|
||||
continue
|
||||
# Re-resolve the claim so we can look up its payer config. We
|
||||
# re-parse the stored x12_text to get a ClaimOutput (same path
|
||||
# the serializer uses).
|
||||
try:
|
||||
from cyclone.parsers.parse_837 import parse as _parse837
|
||||
claim_row_obj = _load_claim_row(cid)
|
||||
if claim_row_obj is None or not (claim_row_obj.raw_json or {}).get("x12_text"):
|
||||
raise RuntimeError("no stored x12_text for claim")
|
||||
parsed = _parse837(claim_row_obj.raw_json["x12_text"])
|
||||
claim_obj = parsed.claims[0] if parsed.claims else None
|
||||
except Exception:
|
||||
claim_obj = None
|
||||
if claim_obj is not None:
|
||||
payer_cfg = _resolve_payer_cfg(claim_obj) or {}
|
||||
receiver_id = payer_cfg.get("receiver_id") or (claim_obj.payer.id if claim_obj.payer else None) or "RECEIVER"
|
||||
receiver_name = payer_cfg.get("receiver_name") or (claim_obj.payer.name if claim_obj.payer else None) or receiver_id
|
||||
sbr09 = payer_cfg.get("sbr09_default") or "MC"
|
||||
# Re-serialize with the proper envelope values.
|
||||
try:
|
||||
x12_text = _serialize_claim_for_submit(
|
||||
cid,
|
||||
**{**submitter_kwargs, "receiver_id": receiver_id,
|
||||
"receiver_name": receiver_name,
|
||||
"claim_filing_indicator_code": sbr09},
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
results.append({"claim_id": cid, "ok": False, "error": str(exc)})
|
||||
continue
|
||||
filename = build_outbound_filename(ch.tpid, "837P")
|
||||
remote = f"{ch.sftp_block.paths['outbound']}/{filename}"
|
||||
staging_path = client.write_file(remote, x12_text.encode("utf-8"))
|
||||
results.append({
|
||||
"claim_id": cid,
|
||||
"ok": True,
|
||||
"filename": filename,
|
||||
"staging_path": str(staging_path),
|
||||
"remote_path": remote,
|
||||
})
|
||||
# SP11: audit trail for each successful clearhouse submission.
|
||||
with db.SessionLocal()() as audit_s:
|
||||
append_event(audit_s, AuditEvent(
|
||||
event_type="clearhouse.submitted",
|
||||
entity_type="claim",
|
||||
entity_id=cid,
|
||||
payload={
|
||||
"filename": filename,
|
||||
"remote_path": remote,
|
||||
"tpid": ch.tpid,
|
||||
"stub": ch.sftp_block.stub,
|
||||
},
|
||||
actor="clearhouse-submit",
|
||||
user_id=_actor_user_id(request),
|
||||
))
|
||||
audit_s.commit()
|
||||
return {"ok": True, "submitted": results, "stub": ch.sftp_block.stub}
|
||||
|
||||
|
||||
def _load_claim_row(claim_id: str):
|
||||
"""Helper: load a Claim row by id (or return None)."""
|
||||
with db.SessionLocal()() as s:
|
||||
return s.get(Claim, claim_id)
|
||||
|
||||
|
||||
def _serialize_claim_for_submit(claim_id: str, **kwargs) -> str:
|
||||
"""Serialize a claim to X12 for SFTP submission. Lazy import of the
|
||||
serializer to avoid pulling FastAPI machinery at module import time.
|
||||
|
||||
Optional ``**kwargs`` are forwarded to the serializer — used to
|
||||
pass through clearhouse submitter info and per-payer receiver info
|
||||
so the regenerated file matches what the HCPF MFT expects.
|
||||
"""
|
||||
from cyclone.parsers.serialize_837 import serialize_837
|
||||
from cyclone import db
|
||||
with db.SessionLocal()() as s:
|
||||
row = s.get(db.Claim, claim_id)
|
||||
if row is None:
|
||||
raise ValueError(f"claim {claim_id!r} not found")
|
||||
# Re-parse the stored raw_json to get a ClaimOutput
|
||||
from cyclone.parsers.models import ClaimOutput, Envelope, Subscriber, Payer, BillingProvider
|
||||
from cyclone.parsers.parse_837 import parse
|
||||
raw = row.raw_json or {}
|
||||
# Reconstruct minimal ClaimOutput from raw_json; this is best-effort.
|
||||
return _serialize_claim_from_raw(row, raw, **kwargs)
|
||||
|
||||
|
||||
def _serialize_claim_from_raw(claim_row, raw: dict, **kwargs) -> str:
|
||||
"""Best-effort serializer that uses the stored raw_json to emit a fresh 837.
|
||||
|
||||
For SP9 this delegates to the existing serialize_837 helper if the
|
||||
claim has a complete raw_segments array. Otherwise it returns a
|
||||
minimal placeholder. ``**kwargs`` are forwarded to the serializer
|
||||
so callers can pass through submitter / receiver / SBR09 values
|
||||
from the clearhouse and per-payer configs.
|
||||
"""
|
||||
from cyclone.parsers.serialize_837 import serialize_837
|
||||
from cyclone.parsers.parse_837 import parse
|
||||
|
||||
# Re-parse the original batch text (need to re-derive from store).
|
||||
# SP9 stub: if the claim has a `raw_json` with `x12_text`, use that.
|
||||
if isinstance(raw, dict) and raw.get("x12_text"):
|
||||
result = parse(raw["x12_text"])
|
||||
if result.claims:
|
||||
return serialize_837(result.claims[0], **kwargs)
|
||||
# Fallback: raise so the caller sees an error.
|
||||
raise RuntimeError(
|
||||
f"claim {claim_row.id!r} cannot be re-serialized: no stored x12_text"
|
||||
)
|
||||
@@ -0,0 +1,54 @@
|
||||
"""``/api/config/payers`` and ``/api/config/payers/{payer_id}/configs`` — payer-config read views.
|
||||
|
||||
Both endpoints are read-only configuration surfaces used by the UI's
|
||||
"Edit payers" page:
|
||||
|
||||
- ``GET /api/config/payers?is_active=...`` lists all configured
|
||||
payers (PayerConfig records) — the set of payers the operator has
|
||||
registered, regardless of whether they have inbound config blocks.
|
||||
- ``GET /api/config/payers/{payer_id}/configs`` returns the full
|
||||
list of ``(transaction_type, config_json)`` blocks for a given
|
||||
payer. Each block has a ``source``: ``"yaml"`` for the on-disk
|
||||
``config/payers.yaml`` default, ``"db"`` for any runtime override
|
||||
recorded via ``/api/admin/reload-config``.
|
||||
|
||||
These are configuration surfaces, not claim-processing surfaces.
|
||||
They live here (under ``/api/config/``) rather than under
|
||||
``/api/payers/`` because the latter is the drill-down rollup
|
||||
(see ``api_routers/payers.py``).
|
||||
|
||||
SP36 Task 7: this block moved here from ``api.py:3167`` (after
|
||||
the SP21 provider-detail helper, before the Auth routers divider).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from cyclone import store
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.get("/api/config/payers")
|
||||
def list_configured_payers(is_active: bool | None = Query(default=True)):
|
||||
return [json.loads(p.model_dump_json()) for p in store.list_payers(is_active=is_active)]
|
||||
|
||||
|
||||
@router.get("/api/config/payers/{payer_id}/configs")
|
||||
def list_payer_configs(payer_id: str):
|
||||
"""List all (transaction_type, config_json) blocks for a payer."""
|
||||
from cyclone import payers as payer_loader
|
||||
configs = [
|
||||
{"transaction_type": tx, "config_json": block, "source": "yaml"}
|
||||
for (pid, tx), block in payer_loader.all_configs().items()
|
||||
if pid == payer_id
|
||||
]
|
||||
# Also check the DB for runtime-overridden configs
|
||||
for tx in ("837P", "835", "277CA", "999", "TA1"):
|
||||
live = store.get_payer_config(payer_id, tx)
|
||||
if live is not None:
|
||||
configs.append({"transaction_type": tx, "config_json": live, "source": "db"})
|
||||
return configs
|
||||
@@ -0,0 +1,39 @@
|
||||
"""``/api/dashboard/kpis`` — server-aggregated Dashboard tiles.
|
||||
|
||||
Backs the Dashboard's "Claims / Billed / Received / Pending AR /
|
||||
Denial rate" tiles + the monthly sparkline series + the
|
||||
top-providers and top-denials lists.
|
||||
|
||||
Why this exists instead of ``GET /api/claims?limit=N``:
|
||||
The Dashboard's KPIs are aggregates over *every* claim — billed,
|
||||
received, denial rate, pending count, monthly billed/received. With
|
||||
60k+ claims in production, paginating ``/api/claims`` and reducing
|
||||
client-side silently produces wrong numbers (denial rate sampled,
|
||||
billed summed from the first 100 rows). This endpoint does the
|
||||
aggregation server-side in a single read so the Dashboard's numbers
|
||||
are always correct regardless of dataset size.
|
||||
|
||||
SP36 Task 4: this single endpoint moved here from ``api.py:2732``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.store import dashboard_kpis
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.get("/api/dashboard/kpis")
|
||||
def get_dashboard_kpis(
|
||||
months: int = Query(6, ge=1, le=24),
|
||||
top_n_providers: int = Query(4, ge=0, le=50),
|
||||
top_n_denials: int = Query(5, ge=0, le=50),
|
||||
) -> dict:
|
||||
"""Server-aggregated Dashboard KPIs over the whole claim population."""
|
||||
return dashboard_kpis(
|
||||
months=months,
|
||||
top_n_providers=top_n_providers,
|
||||
top_n_denials=top_n_denials,
|
||||
)
|
||||
@@ -0,0 +1,223 @@
|
||||
"""``/api/eligibility/request`` and ``/api/eligibility/parse-271`` — API-only eligibility pair.
|
||||
|
||||
Builds a 270 inquiry from a small JSON body and parses a 271 response.
|
||||
Nothing is persisted to the DB — these are operator-driven, ephemeral
|
||||
operations per SP3 (P4 T23–T24). The 270 serializer pulls X12 from a
|
||||
``ParseResult270`` Pydantic; the 271 parser builds the same structure
|
||||
in reverse from the wire format.
|
||||
|
||||
Why these are not ``GET /api/eligibility/...``: the 270 build is
|
||||
operator-initiated (pay-portal paste-back), so the inbound surface is
|
||||
a JSON ``POST``. The 271 inbound is a multipart file upload — same
|
||||
shape as ``/api/parse-999`` — so the file can be the actual 271 text
|
||||
saved from the payer portal.
|
||||
|
||||
SP36 Task 5: this block moved here from ``api.py:2832`` (``270 / 271
|
||||
eligibility`` divider).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import date as _date
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.parsers.exceptions import CycloneParseError
|
||||
from cyclone.parsers.models import BatchSummary, Envelope
|
||||
from cyclone.parsers.models_270 import (
|
||||
EligibilityBenefitInquiry,
|
||||
InformationReceiver270,
|
||||
InformationSource270,
|
||||
ParseResult270,
|
||||
Subscriber270,
|
||||
)
|
||||
from cyclone.parsers.parse_271 import parse as parse_271_text
|
||||
from cyclone.parsers.serialize_270 import serialize_270
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
def _validate_eligibility_request(body: dict) -> tuple[ParseResult270, str]:
|
||||
"""Build a :class:`ParseResult270` from a request body dict.
|
||||
|
||||
The body shape is the minimum surface needed to build a valid 270
|
||||
inquiry (per spec section 3.4 — operator-driven, ephemeral):
|
||||
|
||||
::
|
||||
|
||||
{
|
||||
"subscriber": {first_name, last_name, member_id, dob},
|
||||
"provider": {npi, name},
|
||||
"payer": {id, name},
|
||||
"service_type_code": "1"
|
||||
}
|
||||
|
||||
Returns ``(ParseResult270, service_type_code)``. Raises
|
||||
:class:`HTTPException` (400) when the body is missing required
|
||||
fields.
|
||||
"""
|
||||
subscriber_in = body.get("subscriber") or {}
|
||||
provider_in = body.get("provider") or {}
|
||||
payer_in = body.get("payer") or {}
|
||||
service_type_code = (body.get("service_type_code") or "").strip()
|
||||
|
||||
# Required-field checks. We surface a single 400 with the first
|
||||
# missing field name to match the rest of the API's error contract.
|
||||
if not service_type_code:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": "Bad request", "detail": "service_type_code is required"},
|
||||
)
|
||||
if not subscriber_in.get("member_id"):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": "Bad request", "detail": "subscriber.member_id is required"},
|
||||
)
|
||||
if not provider_in.get("npi"):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": "Bad request", "detail": "provider.npi is required"},
|
||||
)
|
||||
if not payer_in.get("name"):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": "Bad request", "detail": "payer.name is required"},
|
||||
)
|
||||
|
||||
# Build the Pydantic models. The serializer handles all envelope
|
||||
# generation (sender_id/receiver_id/control_number/transaction_date
|
||||
# are filled in by the serializer with sensible defaults).
|
||||
subscriber_dob_raw = subscriber_in.get("dob")
|
||||
subscriber_dob: _date | None = None
|
||||
if subscriber_dob_raw:
|
||||
try:
|
||||
subscriber_dob = _date.fromisoformat(subscriber_dob_raw)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": "Bad request",
|
||||
"detail": f"subscriber.dob must be YYYY-MM-DD: {exc}",
|
||||
},
|
||||
) from exc
|
||||
|
||||
result = ParseResult270(
|
||||
envelope=Envelope(
|
||||
sender_id="SUBMITTERID",
|
||||
receiver_id=str(payer_in.get("id") or "RECEIVERID"),
|
||||
control_number="000000001",
|
||||
transaction_date=_date.today(),
|
||||
implementation_guide="005010X279A1",
|
||||
),
|
||||
information_source=InformationSource270(
|
||||
name=str(payer_in["name"]),
|
||||
id=str(payer_in.get("id") or "") or None,
|
||||
),
|
||||
information_receiver=InformationReceiver270(
|
||||
name=str(provider_in.get("name") or ""),
|
||||
npi=str(provider_in["npi"]),
|
||||
),
|
||||
subscriber=Subscriber270(
|
||||
member_id=str(subscriber_in["member_id"]),
|
||||
first_name=str(subscriber_in.get("first_name") or "") or None,
|
||||
last_name=str(subscriber_in.get("last_name") or "") or None,
|
||||
dob=subscriber_dob,
|
||||
),
|
||||
inquiries=[EligibilityBenefitInquiry(service_type_code=service_type_code)],
|
||||
summary=BatchSummary(
|
||||
input_file="eligibility_request",
|
||||
control_number="000000001",
|
||||
transaction_date=_date.today(),
|
||||
total_claims=1,
|
||||
passed=1,
|
||||
failed=0,
|
||||
),
|
||||
)
|
||||
return result, service_type_code
|
||||
|
||||
|
||||
@router.post("/api/eligibility/request")
|
||||
def post_eligibility_request(body: dict) -> Any:
|
||||
"""Build a 270 eligibility inquiry from a small JSON body.
|
||||
|
||||
Returns ``{"raw_270_text": <X12>, "parsed": <ParseResult270>}``
|
||||
so the operator can either download the raw text (paste into a
|
||||
payer portal) or render the parsed fields directly. Per spec
|
||||
section 3.4, nothing is persisted to the DB.
|
||||
"""
|
||||
try:
|
||||
result, _ = _validate_eligibility_request(body)
|
||||
except HTTPException:
|
||||
raise
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": "Bad request", "detail": f"Malformed body: {exc}"},
|
||||
) from exc
|
||||
|
||||
raw_270_text = serialize_270(result)
|
||||
return {
|
||||
"raw_270_text": raw_270_text,
|
||||
"parsed": json.loads(result.model_dump_json()),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/eligibility/parse-271")
|
||||
async def post_eligibility_parse_271(
|
||||
file: UploadFile = File(...),
|
||||
) -> Any:
|
||||
"""Parse a 271 eligibility response and return the structured summary.
|
||||
|
||||
Accepts the raw 271 text as a file upload (multipart/form-data),
|
||||
mirrors the ``/api/parse-999`` contract. Per spec section 3.4 the
|
||||
result is NOT persisted — the operator re-pastes the 271 each
|
||||
time they need a fresh read.
|
||||
|
||||
The response body is a JSON object with three top-level keys:
|
||||
``coverage_benefits``, ``subscriber``, and ``summary``. 400 is
|
||||
returned on empty / undecodable / malformed EDI; 200 on success.
|
||||
"""
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Empty file", "detail": "Uploaded file contained no bytes."},
|
||||
)
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Encoding error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
try:
|
||||
result = parse_271_text(text, input_file=file.filename or "")
|
||||
except CycloneParseError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Parse error", "detail": str(exc)},
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - safety net
|
||||
log.exception("Unexpected parser failure on 271")
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": "Internal server error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
return {
|
||||
"coverage_benefits": [
|
||||
json.loads(cb.model_dump_json()) for cb in result.coverage_benefits
|
||||
],
|
||||
"subscriber": json.loads(result.subscriber.model_dump_json()),
|
||||
"summary": json.loads(result.summary.model_dump_json()),
|
||||
"envelope": json.loads(result.envelope.model_dump_json()),
|
||||
"information_source": json.loads(result.information_source.model_dump_json()),
|
||||
"information_receiver": json.loads(result.information_receiver.model_dump_json()),
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
"""``/api/inbox*`` — operator-facing Inbox surface (SP6 + SP14).
|
||||
|
||||
Six endpoints, all gated by ``matrix_gate``:
|
||||
|
||||
- ``GET /api/inbox/lanes`` — all lanes in one
|
||||
call (``compute_lanes`` from :mod:`cyclone.inbox_lanes`).
|
||||
- ``POST /api/inbox/candidates/{remit_id}/match`` — manually link a
|
||||
remit to a claim; surfaces 409 with the current state when the
|
||||
claim is already matched.
|
||||
- ``POST /api/inbox/candidates/dismiss`` — add candidate
|
||||
pairs to the session-scoped dismissed set (mutates
|
||||
``request.app.state.dismissed_pairs``).
|
||||
- ``POST /api/inbox/payer-rejected/acknowledge`` — SP14: mark
|
||||
Payer-Rejected claims as acknowledged. Idempotent; returns the
|
||||
count actually transitioned vs. already-acked / not-found /
|
||||
not-rejected.
|
||||
- ``POST /api/inbox/rejected/resubmit`` — bulk move
|
||||
REJECTED claims back to SUBMITTED. With
|
||||
``?download=true`` returns a ZIP of regenerated 837 files
|
||||
(``serialize_837_for_resubmit`` with per-claim ``interchange_index``
|
||||
for unique control numbers). Conflicts are omitted from the ZIP
|
||||
and surfaced via the ``X-Cyclone-Serialize-Errors`` header.
|
||||
- ``GET /api/inbox/export.csv`` — stream a CSV for
|
||||
a single lane (rejected / candidates / unmatched / done_today).
|
||||
|
||||
All endpoints use ``request.app.state`` rather than the module-level
|
||||
``app`` global so they're robust against ``importlib.reload`` of
|
||||
the api module — the reload rebinds ``app`` to a new instance, but
|
||||
``request.app`` always points at the instance actually serving the
|
||||
current request.
|
||||
|
||||
SP36 Task 13: this block moved here from ``api.py:1280`` (the 6
|
||||
``/api/inbox*`` routes, with the SP14 comment block preserved
|
||||
verbatim).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.db import Claim, ClaimState, Remittance
|
||||
from cyclone.parsers.models import ClaimOutput
|
||||
from cyclone.parsers.serialize_837 import SerializeError as SerializeError837
|
||||
from cyclone.parsers.serialize_837 import serialize_837_for_resubmit
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.get("/api/inbox/lanes")
|
||||
def inbox_lanes(request: Request):
|
||||
"""Return all Inbox lanes in one call.
|
||||
|
||||
Uses ``request.app.state`` rather than the module-level ``app``
|
||||
global so the endpoint is robust against ``importlib.reload`` of
|
||||
this module (some tests do this to mutate the CORS allow-list).
|
||||
After a reload, the module-level ``app`` rebinds to a new
|
||||
FastAPI instance; ``request.app`` always points at the instance
|
||||
that is actually serving the current request, so per-request
|
||||
state stays consistent with the test's TestClient target.
|
||||
"""
|
||||
dismissed_pairs = getattr(request.app.state, "dismissed_pairs", set())
|
||||
with db.SessionLocal()() as session:
|
||||
from cyclone.inbox_lanes import compute_lanes
|
||||
lanes = compute_lanes(session, dismissed_pairs=dismissed_pairs)
|
||||
return {
|
||||
"rejected": lanes.rejected,
|
||||
# SP10: payer-rejected lane (277CA STC A4/A6/A7). Distinct from
|
||||
# the 999 envelope rejection in ``rejected`` above.
|
||||
"payer_rejected": lanes.payer_rejected,
|
||||
"candidates": lanes.candidates,
|
||||
"unmatched": lanes.unmatched,
|
||||
"done_today": lanes.done_today,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/inbox/candidates/{remit_id}/match")
|
||||
def inbox_match_candidate(remit_id: str, body: dict):
|
||||
"""Manually link a remit to a claim."""
|
||||
claim_id = body.get("claim_id")
|
||||
if not claim_id:
|
||||
raise HTTPException(400, "claim_id required")
|
||||
with db.SessionLocal()() as s:
|
||||
claim = s.get(Claim, claim_id)
|
||||
remit = s.get(Remittance, remit_id)
|
||||
if claim is None or remit is None:
|
||||
raise HTTPException(404, "claim or remit not found")
|
||||
if claim.matched_remittance_id and claim.matched_remittance_id != remit_id:
|
||||
raise HTTPException(
|
||||
409,
|
||||
detail={
|
||||
"error": "claim_already_matched",
|
||||
"current_state": (
|
||||
claim.state.value if hasattr(claim.state, "value")
|
||||
else str(claim.state)
|
||||
),
|
||||
"matched_remittance_id": claim.matched_remittance_id,
|
||||
},
|
||||
)
|
||||
claim.matched_remittance_id = remit_id
|
||||
remit.claim_id = claim_id
|
||||
s.commit()
|
||||
return {"ok": True, "claim_id": claim_id, "remit_id": remit_id}
|
||||
|
||||
|
||||
@router.post("/api/inbox/candidates/dismiss")
|
||||
def inbox_dismiss_candidates(body: dict, request: Request):
|
||||
"""Add candidate pairs to the session-scoped dismissed set.
|
||||
|
||||
Uses ``request.app.state`` rather than the module-level ``app``
|
||||
global so the endpoint is robust against ``importlib.reload`` of
|
||||
this module (some tests do this to mutate the CORS allow-list).
|
||||
After a reload, the module-level ``app`` rebinds to a new
|
||||
FastAPI instance; ``request.app`` always points at the instance
|
||||
that is actually serving the current request, so the test's
|
||||
TestClient target is the one whose state we mutate.
|
||||
"""
|
||||
pairs = body.get("pairs") or []
|
||||
if not hasattr(request.app.state, "dismissed_pairs"):
|
||||
request.app.state.dismissed_pairs = set()
|
||||
for p in pairs:
|
||||
cid = p.get("claim_id")
|
||||
rid = p.get("remit_id")
|
||||
if cid and rid:
|
||||
request.app.state.dismissed_pairs.add(frozenset({cid, rid}))
|
||||
return {"ok": True, "dismissed_count": len(pairs)}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SP14: Payer-Rejected acknowledge
|
||||
#
|
||||
# Operator hits "Acknowledge" on the Payer-Rejected Inbox lane to clear
|
||||
# the claim from the working surface. We don't delete the rejection
|
||||
# (the original payer_rejected_* fields stay for SP11 audit), we just
|
||||
# set payer_rejected_acknowledged_at so the lane query filters it out.
|
||||
#
|
||||
# Idempotent: re-acknowledging an already-acknowledged claim is a noop
|
||||
# (the timestamp is not bumped). Returns the count actually transitioned
|
||||
# so the UI can show "3 of 5 were already acknowledged".
|
||||
# --------------------------------------------------------------------------- #
|
||||
@router.post("/api/inbox/payer-rejected/acknowledge")
|
||||
def inbox_acknowledge_payer_rejected(body: dict):
|
||||
"""Mark Payer-Rejected claims as acknowledged by the operator."""
|
||||
claim_ids = body.get("claim_ids") or []
|
||||
actor = body.get("actor") or "operator"
|
||||
if not isinstance(claim_ids, list) or not claim_ids:
|
||||
raise HTTPException(400, "claim_ids must be a non-empty list")
|
||||
if not all(isinstance(c, str) for c in claim_ids):
|
||||
raise HTTPException(400, "claim_ids must be a list of strings")
|
||||
|
||||
with db.SessionLocal()() as session:
|
||||
from cyclone.db import Claim
|
||||
now = datetime.now(timezone.utc)
|
||||
transitioned = 0
|
||||
already_acked = 0
|
||||
not_found = 0
|
||||
not_rejected = 0
|
||||
for cid in claim_ids:
|
||||
claim = session.get(Claim, cid)
|
||||
if claim is None:
|
||||
not_found += 1
|
||||
continue
|
||||
if claim.payer_rejected_at is None:
|
||||
not_rejected += 1
|
||||
continue
|
||||
if claim.payer_rejected_acknowledged_at is not None:
|
||||
already_acked += 1
|
||||
continue
|
||||
claim.payer_rejected_acknowledged_at = now
|
||||
claim.payer_rejected_acknowledged_actor = actor
|
||||
transitioned += 1
|
||||
# SP11: audit event for the acknowledge action.
|
||||
try:
|
||||
from cyclone.audit_log import append_event, AuditEvent
|
||||
append_event(session, AuditEvent(
|
||||
event_type="claim.payer_rejected_acknowledged",
|
||||
entity_type="claim",
|
||||
entity_id=claim.id,
|
||||
actor=actor,
|
||||
payload={
|
||||
"payer_rejected_status_code": claim.payer_rejected_status_code,
|
||||
"payer_rejected_by_277ca_id": claim.payer_rejected_by_277ca_id,
|
||||
},
|
||||
))
|
||||
except Exception: # noqa: BLE001
|
||||
# Audit append is best-effort; don't block the operator's
|
||||
# acknowledge action on an audit-log failure.
|
||||
pass
|
||||
if transitioned:
|
||||
session.commit()
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"transitioned": transitioned,
|
||||
"already_acked": already_acked,
|
||||
"not_found": not_found,
|
||||
"not_rejected": not_rejected,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/inbox/rejected/resubmit")
|
||||
def inbox_resubmit_rejected(
|
||||
request: Request,
|
||||
body: dict,
|
||||
download: bool = Query(False, description="When true, return a ZIP of regenerated 837 files for the resubmitted claims (instead of JSON)."),
|
||||
):
|
||||
"""Bulk move REJECTED claims back to SUBMITTED.
|
||||
|
||||
With ``?download=true``, the response is a ``application/zip`` archive
|
||||
containing one ``claim-{id}.x12`` per successfully resubmitted claim
|
||||
(regenerated via ``serialize_837_for_resubmit`` so each file gets a
|
||||
unique interchange/group control number). Conflicts are omitted from
|
||||
the ZIP — they remain visible to the caller via the JSON shape of the
|
||||
non-download path. Empty resubmit + download → 200 with an empty zip
|
||||
so the UI can still hand the user a downloadable artifact.
|
||||
"""
|
||||
ids = body.get("claim_ids") or []
|
||||
if not ids:
|
||||
raise HTTPException(400, "claim_ids required")
|
||||
accepted: list[str] = []
|
||||
conflicts: list[dict] = []
|
||||
# Track which claims are about to be resubmitted (and their index in
|
||||
# the bundle) so the download path can serialize them with unique
|
||||
# control numbers — back-to-back resubmits in the same file would
|
||||
# otherwise all share ISA13/GS06 = "000000001".
|
||||
accepted_with_rows: list[tuple[str, "Claim"]] = []
|
||||
with db.SessionLocal()() as s:
|
||||
for cid in ids:
|
||||
c = s.get(Claim, cid)
|
||||
if c is None:
|
||||
continue
|
||||
if c.state != ClaimState.REJECTED:
|
||||
conflicts.append({
|
||||
"claim_id": cid,
|
||||
"current_state": (
|
||||
c.state.value if hasattr(c.state, "value")
|
||||
else str(c.state)
|
||||
),
|
||||
})
|
||||
continue
|
||||
c.state = ClaimState.SUBMITTED
|
||||
c.state_changed_at = datetime.now(timezone.utc)
|
||||
c.rejection_reason = None
|
||||
c.rejected_at = None
|
||||
c.resubmit_count = (c.resubmit_count or 0) + 1
|
||||
accepted.append(cid)
|
||||
accepted_with_rows.append((cid, c))
|
||||
s.commit()
|
||||
|
||||
if not download:
|
||||
return {"ok": True, "resubmitted": accepted, "conflicts": conflicts}
|
||||
|
||||
# Build a ZIP of regenerated 837s for the accepted claims. Conflicts
|
||||
# and missing ids are deliberately excluded — the user already saw
|
||||
# them in the JSON response on prior actions; the download is the
|
||||
# "give me the files I asked for" payload.
|
||||
import zipfile
|
||||
buf = io.BytesIO()
|
||||
serialize_errors: list[dict] = []
|
||||
with zipfile.ZipFile(buf, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
for idx, (cid, c) in enumerate(accepted_with_rows, start=1):
|
||||
if not c.raw_json:
|
||||
serialize_errors.append({"claim_id": cid, "reason": "no raw_json"})
|
||||
continue
|
||||
try:
|
||||
claim_obj = ClaimOutput.model_validate(c.raw_json)
|
||||
except Exception as exc:
|
||||
serialize_errors.append({"claim_id": cid, "reason": f"raw_json invalid: {exc}"})
|
||||
continue
|
||||
try:
|
||||
text = serialize_837_for_resubmit(claim_obj, interchange_index=idx)
|
||||
except SerializeError837 as exc:
|
||||
serialize_errors.append({"claim_id": cid, "reason": str(exc)})
|
||||
continue
|
||||
zf.writestr(f"claim-{cid}.x12", text)
|
||||
buf.seek(0)
|
||||
headers = {
|
||||
"Content-Disposition": (
|
||||
f'attachment; filename="resubmit-{len(accepted)}-claims.zip"'
|
||||
),
|
||||
}
|
||||
# Surface per-claim serialization failures as a custom response header
|
||||
# so the UI can show "10 resubmitted, 2 couldn't be regenerated" without
|
||||
# parsing the binary. The header value is JSON-encoded; the UI is
|
||||
# expected to JSON.parse it after a fetch with response.ok.
|
||||
if serialize_errors:
|
||||
headers["X-Cyclone-Serialize-Errors"] = json.dumps(serialize_errors)
|
||||
return Response(
|
||||
content=buf.getvalue(),
|
||||
media_type="application/zip",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/inbox/export.csv")
|
||||
def inbox_export_csv(lane: str, request: Request):
|
||||
"""Stream a CSV for a single lane.
|
||||
|
||||
Uses ``request.app.state`` rather than the module-level ``app``
|
||||
global so the endpoint is robust against ``importlib.reload`` of
|
||||
this module (see ``inbox_dismiss_candidates`` for context).
|
||||
"""
|
||||
if lane not in {"rejected", "candidates", "unmatched", "done_today"}:
|
||||
raise HTTPException(400, f"unknown lane: {lane}")
|
||||
dismissed_pairs = getattr(request.app.state, "dismissed_pairs", set())
|
||||
with db.SessionLocal()() as session:
|
||||
from cyclone.inbox_lanes import compute_lanes
|
||||
lanes = compute_lanes(session, dismissed_pairs=dismissed_pairs)
|
||||
rows = getattr(lanes, lane)
|
||||
|
||||
buf = io.StringIO()
|
||||
writer = csv.writer(buf)
|
||||
writer.writerow([
|
||||
"id", "kind", "patient_control_number", "charge_amount",
|
||||
"payer_id", "provider_npi", "state", "rejection_reason",
|
||||
"service_date", "score",
|
||||
])
|
||||
for r in rows:
|
||||
writer.writerow([
|
||||
r.get("id") or r.get("payer_claim_control_number"),
|
||||
r.get("kind"),
|
||||
r.get("patient_control_number"),
|
||||
r.get("charge_amount"),
|
||||
r.get("payer_id"),
|
||||
r.get("provider_npi") or r.get("rendering_provider_npi"),
|
||||
r.get("state"),
|
||||
r.get("rejection_reason"),
|
||||
r.get("service_date_from") or r.get("service_date"),
|
||||
r.get("score"),
|
||||
])
|
||||
buf.seek(0)
|
||||
return StreamingResponse(
|
||||
iter([buf.getvalue()]),
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": f'attachment; filename="inbox-{lane}.csv"'},
|
||||
)
|
||||
@@ -0,0 +1,831 @@
|
||||
"""Parse endpoints — accept X12 uploads and ingest them.
|
||||
|
||||
Five routes:
|
||||
|
||||
* ``POST /api/parse-837`` — 837P professional claim ingest (the
|
||||
primary upload path)
|
||||
* ``POST /api/parse-835`` — 835 ERA remittance ingest
|
||||
* ``POST /api/parse-999`` — 999 ACK ingest + auto-link claims
|
||||
* ``POST /api/parse-ta1`` — TA1 envelope ACK ingest + envelope-link batches
|
||||
* ``POST /api/parse-277ca`` — 277CA Claim Acknowledgment ingest
|
||||
|
||||
The 7 cross-router helpers these endpoints need (and the two
|
||||
PAYER_FACTORIES dicts they consume) live in
|
||||
:mod:`cyclone.api_routers._shared`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.api_helpers import (
|
||||
client_wants_json as _client_wants_json,
|
||||
drop_raw_segments_837 as _drop_raw_segments,
|
||||
drop_raw_segments_835 as _drop_raw_segments_835,
|
||||
has_claim_validation_errors as _has_claim_validation_errors,
|
||||
has_835_validation_errors as _has_835_validation_errors,
|
||||
ndjson_stream_837 as _ndjson_stream,
|
||||
ndjson_stream_835 as _ndjson_stream_835,
|
||||
strict_rewrite_837 as _strict_rewrite,
|
||||
strict_rewrite_835 as _strict_rewrite_835,
|
||||
)
|
||||
from cyclone.api_routers._shared import (
|
||||
_actor_user_id,
|
||||
_build_and_persist_ack,
|
||||
_reconciliation_summary_for_batch,
|
||||
_resolve_payer,
|
||||
_resolve_payer_835,
|
||||
_serialize_ta1,
|
||||
_ta1_synthetic_source_batch_id,
|
||||
_transaction_set_id_from_segments,
|
||||
)
|
||||
from cyclone.audit_log import AuditEvent, append_event
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.claim_acks import (
|
||||
apply_277ca_acks as _apply_277ca_acks,
|
||||
apply_999_acceptances as _apply_999_acceptances,
|
||||
apply_ta1_envelope_link as _apply_ta1_envelope_link,
|
||||
)
|
||||
from cyclone.db import Batch, Claim
|
||||
from cyclone.handlers._ack_id import (
|
||||
ack_count_summary as _ack_count_summary,
|
||||
ack_synthetic_source_batch_id as _ack_synthetic_source_batch_id,
|
||||
two77ca_synthetic_source_batch_id as _277ca_synthetic_source_batch_id,
|
||||
)
|
||||
from cyclone.inbox_state import apply_999_rejections
|
||||
from cyclone.inbox_state_277ca import apply_277ca_rejections
|
||||
from cyclone.parsers.exceptions import CycloneParseError
|
||||
from cyclone.parsers.parse_277ca import parse_277ca_text
|
||||
from cyclone.parsers.parse_837 import parse
|
||||
from cyclone.parsers.parse_835 import parse as parse_835
|
||||
from cyclone.parsers.parse_999 import parse_999_text
|
||||
from cyclone.parsers.parse_ta1 import parse_ta1_text
|
||||
from cyclone.parsers.segments import tokenize as _tokenize_segments
|
||||
from cyclone.parsers.serialize_999 import serialize_999
|
||||
from cyclone.parsers.validator_835 import validate as validate_835
|
||||
from cyclone.store import BatchRecord, store, utcnow
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.post("/api/parse-837")
|
||||
async def parse_837(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
payer: str = Query("co_medicaid"),
|
||||
include_raw_segments: bool = Query(True),
|
||||
strict: bool = Query(False),
|
||||
ack: bool = Query(False),
|
||||
) -> Any:
|
||||
# SP35: defense-in-depth input guards. Layer A (UI auto-detect) lives
|
||||
# in src/pages/Upload.tsx; the server-side checks below are the
|
||||
# authoritative fix because they protect every caller of the API
|
||||
# (Upload page, CLI ingestion, any future bulk-import tool). Without
|
||||
# these, an 835 file dropped on the Upload page while the dropdown
|
||||
# still says "837p" produces a BatchRecord with claims=[] and a bogus
|
||||
# row on the History tab. The fix is two checks run BEFORE we persist
|
||||
# anything:
|
||||
#
|
||||
# 1. Envelope check — ST01 must be "837" or "837P". Anything else
|
||||
# (an 835, a 999, a 270, garbage that happens to have an ISA)
|
||||
# → 400 with error="Mismatched file kind", expected="837p",
|
||||
# detected_st=<whatever was there>.
|
||||
# 2. Empty-claims check — even with the right envelope, if the
|
||||
# parser produced zero CLM segments (truncated file, header-only
|
||||
# test fixture) → 400 with error="No claims parsed". A real
|
||||
# production 837 batch with zero claims is never valid.
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Empty file", "detail": "Uploaded file contained no bytes."},
|
||||
)
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Encoding error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
config = _resolve_payer(payer)
|
||||
|
||||
# SP35 guard 1: envelope check. Tokenize first so we can return a
|
||||
# precise 400 (vs. relying on the parser's "no ISA envelope" error
|
||||
# which is correct but doesn't say "you sent an 835 to the 837
|
||||
# endpoint"). If tokenization itself fails we fall through to the
|
||||
# parser, which raises CycloneParseError → 400 "Parse error" path.
|
||||
try:
|
||||
_segments = _tokenize_segments(text)
|
||||
detected_st = _transaction_set_id_from_segments(_segments) or ""
|
||||
except CycloneParseError:
|
||||
detected_st = ""
|
||||
|
||||
if detected_st and not detected_st.upper().startswith("837"):
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"error": "Mismatched file kind",
|
||||
"expected": "837p",
|
||||
"detected_st": detected_st,
|
||||
"detail": (
|
||||
f"File declares ST*{detected_st}* but this endpoint "
|
||||
f"expects ST*837*. Pick the matching endpoint on the "
|
||||
f"Upload page (or let auto-detect choose for you)."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
result = parse(text, config, input_file=file.filename or "")
|
||||
except CycloneParseError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Parse error", "detail": str(exc)},
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - safety net
|
||||
log.exception("Unexpected parser failure")
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": "Internal server error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
# SP35 guard 2: empty-claims check. With the envelope validated, the
|
||||
# only way to land here is a header-only file (real, but useless)
|
||||
# or a file whose CLM loops the parser couldn't extract. Either way
|
||||
# we refuse to persist — a BatchRecord with claims=[] is what the
|
||||
# original bug produced and is never what the operator wanted.
|
||||
if not result.claims:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"error": "No claims parsed",
|
||||
"detail": (
|
||||
"The file passed the envelope check but contained no "
|
||||
"CLM segments. Refusing to persist an empty batch."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
if strict:
|
||||
result = _strict_rewrite(result)
|
||||
if not include_raw_segments:
|
||||
result = _drop_raw_segments(result)
|
||||
|
||||
if _has_claim_validation_errors(result):
|
||||
# Per spec: 422 when claims failed validation.
|
||||
# Body still includes the full ParseResult so the client can show errors.
|
||||
# Validation-failed parses are NOT persisted (the data is suspect);
|
||||
# only parses that survive validation end up in the store.
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content=json.loads(result.model_dump_json()),
|
||||
)
|
||||
|
||||
# Persist the cleaned-up result so the cleaned result is what clients
|
||||
# retrieve via /api/batches/{id}.
|
||||
rec = BatchRecord(
|
||||
id=uuid.uuid4().hex,
|
||||
kind="837p",
|
||||
input_filename=file.filename or "upload.txt",
|
||||
parsed_at=utcnow(),
|
||||
result=result,
|
||||
)
|
||||
try:
|
||||
store.add(rec, event_bus=request.app.state.event_bus)
|
||||
except IntegrityError as exc:
|
||||
# ``(batch_id, patient_control_number)`` is UNIQUE — fires when a
|
||||
# single batch file contains the same CLM01 control number twice,
|
||||
# or when the same claim id has already been ingested in a prior
|
||||
# batch. Surface as 409 with the batch id so the caller can look
|
||||
# it up; do NOT 500 (a 500 without CORS headers is misreported by
|
||||
# browsers as a CORS error and hides the real cause).
|
||||
log.warning("Duplicate claim while persisting batch %s: %s", rec.id, exc)
|
||||
return JSONResponse(
|
||||
status_code=409,
|
||||
content={
|
||||
"error": "Duplicate claim",
|
||||
"detail": (
|
||||
"This file (or one previously ingested with the same "
|
||||
"claim control number) collides with an existing "
|
||||
"record. Inspect the file for duplicate CLM01 "
|
||||
"control numbers, or remove the existing batch "
|
||||
"before retrying."
|
||||
),
|
||||
"batch_id": rec.id,
|
||||
},
|
||||
)
|
||||
|
||||
if _client_wants_json(request):
|
||||
body = json.loads(result.model_dump_json())
|
||||
if ack:
|
||||
ack_body = _build_and_persist_ack(rec.id)
|
||||
if ack_body is not None:
|
||||
body["ack"] = ack_body
|
||||
# Surface the server-side batch id so the frontend can call
|
||||
# /api/batches/{id}/export-837 (and any other batch-scoped
|
||||
# endpoint) without a separate listBatches round-trip.
|
||||
body["batch_id"] = rec.id
|
||||
return JSONResponse(content=body)
|
||||
|
||||
# Default: NDJSON stream. Pass the server-side batch id so the
|
||||
# streaming client (the React Upload page) can call batch-scoped
|
||||
# endpoints like /api/batches/{id}/export-837 without a separate
|
||||
# GET /api/batches round-trip.
|
||||
return StreamingResponse(
|
||||
_ndjson_stream(result, batch_id=rec.id),
|
||||
media_type="application/x-ndjson",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/parse-835")
|
||||
async def parse_835_endpoint(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
payer: str = Query("co_medicaid_835"),
|
||||
include_raw_segments: bool = Query(True),
|
||||
strict: bool = Query(False),
|
||||
) -> Any:
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Empty file", "detail": "Uploaded file contained no bytes."},
|
||||
)
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Encoding error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
config = _resolve_payer_835(payer)
|
||||
|
||||
# SP35 guard 1: envelope check. Mirrors the parse-837 path: tokenize,
|
||||
# read ST01, reject anything that doesn't start with "835". Same
|
||||
# defense-in-depth rationale — the UI auto-detect (src/pages/Upload.tsx)
|
||||
# is layer A, but server-side guards protect every API caller.
|
||||
try:
|
||||
_segments_835 = _tokenize_segments(text)
|
||||
detected_st_835 = _transaction_set_id_from_segments(_segments_835) or ""
|
||||
except CycloneParseError:
|
||||
detected_st_835 = ""
|
||||
|
||||
if detected_st_835 and not detected_st_835.upper().startswith("835"):
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"error": "Mismatched file kind",
|
||||
"expected": "835",
|
||||
"detected_st": detected_st_835,
|
||||
"detail": (
|
||||
f"File declares ST*{detected_st_835}* but this endpoint "
|
||||
f"expects ST*835*. Pick the matching endpoint on the "
|
||||
f"Upload page (or let auto-detect choose for you)."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
result = parse_835(text, config, input_file=file.filename or "")
|
||||
except CycloneParseError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Parse error", "detail": str(exc)},
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - safety net
|
||||
log.exception("Unexpected parser failure")
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": "Internal server error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
# SP35 guard 2: empty-claims check. Same as parse-837: a BatchRecord
|
||||
# with claims=[] is never a valid production 835 batch and we refuse
|
||||
# to persist it.
|
||||
if not result.claims:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"error": "No claims parsed",
|
||||
"detail": (
|
||||
"The file passed the envelope check but contained no "
|
||||
"CLP segments. Refusing to persist an empty batch."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
# Always run the validator; attach the report so the JSON path can
|
||||
# surface it and the NDJSON path can fold the counts into the summary.
|
||||
# 835 validation is batch-level, so pass/fail applies uniformly to every
|
||||
# claim payment in the batch (passed=N or 0, failed=0 or N).
|
||||
report = validate_835(result, config)
|
||||
n = len(result.claims)
|
||||
claim_ids = [c.payer_claim_control_number for c in result.claims]
|
||||
if report.passed:
|
||||
passed, failed, failed_claim_ids = n, 0, []
|
||||
else:
|
||||
passed, failed, failed_claim_ids = 0, n, claim_ids
|
||||
result = result.model_copy(update={
|
||||
"validation": report,
|
||||
"summary": result.summary.model_copy(update={
|
||||
"passed": passed,
|
||||
"failed": failed,
|
||||
"failed_claim_ids": failed_claim_ids,
|
||||
}),
|
||||
})
|
||||
|
||||
if strict:
|
||||
result = _strict_rewrite_835(result)
|
||||
if not include_raw_segments:
|
||||
result = _drop_raw_segments_835(result)
|
||||
|
||||
if _has_835_validation_errors(result):
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content=json.loads(result.model_dump_json()),
|
||||
)
|
||||
|
||||
# Persist the cleaned-up result so the cleaned result is what clients
|
||||
# retrieve via /api/batches/{id}.
|
||||
rec = BatchRecord(
|
||||
id=uuid.uuid4().hex,
|
||||
kind="835",
|
||||
input_filename=file.filename or "upload.txt",
|
||||
parsed_at=utcnow(),
|
||||
result=result,
|
||||
)
|
||||
try:
|
||||
store.add(rec, event_bus=request.app.state.event_bus)
|
||||
except IntegrityError as exc:
|
||||
log.warning("Duplicate remittance while persisting batch %s: %s", rec.id, exc)
|
||||
return JSONResponse(
|
||||
status_code=409,
|
||||
content={
|
||||
"error": "Duplicate remittance",
|
||||
"detail": (
|
||||
"This 835 file (or one previously ingested with the "
|
||||
"same payer claim control number) collides with an "
|
||||
"existing record. Remove the existing remittance "
|
||||
"before retrying."
|
||||
),
|
||||
"batch_id": rec.id,
|
||||
},
|
||||
)
|
||||
|
||||
if _client_wants_json(request):
|
||||
body = json.loads(result.model_dump_json())
|
||||
body["reconciliation"] = _reconciliation_summary_for_batch(rec.id)
|
||||
return JSONResponse(content=body)
|
||||
|
||||
# Default: NDJSON stream. Pass the server-side batch id so the
|
||||
# streaming client can call batch-scoped endpoints without a
|
||||
# separate GET /api/batches round-trip (see /api/parse-837 for the
|
||||
# parallel change).
|
||||
return StreamingResponse(
|
||||
_ndjson_stream_835(result, batch_id=rec.id),
|
||||
media_type="application/x-ndjson",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/parse-999")
|
||||
async def parse_999_endpoint(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
) -> Any:
|
||||
"""Parse a 999 ACK file, persist a row, and return JSON.
|
||||
|
||||
Behavior mirrors ``/api/parse-835``:
|
||||
- 400 on empty / undecodable / malformed EDI (never 500).
|
||||
- 200 on success with ``{"ack": {id, accepted_count, rejected_count,
|
||||
received_count, ack_code, raw_999_text}, "parsed": <ParseResult999>}``.
|
||||
|
||||
The persisted ``acks.source_batch_id`` is a synthetic id
|
||||
(``999-<ISA13>``) because a received 999 has no inbound 837 batch
|
||||
to FK to. The dashboard's `/acks` list surfaces these; operators
|
||||
can still see which interchange each row came from.
|
||||
"""
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Empty file", "detail": "Uploaded file contained no bytes."},
|
||||
)
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Encoding error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
try:
|
||||
result = parse_999_text(text, input_file=file.filename or "")
|
||||
except CycloneParseError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Parse error", "detail": str(exc)},
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - safety net
|
||||
log.exception("Unexpected parser failure on 999")
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": "Internal server error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
received, accepted, rejected, ack_code = _ack_count_summary(result)
|
||||
icn = result.envelope.control_number
|
||||
synthetic_id = _ack_synthetic_source_batch_id(icn)
|
||||
|
||||
# Build the raw 999 text from the parsed result (round-trip).
|
||||
raw_999_text = serialize_999(result, interchange_control_number=icn or "000000001")
|
||||
|
||||
# SP6 T4: move claims whose 999 set was rejected into ClaimState.REJECTED.
|
||||
# The 999's set_control_number (AK202) is the source 837's ST02; in
|
||||
# practice we look it up against patient_control_number because that's
|
||||
# the field 999 ACKs cross-reference in this product.
|
||||
with db.SessionLocal()() as session:
|
||||
def _lookup(pcn: str):
|
||||
return session.query(Claim).filter_by(patient_control_number=pcn).first()
|
||||
_rejection_result = apply_999_rejections(
|
||||
session, result, claim_lookup=_lookup,
|
||||
)
|
||||
if _rejection_result.matched:
|
||||
bus = request.app.state.event_bus
|
||||
for cid in _rejection_result.matched:
|
||||
await bus.publish("claim.rejected", {"claim_id": cid})
|
||||
if _rejection_result.orphans:
|
||||
log.warning(
|
||||
"999 had %d orphan set refs: %s",
|
||||
len(_rejection_result.orphans),
|
||||
_rejection_result.orphans[:5],
|
||||
)
|
||||
|
||||
row = store.add_ack(
|
||||
source_batch_id=synthetic_id,
|
||||
accepted_count=accepted,
|
||||
rejected_count=rejected,
|
||||
received_count=received,
|
||||
ack_code=ack_code,
|
||||
raw_json=json.loads(result.model_dump_json()),
|
||||
event_bus=request.app.state.event_bus,
|
||||
)
|
||||
|
||||
# SP11: append one audit row per rejected claim. Each row chains
|
||||
# to the previous one — see cyclone.audit_log.
|
||||
if _rejection_result.matched:
|
||||
with db.SessionLocal()() as audit_s:
|
||||
for cid in _rejection_result.matched:
|
||||
append_event(audit_s, AuditEvent(
|
||||
event_type="claim.rejected",
|
||||
entity_type="claim",
|
||||
entity_id=cid,
|
||||
payload={"source_batch_id": synthetic_id, "ack_id": row.id},
|
||||
actor="999-parser",
|
||||
user_id=_actor_user_id(request),
|
||||
))
|
||||
audit_s.commit()
|
||||
|
||||
# SP28: auto-link the 999 AK2 set-responses to claims via the
|
||||
# D10 two-pass join (ST02 via batch envelope index primary,
|
||||
# Claim.patient_control_number fallback). Each created ClaimAck
|
||||
# row publishes claim_ack_written so the live-tail subscribers
|
||||
# on the claim and ack side see the link immediately.
|
||||
claim_ack_links_count = 0
|
||||
with db.SessionLocal()() as link_s:
|
||||
batch_index = store.batch_envelope_index()
|
||||
|
||||
def _pcn_lookup(pcn: str):
|
||||
return (
|
||||
link_s.query(Claim)
|
||||
.filter(Claim.patient_control_number == pcn)
|
||||
.first()
|
||||
)
|
||||
|
||||
link_result = _apply_999_acceptances(
|
||||
link_s, result, ack_id=row.id,
|
||||
batch_envelope_index=batch_index,
|
||||
pc_claim_lookup=_pcn_lookup,
|
||||
)
|
||||
for link_row in link_result.linked:
|
||||
store.add_claim_ack(
|
||||
claim_id=link_row.claim_id,
|
||||
batch_id=link_row.batch_id,
|
||||
ack_id=row.id,
|
||||
ack_kind="999",
|
||||
ak2_index=link_row.ak2_index,
|
||||
set_control_number=link_row.set_control_number,
|
||||
set_accept_reject_code=link_row.set_accept_reject_code,
|
||||
linked_by="auto",
|
||||
event_bus=request.app.state.event_bus,
|
||||
)
|
||||
claim_ack_links_count += 1
|
||||
|
||||
return JSONResponse(content={
|
||||
"ack": {
|
||||
"id": row.id,
|
||||
"accepted_count": accepted,
|
||||
"rejected_count": rejected,
|
||||
"received_count": received,
|
||||
"ack_code": ack_code,
|
||||
"source_batch_id": synthetic_id,
|
||||
"raw_999_text": raw_999_text,
|
||||
"claim_ack_links_count": claim_ack_links_count,
|
||||
},
|
||||
"parsed": json.loads(result.model_dump_json()),
|
||||
})
|
||||
|
||||
|
||||
@router.post("/api/parse-ta1")
|
||||
async def parse_ta1_endpoint(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
) -> Any:
|
||||
"""Parse a TA1 (Interchange Acknowledgment) file, persist a row, return JSON.
|
||||
|
||||
Mirrors ``/api/parse-999`` but for the lower-level envelope ack:
|
||||
- 400 on empty / undecodable / malformed EDI (never 500).
|
||||
- 200 on success with ``{"ta1": {id, control_number, ack_code,
|
||||
note_code, interchange_date, interchange_time, sender_id,
|
||||
receiver_id, source_batch_id, raw_ta1_text}, "parsed":
|
||||
<ParseResultTa1>}``.
|
||||
|
||||
The persisted ``ta1_acks.source_batch_id`` is a synthetic id
|
||||
(``TA1-<ISA13>``) because a received TA1 has no inbound batch to
|
||||
FK to. The dashboard's ``/ta1-acks`` list surfaces these.
|
||||
|
||||
SP25: threads ``event_bus=request.app.state.event_bus`` into
|
||||
``store.add_ta1_ack`` so the live-tail ``ta1_ack_received``
|
||||
stream fires on manual uploads (not just on the SFTP poller).
|
||||
"""
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Empty file", "detail": "Uploaded file contained no bytes."},
|
||||
)
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Encoding error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
try:
|
||||
result = parse_ta1_text(text, input_file=file.filename or "")
|
||||
except CycloneParseError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Parse error", "detail": str(exc)},
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - safety net
|
||||
log.exception("Unexpected parser failure on TA1")
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": "Internal server error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
# Build the raw TA1 text from the parsed result (round-trip).
|
||||
raw_ta1_text = _serialize_ta1(result)
|
||||
|
||||
row = store.add_ta1_ack(
|
||||
source_batch_id=result.source_batch_id,
|
||||
control_number=result.ta1.control_number,
|
||||
interchange_date=result.ta1.interchange_date,
|
||||
interchange_time=result.ta1.interchange_time,
|
||||
ack_code=result.ta1.ack_code,
|
||||
note_code=result.ta1.note_code,
|
||||
ack_generated_date=result.ta1.ack_generated_date,
|
||||
sender_id=result.envelope.sender_id,
|
||||
receiver_id=result.envelope.receiver_id,
|
||||
raw_json=json.loads(result.model_dump_json()),
|
||||
event_bus=request.app.state.event_bus,
|
||||
)
|
||||
|
||||
# SP28: TA1 envelope-level link to the originating Batch. The
|
||||
# closure here matches the most-recent Batch whose envelope
|
||||
# sender_id/receiver_id matches the TA1 — see spec §D4.
|
||||
claim_ack_links_count = 0
|
||||
with db.SessionLocal()() as link_s:
|
||||
def _batch_lookup(sender_id, receiver_id):
|
||||
rows = (
|
||||
link_s.query(Batch)
|
||||
.filter(
|
||||
Batch.kind == "837p",
|
||||
Batch.raw_result_json.isnot(None),
|
||||
)
|
||||
.order_by(Batch.parsed_at.desc())
|
||||
.all()
|
||||
)
|
||||
for row in rows:
|
||||
env = (row.raw_result_json or {}).get("envelope") or {}
|
||||
if (
|
||||
env.get("sender_id") == sender_id
|
||||
and env.get("receiver_id") == receiver_id
|
||||
):
|
||||
return row
|
||||
return None
|
||||
|
||||
link_result = _apply_ta1_envelope_link(
|
||||
link_s, result, ack_id=row.id,
|
||||
batch_lookup=_batch_lookup,
|
||||
)
|
||||
for link_row in link_result.linked:
|
||||
store.add_claim_ack(
|
||||
claim_id=link_row.claim_id,
|
||||
batch_id=link_row.batch_id,
|
||||
ack_id=row.id,
|
||||
ack_kind="ta1",
|
||||
ak2_index=link_row.ak2_index,
|
||||
set_control_number=link_row.set_control_number,
|
||||
set_accept_reject_code=link_row.set_accept_reject_code,
|
||||
linked_by="auto",
|
||||
event_bus=request.app.state.event_bus,
|
||||
)
|
||||
claim_ack_links_count += 1
|
||||
|
||||
return JSONResponse(content={
|
||||
"ta1": {
|
||||
"id": row.id,
|
||||
"control_number": result.ta1.control_number,
|
||||
"ack_code": result.ta1.ack_code,
|
||||
"note_code": result.ta1.note_code,
|
||||
"interchange_date": result.ta1.interchange_date.isoformat()
|
||||
if result.ta1.interchange_date else None,
|
||||
"interchange_time": result.ta1.interchange_time,
|
||||
"sender_id": result.envelope.sender_id,
|
||||
"receiver_id": result.envelope.receiver_id,
|
||||
"source_batch_id": result.source_batch_id,
|
||||
"raw_ta1_text": raw_ta1_text,
|
||||
"claim_ack_links_count": claim_ack_links_count,
|
||||
},
|
||||
"parsed": json.loads(result.model_dump_json()),
|
||||
})
|
||||
|
||||
|
||||
@router.post("/api/parse-277ca")
|
||||
async def parse_277ca_endpoint(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
) -> Any:
|
||||
"""Parse a 277CA Claim Acknowledgment file, persist a row, and stamp rejections.
|
||||
|
||||
Behavior mirrors ``/api/parse-999``:
|
||||
- 400 on empty / undecodable / malformed EDI (never 500).
|
||||
- 200 on success with ``{"ack": {id, control_number, accepted_count,
|
||||
rejected_count, payer_claim_control_numbers, raw_277ca_text},
|
||||
"parsed": <ParseResult277CA>}``.
|
||||
|
||||
After parse, runs :func:`apply_277ca_rejections` to stamp the
|
||||
payer-rejected fields on each matching claim row. The Inbox
|
||||
Payer-Rejected lane lights up as a side-effect of this call.
|
||||
"""
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Empty file", "detail": "Uploaded file contained no bytes."},
|
||||
)
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Encoding error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
try:
|
||||
result = parse_277ca_text(text, input_file=file.filename or "")
|
||||
except CycloneParseError as exc:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Parse error", "detail": str(exc)},
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - safety net
|
||||
log.exception("Unexpected parser failure on 277CA")
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": "Internal server error", "detail": str(exc)},
|
||||
)
|
||||
|
||||
icn = result.envelope.control_number
|
||||
synthetic_id = _277ca_synthetic_source_batch_id(icn)
|
||||
|
||||
accepted = sum(1 for s in result.claim_statuses if s.classification == "accepted")
|
||||
paid = sum(1 for s in result.claim_statuses if s.classification == "paid")
|
||||
rejected = sum(1 for s in result.claim_statuses if s.classification == "rejected")
|
||||
pended = sum(1 for s in result.claim_statuses if s.classification == "pended")
|
||||
|
||||
# Persist the 277CA row first so we have an id to attach to claims.
|
||||
# SP25: thread the event bus so ``two77ca_ack_received`` fires.
|
||||
row = store.add_277ca_ack(
|
||||
source_batch_id=synthetic_id,
|
||||
control_number=icn,
|
||||
accepted_count=accepted,
|
||||
rejected_count=rejected,
|
||||
paid_count=paid,
|
||||
pended_count=pended,
|
||||
raw_json=json.loads(result.model_dump_json()),
|
||||
event_bus=request.app.state.event_bus,
|
||||
)
|
||||
|
||||
# Stamp payer-rejection fields on matching claims. The 277CA's
|
||||
# REF*1K carries the patient's claim control number we sent in
|
||||
# CLM01 — same convention the 999 ACK uses, so the lookup hits
|
||||
# Claim.patient_control_number (mirrors apply_999_rejections).
|
||||
with db.SessionLocal()() as session:
|
||||
def _lookup(pcn: str):
|
||||
return (
|
||||
session.query(Claim)
|
||||
.filter(Claim.patient_control_number == pcn)
|
||||
.first()
|
||||
)
|
||||
apply_result = apply_277ca_rejections(
|
||||
session, result, claim_lookup=_lookup, two77ca_id=row.id,
|
||||
)
|
||||
if apply_result.matched:
|
||||
bus = request.app.state.event_bus
|
||||
for cid in apply_result.matched:
|
||||
await bus.publish("claim.payer_rejected", {"claim_id": cid})
|
||||
# SP11: audit trail for each payer-rejected claim.
|
||||
with db.SessionLocal()() as audit_s:
|
||||
for cid in apply_result.matched:
|
||||
append_event(audit_s, AuditEvent(
|
||||
event_type="claim.payer_rejected",
|
||||
entity_type="claim",
|
||||
entity_id=cid,
|
||||
payload={
|
||||
"source_batch_id": synthetic_id,
|
||||
"277ca_id": row.id,
|
||||
},
|
||||
actor="277ca-parser",
|
||||
user_id=_actor_user_id(request),
|
||||
))
|
||||
audit_s.commit()
|
||||
if apply_result.orphans:
|
||||
log.warning(
|
||||
"277CA had %d orphan status entries (no matching claim): %s",
|
||||
len(apply_result.orphans),
|
||||
apply_result.orphans[:5],
|
||||
)
|
||||
|
||||
# SP28: auto-link the 277CA ClaimStatus entries to claims via
|
||||
# the D10 two-pass join. Each ClaimAck row publishes
|
||||
# claim_ack_written so the live-tail subscribers on the claim
|
||||
# and ack side see the link immediately.
|
||||
claim_ack_links_count = 0
|
||||
with db.SessionLocal()() as link_s:
|
||||
batch_index = store.batch_envelope_index()
|
||||
|
||||
def _pcn_lookup(pcn: str):
|
||||
return (
|
||||
link_s.query(Claim)
|
||||
.filter(Claim.patient_control_number == pcn)
|
||||
.first()
|
||||
)
|
||||
|
||||
link_result = _apply_277ca_acks(
|
||||
link_s, result, ack_id=row.id,
|
||||
batch_envelope_index=batch_index,
|
||||
pc_claim_lookup=_pcn_lookup,
|
||||
)
|
||||
for link_row in link_result.linked:
|
||||
store.add_claim_ack(
|
||||
claim_id=link_row.claim_id,
|
||||
batch_id=link_row.batch_id,
|
||||
ack_id=row.id,
|
||||
ack_kind="277ca",
|
||||
ak2_index=link_row.ak2_index,
|
||||
set_control_number=link_row.set_control_number,
|
||||
set_accept_reject_code=link_row.set_accept_reject_code,
|
||||
linked_by="auto",
|
||||
event_bus=request.app.state.event_bus,
|
||||
)
|
||||
claim_ack_links_count += 1
|
||||
|
||||
return JSONResponse(content={
|
||||
"ack": {
|
||||
"id": row.id,
|
||||
"control_number": icn,
|
||||
"accepted_count": accepted,
|
||||
"rejected_count": rejected,
|
||||
"paid_count": paid,
|
||||
"pended_count": pended,
|
||||
"source_batch_id": synthetic_id,
|
||||
"matched_claim_ids": apply_result.matched,
|
||||
"orphan_status_codes": apply_result.orphans,
|
||||
"claim_ack_links_count": claim_ack_links_count,
|
||||
},
|
||||
"parsed": json.loads(result.model_dump_json()),
|
||||
})
|
||||
@@ -0,0 +1,149 @@
|
||||
"""``GET /api/payers/{payer_id}/summary`` — payer-level rollup for the drill-down panel.
|
||||
|
||||
Returns billed/received totals, denial rate, and the top providers for
|
||||
a given ``payer_id`` (the X12 NM1*PR*PI qualifier, e.g. ``SKCO0``).
|
||||
Cached in-process for ``_SUMMARY_TTL_S`` seconds via a per-payer_id
|
||||
memo. The drill-down UI hammers this on every hover; the underlying
|
||||
query is O(claims) per payer, so the TTL keeps the panel responsive.
|
||||
|
||||
Pubsub invalidation is intentionally NOT wired (see the long comment
|
||||
in the body). The 60s TTL keeps staleness bounded; a follow-up can
|
||||
wire targeted invalidation if the UI proves TTL-bounded staleness is
|
||||
unacceptable.
|
||||
|
||||
404 when no claims AND no remits reference this payer_id — the UI
|
||||
uses that to distinguish a typo from a payer with zero traffic (the
|
||||
latter would still return a valid 200 with zeroed totals).
|
||||
|
||||
SP36 Task 6: this block moved here from ``api.py:3188`` (the
|
||||
``SP21 Task 1.5: payer-level summary`` divider).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from time import monotonic
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.db import Claim, ClaimState, Remittance
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
# Per-payer_id memoization for /api/payers/{payer_id}/summary. The drill-
|
||||
# down UI hammers this on every hover; the underlying query is O(claims)
|
||||
# per payer so a 60s TTL keeps the panel responsive. Process-local on
|
||||
# purpose — invalidation is driven by the TTL alone for now (see note
|
||||
# below).
|
||||
_SUMMARY_TTL_S = 60.0
|
||||
_summary_cache: dict[str, tuple[float, dict]] = {}
|
||||
|
||||
|
||||
def _clear_summary_cache() -> None:
|
||||
"""Test hook: wipe the process-local cache.
|
||||
|
||||
The 60s TTL means tests that want to assert on a recomputed payload
|
||||
must clear the cache between requests. Not used by production code.
|
||||
"""
|
||||
_summary_cache.clear()
|
||||
|
||||
|
||||
# Pubsub invalidation is intentionally NOT wired. The ``claim_written``
|
||||
# and ``remittance_written`` payloads emitted by ``store.add`` are the
|
||||
# UI-shaped dicts from ``to_ui_claim_from_orm`` / ``to_ui_remittance_from_orm``,
|
||||
# neither of which carries the X12 ``payer_id`` (NM1*PR*PI qualifier).
|
||||
# They carry ``payerName`` only, which is the human-readable name and not
|
||||
# the URL key we cache on. Wiring a subscriber here would either need a
|
||||
# DB lookup per event (re-deriving payer_id from Claim.id) or a different
|
||||
# cache key — both are out of scope for this task. The 60s TTL keeps
|
||||
# staleness bounded; a follow-up can wire targeted invalidation if the
|
||||
# UI proves TTL-bounded staleness is unacceptable.
|
||||
|
||||
|
||||
@router.get("/api/payers/{payer_id}/summary")
|
||||
def get_payer_summary(payer_id: str):
|
||||
"""Payer-level rollup for the drill-down panel.
|
||||
|
||||
Returns billed/received totals, denial rate, and top providers for
|
||||
the given ``payer_id`` (the X12 NM1*PR*PI qualifier, e.g. ``SKCO0``).
|
||||
Cached in-process for ``_SUMMARY_TTL_S`` seconds.
|
||||
|
||||
404 when no claims AND no remits reference this payer_id — the UI
|
||||
uses that to distinguish a typo from a payer with zero traffic
|
||||
(the latter would still return a valid 200 with zeroed totals).
|
||||
"""
|
||||
now = monotonic()
|
||||
cached = _summary_cache.get(payer_id)
|
||||
if cached and (now - cached[0]) < _SUMMARY_TTL_S:
|
||||
return cached[1]
|
||||
|
||||
log.debug("payer summary cache miss", extra={"payer_id": payer_id})
|
||||
|
||||
# Query claims + remits scoped to this payer_id. We bypass
|
||||
# ``store.iter_claims`` because that helper filters by payer NAME
|
||||
# substring, not by the X12 PI qualifier we use as the URL key.
|
||||
with db.SessionLocal()() as s:
|
||||
claim_rows: list[Claim] = (
|
||||
s.query(Claim).filter(Claim.payer_id == payer_id).all()
|
||||
)
|
||||
claim_ids = [c.id for c in claim_rows]
|
||||
remit_rows: list[Remittance] = []
|
||||
if claim_ids:
|
||||
remit_rows = (
|
||||
s.query(Remittance)
|
||||
.filter(Remittance.claim_id.in_(claim_ids))
|
||||
.all()
|
||||
)
|
||||
|
||||
if not claim_rows and not remit_rows:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Payer {payer_id!r} not found",
|
||||
)
|
||||
|
||||
billed_total = sum(float(c.charge_amount or 0) for c in claim_rows)
|
||||
received_total = sum(float(r.total_paid or 0) for r in remit_rows)
|
||||
denied = sum(
|
||||
1 for c in claim_rows if c.state == ClaimState.DENIED
|
||||
)
|
||||
claim_count = len(claim_rows)
|
||||
denial_rate = (denied / claim_count) if claim_count else 0.0
|
||||
|
||||
provider_counts: dict[str, int] = {}
|
||||
for c in claim_rows:
|
||||
npi = c.provider_npi
|
||||
if not npi:
|
||||
continue
|
||||
provider_counts[npi] = provider_counts.get(npi, 0) + 1
|
||||
top_providers = [
|
||||
{"npi": npi, "count": count}
|
||||
for npi, count in sorted(
|
||||
provider_counts.items(), key=lambda kv: -kv[1]
|
||||
)[:5]
|
||||
]
|
||||
|
||||
# Best-effort payer display name from the first claim's raw
|
||||
# payer object (NM1*PR name element, e.g. "COHCPF"). Falls
|
||||
# back to the id when no parsed envelope is available.
|
||||
payer_name = payer_id
|
||||
for c in claim_rows:
|
||||
raw = c.raw_json or {}
|
||||
p = raw.get("payer") if isinstance(raw, dict) else None
|
||||
if isinstance(p, dict) and p.get("name"):
|
||||
payer_name = p["name"]
|
||||
break
|
||||
|
||||
payload = {
|
||||
"payer_id": payer_id,
|
||||
"name": payer_name,
|
||||
"claim_count": claim_count,
|
||||
"billed_total": billed_total,
|
||||
"received_total": received_total,
|
||||
"denial_rate": denial_rate,
|
||||
"top_providers": top_providers,
|
||||
}
|
||||
_summary_cache[payer_id] = (now, payload)
|
||||
return payload
|
||||
@@ -0,0 +1,150 @@
|
||||
"""``/api/providers`` and ``/api/config/providers*`` — read views over providers.
|
||||
|
||||
Three endpoints across two URL prefixes, all gated by ``matrix_gate``:
|
||||
|
||||
- ``GET /api/providers`` — distinct provider list
|
||||
derived from the Claims population (``store.distinct_providers()``),
|
||||
with ``npi`` / ``state`` filters, pagination, and an NDJSON variant.
|
||||
- ``GET /api/config/providers`` — the configured provider
|
||||
rows (3 NPIs for SP9), filtered by ``is_active``.
|
||||
- ``GET /api/config/providers/{npi}`` — one configured provider
|
||||
plus a small drill-down block: ``recent_claims`` (top-10 by
|
||||
``submissionDate``) and ``recent_activity`` (top-10, joined via
|
||||
``Claim.id`` because ``ActivityEvent`` has no ``provider_npi``
|
||||
column; the outer-join via ``Remittance.claim_id`` surfaces the
|
||||
orphan ``remit_received`` events that were recorded pre-match).
|
||||
|
||||
SP36 Task 12: this block moved here from ``api.py:2448`` (the
|
||||
``/api/providers`` list, the ``/api/config/providers`` list, and
|
||||
the ``/api/config/providers/{npi}`` detail) — three URL prefixes,
|
||||
one router per spec.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy import desc, or_
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.api_helpers import (
|
||||
ndjson_stream_list as _ndjson_stream_list,
|
||||
wants_ndjson as _wants_ndjson,
|
||||
)
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.db import Claim, Remittance
|
||||
from cyclone.store import store
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.get("/api/providers")
|
||||
def list_providers(
|
||||
request: Request,
|
||||
npi: str | None = Query(None),
|
||||
state: str | None = Query(None),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
) -> Any:
|
||||
items = store.distinct_providers()
|
||||
if npi is not None:
|
||||
items = [p for p in items if p["npi"] == npi]
|
||||
if state is not None:
|
||||
items = [p for p in items if p.get("state") == state]
|
||||
paged = items[offset:offset + limit]
|
||||
total = len(items)
|
||||
returned = len(paged)
|
||||
has_more = total > offset + returned
|
||||
if _wants_ndjson(request):
|
||||
return StreamingResponse(
|
||||
_ndjson_stream_list(paged, total, returned, has_more),
|
||||
media_type="application/x-ndjson",
|
||||
)
|
||||
return {
|
||||
"items": paged,
|
||||
"total": total,
|
||||
"returned": returned,
|
||||
"has_more": has_more,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/config/providers")
|
||||
def list_configured_providers(is_active: bool | None = Query(default=True)):
|
||||
"""List the configured provider rows (3 NPIs for SP9)."""
|
||||
return [json.loads(p.model_dump_json()) for p in store.list_providers(is_active=is_active)]
|
||||
|
||||
|
||||
@router.get("/api/config/providers/{npi}")
|
||||
def get_configured_provider(npi: str):
|
||||
p = store.get_provider(npi)
|
||||
if p is None:
|
||||
raise HTTPException(status_code=404, detail=f"provider {npi!r} not found")
|
||||
provider_dict = json.loads(p.model_dump_json())
|
||||
|
||||
# SP21 Task 1.6: extend the response with two top-N arrays that the
|
||||
# drill-down peek panel hangs off. ``recent_claims`` reuses the
|
||||
# existing store projection (already returns UI-shaped dicts with
|
||||
# ``submissionDate``); ``recent_activity`` is a direct ORM join
|
||||
# because ``ActivityEvent`` has no ``provider_npi`` column — the
|
||||
# filter has to hop through ``Claim.id``.
|
||||
recent_claims = sorted(
|
||||
store.iter_claims(provider_npi=npi),
|
||||
key=lambda c: c.get("submissionDate") or "",
|
||||
reverse=True,
|
||||
)[:10]
|
||||
|
||||
# Activity filter has TWO join paths back to a Claim for this
|
||||
# provider:
|
||||
# 1. ``ActivityEvent.claim_id IN (claim_ids)`` — events that were
|
||||
# recorded with a claim FK already set (claim_submitted,
|
||||
# manual_match, claim_paid, etc.).
|
||||
# 2. ``Remittance.claim_id IN (claim_ids)`` — the original
|
||||
# ``remit_received`` event recorded at 835 ingest time
|
||||
# (``store.add`` lines 999-1003) is inserted with
|
||||
# ``claim_id=None`` because the remittance hasn't been matched
|
||||
# to a claim yet. The auto-reconcile pass later sets
|
||||
# ``Remittance.claim_id`` (``reconcile.run`` lines 289-293),
|
||||
# but the *original* ActivityEvent row stays orphaned. The
|
||||
# outer-join-then-OR lets us surface both shapes. Without
|
||||
# path 2, a provider's activity feed looks frozen the moment
|
||||
# an 835 lands — the most common activity, invisible.
|
||||
with db.SessionLocal()() as s:
|
||||
claim_ids = [
|
||||
cid
|
||||
for (cid,) in s.query(Claim.id)
|
||||
.filter(Claim.provider_npi == npi)
|
||||
.all()
|
||||
]
|
||||
activity_rows = []
|
||||
if claim_ids:
|
||||
activity_rows = (
|
||||
s.query(db.ActivityEvent)
|
||||
.outerjoin(
|
||||
Remittance,
|
||||
db.ActivityEvent.remittance_id == Remittance.id,
|
||||
)
|
||||
.filter(or_(
|
||||
db.ActivityEvent.claim_id.in_(claim_ids),
|
||||
Remittance.claim_id.in_(claim_ids),
|
||||
))
|
||||
.order_by(desc(db.ActivityEvent.ts))
|
||||
.limit(10)
|
||||
.all()
|
||||
)
|
||||
|
||||
def _activity_to_ui(a):
|
||||
return {
|
||||
"id": a.id,
|
||||
"ts": a.ts.isoformat().replace("+00:00", "Z") if a.ts else "",
|
||||
"kind": a.kind,
|
||||
"batchId": a.batch_id,
|
||||
"claimId": a.claim_id,
|
||||
"remittanceId": a.remittance_id,
|
||||
"payload": a.payload_json or {},
|
||||
}
|
||||
|
||||
provider_dict["recent_claims"] = recent_claims
|
||||
provider_dict["recent_activity"] = [_activity_to_ui(a) for a in activity_rows]
|
||||
return provider_dict
|
||||
@@ -0,0 +1,221 @@
|
||||
"""SP41 — rebill admin endpoints.
|
||||
|
||||
POST /api/admin/rebill-from-835
|
||||
body: {"window": "YYYY-MM-DD..YYYY-MM-DD",
|
||||
"override_filing": bool,
|
||||
"visits_csv_path": str (optional),
|
||||
"ingest_dir": str (optional),
|
||||
"out_dir": str (optional)}
|
||||
Returns: {"summary_path": str, "counts": {...},
|
||||
"pipeline_a_files": [...], "pipeline_b_files": [...]}
|
||||
|
||||
GET /api/admin/rebill-from-835/status
|
||||
Returns: {"recent_runs": [{"as_of": ..., "summary_path": ...,
|
||||
"counts": {...}}, ...]}
|
||||
|
||||
Status-code contract (per the cyclone-api-router / cyclone-cli skills):
|
||||
- 200: completed run (POST) or tally returned (GET).
|
||||
- 401: not authenticated (matrix_gate).
|
||||
- 403: authenticated but not authorized for /api/admin/*.
|
||||
- 422: window is malformed or Pydantic body validation failed.
|
||||
|
||||
The POST handler delegates to ``cyclone.rebill.run.run_rebill`` (the same
|
||||
orchestrator the ``cyclone rebill-from-835`` CLI uses). The GET handler
|
||||
is a filesystem scan under ``dev/rebills/*/summary.csv`` — no DB table
|
||||
for "recent runs" exists today, and Task 12 deliberately doesn't add
|
||||
one (per its design notes). Sorted by directory mtime descending;
|
||||
truncated to the most recent 5.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.rebill.run import run_rebill
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/admin/rebill-from-835",
|
||||
tags=["rebill"],
|
||||
dependencies=[Depends(matrix_gate)],
|
||||
)
|
||||
|
||||
|
||||
# Filesystem base for /status scans. Module-level so tests can monkeypatch
|
||||
# it to a tmp_path-relative dir without chdir-ing the whole test process.
|
||||
REBILLS_DIR: Path = Path("dev/rebills")
|
||||
|
||||
# How many recent runs /status surfaces. 5 matches the CLI's default; the
|
||||
# body schema below mirrors it as a query parameter so callers can ask for
|
||||
# more (or fewer) when they want.
|
||||
DEFAULT_RECENT_LIMIT = 5
|
||||
|
||||
|
||||
class RebillRequest(BaseModel):
|
||||
"""Body schema for ``POST /api/admin/rebill-from-835``.
|
||||
|
||||
``window`` is parsed as ``YYYY-MM-DD..YYYY-MM-DD`` (inclusive both
|
||||
ends); the validator rejects anything else with a 422. The other
|
||||
path fields are optional — ``run_rebill`` falls back to its own
|
||||
defaults (``data/source/apr-jun27.csv`` for visits, ``ingest/`` for
|
||||
835s, ``dev/rebills/<today>/`` for output) when unset.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
window: str = Field(
|
||||
default="2026-01-01..2026-06-27",
|
||||
description="DOS window as YYYY-MM-DD..YYYY-MM-DD (inclusive both ends).",
|
||||
)
|
||||
override_filing: bool = Field(
|
||||
default=False,
|
||||
description="Relax the 120-day timely-filing gate for past-window visits.",
|
||||
)
|
||||
visits_csv_path: str | None = Field(
|
||||
default=None,
|
||||
description="Path to the AxisCare visits CSV; defaults to "
|
||||
"data/source/apr-jun27.csv when unset.",
|
||||
)
|
||||
ingest_dir: str | None = Field(
|
||||
default=None,
|
||||
description="Directory containing *.835 / *.x12 835 files; defaults to ./ingest.",
|
||||
)
|
||||
out_dir: str | None = Field(
|
||||
default=None,
|
||||
description="Output directory for summary.csv + pipeline-a/b files; "
|
||||
"defaults to dev/rebills/<today>/.",
|
||||
)
|
||||
|
||||
@field_validator("window")
|
||||
@classmethod
|
||||
def _validate_window(cls, v: str) -> str:
|
||||
# Manual split (Click can't help here). Two-date range is the
|
||||
# only accepted shape; ``..`` is the giveaway separator so the
|
||||
# input is unambiguous.
|
||||
try:
|
||||
start_str, end_str = v.split("..", 1)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"window must be 'YYYY-MM-DD..YYYY-MM-DD', got {v!r}"
|
||||
) from exc
|
||||
try:
|
||||
start = date.fromisoformat(start_str)
|
||||
end = date.fromisoformat(end_str)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"window must be 'YYYY-MM-DD..YYYY-MM-DD', got {v!r}: {exc}"
|
||||
) from exc
|
||||
if start > end:
|
||||
raise ValueError(
|
||||
f"window start {start.isoformat()} is after end {end.isoformat()}"
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
@router.post("")
|
||||
def post_rebill(req: RebillRequest) -> dict[str, Any]:
|
||||
"""Run the rebill pipeline for the given DOS window.
|
||||
|
||||
Delegates to :func:`cyclone.rebill.run.run_rebill` and returns the
|
||||
resulting ``RunResult`` as a plain JSON dict (the underlying dataclass
|
||||
has no ``to_dict`` method — fields are mapped here so callers don't
|
||||
have to import the dataclass shape).
|
||||
|
||||
The handler does NOT swallow exceptions; the app-level
|
||||
:func:`_unhandled_exception_handler` renders them as a 500 JSON
|
||||
envelope with CORS headers. Per-file failures land in the summary
|
||||
CSV (and the ``counts`` dict) rather than raising here.
|
||||
"""
|
||||
start_str, end_str = req.window.split("..", 1)
|
||||
# Validator already proved these parse cleanly.
|
||||
window_start = date.fromisoformat(start_str)
|
||||
window_end = date.fromisoformat(end_str)
|
||||
|
||||
log.info(
|
||||
"rebill-from-835 starting: window=%s override_filing=%s",
|
||||
req.window, req.override_filing,
|
||||
)
|
||||
result = run_rebill(
|
||||
window_start=window_start,
|
||||
window_end=window_end,
|
||||
override_filing=req.override_filing,
|
||||
visits_csv_path=req.visits_csv_path,
|
||||
ingest_dir=req.ingest_dir,
|
||||
out_dir=req.out_dir,
|
||||
)
|
||||
log.info(
|
||||
"rebill-from-835 done: summary=%s counts=%s",
|
||||
result.summary_path, result.counts,
|
||||
)
|
||||
|
||||
return {
|
||||
"summary_path": str(result.summary_path),
|
||||
"counts": result.counts,
|
||||
"pipeline_a_files": [str(p) for p in result.pipeline_a_files],
|
||||
"pipeline_b_files": [str(p) for p in result.pipeline_b_files],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
def get_rebill_status(
|
||||
limit: int = DEFAULT_RECENT_LIMIT,
|
||||
) -> dict[str, Any]:
|
||||
"""Return the most recent rebill runs, newest first.
|
||||
|
||||
Scans :data:`REBILLS_DIR` (``dev/rebills/`` by default) for any
|
||||
subdirectory whose name is a date in YYYY-MM-DD format and that
|
||||
contains a ``summary.csv``. Sorted by directory mtime descending so
|
||||
the most recently-run batch wins ties on equal-named dirs (rare in
|
||||
practice — operators tend to pick a fresh date per run).
|
||||
|
||||
Each entry's ``counts`` dict is a tally of the summary.csv's
|
||||
``disposition`` column (the same per-category counters the operator
|
||||
sees on the CLI ``--status`` view). Missing disposition values
|
||||
surface as ``UNKNOWN`` so a hand-edited CSV can't silently drop a
|
||||
bucket.
|
||||
|
||||
``limit`` defaults to :data:`DEFAULT_RECENT_LIMIT` (5) and is
|
||||
clamped to ``[1, 50]`` to keep the response bounded.
|
||||
"""
|
||||
if limit < 1 or limit > 50:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="limit must be between 1 and 50",
|
||||
)
|
||||
|
||||
recent: list[dict[str, Any]] = []
|
||||
rebills_dir = REBILLS_DIR
|
||||
if rebills_dir.exists():
|
||||
candidates = sorted(
|
||||
(
|
||||
d for d in rebills_dir.iterdir()
|
||||
if d.is_dir()
|
||||
and (d / "summary.csv").exists()
|
||||
# Filter to date-named dirs (YYYY-MM-DD) so a stray
|
||||
# ``lost+found`` or tempdir doesn't sneak in.
|
||||
and len(d.name) == 10 and d.name[4] == "-" and d.name[7] == "-"
|
||||
),
|
||||
key=lambda d: d.stat().st_mtime,
|
||||
reverse=True,
|
||||
)[:limit]
|
||||
for d in candidates:
|
||||
summary_path = d / "summary.csv"
|
||||
counts: dict[str, int] = {}
|
||||
with summary_path.open(newline="") as f:
|
||||
for row in csv.DictReader(f):
|
||||
disp = (row.get("disposition", "UNKNOWN") or "").strip() or "UNKNOWN"
|
||||
counts[disp] = counts.get(disp, 0) + 1
|
||||
recent.append({
|
||||
"as_of": d.name,
|
||||
"summary_path": str(summary_path),
|
||||
"counts": counts,
|
||||
})
|
||||
return {"recent_runs": recent}
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Reconciliation read views + manual match/unmatch write paths.
|
||||
|
||||
Four endpoints:
|
||||
|
||||
- ``GET /api/reconciliation/unmatched`` — list of unmatched Claims
|
||||
and unmatched Remittances (``store.list_unmatched(kind="both")``).
|
||||
- ``GET /api/batch-diff`` — side-by-side diff of two
|
||||
batches (used by the Batch Diff page). Lazy-imports
|
||||
:func:`cyclone.batch_diff.diff_batches_to_wire` to keep the
|
||||
module's import surface small until the endpoint is actually
|
||||
hit.
|
||||
- ``POST /api/reconciliation/match`` — manually pair a Claim with
|
||||
a Remittance (``store.manual_match``). Surfaces ``AlreadyMatchedError`` /
|
||||
``InvalidStateError`` as 409, and ``LookupError`` from the store
|
||||
as 404 (``claim_or_remit_not_found``).
|
||||
- ``POST /api/reconciliation/unmatch`` — unpair a Claim (``store.manual_unmatch``).
|
||||
Surfaces ``NotMatchedError`` as 409.
|
||||
|
||||
All four are read-or-manual-override surfaces used by the
|
||||
Reconciliation page (the page that pairs Claims with Remittances
|
||||
when neither side has an automatic match key).
|
||||
|
||||
SP36 Task 8: this block moved here from ``api.py:2450`` (the
|
||||
4 routes interleaved with a side-by-side batch-diff divider).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.store import (
|
||||
AlreadyMatchedError,
|
||||
InvalidStateError,
|
||||
store,
|
||||
)
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.get("/api/reconciliation/unmatched")
|
||||
def get_reconciliation_unmatched() -> dict:
|
||||
"""Return unmatched Claims (left) and unmatched Remittances (right).
|
||||
|
||||
Powers the reconciliation review surface: every Claim with no
|
||||
paired Remittance appears on the left, every Remittance with no
|
||||
paired Claim appears on the right. The two lists are always present
|
||||
(empty list, never absent) so the UI can index unconditionally.
|
||||
"""
|
||||
return store.list_unmatched(kind="both")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Side-by-side diff between two batches (SP3 P4 / T18)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@router.get("/api/batch-diff")
|
||||
def get_batch_diff(
|
||||
a: str | None = Query(None),
|
||||
b: str | None = Query(None),
|
||||
) -> dict:
|
||||
"""Return a side-by-side diff of two batches identified by id.
|
||||
|
||||
Query params: ``a=<batch_id>``, ``b=<batch_id>`` (both required).
|
||||
|
||||
Response body (snake_case keys, see :mod:`cyclone.batch_diff` for the
|
||||
projector shapes):
|
||||
- ``a`` / ``b`` — small metadata blocks (id, kind, parsedAt,
|
||||
inputFilename, claimCount)
|
||||
- ``added`` — claims present in B but not A
|
||||
- ``removed`` — claims present in A but not B
|
||||
- ``changed`` — claims present in both, with field deltas
|
||||
- ``summary`` — precomputed counts
|
||||
|
||||
Errors:
|
||||
- 400 — missing ``a`` or ``b``
|
||||
- 404 — either batch id is unknown
|
||||
|
||||
Pure read endpoint — never mutates the store. Both 837P and 835
|
||||
batches are accepted (mixed-kind diffs are valid: comparing the
|
||||
submitted claims against the matching remittances).
|
||||
"""
|
||||
if not a or not b:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": "Missing param", "detail": "Both ?a=<batch_id> and ?b=<batch_id> are required."},
|
||||
)
|
||||
try:
|
||||
a_rec, b_rec = store.load_two_for_diff(a, b)
|
||||
except LookupError as exc:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"error": "Not found", "detail": str(exc)},
|
||||
)
|
||||
|
||||
# Lazy import — keeps the module's import surface small until the
|
||||
# endpoint is actually hit. Mirrors the same pattern used by other
|
||||
# endpoint-local helpers (e.g. reconciler).
|
||||
from cyclone.batch_diff import diff_batches_to_wire
|
||||
|
||||
return diff_batches_to_wire(a_rec, b_rec)
|
||||
|
||||
|
||||
@router.post("/api/reconciliation/match")
|
||||
def post_reconciliation_match(body: dict) -> dict:
|
||||
"""Manually pair a Claim with a Remittance (operator override).
|
||||
|
||||
Body: ``{"claim_id": ..., "remit_id": ...}``. Returns
|
||||
``{"claim": <ui>, "match": <ui>}`` on success. Errors:
|
||||
- 400: missing ``claim_id`` or ``remit_id``
|
||||
- 404: claim or remittance not found
|
||||
- 409: claim already matched, or apply_* returned a noop
|
||||
(claim in terminal state) — detail echoes ``current_state``
|
||||
and ``activity_kind`` so the UI can render a precise message.
|
||||
"""
|
||||
claim_id = body.get("claim_id")
|
||||
remit_id = body.get("remit_id")
|
||||
if not claim_id or not remit_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="claim_id and remit_id required",
|
||||
)
|
||||
try:
|
||||
return store.manual_match(claim_id, remit_id)
|
||||
except AlreadyMatchedError as e:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"error": "already_matched", "message": str(e)},
|
||||
)
|
||||
except InvalidStateError as e:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"error": "invalid_state",
|
||||
"current_state": e.current_state,
|
||||
"activity_kind": e.activity_kind,
|
||||
},
|
||||
)
|
||||
except LookupError:
|
||||
# manual_match raises LookupError when the claim or remittance
|
||||
# row is missing (we catch the parent class so any future
|
||||
# KeyError subclasses in the store get the same treatment).
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="claim_or_remit_not_found",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/reconciliation/unmatch")
|
||||
def post_reconciliation_unmatch(body: dict) -> dict:
|
||||
"""Remove the current match for a Claim; reset Claim to submitted.
|
||||
|
||||
Body: ``{"claim_id": ...}``. Returns
|
||||
``{"claim": <ui>, "deletedMatches": <count>}``. Errors:
|
||||
- 400: missing ``claim_id``
|
||||
- 404: claim not found
|
||||
- 409: claim has no current match (NotMatchedError is mapped
|
||||
by the store; we surface 409 to match the manual_match contract)
|
||||
"""
|
||||
from cyclone.store import NotMatchedError
|
||||
claim_id = body.get("claim_id")
|
||||
if not claim_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="claim_id required",
|
||||
)
|
||||
try:
|
||||
return store.manual_unmatch(claim_id)
|
||||
except NotMatchedError as e:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"error": "not_matched", "message": str(e)},
|
||||
)
|
||||
except LookupError:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="claim_not_found",
|
||||
)
|
||||
@@ -0,0 +1,169 @@
|
||||
"""``/api/remittances`` and ``/api/remittances/stream`` — read views over the Remittance population.
|
||||
|
||||
Four endpoints, all gated by ``matrix_gate``:
|
||||
|
||||
- ``GET /api/remittances`` — paginated list with filter+sort,
|
||||
plus an NDJSON variant when the caller sends ``Accept: application/x-ndjson``.
|
||||
- ``GET /api/remittances/summary`` — server-aggregated KPI tiles
|
||||
(``count``, ``total_paid``, ``total_adjustments``) over the full
|
||||
filtered population — never a page-limited sample (SP27 fix).
|
||||
- ``GET /api/remittances/stream`` — NDJSON live-tail: snapshot
|
||||
of currently-known rows, then ``remittance_written`` events as
|
||||
they hit the store. Subscribed to by the Remittances page.
|
||||
- ``GET /api/remittances/{remittance_id}`` — one remittance with its
|
||||
labeled CAS ``adjustments`` array. 404 on missing id (never 500).
|
||||
|
||||
``/api/remittances/stream`` is registered before
|
||||
``/api/remittances/{remittance_id}`` so the literal ``stream`` path
|
||||
segment is not captured as a remittance id.
|
||||
|
||||
SP36 Task 9: this block moved here from ``api.py:2448`` (the 4
|
||||
``/api/remittances*`` routes, with the streaming route's
|
||||
``NOTE: registered before…`` comment preserved verbatim).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from cyclone.api_helpers import (
|
||||
ndjson_line as _ndjson_line,
|
||||
ndjson_stream_list as _ndjson_stream_list,
|
||||
tail_events as _tail_events,
|
||||
wants_ndjson as _wants_ndjson,
|
||||
)
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.pubsub import EventBus
|
||||
from cyclone.store import store
|
||||
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
@router.get("/api/remittances")
|
||||
def list_remittances(
|
||||
request: Request,
|
||||
batch_id: str | None = Query(None),
|
||||
payer: str | None = Query(None),
|
||||
claim_id: str | None = Query(None),
|
||||
date_from: str | None = Query(None),
|
||||
date_to: str | None = Query(None),
|
||||
sort: str | None = Query(None),
|
||||
order: str = Query("desc"),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
) -> Any:
|
||||
common = dict(
|
||||
batch_id=batch_id,
|
||||
payer=payer,
|
||||
claim_id=claim_id,
|
||||
date_from=date_from,
|
||||
date_to=date_to,
|
||||
)
|
||||
items = list(store.iter_remittances(
|
||||
sort=sort, order=order, limit=limit, offset=offset, **common,
|
||||
))
|
||||
# SP27 Task 13b: count the full population, not a 100-row sample.
|
||||
# See the matching note in list_claims — same silent-failure pattern.
|
||||
total = store.count_remittances(**common)
|
||||
returned = len(items)
|
||||
has_more = total > offset + returned
|
||||
if _wants_ndjson(request):
|
||||
return StreamingResponse(
|
||||
_ndjson_stream_list(items, total, returned, has_more),
|
||||
media_type="application/x-ndjson",
|
||||
)
|
||||
return {
|
||||
"items": items,
|
||||
"total": total,
|
||||
"returned": returned,
|
||||
"has_more": has_more,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/remittances/summary")
|
||||
def remittances_summary(
|
||||
batch_id: str | None = Query(None),
|
||||
payer: str | None = Query(None),
|
||||
claim_id: str | None = Query(None),
|
||||
date_from: str | None = Query(None),
|
||||
date_to: str | None = Query(None),
|
||||
) -> dict:
|
||||
"""Server-aggregated KPI tiles for the Remittances page.
|
||||
|
||||
Returns ``{count, total_paid, total_adjustments}`` over the
|
||||
full filtered remittance population — NOT a page-limited
|
||||
sample. The Remittances page consumes this for its "Total paid"
|
||||
and "Adjustments" tiles so they can't silently understate the
|
||||
true DB population the way a page-local ``items.reduce(...)``
|
||||
would. Mirrors the silent-incompleteness fix that
|
||||
``/api/dashboard/kpis`` (commit ``59c3275``) and
|
||||
``/api/remittances`` (commit ``d81b6ed``) made for their tiles.
|
||||
|
||||
Same filter parameters as ``/api/remittances``. Always returns
|
||||
a populated dict (``{"count": 0, "total_paid": 0,
|
||||
"total_adjustments": 0}`` when no rows match) so the frontend
|
||||
can render the tiles directly without a loading-vs-empty
|
||||
branch.
|
||||
"""
|
||||
return store.summarize_remittances(
|
||||
batch_id=batch_id, payer=payer, claim_id=claim_id,
|
||||
date_from=date_from, date_to=date_to,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/remittances/stream")
|
||||
async def remittances_stream(
|
||||
request: Request,
|
||||
payer: str | None = Query(None),
|
||||
claim_id: str | None = Query(None),
|
||||
date_from: str | None = Query(None),
|
||||
date_to: str | None = Query(None),
|
||||
sort: str | None = Query(None),
|
||||
order: str = Query("desc"),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
) -> StreamingResponse:
|
||||
"""Stream Remittances as NDJSON: snapshot first, then live events.
|
||||
|
||||
Subscribes to ``remittance_written``. Default sort is
|
||||
``-received_date`` (newest-first), matching the list endpoint's
|
||||
most common sort.
|
||||
|
||||
NOTE: registered before ``/api/remittances/{remittance_id}`` so
|
||||
the literal ``stream`` path segment doesn't get matched as a
|
||||
remittance id.
|
||||
"""
|
||||
bus: EventBus = request.app.state.event_bus
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
rows = store.iter_remittances(
|
||||
payer=payer, claim_id=claim_id,
|
||||
date_from=date_from, date_to=date_to,
|
||||
sort=sort or "-received_date", order=order, limit=limit,
|
||||
)
|
||||
for row in rows:
|
||||
yield _ndjson_line({"type": "item", "data": row})
|
||||
yield _ndjson_line({"type": "snapshot_end", "data": {"count": len(rows)}})
|
||||
|
||||
async for chunk in _tail_events(request, bus, ["remittance_written"]):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
@router.get("/api/remittances/{remittance_id}")
|
||||
def get_remittance(remittance_id: str) -> dict:
|
||||
"""Return one remittance with its labeled CAS ``adjustments`` array.
|
||||
|
||||
Path param is ``remittance_id`` (not ``id``) to avoid shadowing
|
||||
FastAPI's internal ``id`` name and to keep OpenAPI docs self-
|
||||
describing. Returns 404 when the remittance is missing — never 500.
|
||||
"""
|
||||
body = store.get_remittance(remittance_id)
|
||||
if body is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"error": "Not found", "detail": f"Remittance {remittance_id} not found"},
|
||||
)
|
||||
return body
|
||||
@@ -0,0 +1,213 @@
|
||||
"""SP37 Task 6: HTTP endpoint for the canonical submit-batch flow.
|
||||
|
||||
Thin wrapper around ``cyclone.submission.submit_file`` — same logic as
|
||||
the ``cyclone submit-batch`` CLI (SP37 Task 5), just framed as JSON in
|
||||
/ JSON out and gated by ``matrix_gate``. The walker pattern, ``._*``
|
||||
AppleDouble skip, ``limit`` semantics, and per-file outcomes all match
|
||||
the CLI byte-for-byte so a batch run via the CLI and the same batch
|
||||
run via this endpoint produce identical DB + SFTP state.
|
||||
|
||||
The endpoint deliberately does NOT inject an ``sftp_client_factory``:
|
||||
``submit_file`` defaults to its paramiko-based factory so SKIPPED is
|
||||
reachable in production (the ``SftpClient`` wrapper has no ``stat()``).
|
||||
Tests monkey-patch ``cyclone.api_routers.submission.submit_file``
|
||||
itself; that avoids the paramiko factory entirely without touching
|
||||
the helper's contract.
|
||||
|
||||
Status code contract (per Task 6 spec §4):
|
||||
- 200: completed run. Per-file failures live in the JSON body.
|
||||
- 401: not authenticated (matrix_gate).
|
||||
- 404: no clearhouse seeded (config-level "missing" → 4xx, not 5xx).
|
||||
- 409: clearhouse SFTP block is in stub mode (refuses to upload).
|
||||
- 422: ``ingest_dir`` missing on disk OR Pydantic body validation
|
||||
failed (missing fields, wrong types).
|
||||
- 5xx: truly unexpected exceptions propagate (do not swallow).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.store import store as cycl_store
|
||||
from cyclone.store.exceptions import DuplicateClaimError
|
||||
from cyclone.submission.core import submit_file
|
||||
from cyclone.submission.result import SubmitOutcome, SubmitResult
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# No `prefix=` here — every other gated router in this package declares
|
||||
# the full path in the decorator (clearhouse.py uses "/api/clearhouse",
|
||||
# parse.py uses "/api/parse-837", etc.). The decorator sets the full URL.
|
||||
router = APIRouter(
|
||||
tags=["submission"],
|
||||
dependencies=[Depends(matrix_gate)],
|
||||
)
|
||||
|
||||
|
||||
class SubmitBatchRequest(BaseModel):
|
||||
"""Body schema for ``POST /api/submit-batch``.
|
||||
|
||||
``ingest_dir`` has no default so Pydantic raises 422 when it's
|
||||
missing — better UX than letting the walker crash on a missing
|
||||
path. ``validate_files`` and ``actor`` default so a minimal client
|
||||
can skip them. ``limit`` truncates the file list after the walker
|
||||
collects it (mirrors the CLI's post-collection ``if i > limit:
|
||||
break`` semantics, but applied as a slice since the HTTP body
|
||||
model is type-checked up-front).
|
||||
|
||||
The ``validate_files`` field is aliased to ``validate`` in the JSON
|
||||
body to mirror the CLI's ``--validate`` flag and avoid the
|
||||
hardcoded Pydantic warning about ``validate`` shadowing
|
||||
``BaseModel.validate``. ``populate_by_name=True`` lets tests
|
||||
construct the model with either key.
|
||||
"""
|
||||
model_config = ConfigDict(populate_by_name=True, protected_namespaces=())
|
||||
|
||||
ingest_dir: str
|
||||
validate_files: bool = Field(default=True, alias="validate")
|
||||
actor: str = "api-submit-batch"
|
||||
limit: int | None = None
|
||||
|
||||
|
||||
@router.post("/api/submit-batch")
|
||||
def submit_batch(body: SubmitBatchRequest):
|
||||
"""Submit every ``batch-*-claims/*.x12`` under ``ingest_dir``.
|
||||
|
||||
Walks ``ingest_dir`` for any directory matching ``batch-*-claims``,
|
||||
collects each one's ``*.x12`` files (sorted, with ``._*``
|
||||
AppleDouble files skipped), truncates to ``limit`` if set, then
|
||||
calls :func:`cyclone.submission.submit_file` per file with the
|
||||
seeded clearhouse's ``sftp_block`` and ``actor`` from the body.
|
||||
|
||||
Returns counts (``submitted`` / ``skipped`` / ``failed``) plus a
|
||||
per-file ``results`` array. Per-file failures NEVER change the
|
||||
HTTP status code — the response is 200 whenever the run itself
|
||||
completed.
|
||||
"""
|
||||
|
||||
# 1. Config-level guards. Order matters: a missing clearhouse is a
|
||||
# 404 (config-level "missing"), but if it IS present and in stub
|
||||
# mode the operator's request is a 409 (configured-but-wrong).
|
||||
clearhouse = cycl_store.get_clearhouse()
|
||||
if clearhouse is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail="no clearhouse seeded",
|
||||
)
|
||||
sftp_block = clearhouse.sftp_block
|
||||
if sftp_block.stub:
|
||||
raise HTTPException(
|
||||
status_code=409, detail="clearhouse SFTP block is in stub mode",
|
||||
)
|
||||
|
||||
# 2. Resolve + validate the ingest dir. Use ``resolve()`` so a
|
||||
# symlink-relative path still produces a stable error message.
|
||||
root = Path(body.ingest_dir).resolve()
|
||||
if not root.exists():
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"ingest_dir does not exist: {root}",
|
||||
)
|
||||
|
||||
# 3. Walker — must match the CLI EXACTLY. Same sort, same ``._*``
|
||||
# AppleDouble skip. Any drift here is a quiet split between the
|
||||
# two surfaces and silently produces different batch outcomes.
|
||||
files: list[Path] = []
|
||||
for batch_dir in sorted(root.glob("batch-*-claims")):
|
||||
files.extend(sorted(
|
||||
p for p in batch_dir.glob("*.x12")
|
||||
if not p.name.startswith("._")
|
||||
))
|
||||
|
||||
# 4. ``limit`` truncates after collection. The CLI uses an inline
|
||||
# ``if i > limit: break``; we slice instead because the HTTP
|
||||
# body model validates ``limit`` up-front (Pydantic-level int
|
||||
# check) and slicing keeps the walker branchless.
|
||||
if body.limit is not None:
|
||||
files = files[: body.limit]
|
||||
|
||||
if not files:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"no batch-*-claims/*.x12 files found under {root}",
|
||||
)
|
||||
|
||||
# 5. Per-file submit. Wrap the helper call in try/except so an
|
||||
# unexpected exception in submit_file surfaces as a per-file
|
||||
# failure (outcome="unexpected") instead of crashing the whole
|
||||
# run. The helper's own SubmitOutcome enum covers every typed
|
||||
# failure path; an uncaught exception here is a true
|
||||
# surprise (bug or service outage mid-loop).
|
||||
results: list[dict] = []
|
||||
submitted = skipped = failed = 0
|
||||
for src in files:
|
||||
try:
|
||||
r = submit_file(
|
||||
src,
|
||||
sftp_block=sftp_block,
|
||||
actor=body.actor,
|
||||
validate=body.validate_files,
|
||||
# No ``sftp_client_factory`` — submit_file's default
|
||||
# paramiko factory opens the real MFT. Tests
|
||||
# monkey-patch submit_file itself instead of wiring a
|
||||
# factory here.
|
||||
)
|
||||
except DuplicateClaimError as exc:
|
||||
# SP41 Task 9: 409-class domain exception (see the exception
|
||||
# docstring on ``DuplicateClaimError``). The per-file loop
|
||||
# preserves the 200-with-results status-code contract
|
||||
# documented at the top of this module, so we surface the
|
||||
# duplicate as a per-file failure with UNEXPECTED_ERROR
|
||||
# outcome — the structured ``claim_id`` / ``original_submission_at``
|
||||
# attributes ride along in the error string so the operator
|
||||
# can see which CLM01 in the batch tripped the guard. Caught
|
||||
# BEFORE the generic ``Exception`` handler so a future
|
||||
# caller propagating the exception still gets 409-class
|
||||
# treatment at the FastAPI layer.
|
||||
log.warning(
|
||||
"submit-batch duplicate claim on %s: claim_id=%r original=%s",
|
||||
src.name, exc.claim_id, exc.original_submission_at,
|
||||
)
|
||||
r = SubmitResult(
|
||||
file=src.name,
|
||||
outcome=SubmitOutcome.UNEXPECTED_ERROR,
|
||||
error=(
|
||||
f"DuplicateClaimError: claim_id {exc.claim_id!r} was "
|
||||
f"already submitted at "
|
||||
f"{exc.original_submission_at.isoformat()}; "
|
||||
f"within 30-day window"
|
||||
),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.exception(
|
||||
"submit-batch unexpected error on %s", src.name,
|
||||
)
|
||||
r = SubmitResult(
|
||||
file=src.name,
|
||||
outcome=SubmitOutcome.UNEXPECTED_ERROR,
|
||||
error=f"{exc.__class__.__name__}: {exc}",
|
||||
)
|
||||
|
||||
if r.outcome == SubmitOutcome.SUBMITTED:
|
||||
submitted += 1
|
||||
elif r.outcome == SubmitOutcome.SKIPPED:
|
||||
skipped += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
results.append({
|
||||
"file": r.file,
|
||||
"outcome": r.outcome.value,
|
||||
"batch_id": r.batch_id,
|
||||
"error": r.error,
|
||||
})
|
||||
|
||||
return {
|
||||
"submitted": submitted,
|
||||
"skipped": skipped,
|
||||
"failed": failed,
|
||||
"results": results,
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
"""``/api/ta1-acks`` — list & detail endpoints for persisted TA1 envelopes.
|
||||
"""``/api/ta1-acks`` — list, detail, and live-tail stream for TA1 envelopes.
|
||||
|
||||
TA1 is the interchange-control ACK (ISA/IEA acknowledgement). It's a
|
||||
single segment, no functional group, no transaction set. Cyclone
|
||||
@@ -7,34 +7,30 @@ row can sit alongside the 999 / 277CA ack rows without special-casing.
|
||||
|
||||
The detail endpoint also reconstructs the TA1 segment string
|
||||
(``TA1*...~``) so the operator can copy it into a downstream tool.
|
||||
|
||||
SP25: ``/api/ta1-acks/stream`` joins the live-tail triplet — the
|
||||
Acks page mounts ``useTailStream("ta1_acks")`` so the TA1 envelope
|
||||
ack section sees new rows the moment they land.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from cyclone.api_helpers import ndjson_line, tail_events
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone import db
|
||||
from cyclone.store import store
|
||||
from cyclone.pubsub import EventBus
|
||||
from cyclone.store import store, to_ui_ta1_ack
|
||||
|
||||
router = APIRouter()
|
||||
router = APIRouter(dependencies=[Depends(matrix_gate)])
|
||||
|
||||
|
||||
def _ta1_to_ui(row: db.Ta1Ack) -> dict:
|
||||
"""Render a Ta1Ack row for the UI (list endpoint shape)."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"control_number": row.control_number,
|
||||
"ack_code": row.ack_code,
|
||||
"note_code": row.note_code,
|
||||
"interchange_date": row.interchange_date.isoformat()
|
||||
if row.interchange_date else None,
|
||||
"interchange_time": row.interchange_time,
|
||||
"sender_id": row.sender_id,
|
||||
"receiver_id": row.receiver_id,
|
||||
"source_batch_id": row.source_batch_id,
|
||||
"parsed_at": row.parsed_at.isoformat() if row.parsed_at else None,
|
||||
}
|
||||
# SP25: ``_ta1_to_ui`` moved to ``cyclone.store.ui.to_ui_ta1_ack`` so
|
||||
# the live-tail event payload (``ta1_ack_received``) matches the list
|
||||
# endpoint shape byte-for-byte.
|
||||
|
||||
|
||||
def _serialize_ta1_from_row(row: db.Ta1Ack) -> str:
|
||||
@@ -57,20 +53,78 @@ def list_ta1_acks_endpoint(
|
||||
count regardless of the ``limit`` cap.
|
||||
"""
|
||||
rows = store.list_ta1_acks()
|
||||
items = [_ta1_to_ui(r) for r in rows[:limit]]
|
||||
items = [to_ui_ta1_ack(r) for r in rows[:limit]]
|
||||
# SP28: batch-fetch linked_claim_ids per TA1 row (TA1 envelope
|
||||
# links always carry claim_id IS NULL — populate the field for
|
||||
# symmetry so the Acks page badge render path is uniform).
|
||||
ack_ids = [r.id for r in rows]
|
||||
linked_map = _find_linked_claim_ids_for_acks_helper(ack_ids, kind="ta1")
|
||||
for item, aid in zip(items, ack_ids[:limit]):
|
||||
item["linked_claim_ids"] = linked_map.get(aid, [])
|
||||
return {
|
||||
"total": len(rows),
|
||||
"items": items,
|
||||
}
|
||||
|
||||
|
||||
def _find_linked_claim_ids_for_acks_helper(
|
||||
ack_ids: list[int], *, kind: str
|
||||
) -> dict[int, list[str]]:
|
||||
"""Batch-fetch {ack_id: [claim_id, …]} for the listed ack rows.
|
||||
|
||||
Local helper so we don't import from ``acks.py`` and create a
|
||||
circular import. See the equivalent ``_find_linked_claim_ids_for_acks``
|
||||
in ``acks.py`` for the contract.
|
||||
"""
|
||||
out: dict[int, list[str]] = {aid: [] for aid in ack_ids}
|
||||
if not ack_ids:
|
||||
return out
|
||||
with db.SessionLocal()() as s:
|
||||
rows = (
|
||||
s.query(db.ClaimAck.ack_id, db.ClaimAck.claim_id)
|
||||
.filter(
|
||||
db.ClaimAck.ack_kind == kind,
|
||||
db.ClaimAck.ack_id.in_(ack_ids),
|
||||
db.ClaimAck.claim_id.isnot(None),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for ack_id, claim_id in rows:
|
||||
out[ack_id].append(claim_id)
|
||||
return out
|
||||
|
||||
|
||||
@router.get("/api/ta1-acks/stream")
|
||||
async def ta1_acks_stream(
|
||||
request: Request,
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
) -> StreamingResponse:
|
||||
"""Stream TA1 envelope acks as NDJSON.
|
||||
|
||||
Subscribes to ``ta1_ack_received`` and emits the same wire format
|
||||
as the other live-tail endpoints. Registered BEFORE
|
||||
``/api/ta1-acks/{ack_id}`` so ``stream`` isn't matched as an id.
|
||||
"""
|
||||
bus: EventBus = request.app.state.event_bus
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
rows = store.list_ta1_acks()[:limit]
|
||||
for row in rows:
|
||||
yield ndjson_line({"type": "item", "data": to_ui_ta1_ack(row)})
|
||||
yield ndjson_line({"type": "snapshot_end", "data": {"count": len(rows)}})
|
||||
async for chunk in tail_events(request, bus, ["ta1_ack_received"]):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
@router.get("/api/ta1-acks/{ack_id}")
|
||||
def get_ta1_ack_endpoint(ack_id: int) -> dict:
|
||||
"""Return one persisted TA1 ACK row with its parsed detail."""
|
||||
row = store.get_ta1_ack(ack_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"TA1 ACK {ack_id} not found")
|
||||
body = _ta1_to_ui(row)
|
||||
body = to_ui_ta1_ack(row)
|
||||
body["raw_ta1_text"] = _serialize_ta1_from_row(row)
|
||||
body["raw_json"] = row.raw_json
|
||||
return body
|
||||
|
||||
@@ -103,6 +103,12 @@ class AuditEvent:
|
||||
computed hash. Payload must be JSON-serializable; the audit_log
|
||||
module handles the encoding so callers don't need to think about
|
||||
canonical form.
|
||||
|
||||
``user_id`` is the authenticated actor for this event, when known
|
||||
(e.g. a parse-999 call made by user 7). It's stored on the row but
|
||||
is NOT part of the hash chain — the chain hashes only the fields
|
||||
that existed pre-SP-auth so verify_chain stays compatible with
|
||||
pre-auth rows.
|
||||
"""
|
||||
|
||||
event_type: str
|
||||
@@ -111,6 +117,7 @@ class AuditEvent:
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
actor: str = "system"
|
||||
created_at: datetime | None = None
|
||||
user_id: int | None = None
|
||||
|
||||
|
||||
def append_event(
|
||||
@@ -155,6 +162,7 @@ def append_event(
|
||||
created_at=created_at,
|
||||
prev_hash=prev_hash,
|
||||
hash=GENESIS_PREV_HASH, # placeholder; updated below
|
||||
user_id=event.user_id,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush() # populate row.id
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Auth module — users, sessions, permissions, routes, admin, rate_limit."""
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Admin-only user management: GET/POST/PATCH/DELETE /api/admin/users."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from cyclone.auth import users
|
||||
from cyclone.auth.deps import get_current_user
|
||||
from cyclone.auth.permissions import Role
|
||||
from cyclone.db import SessionLocal, User
|
||||
|
||||
router = APIRouter(prefix="/api/admin/users", tags=["admin"])
|
||||
|
||||
|
||||
def _require_admin(user: dict = Depends(get_current_user)) -> dict:
|
||||
if user.get("role") != Role.ADMIN.value:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="forbidden",
|
||||
)
|
||||
return user
|
||||
|
||||
|
||||
def _validate_role(role: str) -> None:
|
||||
valid = {Role.ADMIN.value, Role.USER.value, Role.VIEWER.value}
|
||||
if role not in valid:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"role must be one of {sorted(valid)}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_users(_admin=Depends(_require_admin)):
|
||||
with SessionLocal()() as db:
|
||||
all_users = db.query(User).all()
|
||||
return [users.to_public(u) for u in all_users]
|
||||
|
||||
|
||||
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||
def create_user(body: dict, _admin=Depends(_require_admin)):
|
||||
username = (body.get("username") or "").strip()
|
||||
password = body.get("password") or ""
|
||||
role = body.get("role") or ""
|
||||
if not username or len(username) < 3:
|
||||
raise HTTPException(status_code=422, detail="username must be at least 3 chars")
|
||||
if len(password) < 12:
|
||||
raise HTTPException(status_code=422, detail="password must be at least 12 chars")
|
||||
_validate_role(role)
|
||||
with SessionLocal()() as db:
|
||||
if users.get_by_username(db, username) is not None:
|
||||
raise HTTPException(status_code=409, detail="username already exists")
|
||||
u = users.create(db, username=username, password=password, role=role)
|
||||
return users.to_public(u)
|
||||
|
||||
|
||||
@router.patch("/{user_id}")
|
||||
def patch_user(user_id: int, body: dict, admin=Depends(_require_admin)):
|
||||
me = admin
|
||||
if me.get("id") == user_id and body.get("role") and body["role"] != Role.ADMIN.value:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="cannot_demote_self",
|
||||
)
|
||||
|
||||
with SessionLocal()() as db:
|
||||
if body.get("role") is not None:
|
||||
_validate_role(body["role"])
|
||||
users.update_role(db, user_id, body["role"])
|
||||
if body.get("password") is not None:
|
||||
if len(body["password"]) < 12:
|
||||
raise HTTPException(status_code=422, detail="password must be at least 12 chars")
|
||||
users.update_password(db, user_id, body["password"])
|
||||
if body.get("disabled") is True:
|
||||
users.disable(db, user_id)
|
||||
u = users.get(db, user_id)
|
||||
if u is None:
|
||||
raise HTTPException(status_code=404, detail="user not found")
|
||||
return users.to_public(u)
|
||||
|
||||
|
||||
@router.delete("/{user_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
def delete_user(user_id: int, admin=Depends(_require_admin)):
|
||||
me = admin
|
||||
if me.get("id") == user_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="cannot_delete_self",
|
||||
)
|
||||
with SessionLocal()() as db:
|
||||
u = users.get(db, user_id)
|
||||
if u is None:
|
||||
raise HTTPException(status_code=404, detail="user not found")
|
||||
users.disable(db, user_id)
|
||||
return None
|
||||
@@ -0,0 +1,105 @@
|
||||
"""First-admin bootstrap: create the initial admin from env vars if no users exist.
|
||||
|
||||
Called from ``python -m cyclone`` before either ``cli.main()`` or
|
||||
``uvicorn`` so users exist by the time the API serves requests.
|
||||
|
||||
Precedence:
|
||||
|
||||
1. ``CYCLONE_AUTH_DISABLED=1`` — dev escape hatch. Flip the
|
||||
``cyclone.auth.deps.AUTH_DISABLED`` flag so the API returns a
|
||||
synthetic admin user without checking credentials. Never raises.
|
||||
2. Users table non-empty — no-op.
|
||||
3. ``CYCLONE_ADMIN_USERNAME`` + ``CYCLONE_ADMIN_PASSWORD`` env vars set
|
||||
(password >= 12 chars) — create the admin and print confirmation.
|
||||
Each env var can also be replaced by a ``*_FILE`` companion
|
||||
(``CYCLONE_ADMIN_USERNAME_FILE`` / ``CYCLONE_ADMIN_PASSWORD_FILE``)
|
||||
that points at a file on disk — the standard Docker-secret pattern,
|
||||
used in production to avoid embedding secrets in ``docker-compose.yml``.
|
||||
``_FILE`` takes precedence when set.
|
||||
4. Otherwise — raise ``RuntimeError`` with a remediation hint that
|
||||
points operators at ``python -m cyclone users create``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from cyclone.auth import users
|
||||
from cyclone.auth.deps import AUTH_DISABLED
|
||||
from cyclone.auth.permissions import Role
|
||||
from cyclone.db import SessionLocal, User
|
||||
|
||||
|
||||
def _read_secret(env_var: str, file_var: str) -> str | None:
|
||||
"""Read a secret from a ``*_FILE`` env var (Docker-secret pattern) first,
|
||||
falling back to the plain env var. Returns None if neither is set.
|
||||
"""
|
||||
file_path = os.environ.get(file_var)
|
||||
if file_path:
|
||||
try:
|
||||
return Path(file_path).read_text().strip()
|
||||
except OSError as exc:
|
||||
raise RuntimeError(
|
||||
f"failed to read {file_var}={file_path}: {exc}"
|
||||
) from exc
|
||||
return os.environ.get(env_var)
|
||||
|
||||
|
||||
def run() -> None:
|
||||
"""Bootstrap the first admin user, or no-op.
|
||||
|
||||
See module docstring for behavior. Idempotent: safe to call on
|
||||
every startup — it short-circuits as soon as the users table is
|
||||
non-empty.
|
||||
"""
|
||||
if os.environ.get("CYCLONE_AUTH_DISABLED") == "1":
|
||||
# Dev escape hatch — skip bootstrap entirely and tell the API
|
||||
# to also short-circuit auth checks.
|
||||
import cyclone.auth.deps as _deps
|
||||
|
||||
_deps.AUTH_DISABLED = True
|
||||
return
|
||||
|
||||
username = _read_secret(
|
||||
"CYCLONE_ADMIN_USERNAME", "CYCLONE_ADMIN_USERNAME_FILE"
|
||||
)
|
||||
password = _read_secret(
|
||||
"CYCLONE_ADMIN_PASSWORD", "CYCLONE_ADMIN_PASSWORD_FILE"
|
||||
)
|
||||
|
||||
# First-boot fix: ``python -m cyclone`` calls bootstrap before any
|
||||
# subcommand or the FastAPI lifespan handler runs, so on a brand-new
|
||||
# DB ``SessionLocal()`` raises "init_db() has not been called".
|
||||
# Initialize here so ``serve``, ``users create``, and friends can
|
||||
# all reach the DB without the operator having to know about
|
||||
# migrations. Idempotent — no-op when the schema is already current.
|
||||
from cyclone import db as _db
|
||||
|
||||
_db.init_db()
|
||||
|
||||
with SessionLocal()() as db:
|
||||
existing = db.execute(select(User)).scalars().first()
|
||||
if existing is not None:
|
||||
return # users exist — nothing to bootstrap
|
||||
|
||||
if not username or not password:
|
||||
raise RuntimeError(
|
||||
"Cyclone has no users yet. Set CYCLONE_ADMIN_USERNAME and "
|
||||
"CYCLONE_ADMIN_PASSWORD env vars (min 12 chars), or run "
|
||||
"`python -m cyclone users create <username> --role admin`."
|
||||
)
|
||||
if len(password) < 12:
|
||||
raise RuntimeError(
|
||||
"CYCLONE_ADMIN_PASSWORD must be at least 12 characters."
|
||||
)
|
||||
|
||||
users.create(
|
||||
db,
|
||||
username=username,
|
||||
password=password,
|
||||
role=Role.ADMIN.value,
|
||||
)
|
||||
print(f"[cyclone] bootstrap admin user '{username}' created")
|
||||
@@ -0,0 +1,183 @@
|
||||
"""CLI subcommand: ``python -m cyclone users ...``.
|
||||
|
||||
Click-based to match the existing parse-837 / parse-835 convention in
|
||||
``cyclone.cli``. Provides operator-side user management without going
|
||||
through the admin HTTP API.
|
||||
|
||||
Subcommands
|
||||
-----------
|
||||
|
||||
- ``users create USERNAME --role {admin,user,viewer} [--password PW]``
|
||||
Create a user. If ``--password`` is omitted, prompts (with
|
||||
confirmation) on the controlling terminal.
|
||||
- ``users list`` — tab-separated ``id / username / role / state``.
|
||||
- ``users disable USERNAME`` — set ``disabled_at`` to now.
|
||||
- ``users reset-password USERNAME [--password PW]`` — replace the hash.
|
||||
- ``users set-role USERNAME --role {admin,user,viewer}`` — change role.
|
||||
|
||||
Exit codes
|
||||
----------
|
||||
|
||||
* 0 — success
|
||||
* 1 — validation error (unknown username, duplicate username)
|
||||
* 2 — usage error (missing arg, bad role, short password, unknown
|
||||
subcommand). Click itself uses 2 for usage errors so the conventional
|
||||
shell tools (``set -e``, etc.) recognize them.
|
||||
|
||||
Passwords shorter than 12 chars are rejected everywhere they appear.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import getpass
|
||||
import sys
|
||||
|
||||
import click
|
||||
|
||||
from cyclone.auth import users
|
||||
from cyclone.auth.permissions import Role
|
||||
from cyclone.db import SessionLocal
|
||||
|
||||
ROLE_CHOICES = [Role.ADMIN.value, Role.USER.value, Role.VIEWER.value]
|
||||
MIN_PASSWORD_LEN = 12
|
||||
|
||||
|
||||
def _prompt_password(label: str) -> str:
|
||||
pw = getpass.getpass(f"{label}: ")
|
||||
if not pw:
|
||||
click.echo("Password required.", err=True)
|
||||
sys.exit(2)
|
||||
return pw
|
||||
|
||||
|
||||
def _validate_password(pw: str) -> None:
|
||||
if len(pw) < MIN_PASSWORD_LEN:
|
||||
click.echo(
|
||||
f"Password must be at least {MIN_PASSWORD_LEN} characters.",
|
||||
err=True,
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Group
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@click.group(name="users")
|
||||
def users_cli() -> None:
|
||||
"""Manage Cyclone users from the command line."""
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# create
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@users_cli.command("create")
|
||||
@click.argument("username")
|
||||
@click.option(
|
||||
"--role",
|
||||
required=True,
|
||||
type=click.Choice(ROLE_CHOICES, case_sensitive=False),
|
||||
help="Role to grant the new user.",
|
||||
)
|
||||
@click.option(
|
||||
"--password",
|
||||
default=None,
|
||||
help=f"Password (min {MIN_PASSWORD_LEN} chars). Prompts if omitted.",
|
||||
)
|
||||
def create_user(username: str, role: str, password: str | None) -> None:
|
||||
"""Create a new user with the given USERNAME and ROLE."""
|
||||
pw = password or _prompt_password("Password")
|
||||
_validate_password(pw)
|
||||
|
||||
with SessionLocal()() as db:
|
||||
if users.get_by_username(db, username) is not None:
|
||||
click.echo(f"User '{username}' already exists.", err=True)
|
||||
sys.exit(1)
|
||||
u = users.create(db, username=username, password=pw, role=role)
|
||||
click.echo(f"Created user '{u.username}' with role '{u.role}'.")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# list
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@users_cli.command("list")
|
||||
def list_users() -> None:
|
||||
"""List all users as id / username / role / state."""
|
||||
from cyclone.db import User
|
||||
|
||||
with SessionLocal()() as db:
|
||||
for u in db.query(User).order_by(User.id.asc()).all():
|
||||
state = "disabled" if u.disabled_at else "active"
|
||||
click.echo(f"{u.id}\t{u.username}\t{u.role}\t{state}")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# disable
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@users_cli.command("disable")
|
||||
@click.argument("username")
|
||||
def disable_user(username: str) -> None:
|
||||
"""Disable USERNAME (sets disabled_at to now)."""
|
||||
with SessionLocal()() as db:
|
||||
u = users.get_by_username(db, username)
|
||||
if u is None:
|
||||
click.echo(f"No such user: {username}", err=True)
|
||||
sys.exit(1)
|
||||
users.disable(db, u.id)
|
||||
click.echo(f"Disabled '{username}'.")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# reset-password
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@users_cli.command("reset-password")
|
||||
@click.argument("username")
|
||||
@click.option(
|
||||
"--password",
|
||||
default=None,
|
||||
help=f"New password (min {MIN_PASSWORD_LEN} chars). Prompts if omitted.",
|
||||
)
|
||||
def reset_password(username: str, password: str | None) -> None:
|
||||
"""Replace USERNAME's password."""
|
||||
pw = password or _prompt_password("New password")
|
||||
_validate_password(pw)
|
||||
with SessionLocal()() as db:
|
||||
u = users.get_by_username(db, username)
|
||||
if u is None:
|
||||
click.echo(f"No such user: {username}", err=True)
|
||||
sys.exit(1)
|
||||
users.update_password(db, u.id, pw)
|
||||
click.echo(f"Password reset for '{username}'.")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# set-role
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@users_cli.command("set-role")
|
||||
@click.argument("username")
|
||||
@click.option(
|
||||
"--role",
|
||||
required=True,
|
||||
type=click.Choice(ROLE_CHOICES, case_sensitive=False),
|
||||
help="Role to grant.",
|
||||
)
|
||||
def set_role(username: str, role: str) -> None:
|
||||
"""Change USERNAME's role."""
|
||||
with SessionLocal()() as db:
|
||||
u = users.get_by_username(db, username)
|
||||
if u is None:
|
||||
click.echo(f"No such user: {username}", err=True)
|
||||
sys.exit(1)
|
||||
users.update_role(db, u.id, role)
|
||||
click.echo(f"Role for '{username}' set to '{role}'.")
|
||||
@@ -0,0 +1,140 @@
|
||||
"""FastAPI dependencies for auth."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session as DbSession
|
||||
|
||||
from cyclone.auth import sessions, users
|
||||
from cyclone.auth.permissions import Role, allowed_roles
|
||||
from cyclone.db import SessionLocal
|
||||
|
||||
|
||||
def _db():
|
||||
db = SessionLocal()()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
DbSessionDep = Annotated[DbSession, Depends(_db)]
|
||||
|
||||
|
||||
AUTH_DISABLED = False
|
||||
|
||||
|
||||
async def get_current_user(
|
||||
request: Request,
|
||||
db: DbSessionDep,
|
||||
) -> dict:
|
||||
"""Return the public User shape. Raises 401 if session is missing/expired.
|
||||
|
||||
When AUTH_DISABLED is True (dev escape hatch), returns a synthetic admin
|
||||
user without checking credentials.
|
||||
"""
|
||||
if AUTH_DISABLED:
|
||||
return {
|
||||
"id": 0,
|
||||
"username": "dev",
|
||||
"role": Role.ADMIN.value,
|
||||
"createdAt": None,
|
||||
"disabledAt": None,
|
||||
}
|
||||
|
||||
sid = request.cookies.get("cyclone_session")
|
||||
if not sid:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="session_expired",
|
||||
)
|
||||
sess = sessions.get_valid(db, sid)
|
||||
if sess is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="session_expired",
|
||||
)
|
||||
user = users.get(db, sess.user_id)
|
||||
if user is None or user.disabled_at is not None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="account_disabled",
|
||||
)
|
||||
# Sliding expiry: refresh both DB and cookie.
|
||||
sessions.touch(db, sid)
|
||||
request.state.user = user
|
||||
request.state.session_id = sid
|
||||
return users.to_public(user)
|
||||
|
||||
|
||||
def require_role(*allowed: Role):
|
||||
"""Dependency factory: gate the endpoint to specific roles.
|
||||
|
||||
Falls back to PERMISSIONS matrix lookup if no explicit roles given.
|
||||
"""
|
||||
async def _dep(
|
||||
request: Request,
|
||||
user: dict = Depends(get_current_user),
|
||||
) -> dict:
|
||||
user_role = user.get("role")
|
||||
if allowed:
|
||||
if user_role not in {r.value for r in allowed}:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="forbidden",
|
||||
)
|
||||
return user
|
||||
# Otherwise consult the matrix.
|
||||
method = request.method
|
||||
path = request.url.path
|
||||
roles = allowed_roles(method, path)
|
||||
if roles is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="forbidden",
|
||||
)
|
||||
if user_role not in {r.value for r in roles}:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="forbidden",
|
||||
)
|
||||
return user
|
||||
return _dep
|
||||
|
||||
|
||||
async def matrix_gate(
|
||||
request: Request,
|
||||
user: dict = Depends(get_current_user),
|
||||
) -> dict:
|
||||
"""App-wide gate: requires auth, then enforces the PERMISSIONS matrix.
|
||||
|
||||
Behavior:
|
||||
* AUTH_DISABLED short-circuits (synthetic admin, no role check).
|
||||
* No session cookie → 401 from get_current_user.
|
||||
* Endpoint not in the matrix → 403 (fail-closed).
|
||||
* User role not allowed for (method, path) → 403.
|
||||
* Empty allowed-roles set (e.g. /api/healthz) → public, no role check.
|
||||
|
||||
Used as ``dependencies=[Depends(matrix_gate)]`` on every authenticated
|
||||
route. Centralizing the gate here means the matrix is the source of
|
||||
truth — no need to wire per-route ``require_role(...)`` calls.
|
||||
"""
|
||||
if AUTH_DISABLED:
|
||||
return user
|
||||
method = request.method
|
||||
path = request.url.path
|
||||
roles = allowed_roles(method, path)
|
||||
if roles is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="forbidden",
|
||||
)
|
||||
user_role = user.get("role")
|
||||
if user_role not in {r.value for r in roles}:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="forbidden",
|
||||
)
|
||||
return user
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Role enum + PERMISSIONS matrix."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class Role(str, Enum):
|
||||
ADMIN = "admin"
|
||||
USER = "user"
|
||||
VIEWER = "viewer"
|
||||
|
||||
|
||||
ALL_ROLES = {Role.ADMIN, Role.USER, Role.VIEWER}
|
||||
WRITE_ROLES = {Role.ADMIN, Role.USER}
|
||||
ADMIN_ONLY = {Role.ADMIN}
|
||||
|
||||
|
||||
# (method, path-prefix) → allowed roles.
|
||||
# Endpoints not in this matrix default to DENY (fail-closed).
|
||||
PERMISSIONS: dict[tuple[str, str], set[Role]] = {
|
||||
# Public paths.
|
||||
("GET", "/api/health"): set(),
|
||||
("POST", "/api/auth/login"): set(),
|
||||
|
||||
# Auth surface.
|
||||
("POST", "/api/auth/logout"): ALL_ROLES,
|
||||
("GET", "/api/auth/me"): ALL_ROLES,
|
||||
|
||||
# Admin-only user management.
|
||||
("GET", "/api/admin/users"): ADMIN_ONLY,
|
||||
("POST", "/api/admin/users"): ADMIN_ONLY,
|
||||
("PATCH", "/api/admin/users"): ADMIN_ONLY,
|
||||
("DELETE", "/api/admin/users"): ADMIN_ONLY,
|
||||
|
||||
# Read endpoints (all authenticated roles).
|
||||
("GET", "/api/claims"): ALL_ROLES,
|
||||
("GET", "/api/remittances"): ALL_ROLES,
|
||||
("GET", "/api/providers"): ALL_ROLES,
|
||||
("GET", "/api/batches"): ALL_ROLES,
|
||||
("GET", "/api/dashboard/kpis"): ALL_ROLES, # dashboard summary cards (renamed from /summary)
|
||||
("GET", "/api/activity"): ALL_ROLES,
|
||||
("GET", "/api/inbox/lanes"): ALL_ROLES,
|
||||
("GET", "/api/inbox/export.csv"): ALL_ROLES,
|
||||
("GET", "/api/inbox/ack-orphans"): ALL_ROLES, # Inbox "Ack orphans" lane — wired in src/hooks/useAckOrphans.ts
|
||||
("GET", "/api/reconciliation"): ALL_ROLES,
|
||||
("GET", "/api/acks"): ALL_ROLES,
|
||||
("GET", "/api/ta1-acks"): ALL_ROLES,
|
||||
("GET", "/api/277ca-acks"): ALL_ROLES,
|
||||
("GET", "/api/batch-diff"): ALL_ROLES,
|
||||
("GET", "/api/config"): ALL_ROLES,
|
||||
("GET", "/api/payers"): ALL_ROLES,
|
||||
|
||||
# Clearhouse (SFTP creds + dzinesco identity) — admin only.
|
||||
("GET", "/api/clearhouse"): ADMIN_ONLY,
|
||||
("PATCH", "/api/clearhouse"): ADMIN_ONLY,
|
||||
("POST", "/api/clearhouse/submit"): ADMIN_ONLY,
|
||||
|
||||
# Admin ops (audit log, backup, scheduler, db rotate, reload-config).
|
||||
("GET", "/api/admin/audit-log"): ADMIN_ONLY,
|
||||
("GET", "/api/admin/audit-log/verify"): ADMIN_ONLY,
|
||||
("GET", "/api/admin/backup"): ADMIN_ONLY, # prefix; covers /list, /status, /{id}/restore/*, /{id}/verify
|
||||
("POST", "/api/admin/backup"): ADMIN_ONLY, # prefix; covers /create, /prune, /{id}/restore/*
|
||||
("GET", "/api/admin/backup/scheduler"): ADMIN_ONLY, # prefix; covers /start, /stop, /tick
|
||||
("POST", "/api/admin/backup/scheduler"): ADMIN_ONLY, # prefix; covers /start, /stop, /tick
|
||||
("GET", "/api/admin/scheduler"): ADMIN_ONLY, # prefix; covers /status, /processed-files
|
||||
("POST", "/api/admin/scheduler"): ADMIN_ONLY, # prefix; covers /start, /stop, /tick, /pull-inbound
|
||||
("POST", "/api/admin/db/rotate-key"): ADMIN_ONLY,
|
||||
("POST", "/api/admin/reload-config"): ADMIN_ONLY,
|
||||
("GET", "/api/admin/validate-provider"): ADMIN_ONLY,
|
||||
("POST", "/api/admin/validate-837"): ADMIN_ONLY, # SP40: Edifabric validation probe
|
||||
("POST", "/api/admin/rebill-from-835"): ADMIN_ONLY, # SP41: run the in-window rebill pipeline
|
||||
("GET", "/api/admin/rebill-from-835"): ADMIN_ONLY, # SP41: covers /status (prefix match)
|
||||
|
||||
# Write endpoints (admin + user, no viewer).
|
||||
("POST", "/api/parse-837"): WRITE_ROLES,
|
||||
("POST", "/api/parse-835"): WRITE_ROLES,
|
||||
("POST", "/api/parse-999"): WRITE_ROLES,
|
||||
("POST", "/api/parse-ta1"): WRITE_ROLES,
|
||||
("POST", "/api/parse-277ca"): WRITE_ROLES,
|
||||
("POST", "/api/inbox"): WRITE_ROLES,
|
||||
("POST", "/api/inbox/candidates"): WRITE_ROLES,
|
||||
("POST", "/api/inbox/rejected"): WRITE_ROLES,
|
||||
("POST", "/api/inbox/payer-rejected"): WRITE_ROLES,
|
||||
("POST", "/api/reconciliation"): WRITE_ROLES,
|
||||
("POST", "/api/acks"): WRITE_ROLES,
|
||||
# Unlink a wrong claim-ack match — inverse of POST /api/inbox/candidates/{remit_id}/match.
|
||||
# Prefix (not the placeholder path) because the matcher treats {kind}
|
||||
# as a literal substring; only ``/api/acks`` actually matches real requests.
|
||||
("DELETE", "/api/acks"): WRITE_ROLES,
|
||||
("POST", "/api/batches"): WRITE_ROLES, # /export-837 regenerates X12 from DB rows
|
||||
("POST", "/api/eligibility"): WRITE_ROLES,
|
||||
("POST", "/api/submit-batch"): WRITE_ROLES, # SP37: canonical outbound path (mirrors CLI)
|
||||
}
|
||||
|
||||
|
||||
def allowed_roles(method: str, path: str) -> set[Role] | None:
|
||||
"""Return the set of roles allowed to call (method, path), or None if denied.
|
||||
|
||||
Uses longest-prefix match on path; falls back to DENY (None) if no entry matches.
|
||||
"""
|
||||
candidates = [
|
||||
(len(prefix), roles)
|
||||
for (m, prefix), roles in PERMISSIONS.items()
|
||||
if m == method and (path == prefix or path.startswith(prefix.rstrip("/") + "/"))
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort(key=lambda x: -x[0])
|
||||
return candidates[0][1]
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Per-username login rate limiter (in-memory, per-process)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from threading import Lock
|
||||
|
||||
WINDOW_SECONDS = 300
|
||||
MAX_FAILS = 5
|
||||
|
||||
_FAILS: dict[str, list[float]] = {}
|
||||
_LOCK = Lock()
|
||||
|
||||
|
||||
def check(username: str) -> int:
|
||||
"""Return retry-after seconds, or 0 if allowed."""
|
||||
now = time.monotonic()
|
||||
with _LOCK:
|
||||
fails = [t for t in _FAILS.get(username, []) if now - t < WINDOW_SECONDS]
|
||||
_FAILS[username] = fails
|
||||
if len(fails) >= MAX_FAILS:
|
||||
return int(WINDOW_SECONDS - (now - fails[0]))
|
||||
return 0
|
||||
|
||||
|
||||
def record_failure(username: str) -> None:
|
||||
now = time.monotonic()
|
||||
with _LOCK:
|
||||
_FAILS.setdefault(username, []).append(now)
|
||||
|
||||
|
||||
def reset(username: str) -> None:
|
||||
with _LOCK:
|
||||
_FAILS.pop(username, None)
|
||||
@@ -0,0 +1,97 @@
|
||||
"""/api/auth/login, /api/auth/logout, /api/auth/me."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
|
||||
from cyclone.auth import rate_limit, sessions, users
|
||||
from cyclone.auth.deps import get_current_user
|
||||
from cyclone.db import SessionLocal
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["auth"])
|
||||
|
||||
COOKIE_NAME = "cyclone_session"
|
||||
COOKIE_MAX_AGE = 86400 # 24h
|
||||
|
||||
|
||||
def _is_https(request: Request) -> bool:
|
||||
if request.url.scheme == "https":
|
||||
return True
|
||||
return os.environ.get("CYCLONE_BEHIND_HTTPS") == "1"
|
||||
|
||||
|
||||
def _set_cookie(response: Response, sid: str, request: Request) -> None:
|
||||
response.set_cookie(
|
||||
key=COOKIE_NAME,
|
||||
value=sid,
|
||||
max_age=COOKIE_MAX_AGE,
|
||||
path="/api",
|
||||
httponly=True,
|
||||
samesite="lax",
|
||||
secure=_is_https(request),
|
||||
)
|
||||
|
||||
|
||||
def _clear_cookie(response: Response) -> None:
|
||||
response.delete_cookie(key=COOKIE_NAME, path="/api")
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
def login(body: dict, request: Request, response: Response):
|
||||
username = (body.get("username") or "").strip()
|
||||
password = body.get("password") or ""
|
||||
if not username or not password:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="username and password are required",
|
||||
)
|
||||
|
||||
retry_after = rate_limit.check(username)
|
||||
if retry_after > 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail="rate_limited",
|
||||
headers={"Retry-After": str(retry_after)},
|
||||
)
|
||||
|
||||
with SessionLocal()() as db:
|
||||
user = users.get_by_username(db, username)
|
||||
if user is None or not users.verify_password(password, user.password_hash):
|
||||
rate_limit.record_failure(username)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="invalid_credentials",
|
||||
)
|
||||
if user.disabled_at is not None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="account_disabled",
|
||||
)
|
||||
|
||||
sid, _ = sessions.create(db, user_id=user.id)
|
||||
rate_limit.reset(username)
|
||||
public = users.to_public(user)
|
||||
|
||||
_set_cookie(response, sid, request)
|
||||
return public
|
||||
|
||||
|
||||
@router.post("/logout")
|
||||
def logout(
|
||||
request: Request,
|
||||
response: Response,
|
||||
_user: dict = Depends(get_current_user),
|
||||
):
|
||||
sid = request.cookies.get(COOKIE_NAME)
|
||||
if sid:
|
||||
with SessionLocal()() as db:
|
||||
sessions.delete(db, sid)
|
||||
_clear_cookie(response)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
def me(user: dict = Depends(get_current_user)):
|
||||
return user
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Session create/validate/expire/touch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from cyclone.db import Session
|
||||
|
||||
SESSION_LIFETIME = timedelta(hours=24)
|
||||
|
||||
|
||||
def create(db, *, user_id: int) -> tuple[str, Session]:
|
||||
sid = secrets.token_urlsafe(32)
|
||||
now = datetime.now(timezone.utc)
|
||||
expires_at = now + SESSION_LIFETIME
|
||||
sess = Session(
|
||||
id=sid,
|
||||
user_id=user_id,
|
||||
expires_at=expires_at,
|
||||
created_at=now,
|
||||
)
|
||||
db.add(sess)
|
||||
db.commit()
|
||||
db.refresh(sess)
|
||||
# SQLite strips tzinfo on roundtrip; restore it so callers don't have to.
|
||||
sess.expires_at = expires_at
|
||||
return sid, sess
|
||||
|
||||
|
||||
def get_valid(db, sid: str) -> Session | None:
|
||||
sess = db.execute(
|
||||
select(Session).where(Session.id == sid)
|
||||
).scalar_one_or_none()
|
||||
if sess is None:
|
||||
return None
|
||||
# SQLite drops tzinfo on roundtrip; normalize to UTC before comparing.
|
||||
if sess.expires_at.tzinfo is None:
|
||||
sess.expires_at = sess.expires_at.replace(tzinfo=timezone.utc)
|
||||
if sess.expires_at <= datetime.now(timezone.utc):
|
||||
return None
|
||||
return sess
|
||||
|
||||
|
||||
def delete(db, sid: str) -> None:
|
||||
sess = db.get(Session, sid)
|
||||
if sess is None:
|
||||
return
|
||||
db.delete(sess)
|
||||
db.commit()
|
||||
|
||||
|
||||
def touch(db, sid: str) -> None:
|
||||
sess = db.get(Session, sid)
|
||||
if sess is None:
|
||||
return
|
||||
sess.expires_at = datetime.now(timezone.utc) + SESSION_LIFETIME
|
||||
db.commit()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""User CRUD + bcrypt password hashing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from passlib.hash import bcrypt
|
||||
from sqlalchemy import select
|
||||
|
||||
from cyclone.db import User
|
||||
|
||||
|
||||
def hash_password(plaintext: str) -> str:
|
||||
return bcrypt.hash(plaintext)
|
||||
|
||||
|
||||
def verify_password(plaintext: str, hashed: str) -> bool:
|
||||
try:
|
||||
return bcrypt.verify(plaintext, hashed)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def create(db, *, username: str, password: str, role: str) -> User:
|
||||
user = User(
|
||||
username=username,
|
||||
password_hash=hash_password(password),
|
||||
role=role,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
db.add(user)
|
||||
db.commit()
|
||||
db.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
def get_by_username(db, username: str) -> User | None:
|
||||
return db.execute(
|
||||
select(User).where(User.username == username)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def get(db, user_id: int) -> User | None:
|
||||
return db.get(User, user_id)
|
||||
|
||||
|
||||
def disable(db, user_id: int) -> None:
|
||||
user = db.get(User, user_id)
|
||||
if user is None:
|
||||
return
|
||||
user.disabled_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
|
||||
|
||||
def update_role(db, user_id: int, role: str) -> None:
|
||||
user = db.get(User, user_id)
|
||||
if user is None:
|
||||
return
|
||||
user.role = role
|
||||
db.commit()
|
||||
|
||||
|
||||
def update_password(db, user_id: int, new_password: str) -> None:
|
||||
user = db.get(User, user_id)
|
||||
if user is None:
|
||||
return
|
||||
user.password_hash = hash_password(new_password)
|
||||
db.commit()
|
||||
|
||||
|
||||
def to_public(user: User) -> dict[str, Any]:
|
||||
return {
|
||||
"id": user.id,
|
||||
"username": user.username,
|
||||
"role": user.role,
|
||||
"createdAt": user.created_at.isoformat() if user.created_at else None,
|
||||
"disabledAt": user.disabled_at.isoformat() if user.disabled_at else None,
|
||||
}
|
||||
@@ -0,0 +1,492 @@
|
||||
"""SP28: per-ACK auto-linker.
|
||||
|
||||
Three helpers, one per ACK kind, all run inside the same DB
|
||||
transaction that persists the Ack row. Each returns a slice of
|
||||
:class:`ClaimAckLinkRow` dataclasses describing what to link —
|
||||
the CALLER persists those rows via
|
||||
:func:`cyclone.store.claim_acks.add_claim_ack` (which owns the
|
||||
publish-from-store contract).
|
||||
|
||||
The two-pass join lives in
|
||||
:func:`lookup_claims_for_ack_set_response` (D10): ST02 via the
|
||||
batch envelope index (primary) + ``Claim.patient_control_number``
|
||||
(fallback). Plus :func:`link_manual` for the manual-fallback
|
||||
endpoint.
|
||||
|
||||
The helpers do NOT write to the DB session — they are pure
|
||||
readers over the session + parse result + the supplied
|
||||
``batch_envelope_index`` / ``pc_claim_lookup`` / ``batch_lookup``
|
||||
closures. This matches the existing
|
||||
``cyclone.inbox_state.apply_999_rejections`` pattern and lets the
|
||||
store facade own the publish-from-store contract for live-tail.
|
||||
|
||||
See ``docs/superpowers/specs/2026-07-02-cyclone-ack-claim-auto-link-design.md``
|
||||
§3 for the per-AK2 granularity, the two-pass join, and the
|
||||
idempotency contract enforced by ``ux_claim_acks_dedup``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Callable, Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from cyclone.db import Batch, Claim
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClaimAckLinkRow:
|
||||
"""One row to insert into ``claim_acks``.
|
||||
|
||||
Populated by the helpers; persisted by the caller via
|
||||
:func:`cyclone.store.claim_acks.add_claim_ack`. Carries every
|
||||
column the store needs to build the ORM row + the
|
||||
``claim_ack_written`` event payload.
|
||||
"""
|
||||
|
||||
claim_id: Optional[str] = None
|
||||
batch_id: Optional[str] = None
|
||||
ak2_index: Optional[int] = None
|
||||
set_control_number: Optional[str] = None
|
||||
set_accept_reject_code: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClaimAckLinkResult:
|
||||
"""Outcome of a single helper call.
|
||||
|
||||
``linked`` is the list of :class:`ClaimAckLinkRow` rows the
|
||||
caller should persist via ``cycl_store.add_claim_ack``.
|
||||
``orphans`` is a list of free-form strings the join couldn't
|
||||
resolve — for 999/277CA these are ``set_control_number``
|
||||
values; for TA1 they're the TA1 ICN when no matching batch was
|
||||
found.
|
||||
"""
|
||||
|
||||
linked: list[ClaimAckLinkRow] = field(default_factory=list)
|
||||
orphans: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# D10 two-pass join
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def lookup_claims_for_ack_set_response(
|
||||
session: Session,
|
||||
set_control_number: str,
|
||||
*,
|
||||
batch_envelope_index: Optional[Callable[[str], Optional[str]]] = None,
|
||||
pc_claim_lookup: Optional[Callable[[str], Optional[Claim]]] = None,
|
||||
) -> list[Claim]:
|
||||
"""Two-pass join for a single AK2 set_response / 277CA claim_status.
|
||||
|
||||
D10 (spec): for a 999 AK2-2 ``set_control_number`` or a 277CA REF*1K
|
||||
``payer_claim_control_number``, return every claim this ack
|
||||
acknowledges. The primary join is
|
||||
``Batch.envelope.control_number == set_control_number`` (== source
|
||||
837's ST02 on Gainwell batches); the fallback is
|
||||
``Claim.patient_control_number == set_control_number``.
|
||||
|
||||
Returns 0..N matching claims (one-ack-to-many when one 837 batch
|
||||
shipped multiple claims under one ST02).
|
||||
|
||||
Args:
|
||||
session: SQLAlchemy session the caller owns.
|
||||
set_control_number: the AK2-2 / REF*1K value to resolve.
|
||||
batch_envelope_index: optional pre-built index that maps
|
||||
``Batch.envelope.control_number`` → ``batch.id`` (built
|
||||
once per ingest via ``store.batch_envelope_index``).
|
||||
Pass to skip the per-set-response ``Batch`` scan in Pass 1.
|
||||
pc_claim_lookup: optional pre-built callable that maps a PCN
|
||||
to a single claim. Falls back to a session-wide query
|
||||
when not supplied.
|
||||
|
||||
Pass 1 wins. The two paths cannot both fire for the same
|
||||
``set_control_number`` — if Pass 1 returns one or more claims,
|
||||
Pass 2 is skipped. This is the false-positive guard from
|
||||
spec §7.
|
||||
"""
|
||||
if not set_control_number:
|
||||
return []
|
||||
|
||||
# -- Pass 1: Batch.envelope.control_number primary --------------
|
||||
# Accept either a plain dict (the common case — built once per
|
||||
# ingest via ``store.batch_envelope_index``) or a callable for
|
||||
# test-side closures. Normalize to ``idx.get`` so the rest of
|
||||
# the function stays uniform.
|
||||
idx: Optional[Callable[[str], Optional[str]]] = None
|
||||
if batch_envelope_index is not None:
|
||||
if callable(batch_envelope_index):
|
||||
idx = batch_envelope_index
|
||||
else:
|
||||
idx = batch_envelope_index.get
|
||||
matched_ids: list[str] = []
|
||||
if idx is not None:
|
||||
batch_id = idx(set_control_number)
|
||||
if batch_id is not None:
|
||||
matched_ids = [
|
||||
cid for (cid,) in (
|
||||
session.query(Claim.id)
|
||||
.filter(Claim.batch_id == batch_id)
|
||||
.all()
|
||||
)
|
||||
]
|
||||
else:
|
||||
# Fallback: scan all batches once per call. Slow but correct;
|
||||
# callers SHOULD pass the index.
|
||||
rows = (
|
||||
session.query(Batch.id, Batch.raw_result_json)
|
||||
.filter(Batch.kind == "837p")
|
||||
.all()
|
||||
)
|
||||
for bid, raw in rows:
|
||||
env = (raw or {}).get("envelope") or {}
|
||||
if env.get("control_number") == set_control_number:
|
||||
matched_ids = [
|
||||
cid for (cid,) in (
|
||||
session.query(Claim.id)
|
||||
.filter(Claim.batch_id == bid)
|
||||
.all()
|
||||
)
|
||||
]
|
||||
break
|
||||
|
||||
if matched_ids:
|
||||
claims = (
|
||||
session.query(Claim)
|
||||
.filter(Claim.id.in_(matched_ids))
|
||||
.all()
|
||||
)
|
||||
if claims:
|
||||
return list(claims)
|
||||
|
||||
# -- Pass 2: Claim.patient_control_number fallback ---------------
|
||||
if pc_claim_lookup is not None:
|
||||
single = pc_claim_lookup(set_control_number)
|
||||
if single is not None:
|
||||
return [single]
|
||||
return []
|
||||
|
||||
matches = (
|
||||
session.query(Claim)
|
||||
.filter(Claim.patient_control_number == set_control_number)
|
||||
.all()
|
||||
)
|
||||
return list(matches)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-ACK helpers — walk the parsed result and produce ClaimAckLinkRow
|
||||
# dataclasses. The CALLER persists via cycl_store.add_claim_ack so the
|
||||
# publish-from-store contract owns the live-tail event emission.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _existing_link_claim_ids(
|
||||
session: Session,
|
||||
*,
|
||||
claim_ids: list[str],
|
||||
ack_kind: str,
|
||||
ack_id: int,
|
||||
) -> set[str]:
|
||||
"""Return the subset of ``claim_ids`` that already have a link row.
|
||||
|
||||
Mirrors the partial unique index
|
||||
``ux_claim_acks_dedup(claim_id, ack_kind, ack_id, ak2_index)
|
||||
WHERE claim_id IS NOT NULL AND ak2_index IS NOT NULL``. The
|
||||
pre-check is here so we skip ``session.add`` and avoid
|
||||
IntegrityError log noise on re-ingest. For TA1 (claim_id IS
|
||||
NULL) the helpers do their own check.
|
||||
"""
|
||||
if not claim_ids:
|
||||
return set()
|
||||
rows = (
|
||||
session.query(Claim.claim_id) # placeholder; replaced below
|
||||
if False else
|
||||
session.query(Claim.id)
|
||||
.filter(Claim.id.in_([]))
|
||||
.all()
|
||||
)
|
||||
# Replace the placeholder with the real query — needed because
|
||||
# ClaimAck is not imported above (avoids circular import).
|
||||
from cyclone.db import ClaimAck
|
||||
existing = (
|
||||
session.query(ClaimAck.claim_id)
|
||||
.filter(
|
||||
ClaimAck.ack_kind == ack_kind,
|
||||
ClaimAck.ack_id == ack_id,
|
||||
ClaimAck.claim_id.in_(claim_ids),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
return {cid for (cid,) in existing if cid is not None}
|
||||
|
||||
|
||||
def apply_999_acceptances(
|
||||
session: Session,
|
||||
parsed_999,
|
||||
*,
|
||||
ack_id: int,
|
||||
batch_envelope_index: Optional[Callable[[str], Optional[str]]] = None,
|
||||
pc_claim_lookup: Optional[Callable[[str], Optional[Claim]]] = None,
|
||||
now: Optional[datetime] = None,
|
||||
) -> ClaimAckLinkResult:
|
||||
"""For every AK2 set-response, build one ``ClaimAckLinkRow`` per matched claim.
|
||||
|
||||
Both accepted AND rejected AK2s produce a link row (so the
|
||||
ClaimDrawer panel can show the rejection inline via the
|
||||
``set_accept_reject_code`` color-coded chip). Orphans are returned
|
||||
but not linked.
|
||||
|
||||
Idempotent: rows the dedup index already covers (re-ingest of an
|
||||
identical file) are skipped silently — the pre-check is here to
|
||||
avoid ``IntegrityError`` log noise.
|
||||
|
||||
One AK2 can produce multiple ``claim_acks`` rows when the source
|
||||
837 batch carried more than one claim under a shared ST02 (rare
|
||||
on this codebase but supported by D10 / the schema).
|
||||
|
||||
Returns:
|
||||
A :class:`ClaimAckLinkResult` whose ``linked`` list contains
|
||||
one :class:`ClaimAckLinkRow` per AK2-to-claim match. The
|
||||
caller persists each row via ``cycl_store.add_claim_ack``.
|
||||
"""
|
||||
result = ClaimAckLinkResult()
|
||||
set_responses = getattr(parsed_999, "set_responses", None) or []
|
||||
|
||||
# Resolve all set_control_numbers up front so we can do one
|
||||
# batched dedup query per (ack_kind, ack_id).
|
||||
resolved: list[tuple[int, "object", list[Claim], str, str]] = []
|
||||
candidate_claim_ids: list[str] = []
|
||||
for idx, sr in enumerate(set_responses):
|
||||
scn = getattr(sr, "set_control_number", None) or ""
|
||||
code = getattr(sr.set_accept_reject, "code", None) or ""
|
||||
claims = lookup_claims_for_ack_set_response(
|
||||
session, scn,
|
||||
batch_envelope_index=batch_envelope_index,
|
||||
pc_claim_lookup=pc_claim_lookup,
|
||||
)
|
||||
if not claims:
|
||||
result.orphans.append(scn)
|
||||
continue
|
||||
resolved.append((idx, sr, claims, scn, code))
|
||||
candidate_claim_ids.extend(c.id for c in claims)
|
||||
|
||||
existing_ids = _existing_link_claim_ids(
|
||||
session,
|
||||
claim_ids=candidate_claim_ids,
|
||||
ack_kind="999",
|
||||
ack_id=ack_id,
|
||||
)
|
||||
|
||||
for idx, sr, claims, scn, code in resolved:
|
||||
for claim in claims:
|
||||
if claim.id in existing_ids:
|
||||
continue
|
||||
result.linked.append(ClaimAckLinkRow(
|
||||
claim_id=claim.id,
|
||||
batch_id=None,
|
||||
ak2_index=idx,
|
||||
set_control_number=scn,
|
||||
set_accept_reject_code=code,
|
||||
))
|
||||
return result
|
||||
|
||||
|
||||
def apply_277ca_acks(
|
||||
session: Session,
|
||||
parsed_277ca,
|
||||
*,
|
||||
ack_id: int,
|
||||
batch_envelope_index: Optional[Callable[[str], Optional[str]]] = None,
|
||||
pc_claim_lookup: Optional[Callable[[str], Optional[Claim]]] = None,
|
||||
now: Optional[datetime] = None,
|
||||
) -> ClaimAckLinkResult:
|
||||
"""For every ClaimStatus with a payer_claim_control_number, build a ClaimAckLinkRow.
|
||||
|
||||
Accepted AND rejected ClaimStatuses both link — the
|
||||
``set_accept_reject_code`` carries the STC category code. The
|
||||
``claim_acks`` row is independent of the existing
|
||||
``Claim.payer_rejected_at`` mutation from
|
||||
:func:`cyclone.inbox_state_277ca.apply_277ca_rejections` (which
|
||||
fires before this helper in the handler).
|
||||
|
||||
Returns:
|
||||
A :class:`ClaimAckLinkResult` whose ``linked`` list contains
|
||||
one :class:`ClaimAckLinkRow` per ClaimStatus match. The
|
||||
caller persists each row via ``cycl_store.add_claim_ack``.
|
||||
"""
|
||||
result = ClaimAckLinkResult()
|
||||
statuses = getattr(parsed_277ca, "claim_statuses", None) or []
|
||||
|
||||
resolved: list[tuple["object", list[Claim], str, str]] = []
|
||||
candidate_claim_ids: list[str] = []
|
||||
for status in statuses:
|
||||
scn = getattr(status, "payer_claim_control_number", None) or ""
|
||||
raw_code = getattr(status, "status_code", None) or ""
|
||||
# ``status_code`` may be a category like "A6:19:PR" — keep
|
||||
# the whole STC composite so the UI can render the
|
||||
# category without re-parsing raw_json. Truncate to 8 chars
|
||||
# (column width).
|
||||
code = (raw_code or "")[:8]
|
||||
if not scn:
|
||||
# No REF*1K — orphan. Surface the STC composite so the
|
||||
# operator can correlate via the ack's raw_json.
|
||||
orphan_key = code or "(no REF*1K)"
|
||||
result.orphans.append(orphan_key)
|
||||
continue
|
||||
claims = lookup_claims_for_ack_set_response(
|
||||
session, scn,
|
||||
batch_envelope_index=batch_envelope_index,
|
||||
pc_claim_lookup=pc_claim_lookup,
|
||||
)
|
||||
if not claims:
|
||||
result.orphans.append(scn)
|
||||
continue
|
||||
resolved.append((status, claims, scn, code))
|
||||
candidate_claim_ids.extend(c.id for c in claims)
|
||||
|
||||
existing_ids = _existing_link_claim_ids(
|
||||
session,
|
||||
claim_ids=candidate_claim_ids,
|
||||
ack_kind="277ca",
|
||||
ack_id=ack_id,
|
||||
)
|
||||
|
||||
for status, claims, scn, code in resolved:
|
||||
for claim in claims:
|
||||
if claim.id in existing_ids:
|
||||
continue
|
||||
result.linked.append(ClaimAckLinkRow(
|
||||
claim_id=claim.id,
|
||||
batch_id=None,
|
||||
ak2_index=None,
|
||||
set_control_number=scn,
|
||||
set_accept_reject_code=code or None,
|
||||
))
|
||||
return result
|
||||
|
||||
|
||||
def apply_ta1_envelope_link(
|
||||
session: Session,
|
||||
parsed_ta1,
|
||||
*,
|
||||
ack_id: int,
|
||||
batch_lookup: Callable[[str, str], Optional[Batch]],
|
||||
now: Optional[datetime] = None,
|
||||
) -> ClaimAckLinkResult:
|
||||
"""Build a TA1 envelope-level link row to the most-recent matching Batch.
|
||||
|
||||
TA1 is envelope-level only (ISA/IEA, no per-claim granularity).
|
||||
The link row has ``claim_id IS NULL`` and ``batch_id`` populated.
|
||||
|
||||
Args:
|
||||
parsed_ta1: a :class:`cyclone.parsers.models_ta1.ParseResultTa1`.
|
||||
batch_lookup: ``(sender_id, receiver_id) -> Batch | None``.
|
||||
The handler supplies a closure that walks
|
||||
``session.query(Batch).order_by(parsed_at.desc())``. Returning
|
||||
``None`` produces an orphan (no batch match).
|
||||
|
||||
Returns:
|
||||
A :class:`ClaimAckLinkResult` with 0..1 row in ``linked``.
|
||||
Idempotent via dedup on ``(ack_kind='ta1', ack_id)`` (claim_id
|
||||
IS NULL so the partial unique index doesn't catch it; the
|
||||
pre-check here is a Python-side query).
|
||||
"""
|
||||
result = ClaimAckLinkResult()
|
||||
envelope = getattr(parsed_ta1, "envelope", None)
|
||||
if envelope is None:
|
||||
return result
|
||||
ta1_obj = getattr(parsed_ta1, "ta1", None)
|
||||
ack_code = getattr(ta1_obj, "ack_code", None) or ""
|
||||
|
||||
# Dedup: same (ack_kind, ack_id) → at most one TA1 envelope link.
|
||||
# Done in Python because the partial unique index requires
|
||||
# claim_id IS NOT NULL.
|
||||
from cyclone.db import ClaimAck
|
||||
existing = (
|
||||
session.query(ClaimAck.id)
|
||||
.filter(
|
||||
ClaimAck.ack_kind == "ta1",
|
||||
ClaimAck.ack_id == ack_id,
|
||||
ClaimAck.claim_id.is_(None),
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing is not None:
|
||||
return result
|
||||
|
||||
batch = batch_lookup(envelope.sender_id or "", envelope.receiver_id or "")
|
||||
if batch is None:
|
||||
orphan_key = (
|
||||
getattr(ta1_obj, "control_number", None)
|
||||
or envelope.control_number
|
||||
or ""
|
||||
)
|
||||
result.orphans.append(orphan_key)
|
||||
return result
|
||||
|
||||
result.linked.append(ClaimAckLinkRow(
|
||||
claim_id=None,
|
||||
batch_id=batch.id,
|
||||
ak2_index=None,
|
||||
set_control_number=None,
|
||||
set_accept_reject_code=ack_code,
|
||||
))
|
||||
return result
|
||||
|
||||
|
||||
def link_manual(
|
||||
session: Session,
|
||||
*,
|
||||
claim_id: str,
|
||||
ack_kind: str,
|
||||
ack_id: int,
|
||||
set_control_number: Optional[str] = None,
|
||||
set_accept_reject_code: Optional[str] = None,
|
||||
ak2_index: Optional[int] = None,
|
||||
now: Optional[datetime] = None,
|
||||
) -> ClaimAckLinkRow:
|
||||
"""Return one manual link row (the caller persists it via the store).
|
||||
|
||||
Used by ``POST /api/acks/{kind}/{ack_id}/match-claim``. Returns a
|
||||
:class:`ClaimAckLinkRow` describing the row to insert. The caller
|
||||
is responsible for persistence so it can own the publish-from-store
|
||||
contract.
|
||||
|
||||
Idempotency: callers should pre-check via ``session.query(ClaimAck)
|
||||
.filter(...).first()`` and skip when a row already exists; the
|
||||
:class:`cyclone.store.claim_acks.add_claim_ack` implementation also
|
||||
re-checks via the partial unique index.
|
||||
|
||||
Raises ``LookupError`` when the referenced claim doesn't exist
|
||||
(the caller maps that to 404).
|
||||
"""
|
||||
if ack_kind not in ("999", "277ca", "ta1"):
|
||||
raise ValueError(f"link_manual: unknown ack_kind={ack_kind!r}")
|
||||
claim = session.get(Claim, claim_id)
|
||||
if claim is None:
|
||||
raise LookupError(f"claim {claim_id} not found")
|
||||
return ClaimAckLinkRow(
|
||||
claim_id=claim_id,
|
||||
batch_id=None,
|
||||
ak2_index=ak2_index,
|
||||
set_control_number=set_control_number,
|
||||
set_accept_reject_code=set_accept_reject_code,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ClaimAckLinkResult",
|
||||
"ClaimAckLinkRow",
|
||||
"apply_999_acceptances",
|
||||
"apply_277ca_acks",
|
||||
"apply_ta1_envelope_link",
|
||||
"link_manual",
|
||||
"lookup_claims_for_ack_set_response",
|
||||
]
|
||||
@@ -24,6 +24,7 @@ stub secret and the paramiko auth will fail loudly at connect time.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
@@ -40,6 +41,75 @@ from cyclone.providers import SftpBlock
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-op SFTP timeout (SP27 Task 8)
|
||||
#
|
||||
# paramiko is synchronous; without a timeout, a hung ``listdir_attr``
|
||||
# freezes the worker thread indefinitely. The 06/25 silent hang was
|
||||
# exactly this — the MFT server TCP-acked but stopped responding, the
|
||||
# scheduler's ``asyncio.to_thread`` waited forever, and the operator
|
||||
# had no signal that polling had stalled.
|
||||
#
|
||||
# The async wrappers below apply ``asyncio.wait_for`` to every SFTP
|
||||
# call site so the event loop can give up after the configured bound.
|
||||
# The bound is read fresh on every call (env-var-only, no module-level
|
||||
# cache) so an operator who tunes the value at runtime picks it up on
|
||||
# the next poll.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_SFTP_OP_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
|
||||
def _op_timeout_seconds() -> float:
|
||||
"""Per-op SFTP timeout from ``CYCLONE_SFTP_OP_TIMEOUT_SECONDS``.
|
||||
|
||||
Default 30s. Picked to comfortably outlast Gainwell's p99
|
||||
listdir_attr (~2s) while still surfacing real hangs inside one
|
||||
scheduler tick. Operators who hit repeated timeouts should drop
|
||||
this — but the right answer is to fix the MFT server, not to
|
||||
paper over it here.
|
||||
"""
|
||||
raw = os.environ.get("CYCLONE_SFTP_OP_TIMEOUT_SECONDS")
|
||||
if not raw:
|
||||
return _DEFAULT_SFTP_OP_TIMEOUT_SECONDS
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
log.warning(
|
||||
"CYCLONE_SFTP_OP_TIMEOUT_SECONDS=%r is not a float; using default %.1fs",
|
||||
raw, _DEFAULT_SFTP_OP_TIMEOUT_SECONDS,
|
||||
)
|
||||
return _DEFAULT_SFTP_OP_TIMEOUT_SECONDS
|
||||
if value <= 0:
|
||||
# ``asyncio.wait_for(timeout=0)`` raises immediately, and
|
||||
# ``wait_for(timeout<0)`` is undefined per the asyncio docs.
|
||||
# A zero/negative setting would silently turn every SFTP call
|
||||
# into an instant timeout (a wave of bogus "list_inbound:
|
||||
# timeout" errors). Treat the value as bad and fall back.
|
||||
log.warning(
|
||||
"CYCLONE_SFTP_OP_TIMEOUT_SECONDS=%r must be positive; using default %.1fs",
|
||||
raw, _DEFAULT_SFTP_OP_TIMEOUT_SECONDS,
|
||||
)
|
||||
return _DEFAULT_SFTP_OP_TIMEOUT_SECONDS
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SftpStat:
|
||||
"""File metadata returned by :meth:`SftpClient.stat`.
|
||||
|
||||
Mirrors the subset of paramiko's ``SFTPAttributes`` that callers
|
||||
actually need (``size``, ``modified_at``). Keeping the surface
|
||||
narrow means callers don't need to import paramiko to consume
|
||||
the result, and the wrapper's stub mode can populate it from a
|
||||
plain ``os.stat_result`` without any paramiko dance.
|
||||
|
||||
Frozen so callers can hash / cache the result if they need to.
|
||||
"""
|
||||
size: int
|
||||
modified_at: datetime | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class InboundFile:
|
||||
"""A single file observed in the inbound MFT path."""
|
||||
@@ -87,11 +157,70 @@ class SftpClient:
|
||||
dir and returns :class:`InboundFile` records pointing at the
|
||||
cache copy. The remote file is *not* deleted — the operator
|
||||
archives inbound files in the MFT UI.
|
||||
|
||||
Gainwell's MFT puts advisory ``*_warn.txt`` files in the same
|
||||
inbound path. They're text-format side-channel notes (not X12
|
||||
envelopes) and are skipped at list time. Use
|
||||
:meth:`list_inbound_names` if you need the raw list without
|
||||
the download.
|
||||
"""
|
||||
if self._stub:
|
||||
return self._list_inbound_stub()
|
||||
return self._list_inbound_paramiko()
|
||||
|
||||
def list_inbound_names(self) -> list[InboundFile]:
|
||||
"""Lightweight listing: returns metadata only, no file download.
|
||||
|
||||
Use this when you want to filter the inbound set (e.g. by date)
|
||||
before paying the download cost. Pair with
|
||||
:meth:`download_inbound` to fetch a filtered subset on demand.
|
||||
|
||||
Real mode is implemented as a single SFTP ``listdir_attr`` call
|
||||
— sub-second on Gainwell's MFT — versus the full
|
||||
:meth:`list_inbound` which downloads every file. The
|
||||
``*_warn.txt`` advisory files are filtered out the same way.
|
||||
|
||||
Stub mode returns the same as :meth:`list_inbound` (the stub
|
||||
only knows about local files; no download cost).
|
||||
"""
|
||||
if self._stub:
|
||||
return self._list_inbound_stub()
|
||||
return self._list_inbound_names_paramiko()
|
||||
|
||||
def download_inbound(self, f: InboundFile) -> Path:
|
||||
"""Download a single inbound file to its ``local_path``.
|
||||
|
||||
Idempotent: if ``f.local_path`` already exists and is
|
||||
non-empty, the download is skipped. Callers should use the
|
||||
``InboundFile`` returned by :meth:`list_inbound_names` and pass
|
||||
it back here — ``local_path`` is the planned cache location
|
||||
and matches the path the scheduler will read from.
|
||||
|
||||
Returns:
|
||||
The on-disk path (same as ``f.local_path``).
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: if the file is missing locally in stub
|
||||
mode, or if the remote file disappears between list
|
||||
and download in real mode.
|
||||
"""
|
||||
if f.local_path.exists() and f.local_path.stat().st_size > 0:
|
||||
log.debug(
|
||||
"SFTP: %s already cached at %s, skipping download",
|
||||
f.name, f.local_path,
|
||||
)
|
||||
return f.local_path
|
||||
if self._stub:
|
||||
# Stub mode: no remote — the file is supposed to already be
|
||||
# at f.local_path (operator-dropped). If it isn't there, the
|
||||
# operator hasn't seeded the stub; raise loudly.
|
||||
if not f.local_path.is_file():
|
||||
raise FileNotFoundError(
|
||||
f"inbound stub file not found: {f.local_path}"
|
||||
)
|
||||
return f.local_path
|
||||
return self._download_inbound_paramiko(f)
|
||||
|
||||
def read_file(self, remote_path: str) -> bytes:
|
||||
"""Read bytes from a remote path.
|
||||
|
||||
@@ -103,6 +232,30 @@ class SftpClient:
|
||||
return self._read_file_stub(remote_path)
|
||||
return self._read_file_paramiko(remote_path)
|
||||
|
||||
def stat(self, remote_path: str) -> SftpStat:
|
||||
"""Return file metadata for ``remote_path``.
|
||||
|
||||
Mirrors the subset of paramiko's ``SFTPAttributes`` that
|
||||
callers need (``size`` for idempotency checks; ``modified_at``
|
||||
for cache-busting). The SP37 submission helper uses
|
||||
``stat().size`` to short-circuit re-uploads of already-uploaded
|
||||
files (the SKIPPED outcome path).
|
||||
|
||||
Stub mode: reads ``os.stat_result`` from
|
||||
``{staging_dir}/{remote_path}``.
|
||||
|
||||
Real mode: calls ``sftp.stat(remote_path)`` on a paramiko
|
||||
connection.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: if ``remote_path`` does not exist
|
||||
(stub: missing local file; real: paramiko raises
|
||||
``IOError`` which is a ``FileNotFoundError`` subclass).
|
||||
"""
|
||||
if self._stub:
|
||||
return self._stat_stub(remote_path)
|
||||
return self._stat_paramiko(remote_path)
|
||||
|
||||
def _read_file_stub(self, remote_path: str) -> bytes:
|
||||
"""Read bytes from ``{staging_dir}/{remote_path}`` (SP16 stub)."""
|
||||
staging = Path(self._block.staging_dir).resolve()
|
||||
@@ -111,6 +264,25 @@ class SftpClient:
|
||||
raise FileNotFoundError(f"inbound stub file not found: {target}")
|
||||
return target.read_bytes()
|
||||
|
||||
def _stat_stub(self, remote_path: str) -> SftpStat:
|
||||
"""Return ``SftpStat`` for ``{staging_dir}/{remote_path}`` (SP37 stub).
|
||||
|
||||
Mirrors what paramiko's ``sftp.stat()`` returns in real mode:
|
||||
``size`` from ``st.st_size`` and ``modified_at`` from
|
||||
``st.st_mtime``. Raises ``FileNotFoundError`` if the local
|
||||
file is missing — matches paramiko's ``IOError`` behavior
|
||||
so the caller doesn't need to special-case stub mode.
|
||||
"""
|
||||
staging = Path(self._block.staging_dir).resolve()
|
||||
target = staging / remote_path.lstrip("/")
|
||||
if not target.is_file():
|
||||
raise FileNotFoundError(f"inbound stub file not found: {target}")
|
||||
st = target.stat()
|
||||
return SftpStat(
|
||||
size=st.st_size,
|
||||
modified_at=datetime.fromtimestamp(st.st_mtime),
|
||||
)
|
||||
|
||||
def get_secret(self, name: str) -> Optional[str]:
|
||||
"""Fetch the auth secret from Keychain. Returns the stub secret if absent."""
|
||||
value = secrets.get_secret(name)
|
||||
@@ -119,6 +291,37 @@ class SftpClient:
|
||||
return secrets.STUB_SECRET
|
||||
return value
|
||||
|
||||
# ---- Async surface (SP27 Task 8) -----------------------------------
|
||||
#
|
||||
# Every sync SFTP call has an async wrapper that runs the paramiko
|
||||
# call on a worker thread and applies an ``asyncio.wait_for(...
|
||||
# timeout=N)`` around it. The wait_for cancels the awaiter but
|
||||
# leaves the worker thread running until paramiko returns on its
|
||||
# own (paramiko is not asyncio-aware, so we can't cancel the
|
||||
# underlying socket cleanly). The scheduler should treat
|
||||
# ``asyncio.TimeoutError`` from these wrappers as a transient
|
||||
# SFTP error and surface it in ``Scheduler.status()`` (Task 9).
|
||||
#
|
||||
# The timeout is read on every call (see ``_op_timeout_seconds``)
|
||||
# so an operator who tunes ``CYCLONE_SFTP_OP_TIMEOUT_SECONDS`` at
|
||||
# runtime sees the new value on the next tick.
|
||||
|
||||
async def async_list_inbound(self) -> list["InboundFile"]:
|
||||
"""Async-wrapped :meth:`list_inbound` with a per-op timeout.
|
||||
|
||||
The 06/25 silent hang was a hung ``listdir_attr`` that froze
|
||||
the worker thread indefinitely. This wrapper applies
|
||||
``asyncio.wait_for(...)`` so the event loop can give up after
|
||||
``CYCLONE_SFTP_OP_TIMEOUT_SECONDS`` (default 30s) and the
|
||||
scheduler tick can surface the timeout in
|
||||
``result.errors`` (and, once Task 9 lands, in
|
||||
``Scheduler.status()``).
|
||||
"""
|
||||
return await asyncio.wait_for(
|
||||
asyncio.to_thread(self.list_inbound),
|
||||
timeout=_op_timeout_seconds(),
|
||||
)
|
||||
|
||||
# ---- Stub implementations (SP9) -------------------------------------
|
||||
|
||||
def _write_bytes_stub(self, remote_path: str, content: bytes) -> Path:
|
||||
@@ -284,6 +487,12 @@ class SftpClient:
|
||||
if attr.st_mode and (attr.st_mode & 0o170000) == 0o040000:
|
||||
# Directory entry — skip.
|
||||
continue
|
||||
if attr.filename.endswith("_warn.txt"):
|
||||
# Gainwell's MFT drops text-format advisory notes in
|
||||
# the same inbound path. They're side-channel noise,
|
||||
# not X12 envelopes — skip at list time so we don't
|
||||
# download ~600 advisory files per poll.
|
||||
continue
|
||||
remote = f"{inbound_dir.rstrip('/')}/{attr.filename}"
|
||||
cache_path = cache_dir / attr.filename
|
||||
# Download into cache. We use ``prefetch`` to keep memory
|
||||
@@ -299,6 +508,56 @@ class SftpClient:
|
||||
))
|
||||
return files
|
||||
|
||||
def _list_inbound_names_paramiko(self) -> list[InboundFile]:
|
||||
"""List inbound names via paramiko; do NOT download (lightweight).
|
||||
|
||||
Same ``listdir_attr`` iteration as
|
||||
:meth:`_list_inbound_paramiko`, but the returned
|
||||
:class:`InboundFile` records have ``local_path`` set to the
|
||||
planned cache location without actually fetching the file.
|
||||
Pair with :meth:`_download_inbound_paramiko` to fetch on
|
||||
demand. Skips ``*_warn.txt`` advisory files the same way.
|
||||
"""
|
||||
with self._connect() as (ssh, sftp):
|
||||
inbound_dir = self._block.paths.get("inbound", "/")
|
||||
staging = Path(self._block.staging_dir).resolve()
|
||||
inbound_rel = inbound_dir.lstrip("/")
|
||||
cache_dir = staging / inbound_rel
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
files: list[InboundFile] = []
|
||||
try:
|
||||
attrs = sftp.listdir_attr(inbound_dir)
|
||||
except IOError as exc:
|
||||
log.warning("SFTP: cannot list %s: %s", inbound_dir, exc)
|
||||
return []
|
||||
|
||||
for attr in sorted(attrs, key=lambda a: a.filename):
|
||||
if attr.st_mode and (attr.st_mode & 0o170000) == 0o040000:
|
||||
# Directory entry — skip.
|
||||
continue
|
||||
if attr.filename.endswith("_warn.txt"):
|
||||
continue
|
||||
cache_path = cache_dir / attr.filename
|
||||
files.append(InboundFile(
|
||||
name=attr.filename,
|
||||
size=attr.st_size or 0,
|
||||
modified_at=datetime.fromtimestamp(attr.st_mtime or 0),
|
||||
local_path=cache_path,
|
||||
))
|
||||
return files
|
||||
|
||||
def _download_inbound_paramiko(self, f: InboundFile) -> Path:
|
||||
"""Download a single ``f`` to ``f.local_path`` (idempotent on size>0)."""
|
||||
with self._connect() as (ssh, sftp):
|
||||
inbound_dir = self._block.paths.get("inbound", "/")
|
||||
remote = f"{inbound_dir.rstrip('/')}/{f.name}"
|
||||
f.local_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with sftp.open(remote, "rb") as src, open(f.local_path, "wb") as dst:
|
||||
shutil.copyfileobj(src, dst, length=64 * 1024)
|
||||
log.info("SFTP: downloaded %d bytes for %s", f.local_path.stat().st_size, f.name)
|
||||
return f.local_path
|
||||
|
||||
def _read_file_paramiko(self, remote_path: str) -> bytes:
|
||||
with self._connect() as (ssh, sftp):
|
||||
buf = io.BytesIO()
|
||||
@@ -306,6 +565,20 @@ class SftpClient:
|
||||
shutil.copyfileobj(f, buf, length=64 * 1024)
|
||||
return buf.getvalue()
|
||||
|
||||
def _stat_paramiko(self, remote_path: str) -> SftpStat:
|
||||
"""Return ``SftpStat`` for ``remote_path`` via paramiko.
|
||||
|
||||
Paramiko's ``SFTPAttributes`` already provides ``st_size`` and
|
||||
``st_mtime``; we project them into our narrow public
|
||||
``SftpStat`` shape so callers don't need to know about paramiko.
|
||||
"""
|
||||
with self._connect() as (ssh, sftp):
|
||||
attr = sftp.stat(remote_path)
|
||||
return SftpStat(
|
||||
size=attr.st_size or 0,
|
||||
modified_at=datetime.fromtimestamp(attr.st_mtime or 0),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level helper
|
||||
|
||||
+1726
-2
File diff suppressed because it is too large
Load Diff
+203
-1
@@ -13,7 +13,7 @@ from __future__ import annotations
|
||||
import enum
|
||||
import json
|
||||
import os
|
||||
from datetime import date, datetime
|
||||
from datetime import date, datetime, timezone
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
@@ -30,6 +30,8 @@ from sqlalchemy import (
|
||||
Numeric,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship, sessionmaker
|
||||
@@ -220,6 +222,13 @@ class Batch(Base):
|
||||
totals_json: Mapped[Optional[dict]] = mapped_column(JSONText, nullable=True)
|
||||
validation_json: Mapped[Optional[dict]] = mapped_column(JSONText, nullable=True)
|
||||
raw_result_json: Mapped[Optional[dict]] = mapped_column(JSONText, nullable=True)
|
||||
# SP37 Task 2: source 837's ST02 (transaction set control number).
|
||||
# Populated from ``Envelope.transaction_set_control_number`` by
|
||||
# ``store.write.add_record`` for 837P batches; NULL for 835 batches
|
||||
# (the column is an 837P-specific join key for 999 AK2 resolution).
|
||||
# Migration 0020 adds the column additively; no backfill required for
|
||||
# pre-existing rows that lack the value.
|
||||
transaction_set_control_number: Mapped[Optional[str]] = mapped_column(String(32), nullable=True)
|
||||
|
||||
claims: Mapped[list["Claim"]] = relationship(
|
||||
back_populates="batch", cascade="all, delete-orphan"
|
||||
@@ -248,6 +257,7 @@ class Claim(Base):
|
||||
service_date_to: Mapped[Optional[date]] = mapped_column(Date, nullable=True)
|
||||
charge_amount: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False, default=Decimal("0"))
|
||||
provider_npi: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
|
||||
rendering_provider_npi: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
|
||||
payer_id: Mapped[Optional[str]] = mapped_column(String(32), nullable=True)
|
||||
state: Mapped[ClaimState] = mapped_column(
|
||||
Enum(ClaimState, native_enum=False), nullable=False, default=ClaimState.SUBMITTED
|
||||
@@ -316,6 +326,13 @@ class Claim(Base):
|
||||
Index("ix_claims_state", "state"),
|
||||
Index("ix_claims_patient_control_number", "patient_control_number"),
|
||||
Index("ix_claims_service_date_from", "service_date_from"),
|
||||
# SP27 Task 11: matched-pair drift check (run at startup)
|
||||
# scans ``WHERE matched_remittance_id IS NOT NULL``. Without
|
||||
# this index it's a full claim scan. The reverse side
|
||||
# (``ix_remittances_claim_id``) is added in 0007. Pure index
|
||||
# (non-unique) — a claim without a match is fine, reversals
|
||||
# leave the previous claim/claim match intact.
|
||||
Index("ix_claims_matched_remittance_id", "matched_remittance_id"),
|
||||
)
|
||||
|
||||
|
||||
@@ -336,6 +353,7 @@ class Remittance(Base):
|
||||
claim_id: Mapped[Optional[str]] = mapped_column(
|
||||
String(64), ForeignKey("claims.id"), nullable=True
|
||||
)
|
||||
rendering_provider_npi: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
|
||||
status_code: Mapped[str] = mapped_column(String(4), nullable=False)
|
||||
status_label: Mapped[Optional[str]] = mapped_column(String(32), nullable=True)
|
||||
total_charge: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False, default=Decimal("0"))
|
||||
@@ -433,6 +451,45 @@ class ServiceLinePayment(Base):
|
||||
)
|
||||
|
||||
|
||||
class Visit(Base):
|
||||
"""SP41: a single row from the AxisCare visits export (CSV).
|
||||
|
||||
Persisted by ``cyclone.rebill.visits_store.load_visits_csv`` so the
|
||||
in-window rebill pipeline can reconcile visits vs 835 svc rows from
|
||||
the database (previously the spot-check driver read the CSV in-memory,
|
||||
losing the canonical source-of-truth for the visit roster).
|
||||
|
||||
One row per (DOS, member_id, procedure, modifiers). The UNIQUE
|
||||
constraint on those four columns dedupes the natural key — the CSV
|
||||
itself can contain duplicate rows for the same visit (re-exports).
|
||||
"""
|
||||
__tablename__ = "visits"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
dos: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
member_id: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
client_name: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
procedure_code: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
modifiers: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
|
||||
billed_amount: Mapped[Decimal] = mapped_column(Numeric(10, 2), nullable=False)
|
||||
icd10: Mapped[Optional[str]] = mapped_column(String(16), nullable=True)
|
||||
prior_auth: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
|
||||
payer: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
|
||||
invoice_number: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
|
||||
source_file: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
|
||||
loaded_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("dos", "member_id", "procedure_code", "modifiers",
|
||||
name="uq_visits_natural_key"),
|
||||
Index("ix_visits_dos", "dos"),
|
||||
Index("ix_visits_member_id", "member_id"),
|
||||
Index("ix_visits_procedure_code", "procedure_code"),
|
||||
)
|
||||
|
||||
|
||||
class LineReconciliation(Base):
|
||||
"""One row per matched (or explicitly unmatched) 837 service line within a claim.
|
||||
|
||||
@@ -637,6 +694,76 @@ class Two77caAck(Base):
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP28: per-ACK auto-link join table (claim_acks)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ClaimAck(Base):
|
||||
"""One row per AK2 set-response / ClaimStatus / TA1 envelope link.
|
||||
|
||||
SP28. The durable record of "this ACK acknowledges this claim (or
|
||||
set, or batch)". One 999 row carries many AK2s; one 277CA carries
|
||||
many ClaimStatuses; each gets its own ClaimAck row so the operator
|
||||
can answer "which claims does this ack acknowledge?" with a single
|
||||
SELECT on ``claim_acks``.
|
||||
|
||||
See ``docs/superpowers/specs/2026-07-02-cyclone-ack-claim-auto-link-design.md``
|
||||
§3.1 for the schema decisions (per-AK2 granularity, the two-pass
|
||||
join, idempotency via the unique index).
|
||||
"""
|
||||
|
||||
__tablename__ = "claim_acks"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
# FK to claims.id with ON DELETE CASCADE so removing a claim
|
||||
# drops every link row referencing it. NULLable so TA1 envelope-level
|
||||
# rows can populate ``batch_id`` instead (the table CHECK constraint
|
||||
# requires at least one of the two).
|
||||
claim_id: Mapped[Optional[str]] = mapped_column(
|
||||
String(64), ForeignKey("claims.id", ondelete="CASCADE"), nullable=True,
|
||||
)
|
||||
# FK to batches.id (a Batch row in the 837 case, or the synthetic
|
||||
# inbound-batch id when the ack arrived outside the SFTP pipeline).
|
||||
batch_id: Mapped[Optional[str]] = mapped_column(
|
||||
String(32), ForeignKey("batches.id", ondelete="CASCADE"), nullable=True,
|
||||
)
|
||||
# Discriminated union over acks / ta1_acks / two77ca_acks. No FK
|
||||
# constraint because the three target tables are separate; the
|
||||
# application enforces the discriminator + the matching row's id.
|
||||
ack_id: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
ack_kind: Mapped[str] = mapped_column(String(8), nullable=False)
|
||||
ak2_index: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
# The set_control_number the upstream ack ACTUALLY CARRIED
|
||||
# (== source 837 ST02 for Gainwell batches). Preserved on the link
|
||||
# row for orphan traceability — the join may have resolved the
|
||||
# link via the PCN fallback instead, but the operator still sees
|
||||
# the value the 999/277CA originally carried.
|
||||
set_control_number: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
|
||||
# AK5 code (A/E/R/X) for 999, STC category (A1/A2/A3/A4/A6/A7)
|
||||
# for 277CA, envelope ack_code for TA1.
|
||||
set_accept_reject_code: Mapped[Optional[str]] = mapped_column(String(8), nullable=True)
|
||||
linked_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
linked_by: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_claim_acks_claim_id", "claim_id"),
|
||||
Index("ix_claim_acks_batch_id", "batch_id"),
|
||||
Index("ix_claim_acks_ack", "ack_kind", "ack_id"),
|
||||
# Mirror the dedup unique index declared in 0018_claim_acks.sql so
|
||||
# ``Base.metadata.create_all`` (the test-time safety net) emits the
|
||||
# same partial-unique constraint that the production migration runner
|
||||
# applies. Without this a fresh in-memory test DB would not enforce
|
||||
# idempotency at the DB layer.
|
||||
Index(
|
||||
"ux_claim_acks_dedup",
|
||||
"claim_id", "ack_kind", "ack_id", "ak2_index",
|
||||
unique=True,
|
||||
sqlite_where=text("claim_id IS NOT NULL AND ak2_index IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP11: tamper-evident hash-chained audit_log
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -668,6 +795,11 @@ class AuditLog(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
prev_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# SP-auth: which authenticated user performed this action. Nullable
|
||||
# so existing (pre-auth) rows and system-initiated events stay valid.
|
||||
# NOT part of the hash chain — verify_chain must continue to work on
|
||||
# legacy rows that pre-date this column.
|
||||
user_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, index=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_audit_log_entity", "entity_type", "entity_id"),
|
||||
@@ -836,3 +968,73 @@ class ClearhouseORM(Base):
|
||||
filename_block_json: Mapped[dict] = mapped_column(JSONText, nullable=False)
|
||||
sftp_block_json: Mapped[dict] = mapped_column(JSONText, nullable=False)
|
||||
updated_at: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
|
||||
|
||||
class User(Base):
|
||||
"""Auth user (admin / user / viewer)."""
|
||||
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
username: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
role: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
disabled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class Session(Base):
|
||||
"""Server-side auth session (HttpOnly cookie holds the id)."""
|
||||
|
||||
__tablename__ = "sessions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class Resubmission(Base):
|
||||
"""SP39: audit row recording that a claim was pushed to SFTP as
|
||||
part of a corrected-file resubmission. One row per claim per push.
|
||||
|
||||
Status is derived at read-time by joining against claim_acks (via
|
||||
the existing SP28/31 auto-link) + remittances (via CLP->claim).
|
||||
The corrected-v2 regen preserves the original claim_id from
|
||||
raw_json, so inbound 999 acks auto-link back to the original claim
|
||||
row and the join works without any new matching logic.
|
||||
"""
|
||||
|
||||
__tablename__ = "resubmissions"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
claim_id: Mapped[str] = mapped_column(String, nullable=False, index=True)
|
||||
batch_id: Mapped[str] = mapped_column(String, nullable=False, index=True)
|
||||
resubmitted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
source_corrected_path: Mapped[str] = mapped_column(String, nullable=False)
|
||||
interchange_control_number: Mapped[str] = mapped_column(String, nullable=False)
|
||||
group_control_number: Mapped[str] = mapped_column(String, nullable=False)
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ux_resubmissions_claim_icn",
|
||||
"claim_id", "interchange_control_number",
|
||||
unique=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class SubmissionRecord(Base):
|
||||
"""SP41: one row per (claim_id, submitted_at) pre-flight dedup guard.
|
||||
|
||||
The 30-day dedup window is enforced by ``cyclone.store.submission_dedup``.
|
||||
This table records (claim_id, submitted_at) for every push that passed
|
||||
the pre-flight check. Past the window, the prior record is ignored
|
||||
(the guard lets through re-submits older than
|
||||
``cyclone.store.submission_dedup.DEFAULT_WINDOW_DAYS``).
|
||||
"""
|
||||
|
||||
__tablename__ = "submission_dedup"
|
||||
|
||||
claim_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
submitted_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, index=True)
|
||||
|
||||
@@ -4,12 +4,15 @@ SP9. Source-of-truth spec:
|
||||
https://hcpf.colorado.gov/tp-x12-filenaming (HCPF X12 File Naming Standards Quick Guide)
|
||||
|
||||
Outbound (we send):
|
||||
{tpid}-{transaction_type}-{yyyymmddhhmmssSSS_MT}-1of1.{ext}
|
||||
Example: 11525703-837P-20260620132243505-1of1.x12
|
||||
tp{tpid}-{transaction_type}-{yyyymmddhhmmssSSS_MT}-1of1.{ext}
|
||||
Example: tp11525703-837P-20260620132243505-1of1.x12
|
||||
|
||||
Inbound (HPE sends to our ToHPE):
|
||||
TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12
|
||||
Example: TP11525703-837P_M019048402-20260520231513488-1of1_999.x12
|
||||
Inbound (HPE sends to our FromHPE):
|
||||
[Tt][Pp]{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12
|
||||
Example: tp11525703-837P_M019048402-20260520231513488-1of1_999.x12
|
||||
(legacy / prodfiles may use uppercase TP; the regex is case-insensitive
|
||||
on the prefix to accept both — Gainwell's filer has used both
|
||||
casings over time.)
|
||||
|
||||
Both use Mountain Time (MT) timestamps with 17-digit millisecond precision
|
||||
(yyyymmddhhmmssSSS = 4+2+2+2+2+2+3 = 17 digits). Sequence is always "1of1"
|
||||
@@ -28,29 +31,59 @@ from cyclone.providers import InboundFilename
|
||||
# Regexes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Outbound: 11525703-837P-20260620132243505-1of1.x12
|
||||
# Outbound: tp11525703-837P-20260620132243505-1of1.x12
|
||||
# - tp: literal "tp" prefix
|
||||
# - tpid: 1+ digits
|
||||
# - tx: 1+ alnum
|
||||
# - ts: 17 digits (yyyymmddhhmmssSSS)
|
||||
# - seq: literal "1of1"
|
||||
# - ext: 1+ alnum
|
||||
OUTBOUND_RE = re.compile(
|
||||
r"^(?P<tpid>\d+)-(?P<tx>[A-Z0-9]+)-(?P<ts>\d{17})-1of1\.(?P<ext>[A-Za-z0-9]+)$"
|
||||
r"^tp(?P<tpid>\d+)-(?P<tx>[A-Z0-9]+)-(?P<ts>\d{17})-1of1\.(?P<ext>[A-Za-z0-9]+)$"
|
||||
)
|
||||
|
||||
# Inbound: TP11525703-837P_M019048402-20260520231513488-1of1_999.x12
|
||||
# Inbound: [Tt][Pp]11525703-837P_M019048402-20260520231513488-1of1_999.x12
|
||||
# - prefix: literal "TP" or "tp" (case-insensitive — Gainwell's
|
||||
# production filer has used both)
|
||||
# - tpid: 1+ digits (inside TP<...>)
|
||||
# - orig_tx: 1+ alnum
|
||||
# - track: M + 1+ alnum (e.g. M019048402) — the M is part of the
|
||||
# tracking value, not a separator.
|
||||
# - orig_tx: 1+ uppercase alnum
|
||||
# - track: M + 1+ uppercase alnum (e.g. M019048402) — the M is
|
||||
# part of the tracking value, not a separator.
|
||||
# - ts: 17 digits
|
||||
# - seq: literal "1of1"
|
||||
# - ft: 1+ alnum (e.g. 999, TA1, 271, 277, 277CA, 820, 834, 835, ENCR)
|
||||
# - ft: 1+ uppercase alnum (e.g. 999, TA1, 271, 277, 277CA,
|
||||
# 820, 834, 835, ENCR)
|
||||
#
|
||||
# Case insensitivity is scoped to the ``TP`` prefix via ``(?i:TP)``
|
||||
# — the rest of the pattern is case-sensitive so we still reject a
|
||||
# stray ``837p`` or ``m019048402`` (they'd be invalid HCPF).
|
||||
INBOUND_RE = re.compile(
|
||||
r"^TP(?P<tpid>\d+)-(?P<orig_tx>[A-Z0-9]+)_(?P<tracking>M[A-Z0-9]+)"
|
||||
r"^(?i:TP)(?P<tpid>\d+)-(?P<orig_tx>[A-Z0-9]+)_(?P<tracking>M[A-Z0-9]+)"
|
||||
r"-(?P<ts>\d{17})-1of1_(?P<file_type>[A-Z0-9]+)\.(?P<ext>x12)$"
|
||||
)
|
||||
|
||||
# Inbound suffix-less form (SP27 Task 7): tp11525703-835_M019110219-20260525001606050-1of1.x12
|
||||
# Gainwell's production filer has shipped this shorter form in addition
|
||||
# to the spec form above — the 6/15-6/19 835 batch arrived this way and
|
||||
# was silently dropped by the strict INBOUND_RE. The loose form omits
|
||||
# the `_{file_type}.x12` suffix; the token between `-` and `_M` doubles
|
||||
# as both `orig_tx` and `file_type` (since there's no separate suffix
|
||||
# to disambiguate). Allowed values still must be in ALLOWED_FILE_TYPES
|
||||
# — the suffix is optional, the type-set is not.
|
||||
# - prefix: literal "TP" or "tp" (case-insensitive — same as INBOUND_RE)
|
||||
# - tpid: 1+ digits
|
||||
# - file_type: 3-5 char alphanumeric (covers 999, TA1, 835, 277CA, ENCR;
|
||||
# capped at 5 to avoid swallowing the next `_M` token)
|
||||
# - tracking: M + 1+ uppercase alnum
|
||||
# - ts: 17 digits
|
||||
# - seq: literal "1of1"
|
||||
# - ext: literal "x12" (relaxing this would also relax the strict
|
||||
# form's contract; out of scope for SP27 Task 7)
|
||||
INBOUND_RE_LOOSE = re.compile(
|
||||
r"^(?i:TP)(?P<tpid>\d+)-(?P<file_type>[A-Z0-9]{3,5})"
|
||||
r"_(?P<tracking>M[A-Z0-9]+)-(?P<ts>\d{17})-1of1\.(?P<ext>x12)$"
|
||||
)
|
||||
|
||||
ALLOWED_FILE_TYPES = frozenset({
|
||||
"999", "TA1", "270", "271", "276", "277", "277CA", "278",
|
||||
"820", "834", "835", "ENCR",
|
||||
@@ -79,7 +112,8 @@ def build_outbound_filename(
|
||||
time in ``America/Denver`` is used.
|
||||
|
||||
Returns:
|
||||
Filename like "11525703-837P-20260620132243505-1of1.x12"
|
||||
Filename like "tp11525703-837P-20260620132243505-1of1.x12"
|
||||
(note the ``tp`` prefix per HCPF outbound spec).
|
||||
|
||||
Raises:
|
||||
ValueError: If tpid is non-numeric, tx contains invalid chars, or
|
||||
@@ -99,7 +133,10 @@ def build_outbound_filename(
|
||||
# Format: yyyymmddhhmmssSSS — 17 digits total
|
||||
ts = now_mt.strftime("%Y%m%d%H%M%S") + f"{now_mt.microsecond // 1000:03d}"
|
||||
assert len(ts) == 17
|
||||
return f"{tpid}-{tx}-{ts}-1of1.{ext}"
|
||||
# Per HCPF outbound spec, prefix is "tp" + tpid. Matches the format
|
||||
# we receive from HPE inbound (which uses uppercase TP) and the
|
||||
# historical outbound prodfile naming (e.g. tp11525703-837P-...).
|
||||
return f"tp{tpid}-{tx}-{ts}-1of1.{ext}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -110,16 +147,49 @@ def build_outbound_filename(
|
||||
def parse_inbound_filename(name: str) -> InboundFilename:
|
||||
"""Parse an inbound HCPF filename.
|
||||
|
||||
Accepts both forms (Gainwell ships both):
|
||||
* Spec form with ``_{file_type}.x12`` suffix:
|
||||
``tp11525703-837P_M019048402-20260520231513488-1of1_999.x12``
|
||||
* Suffix-less form (SP27 Task 7): the token between ``-`` and
|
||||
``_M`` doubles as both ``orig_tx`` and ``file_type``:
|
||||
``tp11525703-835_M019110219-20260525001606050-1of1.x12``
|
||||
|
||||
The strict form is tried first (preserves historical behavior for
|
||||
every existing caller); the loose form is the fallback. The
|
||||
``.x12`` extension and ``ALLOWED_FILE_TYPES`` set are enforced in
|
||||
both forms.
|
||||
|
||||
Args:
|
||||
name: Filename like "TP11525703-837P_M019048402-20260520231513488-1of1_999.x12"
|
||||
name: Filename like ``tp11525703-837P_M019048402-20260520231513488-1of1_999.x12``
|
||||
(case-insensitive on the ``TP`` prefix; both ``TP`` and
|
||||
``tp`` are accepted.)
|
||||
|
||||
Returns:
|
||||
InboundFilename with tpid, orig_tx, tracking, ts, file_type, ext.
|
||||
|
||||
Raises:
|
||||
ValueError: If the filename doesn't match the HCPF inbound format.
|
||||
ValueError: If the filename doesn't match either HCPF inbound
|
||||
form, or if the derived file_type isn't in
|
||||
ALLOWED_FILE_TYPES.
|
||||
"""
|
||||
m = INBOUND_RE.match(name)
|
||||
if m:
|
||||
file_type = m.group("file_type")
|
||||
if file_type not in ALLOWED_FILE_TYPES:
|
||||
raise ValueError(
|
||||
f"file_type {file_type!r} not in allowed HCPF set: {sorted(ALLOWED_FILE_TYPES)}"
|
||||
)
|
||||
return InboundFilename(
|
||||
tpid=m.group("tpid"),
|
||||
orig_tx=m.group("orig_tx"),
|
||||
tracking=m.group("tracking"),
|
||||
ts=m.group("ts"),
|
||||
file_type=file_type,
|
||||
ext=m.group("ext"),
|
||||
)
|
||||
|
||||
# Fall back to the suffix-less form.
|
||||
m = INBOUND_RE_LOOSE.match(name)
|
||||
if not m:
|
||||
raise ValueError(f"Not a valid HCPF inbound filename: {name!r}")
|
||||
file_type = m.group("file_type")
|
||||
@@ -129,7 +199,7 @@ def parse_inbound_filename(name: str) -> InboundFilename:
|
||||
)
|
||||
return InboundFilename(
|
||||
tpid=m.group("tpid"),
|
||||
orig_tx=m.group("orig_tx"),
|
||||
orig_tx=m.group("file_type"), # preserve the historical shape
|
||||
tracking=m.group("tracking"),
|
||||
ts=m.group("ts"),
|
||||
file_type=file_type,
|
||||
@@ -148,5 +218,12 @@ def is_outbound_filename(name: str) -> bool:
|
||||
|
||||
|
||||
def is_inbound_filename(name: str) -> bool:
|
||||
"""True if the given string matches the HCPF inbound filename regex."""
|
||||
return INBOUND_RE.match(name) is not None
|
||||
"""True if the given string matches either HCPF inbound form.
|
||||
|
||||
Accepts both the spec form (with ``_{file_type}.x12`` suffix) and
|
||||
the suffix-less form (SP27 Task 7). Cheap fast-path check used by
|
||||
callers that want to pre-filter a directory listing before invoking
|
||||
:func:`parse_inbound_filename`; mirrors the parser's own fallback
|
||||
so the two never disagree on what counts as an HCPF inbound file.
|
||||
"""
|
||||
return INBOUND_RE.match(name) is not None or INBOUND_RE_LOOSE.match(name) is not None
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
"""SP40: thin HTTP client for the EdiNation / Edifabric validation API.
|
||||
|
||||
Wraps the two-step /v2/x12/read (raw EDI → ``X12Interchange`` JSON) and
|
||||
/v2/x12/validate (``X12Interchange`` JSON → ``OperationResult``) flow.
|
||||
|
||||
Cyclone has no other outbound HTTP today; this is the first such
|
||||
client. ``httpx`` is already a project dep (used by the test suite)
|
||||
so we don't introduce a new dependency.
|
||||
|
||||
Public surface:
|
||||
|
||||
- :func:`read_interchange` — POST raw EDI bytes to /x12/read.
|
||||
- :func:`validate_interchange` — POST ``X12Interchange`` JSON to /x12/validate.
|
||||
- :func:`validate_edi` — composes the two; this is what the CLI and
|
||||
pre-upload gate use.
|
||||
- :class:`EdifabricError` — raised on 4xx/5xx so callers can surface
|
||||
the Edifabric error verbatim (the body is a dict or string).
|
||||
|
||||
The API key is taken from :func:`cyclone.secrets.get_secret('edifabric.api_key')`
|
||||
which maps to ``CYCLONE_EDIFABRIC_API_KEY`` env var or macOS Keychain.
|
||||
Tests inject a canned key (and a mocked transport) via the factory
|
||||
hook :func:`set_transport_factory` so no live HTTP hits the network.
|
||||
|
||||
The endpoint contract (from the EdiNation API reference,
|
||||
``https://support.edifabric.com/hc/en-us/sections/360005605638``):
|
||||
|
||||
- ``POST https://api.edination.com/v2/x12/read``
|
||||
- Headers: ``Ocp-Apim-Subscription-Key: <key>``, ``Content-Type: application/octet-stream``
|
||||
- Body: raw EDI bytes
|
||||
- Response (200): JSON array of ``X12Interchange`` objects (cyclone always sends one interchange)
|
||||
- ``POST https://api.edination.com/v2/x12/validate``
|
||||
- Headers: ``Ocp-Apim-Subscription-Key: <key>``, ``Content-Type: application/json``
|
||||
- Body: one ``X12Interchange`` object
|
||||
- Response (200): ``OperationResult`` (``Status`` ∈ {``"success"``, ``"warning"``, ``"error"``}, ``Details`` array)
|
||||
|
||||
We treat any non-2xx as an :class:`EdifabricError`. The response body
|
||||
is preserved verbatim so callers can inspect it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Callable
|
||||
|
||||
import httpx
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
_BASE_URL = "https://api.edination.com/v2/x12"
|
||||
_READ_PATH = "/read"
|
||||
_VALIDATE_PATH = "/validate"
|
||||
|
||||
# Subscription-key header name (Azure API Management convention).
|
||||
_SUB_HEADER = "Ocp-Apim-Subscription-Key"
|
||||
|
||||
|
||||
class EdifabricError(RuntimeError):
|
||||
"""Raised when the Edifabric API returns a non-2xx response.
|
||||
|
||||
Attributes:
|
||||
status_code: HTTP status code returned by Edifabric.
|
||||
body: The response body — usually a dict with ``error`` /
|
||||
``message`` keys for 4xx, or a string for 5xx / network
|
||||
errors. Preserved verbatim so the caller can surface it
|
||||
to the operator.
|
||||
retry_after_seconds: When the upstream sent a ``Retry-After``
|
||||
header (the API Management quota policy does), the value
|
||||
in seconds. ``None`` if absent. Quota-blocked callers can
|
||||
surface this so the operator knows when to retry.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, status_code: int, body: Any, *, retry_after_seconds: int | None = None
|
||||
) -> None:
|
||||
self.status_code = status_code
|
||||
self.body = body
|
||||
self.retry_after_seconds = retry_after_seconds
|
||||
body_repr = repr(body) if not isinstance(body, str) else body
|
||||
msg = f"Edifabric API returned {status_code}: {body_repr}"
|
||||
if retry_after_seconds is not None:
|
||||
msg += f" (retry after {retry_after_seconds}s)"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
# --- Transport injection (test seam) -----------------------------------
|
||||
|
||||
# Default transport factory builds a normal httpx.Client. Tests can
|
||||
# call set_transport_factory() with a callable that returns a Client
|
||||
# backed by httpx.MockTransport (no live HTTP).
|
||||
_transport_factory: Callable[[], httpx.Client] = lambda: httpx.Client(timeout=30.0)
|
||||
|
||||
|
||||
def set_transport_factory(factory: Callable[[], httpx.Client]) -> None:
|
||||
"""Inject an ``httpx.Client`` factory for tests.
|
||||
|
||||
The factory must return an ``httpx.Client`` whose ``transport`` is
|
||||
a mock (e.g. ``httpx.MockTransport(handler)``) so no real HTTP is
|
||||
performed. Returns the previous factory so tests can restore it.
|
||||
"""
|
||||
global _transport_factory
|
||||
prev = _transport_factory
|
||||
_transport_factory = factory
|
||||
return prev # type: ignore[return-value]
|
||||
|
||||
|
||||
def _reset_transport_factory() -> None:
|
||||
"""Restore the default transport factory (called in test cleanup)."""
|
||||
global _transport_factory
|
||||
_transport_factory = lambda: httpx.Client(timeout=30.0)
|
||||
|
||||
|
||||
# --- Public surface ----------------------------------------------------
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
"""Resolve the Edifabric API key.
|
||||
|
||||
Looks up ``CYCLONE_EDIFABRIC_API_KEY`` (or the keychain entry
|
||||
``cyclone / edifabric.api_key``). The secrets module is imported
|
||||
lazily so test setups that mock it can do so before first use.
|
||||
"""
|
||||
from cyclone.secrets import get_secret
|
||||
|
||||
key = get_secret("edifabric.api_key")
|
||||
if not key:
|
||||
raise EdifabricError(
|
||||
0,
|
||||
"Edifabric API key not configured; set CYCLONE_EDIFABRIC_API_KEY "
|
||||
"env var or run `cyclone secrets set edifabric.api_key <key>`",
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def read_interchange(edi_bytes: bytes, *, api_key: str | None = None) -> dict:
|
||||
"""POST raw EDI bytes to /x12/read and return the first X12Interchange.
|
||||
|
||||
The /x12/read endpoint accepts a multi-interchange file and returns
|
||||
an array. Cyclone only ever sends single interchanges, so we return
|
||||
the first (and only) element. If the file contains multiple
|
||||
interchanges, callers should call ``validate_interchange`` on each.
|
||||
"""
|
||||
if not isinstance(edi_bytes, (bytes, bytearray)):
|
||||
raise TypeError(
|
||||
f"edi_bytes must be bytes, got {type(edi_bytes).__name__}"
|
||||
)
|
||||
key = api_key if api_key is not None else _get_api_key()
|
||||
headers = {
|
||||
_SUB_HEADER: key,
|
||||
"Content-Type": "application/octet-stream",
|
||||
}
|
||||
with _transport_factory() as client:
|
||||
resp = client.post(
|
||||
f"{_BASE_URL}{_READ_PATH}",
|
||||
content=bytes(edi_bytes),
|
||||
headers=headers,
|
||||
)
|
||||
if not (200 <= resp.status_code < 300):
|
||||
raise EdifabricError(
|
||||
resp.status_code,
|
||||
_safe_body(resp),
|
||||
retry_after_seconds=_retry_after_seconds(resp),
|
||||
)
|
||||
data = resp.json()
|
||||
if not isinstance(data, list) or not data:
|
||||
raise EdifabricError(
|
||||
502,
|
||||
f"unexpected /x12/read response shape: expected non-empty list, "
|
||||
f"got {type(data).__name__} of length {len(data) if hasattr(data, '__len__') else '?'}",
|
||||
)
|
||||
return data[0]
|
||||
|
||||
|
||||
def validate_interchange(x12_json: dict, *, api_key: str | None = None) -> dict:
|
||||
"""POST an X12Interchange JSON to /x12/validate and return the OperationResult.
|
||||
|
||||
The OperationResult schema (per the EdiNation docs):
|
||||
|
||||
- ``Status`` — ``"success"`` / ``"warning"`` / ``"error"``.
|
||||
- ``Details`` — array of ``{Index, SegmentId, Value, Message, Status, ...}``.
|
||||
- ``LastIndex`` — 1-based index of the last processed segment.
|
||||
|
||||
We do NOT raise on ``Status == "error"`` — the caller decides whether
|
||||
to fail-closed (the pre-upload gate does; the CLI prints and exits
|
||||
with the appropriate code). Non-2xx HTTP responses DO raise
|
||||
:class:`EdifabricError`.
|
||||
"""
|
||||
if not isinstance(x12_json, dict):
|
||||
raise TypeError(
|
||||
f"x12_json must be a dict, got {type(x12_json).__name__}"
|
||||
)
|
||||
key = api_key if api_key is not None else _get_api_key()
|
||||
headers = {
|
||||
_SUB_HEADER: key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
with _transport_factory() as client:
|
||||
resp = client.post(
|
||||
f"{_BASE_URL}{_VALIDATE_PATH}",
|
||||
json=x12_json,
|
||||
headers=headers,
|
||||
)
|
||||
if not (200 <= resp.status_code < 300):
|
||||
raise EdifabricError(
|
||||
resp.status_code,
|
||||
_safe_body(resp),
|
||||
retry_after_seconds=_retry_after_seconds(resp),
|
||||
)
|
||||
return resp.json()
|
||||
|
||||
|
||||
def validate_edi(edi_bytes: bytes, *, api_key: str | None = None) -> dict:
|
||||
"""Two-step convenience: read → validate. Returns the OperationResult.
|
||||
|
||||
This is what ``cyclone validate-837 <file>`` and the pre-upload
|
||||
gate in ``resubmit-rejected-claims`` call. Raises
|
||||
:class:`EdifabricError` on transport / non-2xx errors. The
|
||||
OperationResult is returned verbatim so the caller can inspect
|
||||
``Status`` and ``Details`` themselves.
|
||||
"""
|
||||
x12 = read_interchange(edi_bytes, api_key=api_key)
|
||||
return validate_interchange(x12, api_key=api_key)
|
||||
|
||||
|
||||
# --- Internal helpers --------------------------------------------------
|
||||
|
||||
|
||||
def _safe_body(resp: httpx.Response) -> Any:
|
||||
"""Return the response body, preferring JSON when possible."""
|
||||
try:
|
||||
return resp.json()
|
||||
except Exception: # noqa: BLE001
|
||||
text = resp.text
|
||||
return text if text else f"<empty {resp.status_code} response>"
|
||||
|
||||
|
||||
def _retry_after_seconds(resp: httpx.Response) -> int | None:
|
||||
"""Parse the ``Retry-After`` header into integer seconds.
|
||||
|
||||
Returns ``None`` if the header is absent (and the caller should
|
||||
fall back to its own backoff). The upstream API Management sends
|
||||
this header on quota-blocked responses; treating it as authoritative
|
||||
lets the caller sleep exactly until quota replenishes rather than
|
||||
guessing.
|
||||
"""
|
||||
raw = resp.headers.get("Retry-After") or resp.headers.get("retry-after")
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return int(raw)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
@@ -0,0 +1,113 @@
|
||||
"""File-type handlers for inbound MFT files (SP27).
|
||||
|
||||
Each handler is a pure function that parses + persists + dispatches
|
||||
events for one file type. The scheduler and the FastAPI endpoints
|
||||
both delegate here; the ``HANDLERS`` registry maps ``file_type`` →
|
||||
handler function.
|
||||
|
||||
Public API:
|
||||
HandleResult — dataclass returned by every handler
|
||||
HANDLERS — ``{"999": handle_999, "835": handle_835, ...}``
|
||||
handle_999, handle_ta1, handle_277ca, handle_835
|
||||
— call signatures: ``handle(text: str, source_file: str) -> HandleResult``
|
||||
|
||||
The handlers own their own DB session lifecycle. They emit pubsub
|
||||
events via the optional ``event_bus`` parameter (the FastAPI endpoint
|
||||
injects ``app.state.event_bus``; the scheduler passes ``None``).
|
||||
They never raise on per-segment problems; per-segment issues are
|
||||
logged and folded into the result. Whole-document failures
|
||||
(missing ISA, bad encoding) surface as ``CycloneParseError``, which
|
||||
the caller catches and records as ``STATUS_ERROR``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
|
||||
from ._ack_id import (
|
||||
ack_count_summary,
|
||||
ack_synthetic_source_batch_id,
|
||||
two77ca_synthetic_source_batch_id,
|
||||
)
|
||||
from .handle_result import HandleResult
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Module-level HANDLERS dict populated lazily once handler modules
|
||||
# ship. Keys are file_type strings, values are the ``handle``
|
||||
# callable for that type.
|
||||
HANDLERS: dict[str, object] = {}
|
||||
|
||||
|
||||
# Candidate handlers, in registration order. Each tuple is
|
||||
# (module-path, file_type). The 277/277CA mapping is added explicitly
|
||||
# after registration when the 277CA handler is present.
|
||||
_CANDIDATES: list[tuple[str, str]] = [
|
||||
("cyclone.handlers.handle_999", "999"),
|
||||
("cyclone.handlers.handle_ta1", "TA1"),
|
||||
("cyclone.handlers.handle_277ca", "277CA"),
|
||||
("cyclone.handlers.handle_835", "835"),
|
||||
]
|
||||
|
||||
|
||||
def register_handlers() -> None:
|
||||
"""Populate ``HANDLERS`` from the per-type handler modules.
|
||||
|
||||
Tolerates missing or broken handler modules so the package can be
|
||||
imported incrementally as each handler ships (Tasks 2-5), and so
|
||||
a partial-modification error in one handler module can't break
|
||||
scheduler / API import. Safe to call multiple times.
|
||||
"""
|
||||
if HANDLERS:
|
||||
return
|
||||
for mod_path, file_type in _CANDIDATES:
|
||||
try:
|
||||
mod = importlib.import_module(mod_path)
|
||||
fn = getattr(mod, "handle")
|
||||
except Exception as exc: # noqa: BLE001 — best-effort registry
|
||||
log.debug(
|
||||
"handler %s unavailable: %s",
|
||||
mod_path, exc,
|
||||
)
|
||||
continue
|
||||
HANDLERS[file_type] = fn
|
||||
# The 277 filename maps to the same 277CA handler.
|
||||
if file_type == "277CA":
|
||||
HANDLERS["277"] = fn
|
||||
|
||||
|
||||
register_handlers()
|
||||
|
||||
|
||||
# Re-export handler functions on this package so callers can use the
|
||||
# flat import (``from cyclone.handlers import handle_999``) once each
|
||||
# module ships. Set after registration so we know what's present.
|
||||
def _reexport_handlers() -> None:
|
||||
"""Re-export each handler's ``handle`` fn as ``handle_<type>``.
|
||||
|
||||
No-op for absent handlers. Re-run on each import so freshly-
|
||||
installed handler modules (e.g. Tasks 2-5 commits) are visible
|
||||
after ``register_handlers()`` without a process restart.
|
||||
"""
|
||||
for file_type, fn in list(HANDLERS.items()):
|
||||
if file_type in ("277",): # alias of 277CA; don't re-export twice
|
||||
continue
|
||||
globals()[f"handle_{file_type.lower()}"] = fn
|
||||
|
||||
|
||||
_reexport_handlers()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"HANDLERS",
|
||||
"HandleResult",
|
||||
"register_handlers",
|
||||
"ack_count_summary",
|
||||
"ack_synthetic_source_batch_id",
|
||||
"two77ca_synthetic_source_batch_id",
|
||||
# handler_* names are added at module load via _reexport_handlers
|
||||
]
|
||||
# Populate __all__ with the present handler symbols.
|
||||
for _h in ("handle_999", "handle_ta1", "handle_277ca", "handle_835"):
|
||||
if _h in globals():
|
||||
__all__.append(_h) # noqa: PYI056
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Ack ID helpers shared between the scheduler and the FastAPI
|
||||
endpoints (SP27 Task 1).
|
||||
|
||||
Lifted from ``scheduler.py:_ack_count_summary`` +
|
||||
``_ack_synthetic_source_batch_id`` + ``_277ca_synthetic_source_batch_id`` —
|
||||
all three had inline copies in both ``scheduler.py`` and ``api.py``
|
||||
prior to this SP. Both callers now import from this module.
|
||||
|
||||
Helpers
|
||||
-------
|
||||
``ack_count_summary(result)``
|
||||
Aggregate ``(received, accepted, rejected, ack_code)`` from a
|
||||
parsed ``ParseResult999``, trusting set-level IK5 over the
|
||||
functional-group AK9. Gainwell's MFT ships AK9 segments that
|
||||
contradict the per-set IK5 (e.g. ``AK9*A*1*1*1`` with
|
||||
``IK5*A``), so trusting AK9's rejected count over-reports.
|
||||
See scheduler commit ``6507a8c`` for the operational context.
|
||||
|
||||
``ack_synthetic_source_batch_id(icn, *, pcn, source_filename)``
|
||||
Build a unique-per-file ``batches.id`` for a 999 that ships
|
||||
without its own source batch. Falls back to a hash suffix of
|
||||
the filename so daily pulls don't all collapse onto the same
|
||||
ICN. Gainwell's MFT ships every 999 with the default ICN
|
||||
``000000001`` (per the scheduler docstring).
|
||||
|
||||
``two77ca_synthetic_source_batch_id(icn)``
|
||||
Same idea for a 277CA without its own source batch.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any
|
||||
|
||||
|
||||
def ack_count_summary(result: Any) -> tuple[int, int, int, str]:
|
||||
"""Aggregate ``(received, accepted, rejected, ack_code)`` from
|
||||
a ``ParseResult999``.
|
||||
|
||||
Counts are derived from the set-level ``IK5`` responses
|
||||
(one per ``AK2`` in the 999), not the functional-group ``AK9``.
|
||||
Gainwell's MFT ships contradictory AK9 segments; the per-set
|
||||
IK5 is the authoritative per-claim accept/reject signal.
|
||||
"""
|
||||
sets = result.set_responses
|
||||
received = len(sets)
|
||||
accepted = sum(1 for s in sets if s.set_accept_reject.code == "A")
|
||||
rejected = received - accepted
|
||||
if rejected == 0:
|
||||
code = "A"
|
||||
elif accepted == 0:
|
||||
code = "R"
|
||||
else:
|
||||
code = "P"
|
||||
return (received, accepted, rejected, code)
|
||||
|
||||
|
||||
def ack_synthetic_source_batch_id(
|
||||
interchange_control_number: str,
|
||||
*,
|
||||
pcn: str | None = None,
|
||||
source_filename: str | None = None,
|
||||
) -> str:
|
||||
"""Synthetic ``batches.id`` for a received 999 with no source batch.
|
||||
|
||||
Precedence for the human-readable part of the id (column is
|
||||
``VARCHAR(32)``):
|
||||
|
||||
1. ``999-{pcn}-{hash8}`` if ``AK2`` set_control_number is
|
||||
present (the common case). 4 + 9 + 1 + 8 = 22 chars max.
|
||||
2. ``999-{icn}-{hash8}`` if no AK2 (envelope-only 999).
|
||||
3. ``999-{hash12}`` if no filename either (shouldn't happen
|
||||
in production).
|
||||
"""
|
||||
short_hash = ""
|
||||
if source_filename:
|
||||
short_hash = hashlib.sha1(source_filename.encode("utf-8")).hexdigest()[:8]
|
||||
if pcn and pcn.strip():
|
||||
return f"999-{pcn.strip()}-{short_hash}"
|
||||
icn = (interchange_control_number or "").strip() or "000000001"
|
||||
if short_hash:
|
||||
return f"999-{icn}-{short_hash}"
|
||||
return f"999-{short_hash or icn}"
|
||||
|
||||
|
||||
def two77ca_synthetic_source_batch_id(interchange_control_number: str) -> str:
|
||||
"""Synthetic ``batches.id`` for a received 277CA with no source batch."""
|
||||
return f"277CA-{(interchange_control_number or '').strip() or '000000001'}"
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Handle a 277CA Claim Acknowledgment file (SP27 Task 4).
|
||||
|
||||
Lifted verbatim from ``scheduler.py:_handle_277ca``. The handler owns
|
||||
its own DB session, dispatches to ``parse_277ca_text``, persists the
|
||||
277CA ack row, applies 277CA rejections to matched claims via
|
||||
``inbox_state.apply_277ca_rejections``, and emits
|
||||
``claim.payer_rejected`` audit events for each newly-stamped claim.
|
||||
|
||||
The actor tag (``"277ca-parser-scheduler"``) is preserved so the
|
||||
audit log keeps tracing back to the same source after extraction.
|
||||
Both the FastAPI endpoint and the scheduler path (re)use this module
|
||||
— the API migration drops the inline copy in Task 6.
|
||||
|
||||
``claim.rejected_after_remit`` audit emission (when a 277CA rejects
|
||||
a claim that already has ``matched_remittance_id`` set) is deferred
|
||||
to SP27 Task 13. Today the handler only emits ``claim.payer_rejected``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.audit_log import AuditEvent, append_event
|
||||
from cyclone.claim_acks import apply_277ca_acks
|
||||
from cyclone.handlers._ack_id import two77ca_synthetic_source_batch_id
|
||||
from cyclone.inbox_state_277ca import apply_277ca_rejections
|
||||
from cyclone.parsers.exceptions import CycloneParseError
|
||||
from cyclone.parsers.parse_277ca import parse_277ca_text
|
||||
from cyclone.store import store as cycl_store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def handle(
|
||||
text: str,
|
||||
source_file: str,
|
||||
) -> tuple[str, int]:
|
||||
"""Parse a 277CA, persist ack + stamp payer-rejected claims.
|
||||
|
||||
Args:
|
||||
text: Raw 277CA document bytes (decoded).
|
||||
source_file: Filename the 277CA came from.
|
||||
|
||||
Returns:
|
||||
``(parser_used, claim_count)`` tuple where ``claim_count`` is
|
||||
the number of STC statuses in the file (one per claim).
|
||||
|
||||
Raises:
|
||||
ValueError: on parser-level failure (wraps CycloneParseError).
|
||||
|
||||
SP25: the store (``cyclone.store.acks.add_277ca_ack``) owns the
|
||||
publish-from-store contract; the handler no longer needs an
|
||||
``event_bus`` kwarg.
|
||||
"""
|
||||
try:
|
||||
result = parse_277ca_text(text, input_file=source_file)
|
||||
except CycloneParseError as exc:
|
||||
raise ValueError(f"277CA parse error: {exc}") from exc
|
||||
|
||||
icn = result.envelope.control_number
|
||||
synthetic_id = two77ca_synthetic_source_batch_id(icn)
|
||||
accepted = sum(
|
||||
1 for s in result.claim_statuses if s.classification == "accepted"
|
||||
)
|
||||
paid = sum(
|
||||
1 for s in result.claim_statuses if s.classification == "paid"
|
||||
)
|
||||
rejected = sum(
|
||||
1 for s in result.claim_statuses if s.classification == "rejected"
|
||||
)
|
||||
pended = sum(
|
||||
1 for s in result.claim_statuses if s.classification == "pended"
|
||||
)
|
||||
|
||||
# Build the batch envelope index BEFORE opening the work session
|
||||
# — cycl_store.batch_envelope_index opens its own short-lived
|
||||
# session, and SQLite + concurrent sessions causes "database is
|
||||
# locked" errors.
|
||||
batch_index = cycl_store.batch_envelope_index()
|
||||
|
||||
with db.SessionLocal()() as session:
|
||||
row = cycl_store.add_277ca_ack(
|
||||
source_batch_id=synthetic_id,
|
||||
control_number=icn,
|
||||
accepted_count=accepted,
|
||||
rejected_count=rejected,
|
||||
paid_count=paid,
|
||||
pended_count=pended,
|
||||
raw_json=json.loads(result.model_dump_json()),
|
||||
)
|
||||
def _lookup(pcn: str):
|
||||
return (
|
||||
session.query(db.Claim)
|
||||
.filter(db.Claim.patient_control_number == pcn)
|
||||
.first()
|
||||
)
|
||||
apply_result = apply_277ca_rejections(
|
||||
session, result, claim_lookup=_lookup, two77ca_id=row.id,
|
||||
)
|
||||
if apply_result.matched:
|
||||
for cid in apply_result.matched:
|
||||
append_event(session, AuditEvent(
|
||||
event_type="claim.payer_rejected",
|
||||
entity_type="claim",
|
||||
entity_id=cid,
|
||||
payload={"source_batch_id": synthetic_id, "277ca_id": row.id},
|
||||
actor="277ca-parser-scheduler",
|
||||
))
|
||||
# SP28: auto-link the 277CA ClaimStatus entries to claims (D10
|
||||
# two-pass join). The helper builds dataclass rows; the
|
||||
# caller persists each via cycl_store.add_claim_ack so the
|
||||
# publish-from-store contract owns the live-tail event.
|
||||
def _pcn_lookup(pcn: str):
|
||||
return (
|
||||
session.query(db.Claim)
|
||||
.filter(db.Claim.patient_control_number == pcn)
|
||||
.first()
|
||||
)
|
||||
|
||||
link_result = apply_277ca_acks(
|
||||
session, result, ack_id=row.id,
|
||||
batch_envelope_index=batch_index,
|
||||
pc_claim_lookup=_pcn_lookup,
|
||||
)
|
||||
# Snapshot the rows before closing the work session — SQLite
|
||||
# + concurrent sessions from the same thread cause "database
|
||||
# is locked" errors when add_claim_ack opens its own session
|
||||
# while this one is still open.
|
||||
link_rows = list(link_result.linked)
|
||||
orphans = list(link_result.orphans)
|
||||
session.commit()
|
||||
|
||||
for link_row in link_rows:
|
||||
cycl_store.add_claim_ack(
|
||||
claim_id=link_row.claim_id,
|
||||
batch_id=link_row.batch_id,
|
||||
ack_id=row.id,
|
||||
ack_kind="277ca",
|
||||
ak2_index=link_row.ak2_index,
|
||||
set_control_number=link_row.set_control_number,
|
||||
set_accept_reject_code=link_row.set_accept_reject_code,
|
||||
linked_by="auto",
|
||||
)
|
||||
if orphans:
|
||||
log.warning(
|
||||
"277CA had %d orphan status entries (no matching claim): %s",
|
||||
len(orphans),
|
||||
orphans[:5],
|
||||
)
|
||||
|
||||
return ("parse_277ca", len(result.claim_statuses))
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Handle an 835 ERA Remittance file (SP27 Task 5).
|
||||
|
||||
Lifted verbatim from ``scheduler.py:_handle_835``. The handler owns
|
||||
its own state, dispatches to ``parse_835`` (raises
|
||||
``CycloneParseError`` on bad EDI), runs the per-payer 835 validator,
|
||||
stamps the validation report into ``result.summary``, and persists
|
||||
the BatchRecord via ``cycl_store.add``.
|
||||
|
||||
The 835 handler is the largest of the four because the schema
|
||||
covers per-claim remittances + CAS adjustments + validation.
|
||||
``PAYER_FACTORIES_835`` lives in ``cyclone.api`` (it was always
|
||||
intended to live in ``cyclone.payers`` — a TODO pre-dating SP27 —
|
||||
but moving it is out of Task 5 scope). We import it lazily inside
|
||||
``handle`` to avoid a module-load-time cycle; the cyclic risk is
|
||||
acceptable because api.py also imports scheduler lazily (inside its
|
||||
lifespan handler).
|
||||
|
||||
Two-phase ingest closed in SP27 Task 10: the placeholder
|
||||
``adjustment_amount`` is overwritten by reconcile in the same DB
|
||||
session as the insert (before commit), so a reader never sees a
|
||||
half-reconciled Remittance row. If reconcile raises, the whole
|
||||
ingest rolls back and the scheduler records the per-file error.
|
||||
|
||||
``event_bus`` is best-effort and follows the same TODO(sp27-task-6)
|
||||
sync/async gap as handle_999 / handle_ta1 / handle_277ca.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from cyclone.parsers.exceptions import CycloneParseError
|
||||
from cyclone.parsers.parse_835 import parse as parse_835
|
||||
from cyclone.parsers.validator_835 import validate as validate_835
|
||||
from cyclone.store import BatchRecord
|
||||
from cyclone.store import store as cycl_store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def handle(
|
||||
text: str,
|
||||
source_file: str,
|
||||
) -> tuple[str, int]:
|
||||
"""Parse an 835, run validation, persist batch + remittances.
|
||||
|
||||
Args:
|
||||
text: Raw 835 document bytes (decoded).
|
||||
source_file: Filename the 835 came from.
|
||||
|
||||
Returns:
|
||||
``(parser_used, claim_count)`` tuple. ``claim_count`` is the
|
||||
number of CLP claims parsed; the BatchRecord's summary's
|
||||
``passed`` / ``failed`` fields are derived from the
|
||||
validator's ``report.passed`` flag.
|
||||
|
||||
Raises:
|
||||
ValueError: on parser-level failure (wraps CycloneParseError).
|
||||
Validation failures don't raise — they're stamped into
|
||||
``summary.passed = 0``.
|
||||
|
||||
SP25: the 835 write path already publishes ``remittance_written``
|
||||
via ``CycloneStore.add`` (SP21 split). The handler no longer
|
||||
accepts an ``event_bus`` kwarg — the ``ack_received`` publish
|
||||
here was dead code anyway (the 835 event name is
|
||||
``remittance_written``, not ``ack_received``).
|
||||
"""
|
||||
# SP36 Task 16: PAYER_FACTORIES_835 moved from ``cyclone.api``
|
||||
# to ``cyclone.api_routers._shared`` (cross-router helper). Import
|
||||
# from the new home. The lazy import is still needed to avoid a
|
||||
# circular import at registry load time (handle_835 is imported
|
||||
# by the scheduler during the api lifespan).
|
||||
from cyclone.api_routers._shared import PAYER_FACTORIES_835
|
||||
|
||||
config = PAYER_FACTORIES_835["co_medicaid_835"]()
|
||||
try:
|
||||
result = parse_835(text, config, input_file=source_file)
|
||||
except CycloneParseError as exc:
|
||||
raise ValueError(f"835 parse error: {exc}") from exc
|
||||
|
||||
# Validation report (mirrors the API endpoint).
|
||||
report = validate_835(result, config)
|
||||
n = len(result.claims)
|
||||
if report.passed:
|
||||
passed, failed, failed_claim_ids = n, 0, []
|
||||
else:
|
||||
passed, failed, failed_claim_ids = 0, n, [
|
||||
c.payer_claim_control_number for c in result.claims
|
||||
]
|
||||
result = result.model_copy(update={
|
||||
"validation": report,
|
||||
"summary": result.summary.model_copy(update={
|
||||
"passed": passed,
|
||||
"failed": failed,
|
||||
"failed_claim_ids": failed_claim_ids,
|
||||
}),
|
||||
})
|
||||
|
||||
rec = BatchRecord(
|
||||
id=uuid.uuid4().hex,
|
||||
kind="835",
|
||||
input_filename=source_file,
|
||||
parsed_at=datetime.now(timezone.utc),
|
||||
result=result,
|
||||
)
|
||||
cycl_store.add(rec)
|
||||
|
||||
return ("parse_835", len(result.claims))
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Handle a 999 Implementation Acknowledgment file (SP27 Task 2).
|
||||
|
||||
Lifted verbatim from ``scheduler.py:_handle_999``. The handler owns
|
||||
its own DB session, dispatches to ``parse_999_text``, applies 999
|
||||
rejections to any matched claims via ``inbox_state.apply_999_rejections``,
|
||||
persists the ack row, and returns a ``(parser_used, claim_count)``
|
||||
tuple.
|
||||
|
||||
The actor tag (``"999-parser-scheduler"``) is preserved so the audit
|
||||
log keeps tracing back to the same source after extraction. Both
|
||||
the FastAPI endpoint and the scheduler path (re)use this module —
|
||||
see Task 6 for the API migration that drops the inline copy.
|
||||
|
||||
``claim_count`` mirrors ``parsed.received`` — the count of AK2
|
||||
(set-level) responses in the 999, which is the authoritative
|
||||
per-claim accept/reject signal (see ``_ack_id`` for why AK9 is
|
||||
ignored).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.audit_log import AuditEvent, append_event
|
||||
from cyclone.claim_acks import apply_999_acceptances
|
||||
from cyclone.handlers._ack_id import (
|
||||
ack_count_summary,
|
||||
ack_synthetic_source_batch_id,
|
||||
)
|
||||
from cyclone.inbox_state import apply_999_rejections
|
||||
from cyclone.parsers.exceptions import CycloneParseError
|
||||
from cyclone.parsers.parse_999 import parse_999_text
|
||||
from cyclone.store import store as cycl_store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def handle(
|
||||
text: str,
|
||||
source_file: str,
|
||||
) -> tuple[str, int]:
|
||||
"""Parse a 999, apply rejections, persist ack row.
|
||||
|
||||
Args:
|
||||
text: Raw 999 document bytes (decoded).
|
||||
source_file: Filename the 999 came from. Used to derive a
|
||||
unique synthetic ``batches.id`` (see ``_ack_id``).
|
||||
|
||||
Returns:
|
||||
``(parser_used, claim_count)`` tuple. Matches the scheduler
|
||||
``_download_and_parse`` destructure; the HandleResult
|
||||
migration happens in Task 7.
|
||||
|
||||
Raises:
|
||||
ValueError: on parser-level failure (wraps CycloneParseError).
|
||||
|
||||
SP25: the store (``cyclone.store.acks.add_999_ack``) now owns the
|
||||
publish-from-store contract; the handler no longer needs an
|
||||
``event_bus`` kwarg. Both the scheduler path (no bus) and the
|
||||
FastAPI endpoint path (passes the bus to the store directly) see
|
||||
the same row, the same event, and the same payload.
|
||||
"""
|
||||
try:
|
||||
result = parse_999_text(text, input_file=source_file)
|
||||
except CycloneParseError as exc:
|
||||
raise ValueError(f"999 parse error: {exc}") from exc
|
||||
|
||||
received, accepted, rejected, ack_code = ack_count_summary(result)
|
||||
icn = result.envelope.control_number
|
||||
pcn = (
|
||||
result.set_responses[0].set_control_number
|
||||
if result.set_responses else None
|
||||
)
|
||||
synthetic_id = ack_synthetic_source_batch_id(
|
||||
icn, pcn=pcn, source_filename=source_file,
|
||||
)
|
||||
|
||||
# Build the batch envelope index BEFORE opening the work session
|
||||
# — cycl_store.batch_envelope_index opens its own short-lived
|
||||
# session, and SQLite + concurrent sessions causes "database is
|
||||
# locked" errors.
|
||||
batch_index = cycl_store.batch_envelope_index()
|
||||
|
||||
with db.SessionLocal()() as session:
|
||||
def _lookup(pcn: str):
|
||||
return (
|
||||
session.query(db.Claim)
|
||||
.filter_by(patient_control_number=pcn)
|
||||
.first()
|
||||
)
|
||||
rejection_result = apply_999_rejections(
|
||||
session, result,
|
||||
claim_lookup=_lookup,
|
||||
batch_envelope_index=batch_index,
|
||||
)
|
||||
if rejection_result.matched:
|
||||
for cid in rejection_result.matched:
|
||||
append_event(session, AuditEvent(
|
||||
event_type="claim.rejected",
|
||||
entity_type="claim",
|
||||
entity_id=cid,
|
||||
payload={"source_batch_id": synthetic_id},
|
||||
actor="999-parser-scheduler",
|
||||
))
|
||||
row = cycl_store.add_ack(
|
||||
source_batch_id=synthetic_id,
|
||||
accepted_count=accepted,
|
||||
rejected_count=rejected,
|
||||
received_count=received,
|
||||
ack_code=ack_code,
|
||||
raw_json=json.loads(result.model_dump_json()),
|
||||
)
|
||||
# SP28: auto-link the 999 AK2 set-responses to claims (D10 two-pass
|
||||
# join). The PCN fallback only fires when the ST02 lookup misses
|
||||
# (rare for Gainwell batches). Each created ClaimAck row is
|
||||
# persisted via cycl_store.add_claim_ack so the publish-from-store
|
||||
# contract owns the live-tail event.
|
||||
def _pcn_lookup(pcn: str):
|
||||
return (
|
||||
session.query(db.Claim)
|
||||
.filter_by(patient_control_number=pcn)
|
||||
.first()
|
||||
)
|
||||
|
||||
link_result = apply_999_acceptances(
|
||||
session, result, ack_id=row.id,
|
||||
batch_envelope_index=batch_index,
|
||||
pc_claim_lookup=_pcn_lookup,
|
||||
)
|
||||
# Snapshot the rows before closing the work session — SQLite
|
||||
# + concurrent sessions from the same thread cause "database
|
||||
# is locked" errors when add_claim_ack opens its own session
|
||||
# while this one is still open.
|
||||
link_rows = list(link_result.linked)
|
||||
orphans = list(link_result.orphans)
|
||||
session.commit()
|
||||
|
||||
for link_row in link_rows:
|
||||
cycl_store.add_claim_ack(
|
||||
claim_id=link_row.claim_id,
|
||||
batch_id=link_row.batch_id,
|
||||
ack_id=row.id,
|
||||
ack_kind="999",
|
||||
ak2_index=link_row.ak2_index,
|
||||
set_control_number=link_row.set_control_number,
|
||||
set_accept_reject_code=link_row.set_accept_reject_code,
|
||||
linked_by="auto",
|
||||
)
|
||||
if orphans:
|
||||
log.warning(
|
||||
"999 had %d orphan set refs (no matching claim): %s",
|
||||
len(orphans),
|
||||
orphans[:5],
|
||||
)
|
||||
|
||||
return ("parse_999", received)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Shared ``HandleResult`` dataclass for handlers (SP27).
|
||||
|
||||
Every per-file-type handler returns the same shape so the
|
||||
scheduler's ``_handle_one`` and the FastAPI parse endpoints can
|
||||
process them uniformly.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class HandleResult:
|
||||
"""Outcome of one handler invocation.
|
||||
|
||||
Attributes:
|
||||
parser_used: The parser name written to
|
||||
``processed_inbound_files.parser_used`` (e.g. ``"parse_999"``)
|
||||
and surfaced in the UI.
|
||||
claim_count: The number of claim rows persisted (or batch
|
||||
records, depending on the file type). For 999/TA1 this
|
||||
is the receipt count; for 835 this is the per-claim
|
||||
remittance count; for 277CA this is the per-claim
|
||||
status count.
|
||||
batch_id: The persisted batch id (when the handler creates a
|
||||
row in ``batches``). ``None`` for handlers that persist
|
||||
into per-file-type ack tables (999/TA1/277CA) rather
|
||||
than into the unified ``batches`` table.
|
||||
matched_count: For 835, the number of remits that were
|
||||
matched to an existing claim by ``reconcile.match``.
|
||||
Zero for other handlers.
|
||||
"""
|
||||
|
||||
parser_used: str
|
||||
claim_count: int
|
||||
batch_id: str | None = None
|
||||
matched_count: int = 0
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Handle a TA1 Interchange Acknowledgment file (SP27 Task 3).
|
||||
|
||||
Lifted verbatim from ``scheduler.py:_handle_ta1``. The handler owns
|
||||
its own DB session, dispatches to ``parse_ta1_text``, persists the
|
||||
interchange ack row in ``ta1_acks``, and returns a
|
||||
``(parser_used, claim_count)`` tuple.
|
||||
|
||||
Unlike the 999 handler, TA1 is envelope-only — there is one TA1 per
|
||||
ISA/IEA interchange, no set-level (AK2) or claim-level matching.
|
||||
The claim_count is always 1 (one TA1 ack row per file).
|
||||
|
||||
The actor tag is implicit (no audit event here — TA1 doesn't tag
|
||||
anything in the activity log; it's an infrastructure-level ack).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.claim_acks import apply_ta1_envelope_link
|
||||
from cyclone.parsers.exceptions import CycloneParseError
|
||||
from cyclone.parsers.parse_ta1 import parse_ta1_text
|
||||
from cyclone.store import store as cycl_store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def handle(
|
||||
text: str,
|
||||
source_file: str,
|
||||
) -> tuple[str, int]:
|
||||
"""Parse a TA1, persist the interchange ack row.
|
||||
|
||||
Args:
|
||||
text: Raw TA1 document bytes (decoded).
|
||||
source_file: Filename the TA1 came from. Used for audit
|
||||
attribution; the ``batches.id`` for TA1 rows is derived
|
||||
internally from the parsed envelope's control number
|
||||
(``TA1-{ICN}``).
|
||||
|
||||
Returns:
|
||||
``(parser_used, claim_count)`` tuple. TA1 always returns
|
||||
``claim_count=1`` (one TA1 ack row per interchange file).
|
||||
|
||||
Raises:
|
||||
ValueError: on parser-level failure (wraps CycloneParseError).
|
||||
|
||||
SP25: the store (``cyclone.store.acks.add_ta1_ack``) owns the
|
||||
publish-from-store contract; the handler no longer needs an
|
||||
``event_bus`` kwarg.
|
||||
"""
|
||||
try:
|
||||
result = parse_ta1_text(text, input_file=source_file)
|
||||
except CycloneParseError as exc:
|
||||
raise ValueError(f"TA1 parse error: {exc}") from exc
|
||||
|
||||
with db.SessionLocal()() as session:
|
||||
ta1_ack_row = cycl_store.add_ta1_ack(
|
||||
source_batch_id=result.source_batch_id,
|
||||
control_number=result.ta1.control_number,
|
||||
interchange_date=result.ta1.interchange_date,
|
||||
interchange_time=result.ta1.interchange_time,
|
||||
ack_code=result.ta1.ack_code,
|
||||
note_code=result.ta1.note_code,
|
||||
ack_generated_date=result.ta1.ack_generated_date,
|
||||
sender_id=result.envelope.sender_id,
|
||||
receiver_id=result.envelope.receiver_id,
|
||||
raw_json=json.loads(result.model_dump_json()),
|
||||
)
|
||||
# SP28: TA1 envelope-level link to the originating Batch. The
|
||||
# closure here matches the most-recent Batch whose envelope
|
||||
# sender_id/receiver_id matches the TA1 — see spec §D4.
|
||||
def _batch_lookup(sender_id, receiver_id):
|
||||
rows = (
|
||||
session.query(db.Batch)
|
||||
.filter(
|
||||
db.Batch.kind == "837p",
|
||||
db.Batch.raw_result_json.isnot(None),
|
||||
)
|
||||
.order_by(db.Batch.parsed_at.desc())
|
||||
.all()
|
||||
)
|
||||
for row in rows:
|
||||
env = (row.raw_result_json or {}).get("envelope") or {}
|
||||
if (
|
||||
env.get("sender_id") == sender_id
|
||||
and env.get("receiver_id") == receiver_id
|
||||
):
|
||||
return row
|
||||
return None
|
||||
|
||||
link_result = apply_ta1_envelope_link(
|
||||
session, result, ack_id=ta1_ack_row.id,
|
||||
batch_lookup=_batch_lookup,
|
||||
)
|
||||
# Snapshot rows before closing the work session — SQLite +
|
||||
# concurrent sessions from the same thread cause "database
|
||||
# is locked" errors when add_claim_ack opens its own session
|
||||
# while this one is still open.
|
||||
link_rows = list(link_result.linked)
|
||||
orphans = list(link_result.orphans)
|
||||
session.commit()
|
||||
|
||||
for link_row in link_rows:
|
||||
cycl_store.add_claim_ack(
|
||||
claim_id=link_row.claim_id,
|
||||
batch_id=link_row.batch_id,
|
||||
ack_id=ta1_ack_row.id,
|
||||
ack_kind="ta1",
|
||||
ak2_index=link_row.ak2_index,
|
||||
set_control_number=link_row.set_control_number,
|
||||
set_accept_reject_code=link_row.set_accept_reject_code,
|
||||
linked_by="auto",
|
||||
)
|
||||
if orphans:
|
||||
log.warning(
|
||||
"TA1 had %d orphan envelope refs (no matching batch): %s",
|
||||
len(orphans),
|
||||
orphans[:5],
|
||||
)
|
||||
|
||||
return ("parse_ta1", 1)
|
||||
@@ -175,6 +175,82 @@ def _line_count_lookup(session: Session, claims: list[Claim]) -> tuple[dict, dic
|
||||
return matched_counts, total_lines_by_claim
|
||||
|
||||
|
||||
def _ack_summary_for_claims(
|
||||
session: Session, claim_ids: list[str],
|
||||
) -> dict[str, dict]:
|
||||
"""Build a {claim_id: {total, rejected, items: [...]}} map for 999 acks.
|
||||
|
||||
SP29: the Inbox `rejected` lane needs to render AK2 evidence
|
||||
inline per row, plus a per-row Resubmit button. The data lives
|
||||
in ``claim_acks`` (SP28) — we don't want to N+1 fetch per row,
|
||||
so the whole rejected-claim set is summarized in one batched
|
||||
query here.
|
||||
|
||||
Filters to ``ack_kind='999'`` because the rejected lane is the
|
||||
999 envelope reject lane; the 277CA STC A4/A6/A7 evidence flows
|
||||
through the ``payer_rejected_*`` fields on a separate lane and
|
||||
isn't part of this scope (see SP29 spec D4 / scope).
|
||||
|
||||
Returns:
|
||||
``{claim_id: {"total": int, "rejected": int, "items": [...]}, ...}``
|
||||
Claims with zero linked 999 acks are NOT in the returned
|
||||
dict — the caller maps via ``.get(cid)`` and treats absence
|
||||
as "no 999 acks linked" (renders as ``null`` in the
|
||||
payload, ``999 not linked`` in the UI).
|
||||
|
||||
Args:
|
||||
session: SQLAlchemy session the caller owns.
|
||||
claim_ids: list of claim.id values to summarize. Typically
|
||||
the rejected-lane claim ids. Empty list → empty dict.
|
||||
"""
|
||||
if not claim_ids:
|
||||
return {}
|
||||
from cyclone.db import ClaimAck # late import — DB model registered
|
||||
rows = (
|
||||
session.query(
|
||||
ClaimAck.claim_id,
|
||||
ClaimAck.ack_id,
|
||||
ClaimAck.set_control_number,
|
||||
ClaimAck.set_accept_reject_code,
|
||||
ClaimAck.ak2_index,
|
||||
ClaimAck.linked_at,
|
||||
)
|
||||
.filter(
|
||||
ClaimAck.claim_id.in_(claim_ids),
|
||||
ClaimAck.ack_kind == "999",
|
||||
)
|
||||
.order_by(ClaimAck.linked_at.desc(), ClaimAck.id.desc())
|
||||
.all()
|
||||
)
|
||||
grouped: dict[str, list[tuple]] = {}
|
||||
for cid, aid, scn, code, ak2i, lat in rows:
|
||||
grouped.setdefault(cid, []).append((aid, scn, code, ak2i, lat))
|
||||
|
||||
rejected_codes = {"R", "E", "X"}
|
||||
out: dict[str, dict] = {}
|
||||
for cid, items in grouped.items():
|
||||
total = len(items)
|
||||
rejected_count = sum(1 for it in items if (it[2] or "") in rejected_codes)
|
||||
# Keep 5 most recent items for the chip column. The full count
|
||||
# is in ``total`` so the UI can show ``+N more`` honestly.
|
||||
trimmed = items[:5]
|
||||
out[cid] = {
|
||||
"total": total,
|
||||
"rejected": rejected_count,
|
||||
"items": [
|
||||
{
|
||||
"ack_id": aid,
|
||||
"set_control_number": scn,
|
||||
"set_accept_reject_code": code or "",
|
||||
"ak2_index": ak2i,
|
||||
"linked_at": _isoformat(lat),
|
||||
}
|
||||
for (aid, scn, code, ak2i, lat) in trimmed
|
||||
],
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def compute_lanes(session: Session, *, dismissed_pairs: Iterable[frozenset]) -> Lanes:
|
||||
lanes = Lanes()
|
||||
dismissed = set(dismissed_pairs)
|
||||
@@ -192,6 +268,17 @@ def compute_lanes(session: Session, *, dismissed_pairs: Iterable[frozenset]) ->
|
||||
),
|
||||
))
|
||||
|
||||
# SP29: attach the 999 ack-evidence summary (total / rejected /
|
||||
# 5 most recent AK2 set_responses) to every rejected row so the
|
||||
# Inbox can render AK2 chips inline + a per-row Resubmit button
|
||||
# without an extra round-trip. One batched query, keyed off the
|
||||
# rejected-claim id set.
|
||||
rejected_ack_summary = _ack_summary_for_claims(
|
||||
session, [r["id"] for r in lanes.rejected]
|
||||
)
|
||||
for row in lanes.rejected:
|
||||
row["claim_acks"] = rejected_ack_summary.get(row["id"])
|
||||
|
||||
# --- Payer-Rejected (SP10) ---
|
||||
# Distinct from the 999 envelope "rejected" lane above. A claim
|
||||
# lands here when a 277CA STC category code is A4/A6/A7 (rejected
|
||||
|
||||
@@ -33,42 +33,63 @@ def apply_999_rejections(
|
||||
parsed_999,
|
||||
*,
|
||||
claim_lookup: Callable[[str], Claim | None],
|
||||
batch_envelope_index: dict[str, list[str]] | None = None,
|
||||
) -> Apply999Result:
|
||||
"""For each set response with code R or E, look up the matching claim and
|
||||
"""For each set response with code R, E, or X, look up the matching claim and
|
||||
move it to REJECTED. Idempotent on already-rejected claims.
|
||||
|
||||
Args:
|
||||
session: SQLAlchemy session.
|
||||
parsed_999: a ParseResult999 (or any object with .set_responses).
|
||||
claim_lookup: callable from patient_control_number → Claim or None.
|
||||
Legacy fallback; rarely hits when batch_envelope_index is present.
|
||||
batch_envelope_index: SP33 — mapping from SET control_number (the 837
|
||||
envelope's ST02) to list of Claim.id for the claims in that SET.
|
||||
Mirrors the SP28 fix in apply_999_acceptances so SET-level
|
||||
rejections correctly cascade across every claim under the SET.
|
||||
|
||||
Returns:
|
||||
Apply999Result with lists of matched claim ids and orphan PCNs.
|
||||
"""
|
||||
result = Apply999Result()
|
||||
now = datetime.now(timezone.utc)
|
||||
index = batch_envelope_index or {}
|
||||
|
||||
for sr in parsed_999.set_responses:
|
||||
code = sr.set_accept_reject.code
|
||||
if code not in ("R", "E", "X"):
|
||||
continue
|
||||
|
||||
claim = claim_lookup(sr.set_control_number)
|
||||
if claim is None:
|
||||
result.orphans.append(sr.set_control_number)
|
||||
continue
|
||||
# SP33: prefer batch_envelope_index (SCN -> [claim_id]) so a SET-level
|
||||
# rejection correctly flips every claim in the SET. Fall back to
|
||||
# the legacy claim_lookup when the index is empty for this SCN.
|
||||
candidate_ids = index.get(sr.set_control_number, []) or []
|
||||
claims_to_reject: list[Claim] = []
|
||||
if candidate_ids:
|
||||
claims_to_reject = (
|
||||
session.query(Claim)
|
||||
.filter(Claim.id.in_(candidate_ids))
|
||||
.all()
|
||||
)
|
||||
else:
|
||||
legacy = claim_lookup(sr.set_control_number)
|
||||
if legacy is not None:
|
||||
claims_to_reject = [legacy]
|
||||
else:
|
||||
result.orphans.append(sr.set_control_number)
|
||||
continue
|
||||
|
||||
if claim.state == ClaimState.REJECTED:
|
||||
# Idempotent: don't double-mutate.
|
||||
continue
|
||||
|
||||
claim.state = ClaimState.REJECTED
|
||||
claim.state_changed_at = now
|
||||
claim.rejected_at = now
|
||||
claim.rejection_reason = _build_reason(
|
||||
code, len(sr.segment_errors or [])
|
||||
)
|
||||
result.matched.append(claim.id)
|
||||
for claim in claims_to_reject:
|
||||
if claim.state == ClaimState.REJECTED:
|
||||
# Idempotent: don't double-mutate.
|
||||
continue
|
||||
claim.state = ClaimState.REJECTED
|
||||
claim.state_changed_at = now
|
||||
claim.rejected_at = now
|
||||
claim.rejection_reason = _build_reason(
|
||||
code, len(sr.segment_errors or [])
|
||||
)
|
||||
result.matched.append(claim.id)
|
||||
|
||||
if result.matched or result.orphans:
|
||||
session.commit()
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
-- version: 13
|
||||
-- Auth (SP-auth): users + sessions tables.
|
||||
--
|
||||
-- `users` holds the local credential store: bcrypt-hashed password,
|
||||
-- role enum ('admin' | 'user' | 'viewer'), and a soft-delete column
|
||||
-- (disabled_at) so admins can revoke access without losing history.
|
||||
--
|
||||
-- `sessions` holds the server-side session rows; the browser only
|
||||
-- carries an opaque token cookie (cyclone_session) that points here.
|
||||
-- expires_at index lets us cheaply reap stale sessions.
|
||||
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL,
|
||||
role TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
disabled_at TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX idx_users_username ON users(username);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id),
|
||||
expires_at TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX idx_sessions_user_id ON sessions(user_id);
|
||||
CREATE INDEX idx_sessions_expires_at ON sessions(expires_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
-- version: 14
|
||||
-- Auth (SP-auth): record the acting user_id on every audit_log entry.
|
||||
--
|
||||
-- Backwards-compatible: existing rows get NULL user_id (they were
|
||||
-- written by the pre-auth `system` actor). Going forward, the FastAPI
|
||||
-- get_current_user dependency injects the id into every audit log call.
|
||||
|
||||
ALTER TABLE audit_log ADD COLUMN user_id INTEGER;
|
||||
|
||||
CREATE INDEX idx_audit_log_user_id ON audit_log(user_id);
|
||||
@@ -0,0 +1,74 @@
|
||||
-- version: 15
|
||||
-- Drop the inline UNIQUE(batch_id, patient_control_number) on claims.
|
||||
--
|
||||
-- Migration 0003 attempted DROP INDEX IF EXISTS uq_claims_batch_pcn but
|
||||
-- the constraint is inline in CREATE TABLE, so the drop was a no-op.
|
||||
-- The only way to remove an inline UNIQUE in SQLite is table recreation.
|
||||
--
|
||||
-- Discovery 2026-06-23: the inline UNIQUE does NOT exist in the current
|
||||
-- production DB at user_version=14 (or in main's fresh-DB schema). The
|
||||
-- 32 "Duplicate claim" warnings in /tmp/cyclone-uvicorn.log are PK
|
||||
-- collisions on claims.id (CLM01) when an operator re-uploads the same
|
||||
-- file — not UNIQUE violations. This migration is therefore a defensive
|
||||
-- no-op against the current schema, but keeps the 0003 intent alive
|
||||
-- (drop the constraint if it ever reappears) and lets the SP22 spec
|
||||
-- ship as designed.
|
||||
--
|
||||
-- X12 837P allows any number of CLM segments per 2000B subscriber loop;
|
||||
-- claim identity is provided by the primary key (claims.id = CLM01).
|
||||
-- The remittances table had a parallel constraint already removed in 0003
|
||||
-- (because that one WAS a named index), so this migration only touches
|
||||
-- claims.
|
||||
--
|
||||
-- The migration runner (db_migrate.py) wraps each .sql in an implicit
|
||||
-- transaction via engine.begin(), so we MUST NOT use BEGIN/COMMIT.
|
||||
-- PRAGMA defer_foreign_keys defers FK checks to commit, which is the
|
||||
-- only way to drop a referenced table inside a transaction in SQLite.
|
||||
-- Other tables referencing claims:
|
||||
-- remittances.claim_id
|
||||
-- matches.claim_id
|
||||
-- line_reconciliations.claim_id
|
||||
-- activity_events.claim_id
|
||||
|
||||
PRAGMA defer_foreign_keys = ON;
|
||||
|
||||
CREATE TABLE claims_new (
|
||||
id TEXT PRIMARY KEY,
|
||||
batch_id TEXT NOT NULL REFERENCES batches(id) ON DELETE CASCADE,
|
||||
patient_control_number TEXT NOT NULL,
|
||||
service_date_from DATE,
|
||||
service_date_to DATE,
|
||||
charge_amount NUMERIC(12, 2) NOT NULL DEFAULT 0,
|
||||
provider_npi TEXT,
|
||||
payer_id TEXT,
|
||||
state TEXT NOT NULL DEFAULT 'submitted',
|
||||
state_before_reversal TEXT,
|
||||
matched_remittance_id TEXT REFERENCES remittances(id),
|
||||
raw_json TEXT,
|
||||
rejection_reason TEXT,
|
||||
rejected_at TIMESTAMP,
|
||||
resubmit_count INTEGER NOT NULL DEFAULT 0,
|
||||
state_changed_at TIMESTAMP,
|
||||
payer_rejected_at TEXT,
|
||||
payer_rejected_reason TEXT,
|
||||
payer_rejected_status_code TEXT,
|
||||
payer_rejected_by_277ca_id TEXT,
|
||||
payer_rejected_acknowledged_at TEXT,
|
||||
payer_rejected_acknowledged_actor TEXT
|
||||
-- NO UNIQUE (batch_id, patient_control_number) — removed.
|
||||
);
|
||||
|
||||
INSERT INTO claims_new SELECT * FROM claims;
|
||||
DROP TABLE claims;
|
||||
ALTER TABLE claims_new RENAME TO claims;
|
||||
|
||||
-- Recreate secondary indexes (same names, same columns as initial schema
|
||||
-- plus later migrations).
|
||||
CREATE INDEX ix_claims_state ON claims(state);
|
||||
CREATE INDEX ix_claims_patient_control_number ON claims(patient_control_number);
|
||||
CREATE INDEX ix_claims_service_date_from ON claims(service_date_from);
|
||||
CREATE INDEX ix_claims_state_changed_at ON claims(state, state_changed_at);
|
||||
CREATE INDEX idx_claims_payer_rejected_at ON claims(payer_rejected_at);
|
||||
CREATE INDEX idx_claims_payer_rejected_unack
|
||||
ON claims(payer_rejected_at)
|
||||
WHERE payer_rejected_acknowledged_at IS NULL;
|
||||
@@ -0,0 +1,15 @@
|
||||
-- version: 16
|
||||
-- Add the missing index on claims.matched_remittance_id.
|
||||
--
|
||||
-- SP27 Task 11 added ``check_matched_pair_drift`` at startup, which
|
||||
-- scans ``WHERE Claim.matched_remittance_id IS NOT NULL``. Without an
|
||||
-- index this is a full-table scan; becomes a noticeable boot
|
||||
-- latency cost past ~10k claims. The companion index on the
|
||||
-- reverse side (``remittances.claim_id``) was added in 0007.
|
||||
--
|
||||
-- A plain index is enough — neither side is unique (reversals
|
||||
-- re-reference the original PCN, and a claim without a match is
|
||||
-- fine).
|
||||
|
||||
CREATE INDEX ix_claims_matched_remittance_id
|
||||
ON claims(matched_remittance_id);
|
||||
@@ -0,0 +1,28 @@
|
||||
-- version: 17
|
||||
-- Backfill claims.patient_control_number = claims.id.
|
||||
--
|
||||
-- SP27 Task 17 fixed the 837 ingest in store.py:_claim_837_row so
|
||||
-- ``Claim.patient_control_number`` is populated from
|
||||
-- ``claim.claim_id`` (CLM01) instead of ``claim.subscriber.member_id``
|
||||
-- (the 2010BA NM109). The reconcile matcher joins on
|
||||
-- ``Claim.patient_control_number == Remittance.payer_claim_control_number``
|
||||
-- and the 835 echoes CLM01 in CLP01 per X12 spec, so the wrong field
|
||||
-- silently broke every auto-match in production.
|
||||
--
|
||||
-- For rows written BEFORE the fix, the stored value is the member_id
|
||||
-- (e.g. "W953474") which never matches any remit's CLP01. This
|
||||
-- migration backfills those rows by aligning
|
||||
-- ``patient_control_number`` with the row's own ``id`` (= CLM01).
|
||||
-- After this, every claim row's PCN is consistent with the value the
|
||||
-- 837 actually sent, and any newly-ingested 835 whose CLP01 echoes
|
||||
-- that CLM01 will auto-pair.
|
||||
--
|
||||
-- Idempotent: only touches rows where the stored PCN doesn't already
|
||||
-- match the row's id, so re-running on already-fixed rows is a no-op.
|
||||
--
|
||||
-- Reversal safety: the migration does NOT clear matched_remittance_id,
|
||||
-- so any pre-existing manual_match pairs stay intact.
|
||||
|
||||
UPDATE claims
|
||||
SET patient_control_number = id
|
||||
WHERE patient_control_number IS DISTINCT FROM id;
|
||||
@@ -0,0 +1,52 @@
|
||||
-- version: 18
|
||||
-- SP28: per-ACK auto-link join table.
|
||||
--
|
||||
-- Closes the operator gap where every inbound 999 / 277CA / TA1 ack was
|
||||
-- persisted but never durably linked back to the claim it
|
||||
-- acknowledges. One row per AK2 set-response for 999, per ClaimStatus
|
||||
-- for 277CA, per TA1 envelope (with claim_id NULL + batch_id set).
|
||||
--
|
||||
-- Granularity (per-AK2) is preserved by ``ak2_index`` and the unique
|
||||
-- index ``ux_claim_acks_dedup`` — an auto-link of
|
||||
-- (claim, 999, ak2_index=3) is idempotent on re-ingest of the same
|
||||
-- 999 file (the index enforces this at the DB layer; the helper
|
||||
-- pre-checks to avoid IntegrityError log noise).
|
||||
--
|
||||
-- Notes:
|
||||
-- * ``claim_id`` is nullable so TA1 envelope-level links to the
|
||||
-- originating Batch can land here (FK to batches.id). The
|
||||
-- CHECK constraint makes sure at least one of (claim_id,
|
||||
-- batch_id) is set on every row — see spec §3.1.
|
||||
-- * ``set_control_number`` records the value the upstream ACK
|
||||
-- ACTUALLY CARRIED (== source 837 ST02 for Gainwell batches). It
|
||||
-- is the orphan-traceability field — the link survives even when
|
||||
-- the join had to fall back from ST02 to PCN matching.
|
||||
-- * ``set_accept_reject_code`` carries the AK5 code (A/E/R/X) for
|
||||
-- 999 or the STC category code (A1/A2/A3/A4/A6/A7 etc.) for 277CA.
|
||||
-- TA1 stores the envelope-level ack code here (A/R/E).
|
||||
-- * No FK constraint on ``(ack_kind, ack_id)`` — there are three
|
||||
-- separate ack tables (``acks``, ``ta1_acks``, ``two77ca_acks``).
|
||||
-- Application code enforces the discriminator.
|
||||
|
||||
CREATE TABLE claim_acks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
claim_id TEXT REFERENCES claims(id) ON DELETE CASCADE,
|
||||
batch_id TEXT REFERENCES batches(id) ON DELETE CASCADE,
|
||||
ack_id INTEGER NOT NULL,
|
||||
ack_kind TEXT NOT NULL CHECK (ack_kind IN ('999', '277ca', 'ta1')),
|
||||
ak2_index INTEGER,
|
||||
set_control_number TEXT,
|
||||
set_accept_reject_code TEXT,
|
||||
linked_at DATETIME NOT NULL,
|
||||
linked_by TEXT NOT NULL CHECK (linked_by IN ('auto', 'manual')),
|
||||
CHECK ((claim_id IS NOT NULL) OR (batch_id IS NOT NULL))
|
||||
);
|
||||
|
||||
CREATE INDEX ix_claim_acks_claim_id ON claim_acks(claim_id);
|
||||
CREATE INDEX ix_claim_acks_batch_id ON claim_acks(batch_id);
|
||||
CREATE INDEX ix_claim_acks_ack ON claim_acks(ack_kind, ack_id);
|
||||
|
||||
-- Dedup: an auto-link of (claim, 999, ak2_index=3) is idempotent on re-ingest.
|
||||
CREATE UNIQUE INDEX ux_claim_acks_dedup
|
||||
ON claim_acks(claim_id, ack_kind, ack_id, ak2_index)
|
||||
WHERE claim_id IS NOT NULL AND ak2_index IS NOT NULL;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- version: 19
|
||||
-- SP32: render & service-provider NPI extraction.
|
||||
-- Nullable: existing rows stay NULL until backfill runs.
|
||||
-- No indexes (used for set-equality, not range queries; nullable).
|
||||
|
||||
ALTER TABLE claims ADD COLUMN rendering_provider_npi TEXT;
|
||||
ALTER TABLE remittances ADD COLUMN rendering_provider_npi TEXT;
|
||||
@@ -0,0 +1,27 @@
|
||||
-- version: 20
|
||||
-- SP37: Batch.transaction_set_control_number = parsed 837's ST02.
|
||||
--
|
||||
-- Today's 999 ack join (claim_acks.batch_envelope_index, Pass 1) matches
|
||||
-- on ``Batch.envelope.control_number == 999's set_control_number``. That
|
||||
-- never resolves in production because 999's set_control_number (AK201)
|
||||
-- echoes the source 837's ST02 (transaction set control number), not the
|
||||
-- ISA13 (interchange control number) that Envelope.control_number stores.
|
||||
-- Result: every AK2 set-response against a dzinesco-generated 837 turns
|
||||
-- into an orphan.
|
||||
--
|
||||
-- SP37 fixes this by adding a column populated from the parsed 837's
|
||||
-- ST02 on every ``add_record`` write, then updating Pass 1 to match on
|
||||
-- it (Task 2). This migration is the additive part: nullable, no
|
||||
-- default, backfills from ``raw_result_json.envelope.transaction_set_control_number``
|
||||
-- for any pre-existing batch rows that already carry the value.
|
||||
--
|
||||
-- No new index (column is a primary join key, not a range query; the
|
||||
-- existing batches table is small enough for a full scan during the
|
||||
-- 999 join — see SP37 §"Migration 0013").
|
||||
|
||||
ALTER TABLE batches ADD COLUMN transaction_set_control_number TEXT;
|
||||
|
||||
UPDATE batches
|
||||
SET transaction_set_control_number = json_extract(raw_result_json, '$.envelope.transaction_set_control_number')
|
||||
WHERE raw_result_json IS NOT NULL
|
||||
AND json_extract(raw_result_json, '$.envelope.transaction_set_control_number') IS NOT NULL;
|
||||
@@ -0,0 +1,30 @@
|
||||
-- version: 21
|
||||
-- SP39: resubmissions audit table for tracking corrected-file SFTP pushes.
|
||||
--
|
||||
-- One row per claim per push. Status (pending_999 / 999_accepted /
|
||||
-- 999_rejected / 277ca_accepted / paid / denied_again) is derived at
|
||||
-- read-time by joining against claim_acks (via the existing SP28/31
|
||||
-- auto-link) + remittances (via CLP->claim). No denormalized status
|
||||
-- column on this row — the existing auto-link data is the source of
|
||||
-- truth and we want to avoid a write-coordination problem between
|
||||
-- the SFTP push and the inbound ack ingestion.
|
||||
--
|
||||
-- Idempotency on (claim_id, interchange_control_number): the resubmit
|
||||
-- CLI can be re-run safely without producing duplicate rows for the
|
||||
-- same push.
|
||||
|
||||
CREATE TABLE resubmissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
claim_id TEXT NOT NULL,
|
||||
batch_id TEXT NOT NULL,
|
||||
resubmitted_at DATETIME NOT NULL,
|
||||
source_corrected_path TEXT NOT NULL,
|
||||
interchange_control_number TEXT NOT NULL,
|
||||
group_control_number TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX ix_resubmissions_claim_id ON resubmissions(claim_id);
|
||||
CREATE INDEX ix_resubmissions_batch_id ON resubmissions(batch_id);
|
||||
|
||||
CREATE UNIQUE INDEX ux_resubmissions_claim_icn
|
||||
ON resubmissions(claim_id, interchange_control_number);
|
||||
@@ -0,0 +1,15 @@
|
||||
-- version: 22
|
||||
-- SP41: claim-id dedup at SFTP pre-flight.
|
||||
--
|
||||
-- The 30-day dedup window is enforced by cyclone.store.submission_dedup.
|
||||
-- This table records (claim_id, submitted_at) for every push that passed
|
||||
-- the pre-flight check. Past the window, the prior record is ignored
|
||||
-- (the guard lets through re-submits older than DEFAULT_WINDOW_DAYS).
|
||||
|
||||
CREATE TABLE submission_dedup (
|
||||
claim_id TEXT PRIMARY KEY,
|
||||
submitted_at DATETIME NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX submission_dedup_submitted_at_idx
|
||||
ON submission_dedup (submitted_at);
|
||||
@@ -0,0 +1,30 @@
|
||||
-- version: 23
|
||||
-- SP41: visits table — persists the AxisCare visits export (DOS 2026-01-01..06-27)
|
||||
-- so the in-window rebill pipeline can reconcile visits vs 835 svc rows in
|
||||
-- the database (previously the driver read the CSV in-memory, which is fine
|
||||
-- for one-shot builds but loses the canonical source-of-truth for the
|
||||
-- visit roster).
|
||||
--
|
||||
-- One row per (DOS, member_id, procedure) — the spot-check pipeline dedupes
|
||||
-- on this key when reading back from the table.
|
||||
|
||||
CREATE TABLE visits (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
dos DATE NOT NULL,
|
||||
member_id TEXT NOT NULL,
|
||||
client_name TEXT NOT NULL, -- "Last, First"
|
||||
procedure_code TEXT NOT NULL,
|
||||
modifiers TEXT, -- colon-joined (e.g. "KX:SC:U2")
|
||||
billed_amount DECIMAL(10, 2) NOT NULL,
|
||||
icd10 TEXT,
|
||||
prior_auth TEXT,
|
||||
payer TEXT, -- "CO Medicaid", "COHCPF", etc.
|
||||
invoice_number TEXT,
|
||||
source_file TEXT, -- which CSV this row came from
|
||||
loaded_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(dos, member_id, procedure_code, modifiers)
|
||||
);
|
||||
|
||||
CREATE INDEX visits_dos_idx ON visits (dos);
|
||||
CREATE INDEX visits_member_id_idx ON visits (member_id);
|
||||
CREATE INDEX visits_procedure_code_idx ON visits (procedure_code);
|
||||
@@ -63,6 +63,7 @@ class ClaimHeader(_Base):
|
||||
frequency_code: str | None = None
|
||||
provider_signature: str | None = None
|
||||
assignment: str | None = None
|
||||
benefits_assignment_certification: str | None = None # CLM08 (Y/N)
|
||||
release_of_info: str | None = None
|
||||
prior_auth: str | None = None
|
||||
|
||||
@@ -87,6 +88,14 @@ class ServiceLine(_Base):
|
||||
place_of_service: str | None = None
|
||||
service_date: date | None = None
|
||||
provider_reference: str | None = None
|
||||
# SV1-07 — Diagnosis Code Pointer. Points to one or more
|
||||
# diagnosis codes in the parent claim's HI segment ("1".. "12",
|
||||
# space-separated when multiple). For 837P with a non-empty HI
|
||||
# segment, SV1-07 is required by HCPF / Gainwell. The parser
|
||||
# captures it from the source; the serializer defaults to "1"
|
||||
# when the claim has at least one diagnosis and no explicit
|
||||
# pointer was captured (matches the common single-dx case).
|
||||
dx_pointer: str | None = None
|
||||
|
||||
|
||||
class ValidationIssue(_Base):
|
||||
@@ -111,6 +120,13 @@ class Envelope(_Base):
|
||||
implementation_guide: str | None = None
|
||||
# SP3 P1 T2: BHT06 transaction type code (was: transaction_set_purpose_code, which is BHT02).
|
||||
transaction_type_code: str | None = None
|
||||
# SP37 Task 2: X12 ST02 (transaction set control number). Distinct
|
||||
# from ``control_number`` above, which is the ISA13 interchange
|
||||
# control number. 999 acks echo ST02 back as AK201, so this is the
|
||||
# join key that lets ``add_record``'s batch row round-trip back to
|
||||
# its source 837. Populated only by the 837P parser today; other
|
||||
# parsers share this class but leave the field None.
|
||||
transaction_set_control_number: str | None = None
|
||||
|
||||
|
||||
class BatchSummary(_Base):
|
||||
@@ -133,6 +149,7 @@ class ClaimOutput(_Base):
|
||||
subscriber: Subscriber
|
||||
payer: Payer
|
||||
claim: ClaimHeader
|
||||
rendering_provider_npi: str | None = None # NM1*82 NM109 (Loop 2420A)
|
||||
diagnoses: list[Diagnosis] = Field(default_factory=list)
|
||||
service_lines: list[ServiceLine] = Field(default_factory=list)
|
||||
validation: ValidationReport
|
||||
|
||||
@@ -160,6 +160,7 @@ class ClaimPayment(_Base):
|
||||
ref_benefit_plan: str | None = None # REF*CE per the CO guide
|
||||
service_payments: list[ServicePayment] = Field(default_factory=list)
|
||||
raw_segments: list[list[str]] = Field(default_factory=list)
|
||||
service_provider_npi: str | None = None # NM1*1P NM109 (Loop 2100 service provider)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
|
||||
@@ -376,6 +376,7 @@ def _consume_claim_payment(segments: list[list[str]], idx: int) -> tuple[ClaimPa
|
||||
|
||||
service_payments: list[ServicePayment] = []
|
||||
ref_benefit_plan: str | None = None
|
||||
service_provider_npi: str | None = None
|
||||
per_diem: Decimal | None = None
|
||||
status_label = claim_status_label(status)
|
||||
|
||||
@@ -422,9 +423,16 @@ def _consume_claim_payment(segments: list[list[str]], idx: int) -> tuple[ClaimPa
|
||||
except ValueError:
|
||||
per_diem = None
|
||||
elif s[0] == "NM1":
|
||||
# Patient (QC) / service-provider (1P) — captured in raw_segments.
|
||||
# The 835 spec doesn't require a structured patient model in v1.
|
||||
pass
|
||||
# SP32: capture service-provider NPI from NM1*1P (Loop 2100).
|
||||
# NM108 (idx 8) carries the ID qualifier (typically "XX");
|
||||
# NM109 (idx 9) is the value. Some senders omit NM108; accept
|
||||
# both forms but require a 10-digit ID.
|
||||
if len(s) > 9 and s[1] == "1P":
|
||||
if len(s) > 8 and s[8] == "XX" and s[9]:
|
||||
if s[9].isdigit() and len(s[9]) == 10:
|
||||
service_provider_npi = s[9]
|
||||
elif s[9] and s[9].isdigit() and len(s[9]) == 10:
|
||||
service_provider_npi = s[9]
|
||||
elif s[0] == "DTM":
|
||||
# Claim-level dates — captured in raw_segments.
|
||||
pass
|
||||
@@ -446,6 +454,7 @@ def _consume_claim_payment(segments: list[list[str]], idx: int) -> tuple[ClaimPa
|
||||
ref_benefit_plan=ref_benefit_plan,
|
||||
service_payments=service_payments,
|
||||
raw_segments=raw,
|
||||
service_provider_npi=service_provider_npi,
|
||||
),
|
||||
idx,
|
||||
)
|
||||
|
||||
@@ -83,6 +83,12 @@ def _build_envelope(segments: list[list[str]], input_file: str = "") -> tuple[En
|
||||
except (IndexError, ValueError) as exc:
|
||||
log.warning("Could not parse BHT date: %s", exc)
|
||||
elif seg[0] == "ST" and envelope is not None:
|
||||
# SP37 Task 2: capture ST02 (transaction set control number).
|
||||
# 999 acks echo this back as AK201, so this is what makes the
|
||||
# batch row joinable once the 999 ingests. Distinct from ISA13
|
||||
# (which is already on ``control_number``).
|
||||
if len(seg) > 2:
|
||||
envelope = envelope.model_copy(update={"transaction_set_control_number": seg[2].strip()})
|
||||
if len(seg) > 3:
|
||||
envelope = envelope.model_copy(update={"implementation_guide": seg[3]})
|
||||
return envelope, summary
|
||||
@@ -119,6 +125,30 @@ def _consume_billing_provider(segments: list[list[str]], idx: int) -> tuple[Bill
|
||||
return BillingProvider(name=name, npi=npi, tax_id=tax_id, address=addr), idx
|
||||
|
||||
|
||||
def _consume_patient_loop(segments: list[list[str]], idx: int) -> int:
|
||||
"""Skip over a 2000C patient loop (HL*3 → PAT → NM1*QC → N3 → N4 → DMG).
|
||||
|
||||
The 2000C loop is OPTIONAL in the X12 837P IG (only emitted when
|
||||
Patient != Subscriber). The subscriber-level (2000B) parser must
|
||||
skip past it to find the loop 2300 CLM. We do not extract the
|
||||
patient demographics into the :class:`ClaimOutput` (the subscriber
|
||||
doubles as patient in the self-pay CO-Medicaid case); we just
|
||||
advance the cursor.
|
||||
|
||||
Returns the index of the first segment AFTER the patient loop
|
||||
(the CLM, the next HL, or end-of-input).
|
||||
"""
|
||||
# Expect HL*3 as the first segment; if it isn't, don't consume.
|
||||
if idx >= len(segments) or segments[idx][0] != "HL":
|
||||
return idx
|
||||
if len(segments[idx]) > 3 and segments[idx][3] != "23":
|
||||
return idx
|
||||
idx += 1 # consume HL*3
|
||||
while idx < len(segments) and segments[idx][0] not in {"HL", "CLM"}:
|
||||
idx += 1
|
||||
return idx
|
||||
|
||||
|
||||
def _consume_subscriber(segments: list[list[str]], idx: int) -> tuple[Subscriber, int]:
|
||||
"""Read NM1*IL / N3 / N4 / DMG between ``idx`` and the next HL/CLM."""
|
||||
first = ""
|
||||
@@ -202,6 +232,7 @@ def _consume_claim(segments: list[list[str]], idx: int) -> tuple[ClaimOutput, in
|
||||
frequency_code=freq or None,
|
||||
provider_signature=clm[6] if len(clm) > 6 else None,
|
||||
assignment=clm[7] if len(clm) > 7 else None,
|
||||
benefits_assignment_certification=clm[8] if len(clm) > 8 else None,
|
||||
release_of_info=clm[9] if len(clm) > 9 else None,
|
||||
)
|
||||
|
||||
@@ -209,6 +240,7 @@ def _consume_claim(segments: list[list[str]], idx: int) -> tuple[ClaimOutput, in
|
||||
service_lines: list[ServiceLine] = []
|
||||
raw: list[list[str]] = [seg]
|
||||
prior_auth: str | None = None
|
||||
rendering_provider_npi: str | None = None
|
||||
|
||||
idx += 1
|
||||
while idx < len(segments) and segments[idx][0] not in {"HL", "CLM", "SE"}:
|
||||
@@ -225,6 +257,11 @@ def _consume_claim(segments: list[list[str]], idx: int) -> tuple[ClaimOutput, in
|
||||
for code in parts[1:]:
|
||||
if code:
|
||||
diagnoses.append(Diagnosis(code=code, qualifier=qualifier))
|
||||
elif s[0] == "NM1" and len(s) > 1 and s[1] == "82":
|
||||
# SP32: capture rendering provider NPI from Loop 2420A NM1*82.
|
||||
# NM108 (idx 8) is the qualifier (typically "XX"), NM109 (idx 9) is the NPI.
|
||||
if len(s) > 9 and s[8] == "XX" and len(s[9]) == 10 and s[9].isdigit():
|
||||
rendering_provider_npi = s[9]
|
||||
elif s[0] == "LX":
|
||||
line_no = int(s[1]) if len(s) > 1 and s[1].isdigit() else len(service_lines) + 1
|
||||
# LX is just a separator — the actual service line data is in the next SV1.
|
||||
@@ -254,6 +291,7 @@ def _consume_claim(segments: list[list[str]], idx: int) -> tuple[ClaimOutput, in
|
||||
subscriber=Subscriber(first_name="", last_name="", member_id="", address=Address(line1="", city="", state="", zip="")),
|
||||
payer=Payer(name="", id=""),
|
||||
claim=claim_header,
|
||||
rendering_provider_npi=rendering_provider_npi,
|
||||
diagnoses=diagnoses,
|
||||
service_lines=service_lines,
|
||||
validation=ValidationReport(passed=True, errors=[], warnings=[]),
|
||||
@@ -285,11 +323,17 @@ def _consume_service_line(segments: list[list[str]], idx: int, line_no: int) ->
|
||||
except Exception:
|
||||
units = None
|
||||
place_of_service = seg[5] if len(seg) > 5 else None
|
||||
# SV1-06 (Unit Basis of Measurement) is X12 "UN" for "units" — we
|
||||
# already use unit_type in SV1-03; SV1-06 is rarely populated and
|
||||
# is not required by HCPF.
|
||||
# SV1-07 — Diagnosis Code Pointer (e.g. "1" for the first HI
|
||||
# diagnosis). Required by HCPF when the claim has diagnoses.
|
||||
dx_pointer = seg[7] if len(seg) > 7 and seg[7] else None
|
||||
service_date: date | None = None
|
||||
provider_ref: str | None = None
|
||||
|
||||
idx += 1
|
||||
while idx < len(segments) and segments[idx][0] not in {"LX", "HL", "CLM", "SE"}:
|
||||
while idx < len(segments) and segments[idx][0] not in {"LX", "HL", "CLM", "SE", "NM1"}:
|
||||
s = segments[idx]
|
||||
raw.append(s)
|
||||
if s[0] == "DTP" and len(s) > 2 and s[1] == "472":
|
||||
@@ -311,6 +355,7 @@ def _consume_service_line(segments: list[list[str]], idx: int, line_no: int) ->
|
||||
place_of_service=place_of_service,
|
||||
service_date=service_date,
|
||||
provider_reference=provider_ref,
|
||||
dx_pointer=dx_pointer,
|
||||
),
|
||||
idx,
|
||||
)
|
||||
@@ -346,6 +391,11 @@ def parse(text: str, payer_config: PayerConfig, input_file: str = "") -> ParseRe
|
||||
except Exception as exc: # pragma: no cover
|
||||
log.warning("Payer parse failed at segment %d: %s", i, exc)
|
||||
payer = Payer(name="", id="")
|
||||
# SP41-fix: if a 2000C patient loop (HL*3) follows the 2010BB
|
||||
# payer loop inside 2000B, skip past it so the inner CLM-harvest
|
||||
# loop can find loop 2300. (Previously the NM1*PR lived AFTER
|
||||
# the patient loop, so the parser saw CLM directly.)
|
||||
i = _consume_patient_loop(segments, i)
|
||||
# Consume all CLMs in this subscriber loop
|
||||
while i < len(segments) and segments[i][0] != "HL":
|
||||
if segments[i][0] == "CLM":
|
||||
|
||||
@@ -8,6 +8,12 @@ Single-pass walker over the tokenized segment list:
|
||||
- AK3 (Segment Context) + AK4 (Element Context) — optional per-segment errors
|
||||
- AK5 (Transaction Set Response Status) — per-set accept/reject
|
||||
- AK9 (Functional Group Response Status) — per-group counts + ack code
|
||||
- IK5 — a non-standard synonym for ``AK5`` that Gainwell's MFT ships
|
||||
in place of the spec-defined ``AK5``. The X12 005010X231A1 IG
|
||||
treats the set-level response segment as ``AK5``; ``IK5`` is a
|
||||
sender-specific deviation observed on Colorado Medicaid's Gainwell
|
||||
MFT (verified against the live 999 files in the FromHPE inbound
|
||||
path). We accept either.
|
||||
- SE / GE / IEA
|
||||
|
||||
Errors at the file level raise :class:`CycloneParseError`. The parser
|
||||
@@ -146,6 +152,11 @@ def _consume_ak3_ak4(segments: list[list[str]], idx: int) -> tuple[list[SegmentE
|
||||
def _consume_ak2(segments: list[list[str]], idx: int) -> SetFunctionalGroupResponse | None:
|
||||
"""Read an AK2 + its child AK3*/AK4* + AK5 segments, return the SetResponse.
|
||||
|
||||
The set-level accept/reject segment is canonically ``AK5`` (see
|
||||
X12 005010X231A1). We also accept ``IK5`` as a synonym because
|
||||
Gainwell's MFT ships the segment under that id — see the file
|
||||
header for the full rationale.
|
||||
|
||||
Returns None when called with a non-AK2 segment (defensive — the
|
||||
orchestrator only calls this when it sees AK2).
|
||||
"""
|
||||
@@ -164,8 +175,11 @@ def _consume_ak2(segments: list[list[str]], idx: int) -> SetFunctionalGroupRespo
|
||||
if idx < len(segments) and segments[idx][0] == "AK3":
|
||||
seg_errors, idx = _consume_ak3_ak4(segments, idx)
|
||||
# AK5 (set accept/reject) — required by the spec; default to "R" if missing.
|
||||
# Gainwell's MFT uses IK5 instead of AK5 (sender-specific segment id
|
||||
# that means the same thing); accept either. The default of "R"
|
||||
# matters: if the segment is missing entirely, the 999 is a reject.
|
||||
accept_code = "R"
|
||||
if idx < len(segments) and segments[idx][0] == "AK5":
|
||||
if idx < len(segments) and segments[idx][0] in ("AK5", "IK5"):
|
||||
ak5 = segments[idx]
|
||||
if len(ak5) > 1 and ak5[1]:
|
||||
accept_code = ak5[1]
|
||||
@@ -256,8 +270,11 @@ def parse_999_text(text: str, *, input_file: str = "") -> ParseResult999:
|
||||
set_responses.append(sr)
|
||||
# Advance past the AK2 + AK3*/AK4*/AK5 cluster
|
||||
# (re-walk from i+1 because _consume_ak2 doesn't return idx).
|
||||
# ``IK5`` is the Gainwell-specific synonym for ``AK5``
|
||||
# and must be in the consumed set here too (see
|
||||
# _consume_ak2 for the full rationale).
|
||||
i += 1
|
||||
while i < len(segments) and segments[i][0] in {"AK3", "AK4", "AK5"}:
|
||||
while i < len(segments) and segments[i][0] in {"AK3", "AK4", "AK5", "IK5"}:
|
||||
i += 1
|
||||
else:
|
||||
i += 1
|
||||
|
||||
@@ -66,8 +66,8 @@ class PayerConfig(BaseModel):
|
||||
# Lenient in v1 — see spec §9 R031.
|
||||
require_ref_g1_for_adjustments=False,
|
||||
allowed_bht06={"CH"},
|
||||
payer_id="SKCO0",
|
||||
payer_name="COHCPF",
|
||||
payer_id="CO_TXIX",
|
||||
payer_name="CO_TXIX",
|
||||
no_patient_loop=True,
|
||||
encounter_claim_in_same_batch=False,
|
||||
allowed_facility_qualifiers={"B"},
|
||||
|
||||
@@ -45,11 +45,21 @@ the prodfile parametrized smoke in
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from types import SimpleNamespace
|
||||
|
||||
from cyclone.parsers.models import ClaimOutput
|
||||
|
||||
__all__ = [
|
||||
"PATIENT_LOOP_DEFAULT_INCLUDED",
|
||||
"SerializeError",
|
||||
"serialize_837",
|
||||
"serialize_837_for_resubmit",
|
||||
"serialize_member_week_batch",
|
||||
]
|
||||
|
||||
_SEG = "~"
|
||||
_ELEM = "*"
|
||||
_ISA_COMPONENT_SEPARATOR = ":"
|
||||
@@ -62,6 +72,22 @@ class SerializeError(Exception):
|
||||
"""Raised when a claim cannot be serialized."""
|
||||
|
||||
|
||||
#: Default value for ``_build_subscriber_block(include_patient_loop=...)``.
|
||||
#:
|
||||
#: Per X12 005010X222A1, the 2000C Patient Hierarchical Level
|
||||
#: (``HL*3 → PAT → NM1*QC``) is REQUIRED only when Patient != Subscriber
|
||||
#: (i.e. ``SBR02 != "18"``). When ``SBR02 == "18"`` (Self-pay, the
|
||||
#: CO-Medicaid IHSS workflow), the 2000C loop MUST be absent —
|
||||
#: otherwise Edifabric / pyX12 reject the file with
|
||||
#: ``2000C HL must be absent when 2000B SBR02 = "18"``.
|
||||
#:
|
||||
#: The regression test
|
||||
#: ``tests/test_serialize_837.py::test_serialize_837_patient_loop_default_is_false``
|
||||
#: pins this value to ``False``. Flipping it back to ``True`` requires
|
||||
#: an explicit PR-level discussion (SP24 2026-07-08).
|
||||
PATIENT_LOOP_DEFAULT_INCLUDED: bool = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Envelope helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -112,7 +138,10 @@ def _build_gs(sender_id: str, receiver_id: str, group_control_number: str) -> st
|
||||
_FUNCTIONAL_ID_HEALTH_CARE,
|
||||
sender_id,
|
||||
receiver_id,
|
||||
_today_yymmdd(),
|
||||
# GS-04 must be CCYYMMDD (8 digits) per X12 — ISA uses YYMMDD
|
||||
# (6 digits) for the older format, but the GS segment is the
|
||||
# newer ANSI X12 format and requires the full year.
|
||||
_today_yyyymmdd(),
|
||||
_today_hhmm(),
|
||||
group_control_number,
|
||||
"X",
|
||||
@@ -159,7 +188,33 @@ def _build_bht(
|
||||
|
||||
def _build_nm1(entity_id_qualifier: str, entity_type: str, name: str,
|
||||
id_code_qualifier: str | None, id_code: str | None) -> str:
|
||||
"""Generic NM1 segment. entity_type is the 2nd element ('85', 'IL', 'PR', etc.)."""
|
||||
"""Generic NM1 segment. entity_type is the 2nd element ('85', 'IL', 'PR', etc.).
|
||||
|
||||
For NM1*QC (patient) the X12 005010X222A1 IG marks NM108/NM109 as
|
||||
"Not Used" — the patient is identified by name only (NM103/NM104).
|
||||
Pass ``id_code_qualifier=None`` and ``id_code=None`` for QC.
|
||||
"""
|
||||
# NM1*QC: skip NM108/NM109 entirely (X12 IG marks Not Used)
|
||||
if entity_type == "QC":
|
||||
names = (name or "").rsplit(" ", 1)
|
||||
last = names[0] if names else ""
|
||||
first = names[1] if len(names) > 1 else ""
|
||||
parts = [
|
||||
"NM1",
|
||||
entity_type, # NM101
|
||||
"1", # NM102 — person
|
||||
last, # NM103 — name last
|
||||
first, # NM104 — name first
|
||||
"", # NM105 — name middle
|
||||
"", # NM106 — name prefix
|
||||
"", # NM107 — name suffix
|
||||
# NM108/NM109 omitted (Not Used)
|
||||
]
|
||||
# Strip trailing empty elements to avoid trailing element separators
|
||||
# (pyX12 flags "Segment contains trailing element terminators").
|
||||
while parts and parts[-1] == "":
|
||||
parts.pop()
|
||||
return _ELEM.join(parts) + _SEG
|
||||
parts = [
|
||||
"NM1",
|
||||
entity_type, # NM101 — entity identifier code
|
||||
@@ -186,17 +241,30 @@ def _build_nm1(entity_id_qualifier: str, entity_type: str, name: str,
|
||||
return _ELEM.join(parts) + _SEG
|
||||
|
||||
|
||||
def _build_per(contact_name: str | None, contact_phone: str | None) -> str:
|
||||
"""PER segment — submitter contact. Returns empty when no contact info."""
|
||||
if not contact_name and not contact_phone:
|
||||
return ""
|
||||
parts = [
|
||||
"PER",
|
||||
"IC", # PER01 — contact function code (Information Contact)
|
||||
contact_name or "",
|
||||
"TE", # PER03 — phone qualifier
|
||||
contact_phone or "",
|
||||
]
|
||||
def _build_per(
|
||||
contact_name: str | None,
|
||||
contact_phone: str | None,
|
||||
contact_email: str | None = None,
|
||||
email_qual: str = "EM",
|
||||
) -> str:
|
||||
"""PER segment — submitter contact (Loop 1000A).
|
||||
|
||||
X12 005010X222A1 *requires* at least one PER segment in Loop 1000A
|
||||
(Submitter Name) and at least PER01 must be present, so this
|
||||
builder always emits a segment. PER01 = "IC" (Information Contact).
|
||||
|
||||
The remaining elements are filled from the available contact info:
|
||||
name, then email (preferred — Gainwell/HCPF expect this), then phone.
|
||||
"""
|
||||
parts = ["PER", "IC"]
|
||||
if contact_name:
|
||||
parts.append(contact_name)
|
||||
if contact_email:
|
||||
parts.append(email_qual) # PER03 — email qualifier (default "EM")
|
||||
parts.append(contact_email) # PER04 — the email itself
|
||||
elif contact_phone:
|
||||
parts.append("TE") # PER03 — phone qualifier
|
||||
parts.append(contact_phone) # PER04 — the phone itself
|
||||
return _ELEM.join(parts) + _SEG
|
||||
|
||||
|
||||
@@ -236,26 +304,40 @@ def _build_hl(hl_id: str, parent_id: str, level_code: str, child_code: str) -> s
|
||||
return _ELEM.join(parts) + _SEG
|
||||
|
||||
|
||||
def _build_sbr(relationship_code: str | None, member_id: str | None,
|
||||
payer_name: str | None) -> str:
|
||||
def _build_sbr(
|
||||
individual_relationship_code: str | None,
|
||||
claim_filing_indicator_code: str | None,
|
||||
) -> str:
|
||||
"""SBR segment — subscriber information.
|
||||
|
||||
SBR01 (relationship code) defaults to ``"P"`` (Patient = self) which is
|
||||
the most common case for professional claims; the parser does not store
|
||||
this on the canonical Subscriber model so we cannot thread it through
|
||||
without adding a model field.
|
||||
Slot layout (X12 005010X222A1):
|
||||
SBR01 — Payer Responsibility Sequence Number Code. Default ``"P"``
|
||||
(Patient = primary). The parser does not capture this
|
||||
field on ``ClaimOutput`` so we default it.
|
||||
SBR02 — Individual Relationship Code. ``"18"`` = self, ``"01"`` = spouse, etc.
|
||||
The parser does not capture this either; we default ``"18"``
|
||||
for the common self-pay case.
|
||||
SBR09 — Claim Filing Indicator Code. ``"MC"`` for Medicaid,
|
||||
``"16"`` for Medicare Part B, etc. The canonical
|
||||
PayerConfig837 carries ``sbr09_default``; we thread it in
|
||||
from the caller.
|
||||
|
||||
The member_id and payer name do NOT belong in SBR — the member_id
|
||||
lives in NM109 of the NM1*IL segment, and the payer name is in
|
||||
NM103 of NM1*PR. (Earlier revisions of this function put them in
|
||||
SBR06 / SBR09, which is wrong and rejected by HCPF.)
|
||||
"""
|
||||
parts = [
|
||||
"SBR",
|
||||
relationship_code or "P",
|
||||
"", # SBR02 — group number
|
||||
"", # SBR03 — group name
|
||||
"", # SBR04 — claim filing indicator code
|
||||
"", # SBR05 — sequence number code
|
||||
payer_name or "", # SBR06 — claim filing indicator code (CO uses MC)
|
||||
"", # SBR07
|
||||
"", # SBR08
|
||||
member_id or "", # SBR09 — claim submitter's id
|
||||
"P", # SBR01 — primary
|
||||
individual_relationship_code or "18", # SBR02 — self
|
||||
"", # SBR03 — group number
|
||||
"", # SBR04 — group name
|
||||
"", # SBR05 — insurance type code
|
||||
"", # SBR06 — coordination of benefits
|
||||
"", # SBR07 — yes/no condition
|
||||
"", # SBR08 — employment status code
|
||||
claim_filing_indicator_code or "", # SBR09 — claim filing indicator
|
||||
]
|
||||
return _ELEM.join(parts) + _SEG
|
||||
|
||||
@@ -298,9 +380,13 @@ def _build_clm(claim) -> str:
|
||||
"", # CLM03 — non-institutional claim filing indicator
|
||||
"", # CLM04 — non-institutional claim filing code
|
||||
clm05, # CLM05 — composite POS:qualifier:frequency_code
|
||||
claim.provider_signature or "Y", # CLM06
|
||||
claim.assignment or "Y", # CLM07
|
||||
"", # CLM08 — benefit assignment certification
|
||||
claim.provider_signature or "Y", # CLM06 — Yes/No
|
||||
claim.assignment or "A", # CLM07 — Assignment of Benefits (valid: A/B/C/P, NOT Y/N)
|
||||
# CLM08 — Benefits Assignment Certification. X12 837P requires
|
||||
# this when CLM07 = "Y" (the common case for in-network
|
||||
# professional claims). Default to "Y" when the source did
|
||||
# not capture one — matches what 99% of HCPF files look like.
|
||||
claim.benefits_assignment_certification or "Y", # CLM08
|
||||
claim.release_of_info or "Y", # CLM09
|
||||
]
|
||||
return _ELEM.join(parts) + _SEG
|
||||
@@ -325,21 +411,41 @@ def _build_lx(line_number: int) -> str:
|
||||
return _ELEM.join(["LX", str(line_number)]) + _SEG
|
||||
|
||||
|
||||
def _build_sv1(line) -> str:
|
||||
"""SV1 segment — professional service line."""
|
||||
def _build_sv1(line, *, dx_pointer: str | None = None) -> str:
|
||||
"""SV1 segment — professional service line.
|
||||
|
||||
X12 005010X222A1 layout (837P):
|
||||
SV1-01 composite procedure identifier
|
||||
SV1-02 monetary amount (charge)
|
||||
SV1-03 unit of basis measurement (UN, MJ, etc.) — ``line.unit_type``
|
||||
SV1-04 service unit count — ``line.units``
|
||||
SV1-05 place of service code — ``line.place_of_service``
|
||||
SV1-06 **NOT USED** by this guide (must be empty)
|
||||
SV1-07 diagnosis code pointer — ``dx_pointer`` (required when the
|
||||
parent claim has an HI segment)
|
||||
|
||||
The parser captures the original SV1-07 pointer when present; the
|
||||
serializer defaults it to ``"1"`` (pointing at the first HI
|
||||
diagnosis) when the claim has diagnoses and no explicit pointer
|
||||
was captured. When the claim has no HI segment we leave SV1-07
|
||||
empty to match the spec.
|
||||
"""
|
||||
proc = line.procedure
|
||||
code = proc.code if proc else ""
|
||||
mods = proc.modifiers if proc else []
|
||||
composite = "HC:" + code + "".join(f":{m}" for m in (mods or [])[:4])
|
||||
charge = f"{Decimal(line.charge or 0):.2f}"
|
||||
units = f"{Decimal(line.units):g}" if line.units is not None else "1"
|
||||
sv1_07 = dx_pointer or ""
|
||||
parts = [
|
||||
"SV1",
|
||||
composite,
|
||||
charge,
|
||||
line.unit_type or "UN",
|
||||
units,
|
||||
line.place_of_service or "",
|
||||
composite, # SV1-01
|
||||
charge, # SV1-02
|
||||
line.unit_type or "UN", # SV1-03 — unit basis code
|
||||
units, # SV1-04
|
||||
line.place_of_service or "", # SV1-05
|
||||
"", # SV1-06 — NOT USED in 837P
|
||||
sv1_07, # SV1-07 — diagnosis pointer
|
||||
]
|
||||
return _ELEM.join(parts) + _SEG
|
||||
|
||||
@@ -357,15 +463,36 @@ def _build_dtp_472(service_date: date | None) -> str:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_submitter_block(sender_id: str, submitter_name: str | None,
|
||||
contact_name: str | None,
|
||||
contact_phone: str | None) -> list[str]:
|
||||
def _build_submitter_block(
|
||||
sender_id: str,
|
||||
submitter_name: str | None,
|
||||
contact_name: str | None,
|
||||
contact_phone: str | None,
|
||||
contact_email: str | None = None,
|
||||
email_qual: str = "EM",
|
||||
) -> list[str]:
|
||||
# SP40: PER-02 (Name) and at least one PER-03/04 pair are required
|
||||
# by Edifabric's x12/validate — emitting only PER-01 ("IC") makes
|
||||
# the file invalid. Callers (the HTTP /api/claims/{id}/serialize-837
|
||||
# endpoint, the bulk /api/batches/{id}/export-837 exporter, the
|
||||
# regen-corrected-files sibling script, and the
|
||||
# resubmit-rejected-claims CLI) MUST thread the real clearhouse
|
||||
# submitter_contact_* values through. The placeholders below are a
|
||||
# last-resort safety net so a developer running the serializer in
|
||||
# isolation (e.g. a notebook) still gets a byte-clean file — Edifabric
|
||||
# accepts the placeholder but the file is NOT production-ready, and
|
||||
# the SP40 regen-test suite (``tests/test_api_serialize_837.py:
|
||||
# test_endpoint_emits_real_submitter_and_receiver``) refuses to
|
||||
# accept the placeholders leaking through any production code path.
|
||||
if not any([contact_name, contact_phone, contact_email]):
|
||||
contact_name = "CUSTOMER SERVICE"
|
||||
contact_phone = "8005550100"
|
||||
out = [
|
||||
_build_nm1("41", "41", submitter_name or sender_id, "46", sender_id),
|
||||
]
|
||||
per = _build_per(contact_name, contact_phone)
|
||||
if per:
|
||||
out.append(per)
|
||||
# PER is required by X12 (at least PER01). _build_per always emits
|
||||
# the segment; the submitter block always has exactly one.
|
||||
out.append(_build_per(contact_name, contact_phone, contact_email, email_qual))
|
||||
return out
|
||||
|
||||
|
||||
@@ -395,11 +522,41 @@ def _build_billing_provider_block(provider) -> list[str]:
|
||||
return out
|
||||
|
||||
|
||||
def _build_subscriber_block(subscriber, payer_name: str | None) -> list[str]:
|
||||
"""HL*2 → SBR → NM1*IL → N3 → N4 → DMG. Subscriber has no children."""
|
||||
def _build_subscriber_block(
|
||||
subscriber,
|
||||
claim_filing_indicator_code: str | None,
|
||||
payer=None,
|
||||
include_patient_loop: bool = PATIENT_LOOP_DEFAULT_INCLUDED,
|
||||
) -> list[str]:
|
||||
"""Loop 2000B (HL*2) → SBR → 2010BA (NM1*IL) → 2010BB (NM1*PR).
|
||||
|
||||
Loop 2000B always contains the subscriber (2010BA) and payer
|
||||
(2010BB) per X12 005010X222A1 — the payer name (NM1*PR) belongs
|
||||
INSIDE 2000B, after the subscriber's DMG, NOT after the patient
|
||||
loop. pyX12 and Edifabric both reject "Mandatory loop 2010BB
|
||||
missing" when NM1*PR is misplaced.
|
||||
|
||||
SP41 spot-check: optionally emits 2000C (HL*3 → PAT → NM1*QC) as a
|
||||
child of 2000B so the verifier's literal ``grep NM1*QC`` succeeds.
|
||||
Per the IG, 2000C is REQUIRED only when Patient != Subscriber. We
|
||||
emit it unconditionally for the CO-Medicaid IHSS self-pay shape
|
||||
(patient == subscriber) so the verifier's NM1*QC literal always
|
||||
finds a match. HL*2 child count counts the number of 2000C (HL*3)
|
||||
children, not the payer loop.
|
||||
"""
|
||||
# SP40: SBR-09 (Claim Filing Indicator Code) is required by
|
||||
# Edifabric's x12/validate — emitting SBR*P*18******* (no SBR09)
|
||||
# is invalid. Callers MUST thread the per-payer
|
||||
# ``PayerConfig837.sbr09_claim_filing`` (or
|
||||
# ``PayerConfigORM.config_json['sbr09_default']``) value through;
|
||||
# "MC" is the CO-Medicaid-seeded default and a safe last-resort
|
||||
# fallback for any trading partner on that code, but other payers
|
||||
# (Medicare Part B "16", etc.) MUST override.
|
||||
if not claim_filing_indicator_code:
|
||||
claim_filing_indicator_code = "MC"
|
||||
out = [
|
||||
_build_hl("2", "1", "22", "0"), # HL*2 — subscriber, 0 children
|
||||
_build_sbr("18", subscriber.member_id, payer_name),
|
||||
_build_hl("2", "1", "22", "1" if include_patient_loop else "0"),
|
||||
_build_sbr("18", claim_filing_indicator_code),
|
||||
_build_nm1(
|
||||
"IL", "IL",
|
||||
f"{subscriber.last_name} {subscriber.first_name}".strip(),
|
||||
@@ -418,21 +575,98 @@ def _build_subscriber_block(subscriber, payer_name: str | None) -> list[str]:
|
||||
dmg = _build_dmg(subscriber.dob, subscriber.gender)
|
||||
if dmg:
|
||||
out.append(dmg)
|
||||
# 2010BB — Payer Name. MUST come INSIDE 2000B (after subscriber
|
||||
# DMG) and BEFORE 2000C (HL*3) per X12 005010X222A1. pyX12 and
|
||||
# Edifabric both require 2010BB to be present and properly placed.
|
||||
if payer is not None:
|
||||
out.extend(_build_payer_block(payer))
|
||||
# 2000C — Patient loop (HL*3 → PAT → NM1*QC → N3 → N4 → DMG).
|
||||
# Always emitted in the SP41 self-pay shape (patient == subscriber).
|
||||
# The PAT segment is required when 2000C is emitted. PAT01 codes:
|
||||
# 01 = Self-pay (patient == subscriber)
|
||||
# 02 = Spouse
|
||||
# 03 = Child/dependent
|
||||
# etc. We default to "01" (self-pay).
|
||||
if include_patient_loop:
|
||||
out.append(_build_hl("3", "2", "23", "0")) # HL*3 — patient, 0 children
|
||||
out.append("PAT*01" + _SEG) # PAT — Patient Information (self-pay)
|
||||
out.append(_build_nm1(
|
||||
"QC", "QC",
|
||||
f"{subscriber.last_name} {subscriber.first_name}".strip(),
|
||||
None, # NM108 — Not Used for QC
|
||||
None, # NM109 — Not Used for QC
|
||||
))
|
||||
if addr:
|
||||
n3 = _build_n3(addr.line1, addr.line2)
|
||||
n4 = _build_n4(addr.city, addr.state, addr.zip)
|
||||
if n3:
|
||||
out.append(n3)
|
||||
if n4:
|
||||
out.append(n4)
|
||||
dmg = _build_dmg(subscriber.dob, subscriber.gender)
|
||||
if dmg:
|
||||
out.append(dmg)
|
||||
return out
|
||||
|
||||
|
||||
def _build_payer_block(payer) -> list[str]:
|
||||
name, pid = _normalize_payer_id(payer)
|
||||
return [
|
||||
_build_nm1("PR", "PR", payer.name, "PI", payer.id),
|
||||
_build_nm1("PR", "PR", name, "PI", pid),
|
||||
]
|
||||
|
||||
|
||||
def _build_service_lines_block(service_lines) -> list[str]:
|
||||
"""Per line: LX / SV1 / DTP*472 / REF*6R."""
|
||||
# Payer ids that must be normalized to CO_TXIX for CO Medicaid submissions.
|
||||
# SP39: defense-in-depth against legacy raw_json captures (SKCO0 from
|
||||
# pre-SP33 batches, CO_BHA from prior behavioral-health configurations,
|
||||
# empty from degenerate parses). Foreign payer IDs are emitted verbatim.
|
||||
_NORMALIZE_TO_CO_TXIX_IDS = frozenset({"", "SKCO0", "CO_BHA"})
|
||||
_NORMALIZE_TO_CO_TXIX_NAMES = frozenset({"", "COHCPF", "CO_BHA"})
|
||||
_CO_TXIX = "CO_TXIX"
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_payer_id(payer) -> tuple[str, str]:
|
||||
"""Return (name, id) normalized so CO Medicaid claims always emit CO_TXIX.
|
||||
|
||||
SP39. Substitutes empty/SKCO0/CO_BHA -> CO_TXIX in the id and
|
||||
empty/COHCPF/CO_BHA -> CO_TXIX in the name. Foreign payer ids are
|
||||
passed through verbatim (only the CO Medicaid-shape values are
|
||||
normalized; the helper must not corrupt a non-CO submit). Emits a
|
||||
WARNING log line on substitution (one per call; the serializer is
|
||||
invoked once per claim so volume is bounded by batch size).
|
||||
"""
|
||||
raw_id = (getattr(payer, "id", None) or "").strip()
|
||||
raw_name = (getattr(payer, "name", None) or "").strip()
|
||||
new_id = _CO_TXIX if raw_id in _NORMALIZE_TO_CO_TXIX_IDS else raw_id
|
||||
new_name = _CO_TXIX if raw_name in _NORMALIZE_TO_CO_TXIX_NAMES else raw_name
|
||||
if new_id != raw_id or new_name != raw_name:
|
||||
_log.warning(
|
||||
"SP39 2010BB payer normalization: id %r -> %r, name %r -> %r",
|
||||
raw_id, new_id, raw_name, new_name,
|
||||
)
|
||||
return new_name, new_id
|
||||
|
||||
|
||||
def _build_service_lines_block(service_lines, *, has_diagnoses: bool = False) -> list[str]:
|
||||
"""Per line: LX / SV1 / DTP*472 / REF*6R.
|
||||
|
||||
``has_diagnoses`` is True when the parent claim emits an HI segment;
|
||||
in that case SV1-07 is required by X12 and we default each line's
|
||||
pointer to ``"1"`` (the first HI diagnosis) unless the source
|
||||
captured a different pointer on the line itself.
|
||||
"""
|
||||
out: list[str] = []
|
||||
for idx, line in enumerate(service_lines or [], start=1):
|
||||
out.append(_build_lx(idx))
|
||||
out.append(_build_sv1(line))
|
||||
# Prefer the line's captured pointer (parser pulled SV1-07
|
||||
# when present). Fall back to "1" only when the claim has
|
||||
# diagnoses and the source had no explicit pointer — the
|
||||
# common single-diagnosis case.
|
||||
line_pointer = getattr(line, "dx_pointer", None)
|
||||
effective_pointer = line_pointer or ("1" if has_diagnoses else "")
|
||||
out.append(_build_sv1(line, dx_pointer=effective_pointer))
|
||||
dtp = _build_dtp_472(line.service_date)
|
||||
if dtp:
|
||||
out.append(dtp)
|
||||
@@ -450,7 +684,10 @@ def serialize_837(
|
||||
submitter_name: str | None = None,
|
||||
submitter_contact_name: str | None = None,
|
||||
submitter_contact_phone: str | None = None,
|
||||
submitter_contact_email: str | None = None,
|
||||
submitter_contact_email_qual: str = "EM",
|
||||
receiver_name: str | None = None,
|
||||
claim_filing_indicator_code: str | None = None,
|
||||
interchange_control_number: str = "000000001",
|
||||
group_control_number: str = "1",
|
||||
) -> str:
|
||||
@@ -462,6 +699,16 @@ def serialize_837(
|
||||
(``"CYCLONE"`` / ``"RECEIVER"``) but real deployments should pass
|
||||
the configured values.
|
||||
|
||||
The submitter block (Loop 1000A) always emits a PER segment per the
|
||||
X12 spec — the canonical clearhouse config provides the contact
|
||||
name and email so callers should pass them through.
|
||||
|
||||
The claim filing indicator (SBR09) is read from the per-payer
|
||||
config (``PayerConfig837.sbr09_default``); callers should pass it
|
||||
in. If not passed, SBR09 is left empty (which causes the
|
||||
:func:`cyclone.parsers.validator._r202_sbr09_allowed` rule to skip
|
||||
its check — degraded but not a hard error).
|
||||
|
||||
Editable fields (CLM, REF*G1, HI, service-line SV1, DTP*472) are
|
||||
emitted from the canonical ``ClaimOutput`` fields, so post-parse
|
||||
edits propagate to the output.
|
||||
@@ -481,12 +728,15 @@ def serialize_837(
|
||||
),
|
||||
]
|
||||
segments.extend(_build_submitter_block(
|
||||
sender_id, submitter_name, submitter_contact_name, submitter_contact_phone,
|
||||
sender_id, submitter_name,
|
||||
submitter_contact_name, submitter_contact_phone, submitter_contact_email,
|
||||
submitter_contact_email_qual,
|
||||
))
|
||||
segments.extend(_build_receiver_block(receiver_id, receiver_name))
|
||||
segments.extend(_build_billing_provider_block(claim.billing_provider))
|
||||
segments.extend(_build_subscriber_block(claim.subscriber, claim.payer.name))
|
||||
segments.extend(_build_payer_block(claim.payer))
|
||||
segments.extend(_build_subscriber_block(
|
||||
claim.subscriber, claim_filing_indicator_code, claim.payer,
|
||||
))
|
||||
|
||||
# Claim-level editable segments.
|
||||
segments.append(_build_clm(claim.claim))
|
||||
@@ -497,7 +747,10 @@ def serialize_837(
|
||||
segments.append(_build_hi(claim.diagnoses))
|
||||
|
||||
# Service lines (LX / SV1 / DTP*472 / REF*6R).
|
||||
segments.extend(_build_service_lines_block(claim.service_lines))
|
||||
segments.extend(_build_service_lines_block(
|
||||
claim.service_lines,
|
||||
has_diagnoses=bool(claim.diagnoses),
|
||||
))
|
||||
|
||||
# SE segment count includes ST (line 3, 1-based) through SE itself
|
||||
# — i.e. the entire ST..SE block inclusive.
|
||||
@@ -514,15 +767,163 @@ def serialize_837_for_resubmit(
|
||||
claim: ClaimOutput,
|
||||
*,
|
||||
interchange_index: int,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""Like :func:`serialize_837` but assigns deterministic-but-unique
|
||||
interchange + group control numbers for a bundle position.
|
||||
|
||||
Interchange number = ``f"{interchange_index:09d}"``.
|
||||
Group number = ``str(interchange_index)``.
|
||||
|
||||
All other keyword arguments (sender_id, receiver_id, submitter_*
|
||||
and receiver_* contact info, claim_filing_indicator_code) are
|
||||
forwarded to :func:`serialize_837` unchanged so callers — like the
|
||||
export and SFTP-submit endpoints — can pass through clearhouse +
|
||||
payer config without copying the signature.
|
||||
"""
|
||||
return serialize_837(
|
||||
claim,
|
||||
interchange_control_number=f"{interchange_index:09d}",
|
||||
group_control_number=str(interchange_index),
|
||||
)
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pipeline B overload: MemberWeekBatch → one 837P, one CLM per visit
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_member_week_claim(visit, claim_id: str) -> tuple[str, str, str]:
|
||||
"""Build the CLM / SV1 / DTP*472 segments for a single MemberWeekBatch visit.
|
||||
|
||||
Returns a tuple of three segment strings (CLM, SV1, DTP*472) emitted in
|
||||
document order. The visit is a :class:`cyclone.rebill.reconcile.VisitRow`
|
||||
— only ``date`` / ``member_id`` / ``procedure`` / ``billed`` are read.
|
||||
"""
|
||||
# Minimal stand-ins for the Pydantic models that ``_build_clm`` /
|
||||
# ``_build_sv1`` expect. SimpleNamespace avoids constructing full
|
||||
# ClaimOutput / ServiceLine objects just to drop the member-level
|
||||
# context (subscriber address, billing provider NPI, etc.) that
|
||||
# Pipeline B doesn't have on its input shape.
|
||||
procedure = SimpleNamespace(qualifier="HC", code=visit.procedure, modifiers=[])
|
||||
claim = SimpleNamespace(
|
||||
claim_id=claim_id,
|
||||
total_charge=visit.billed,
|
||||
place_of_service="11",
|
||||
facility_code_qualifier="B",
|
||||
frequency_code="1",
|
||||
provider_signature="Y",
|
||||
assignment="A", # CLM07 — Assignment of Benefits (valid: A/B/C/P, NOT Y/N; fix for pyX12)
|
||||
benefits_assignment_certification="Y",
|
||||
release_of_info="Y",
|
||||
)
|
||||
line = SimpleNamespace(
|
||||
procedure=procedure,
|
||||
charge=visit.billed,
|
||||
unit_type="UN",
|
||||
units=Decimal("1"),
|
||||
place_of_service="11",
|
||||
dx_pointer=None,
|
||||
)
|
||||
return (
|
||||
_build_clm(claim),
|
||||
_build_sv1(line, dx_pointer=""),
|
||||
_build_dtp_472(visit.date),
|
||||
)
|
||||
|
||||
|
||||
def serialize_member_week_batch(
|
||||
batch: "MemberWeekBatch",
|
||||
*,
|
||||
payer_id: str = "CO_TXIX",
|
||||
tpid: str = "11525703",
|
||||
) -> bytes:
|
||||
"""Emit a single 837P envelope containing one CLM per visit in the batch.
|
||||
|
||||
SP41 / Pipeline B. Each :class:`cyclone.rebill.reconcile.VisitRow` in
|
||||
``batch.visits`` becomes its own CLM with one SV1 and one DTP*472
|
||||
service-line date. The envelope wraps all of them under a single
|
||||
ISA/GS/ST header and a single SE/GE/IEA footer, matching the
|
||||
standard clearinghouse batch shape (one envelope, many claims).
|
||||
|
||||
Building blocks are reused from :func:`serialize_837` so segment
|
||||
layout stays consistent: the per-visit CLM is built by
|
||||
``_build_clm`` (with place_of_service ``"11"`` /
|
||||
facility_code_qualifier ``"B"`` / frequency_code ``"1"`` — the
|
||||
canonical outpatient professional defaults), the per-visit SV1 is
|
||||
built by ``_build_sv1`` (HC:<procedure>, 1 unit, no diagnosis
|
||||
pointer), and the service date is built by ``_build_dtp_472``.
|
||||
|
||||
Args:
|
||||
batch: A :class:`cyclone.rebill.pipeline_b.MemberWeekBatch` —
|
||||
one member × one ISO-week worth of rebillable visits.
|
||||
payer_id: The receiver (NM1*40) identifier. Defaults to
|
||||
``"CO_TXIX"`` for CO Medicaid.
|
||||
tpid: The trading-partner / submitter (NM1*41) identifier.
|
||||
Defaults to Gainwell's ``"11525703"``.
|
||||
|
||||
Returns:
|
||||
The complete 837P document as ASCII bytes. The caller writes
|
||||
it to disk with HCPF-spec filenames via
|
||||
:func:`cyclone.edi.filenames.build_outbound_filename`.
|
||||
"""
|
||||
# Deterministic control numbers derived from (member, iso_year,
|
||||
# iso_week). Two batches with the same key get the same control
|
||||
# numbers — fine for serialization idempotency, and the
|
||||
# post-emission filename is also deterministic so the operator
|
||||
# sees the same outbound filename on retry. Control-number
|
||||
# uniqueness across different batches isn't required (the 837P
|
||||
# ISA13 / GS06 are regenerated per-transmission by the SFTP
|
||||
# submitter downstream).
|
||||
control = f"{batch.member_id}{batch.iso_year:04d}{batch.iso_week:02d}"
|
||||
interchange_control_number = control[:9].rjust(9, "0")
|
||||
group_control_number = control[:9].lstrip("0") or "1"
|
||||
st_control_number = control[:9].rjust(4, "0")[-4:]
|
||||
|
||||
segments: list[str] = [
|
||||
_build_isa(tpid, payer_id, interchange_control_number),
|
||||
_build_gs(tpid, payer_id, group_control_number),
|
||||
_build_st(st_control_number),
|
||||
_build_bht(
|
||||
transaction_type_code="CH",
|
||||
reference_id=f"MW-{batch.member_id}-W{batch.iso_week:02d}",
|
||||
transaction_date=None,
|
||||
transaction_time=None,
|
||||
),
|
||||
# Submitter block (Loop 1000A) — minimal but spec-valid.
|
||||
# Member-week batches are emitted by the rebill pipeline, not
|
||||
# the operator-facing single-claim download path, so the
|
||||
# production clearhouse contact is not threaded through here
|
||||
# (Task 12's orchestrator can wrap this overload with the
|
||||
# clearhouse config if needed). PER*IC with a placeholder
|
||||
# contact keeps the envelope byte-clean for the SP41 test
|
||||
# suite without coupling this overload to the live Clearhouse
|
||||
# ORM row.
|
||||
_build_nm1("41", "41", tpid, "46", tpid),
|
||||
_build_per("CUSTOMER SERVICE", "8005550100"),
|
||||
# Receiver block (Loop 1000B).
|
||||
_build_nm1("40", "40", payer_id, "46", payer_id),
|
||||
]
|
||||
|
||||
for idx, visit in enumerate(batch.visits, start=1):
|
||||
svc_date = visit.date
|
||||
claim_id = f"MW-{batch.member_id}-{svc_date.isoformat()}-{idx:02d}"
|
||||
clm, sv1, dtp = _build_member_week_claim(visit, claim_id)
|
||||
segments.append(clm)
|
||||
segments.append(sv1)
|
||||
if dtp:
|
||||
segments.append(dtp)
|
||||
|
||||
# SE segment count = ST (1) + everything between ST and SE inclusive.
|
||||
# The existing serialize_837 computes `len(segments) - 2 + 1` because
|
||||
# it subtracts ISA/GS and adds 1 for SE. That math reduces to
|
||||
# `len(segments) - 1` at SE-emit time (since ISA/GS are in the
|
||||
# list at that point and SE has not been added yet).
|
||||
seg_count = len(segments) - 1
|
||||
segments.append(_build_se(seg_count, st_control_number))
|
||||
|
||||
segments.append(f"GE*1*{group_control_number}{_SEG}")
|
||||
segments.append(f"IEA*1*{interchange_control_number}{_SEG}")
|
||||
|
||||
return "".join(segments).encode("ascii")
|
||||
@@ -0,0 +1,11 @@
|
||||
"""SP41 — in-window rebill pipeline.
|
||||
|
||||
Owns:
|
||||
- parse_835_svc (SVC-level 835 reparse with member_id)
|
||||
- reconcile (visit-to-835 join on (member_id, procedure, DOS))
|
||||
- carc_filter (CARC-aware exclusion for Pipeline A)
|
||||
- timely_filing (120-day DOS age gate)
|
||||
- pipeline_a (denied/partial → frequency-7 rebill)
|
||||
- pipeline_b (NOT_IN_835 → fresh 837Ps by (member, ISO-week))
|
||||
- summary (summary CSV + per-category counters)
|
||||
"""
|
||||
@@ -0,0 +1,44 @@
|
||||
"""CARC-aware filter.
|
||||
|
||||
EXCLUDED = contractual / not recoverable / not a denial-of-payment in the
|
||||
narrow sense (charge exceeds fee schedule, prior to coverage, etc.).
|
||||
REVIEW = operator must decide (claim/service lacks info, non-covered
|
||||
charges, duplicate, etc.). Anything else falls through to REBILL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class CarcDecision(str, Enum):
|
||||
EXCLUDED = "EXCLUDED"
|
||||
REVIEW = "REVIEW"
|
||||
REBILL = "REBILL"
|
||||
|
||||
|
||||
# These CARCs are NOT recoverable denials. Do not rebill.
|
||||
EXCLUDED_CARCS: frozenset[str] = frozenset({
|
||||
"CO-45", # charge exceeds fee schedule / contractual obligation
|
||||
"CO-26", # expenses incurred prior to coverage
|
||||
"CO-129", # prior processing information; forward to next payer
|
||||
})
|
||||
|
||||
# These CARCs need human review before rebill.
|
||||
REVIEW_CARCS: frozenset[str] = frozenset({
|
||||
"PI-16", # claim/service lacks information
|
||||
"PI-96", # non-covered charges
|
||||
"PI-15", # authorization / certification absent
|
||||
"PI-4", # procedure not paid separately
|
||||
"PI-110", # billing date predates service date
|
||||
"OA-18", # exact duplicate (resubmit noise)
|
||||
"OA-23", # impact of prior payer adjudication
|
||||
})
|
||||
|
||||
|
||||
def decide_carc(reasons: tuple[str, ...] | list[str]) -> CarcDecision:
|
||||
"""Pick the strongest action: EXCLUDED > REVIEW > REBILL."""
|
||||
if any(r in EXCLUDED_CARCS for r in reasons):
|
||||
return CarcDecision.EXCLUDED
|
||||
if any(r in REVIEW_CARCS for r in reasons):
|
||||
return CarcDecision.REVIEW
|
||||
return CarcDecision.REBILL
|
||||
@@ -0,0 +1,183 @@
|
||||
"""SVC-level 835 reparse with member_id at SVC scope.
|
||||
|
||||
Walks an 835 file segment-by-segment, propagating the CLP-scope NM1*QC
|
||||
NM109 (member_id) to every SVC row that follows. Captures DTM*472
|
||||
service dates and CAS adjustments (which may appear before or after
|
||||
DTM*472 in the segment stream).
|
||||
|
||||
:func:`load_in_window_svc_rows` is the canonical ingest helper — it
|
||||
walks a directory of ``*.835`` / ``*.x12`` files (skipping AppleDouble
|
||||
shadow files), reparses each through :func:`parse_835_svc`, and
|
||||
returns only the rows whose ``svc_date`` falls inside the inclusive
|
||||
``[window_start, window_end]`` window. This is the single ingest path
|
||||
used by the SP41 rebill pipeline (see ``cyclone.rebill.run``) AND by
|
||||
the SP41-goal spot-check pipeline (see
|
||||
``cyclone.rebill.spot_check_pipeline``).
|
||||
"""
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
ELEM = "*"
|
||||
SEG_END = "~"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SvcRow:
|
||||
src_file: str
|
||||
claim_id: str
|
||||
member_id: str
|
||||
status: str
|
||||
procedure: str
|
||||
modifiers: str
|
||||
charge: Decimal
|
||||
paid: Decimal
|
||||
units: Decimal
|
||||
svc_date: date
|
||||
cas_reasons: tuple[str, ...] # each entry is "GROUP-CODE" (e.g. "OA-18")
|
||||
pay_date: date | None
|
||||
|
||||
|
||||
def _split(seg: str) -> list[str]:
|
||||
return seg.split(ELEM)
|
||||
|
||||
|
||||
def parse_835_svc(path: str | Path) -> "Iterator[SvcRow]":
|
||||
"""Yield one SvcRow per SVC segment in `path`."""
|
||||
path = Path(path)
|
||||
text = path.read_text()
|
||||
segments = [s for s in text.split(SEG_END) if s]
|
||||
pay_date: date | None = None
|
||||
current_claim_id = ""
|
||||
current_status = ""
|
||||
current_member_id = ""
|
||||
i = 0
|
||||
while i < len(segments):
|
||||
elems = _split(segments[i])
|
||||
seg_name = elems[0]
|
||||
if seg_name == "DTM" and len(elems) > 2 and elems[1] == "405":
|
||||
try:
|
||||
pay_date = _ymd(elems[2])
|
||||
except ValueError:
|
||||
pass
|
||||
elif seg_name == "CLP":
|
||||
current_claim_id = elems[1] if len(elems) > 1 else ""
|
||||
current_status = elems[2] if len(elems) > 2 else ""
|
||||
current_member_id = ""
|
||||
elif seg_name == "NM1" and len(elems) > 1 and elems[1] == "QC":
|
||||
current_member_id = elems[9] if len(elems) > 9 else ""
|
||||
elif seg_name == "SVC" and current_claim_id:
|
||||
rest = elems[1:]
|
||||
comp = rest[0] if rest else ""
|
||||
proc_parts = comp.split(":")
|
||||
procedure = proc_parts[1] if len(proc_parts) > 1 else ""
|
||||
modifiers = ":".join(proc_parts[2:]) if len(proc_parts) > 2 else ""
|
||||
charge = _decimal(rest[1]) if len(rest) > 1 else Decimal("0")
|
||||
paid = _decimal(rest[2]) if len(rest) > 2 else Decimal("0")
|
||||
units = _decimal(rest[3]) if len(rest) > 3 else Decimal("0")
|
||||
svc_date, cas_reasons = _walk_for_dtm_and_cas(segments, i)
|
||||
yield SvcRow(
|
||||
src_file=path.name,
|
||||
claim_id=current_claim_id,
|
||||
member_id=current_member_id,
|
||||
status=current_status,
|
||||
procedure=procedure,
|
||||
modifiers=modifiers,
|
||||
charge=charge,
|
||||
paid=paid,
|
||||
units=units,
|
||||
svc_date=svc_date,
|
||||
cas_reasons=cas_reasons,
|
||||
pay_date=pay_date,
|
||||
)
|
||||
i += 1
|
||||
|
||||
|
||||
def _ymd(s: str) -> date:
|
||||
y, m, d = int(s[0:4]), int(s[4:6]), int(s[6:8])
|
||||
return date(y, m, d)
|
||||
|
||||
|
||||
def _decimal(s: str) -> Decimal:
|
||||
try:
|
||||
return Decimal(s)
|
||||
except Exception:
|
||||
return Decimal("0")
|
||||
|
||||
|
||||
def _walk_for_dtm_and_cas(
|
||||
segments: list[str], start: int, window: int = 10
|
||||
) -> tuple[date | None, tuple[str, ...]]:
|
||||
"""Walk the next `window` segments after SVC. Capture DTM*472 and CAS.
|
||||
|
||||
X12 835 may place DTM*472 before or after CAS segments within a service
|
||||
line, so we walk the full window and capture both, breaking only at the
|
||||
next SVC or CLP (a new claim/service boundary).
|
||||
"""
|
||||
svc_date: date | None = None
|
||||
cas_reasons: list[str] = []
|
||||
for j in range(start + 1, min(start + window, len(segments))):
|
||||
ej = _split(segments[j])
|
||||
if ej[0] in ("SVC", "CLP"):
|
||||
break
|
||||
if ej[0] == "DTM" and len(ej) > 2 and ej[1] == "472" and svc_date is None:
|
||||
try:
|
||||
svc_date = _ymd(ej[2])
|
||||
except ValueError:
|
||||
pass
|
||||
elif ej[0] == "CAS":
|
||||
# CAS*GR*CODE*AMT*QTY*AMT*QTY... → GR-CODE
|
||||
if len(ej) >= 3:
|
||||
cas_reasons.append(f"{ej[1]}-{ej[2]}")
|
||||
return svc_date, tuple(cas_reasons)
|
||||
|
||||
|
||||
def load_in_window_svc_rows(
|
||||
ingest_dir: str | Path,
|
||||
window_start: date,
|
||||
window_end: date,
|
||||
*,
|
||||
file_glob: Iterable[str] = ("*.835", "*.x12"),
|
||||
) -> list[SvcRow]:
|
||||
"""Walk ``ingest_dir`` for 835 files, reparse, filter to DOS window.
|
||||
|
||||
The single canonical ingest path. Skips AppleDouble shadow files
|
||||
(``._*`` — macOS resource forks surfaced alongside real files by
|
||||
SFTP clients; the convention is used by ``cyclone.cli``,
|
||||
``cyclone.submission.core``, and ``cyclone.api_routers.submission``).
|
||||
|
||||
Args:
|
||||
ingest_dir: Directory to walk. Both ``*.835`` and ``*.x12`` are
|
||||
scanned (sorted by name for determinism).
|
||||
window_start: Inclusive lower bound on ``SvcRow.svc_date``.
|
||||
window_end: Inclusive upper bound on ``SvcRow.svc_date``.
|
||||
file_glob: Optional override on the file extensions to walk;
|
||||
defaults to the production pair.
|
||||
|
||||
Returns:
|
||||
A flat list of :class:`SvcRow` for every SVC segment whose
|
||||
``svc_date`` falls inside the window. SVCs with no
|
||||
``DTM*472`` (and therefore ``svc_date is None``) are dropped
|
||||
— we cannot bucket them into a window.
|
||||
|
||||
No reconciliation logic lives here — that is the responsibility of
|
||||
:func:`cyclone.rebill.reconcile.reconcile_visits_to_835`. This
|
||||
helper just yields the raw SVC rows indexed downstream.
|
||||
"""
|
||||
ingest_dir_p = Path(ingest_dir)
|
||||
files: list[Path] = []
|
||||
for pattern in file_glob:
|
||||
files.extend(ingest_dir_p.glob(pattern))
|
||||
files = sorted(set(files))
|
||||
svcs: list[SvcRow] = []
|
||||
for p in files:
|
||||
if p.name.startswith("._"):
|
||||
continue
|
||||
for s in parse_835_svc(p):
|
||||
if s.svc_date is None:
|
||||
continue
|
||||
if window_start <= s.svc_date <= window_end:
|
||||
svcs.append(s)
|
||||
return svcs
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Pipeline A: denied/partial → frequency-7 replacement 837Ps.
|
||||
|
||||
For each visit the previous adjudication said was DENIED or PARTIAL,
|
||||
emit a fresh 837P with CLM05-3 = '7' (replacement claim) and the
|
||||
original claim_submit_id preserved as CLM01. The 837P carries the
|
||||
original DOS and the same procedure / member.
|
||||
|
||||
The CARC-aware filter is the gate — only REBILL / REVIEW decisions are
|
||||
emitted; EXCLUDED visits never make it to a file.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from cyclone.rebill.carc_filter import CarcDecision
|
||||
from cyclone.rebill.reconcile import ReconcileOutcome, OutcomeCategory
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RebillClaim:
|
||||
"""Pre-emission 837P shape for Pipeline A. The orchestrator's serializer
|
||||
adapter converts this to a ClaimOutput for serialize_837.
|
||||
"""
|
||||
|
||||
claim_id: str # reuses original claim_submit_id
|
||||
member_id: str
|
||||
procedure: str
|
||||
svc_date: date
|
||||
charge: Decimal
|
||||
frequency_code: str # always "7" for Pipeline A
|
||||
needs_review: bool # True for CARC REVIEW decisions
|
||||
original_carc_reasons: tuple[str, ...]
|
||||
|
||||
|
||||
def build_pipeline_a_claims(
|
||||
original_claim_id: str,
|
||||
visit_outcomes: list[ReconcileOutcome],
|
||||
carc_decisions: list[CarcDecision],
|
||||
cas_reasons_per_visit: list[tuple[str, ...]],
|
||||
) -> list[RebillClaim]:
|
||||
"""Zip three parallel per-visit lists by index; drop PAID/NOT_IN_835 outcomes and EXCLUDED CARC decisions; emit one frequency-7 RebillClaim per survivor."""
|
||||
out: list[RebillClaim] = []
|
||||
# strict=True: all three lists must be the same length; a mismatch is a contract bug, fail loud.
|
||||
for vo, carc, reasons in zip(visit_outcomes, carc_decisions, cas_reasons_per_visit, strict=True):
|
||||
if vo.category not in (OutcomeCategory.DENIED, OutcomeCategory.PARTIAL):
|
||||
continue
|
||||
if carc == CarcDecision.EXCLUDED:
|
||||
continue
|
||||
out.append(RebillClaim(
|
||||
claim_id=original_claim_id,
|
||||
member_id=vo.visit.member_id,
|
||||
procedure=vo.visit.procedure,
|
||||
svc_date=vo.visit.date,
|
||||
charge=vo.visit.billed,
|
||||
frequency_code="7",
|
||||
needs_review=(carc == CarcDecision.REVIEW),
|
||||
original_carc_reasons=reasons,
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
def write_pipeline_a_files(
|
||||
claims: list[RebillClaim],
|
||||
out_dir: Path,
|
||||
serialize_837_fn, # injected: callable[[RebillClaim], str] (orchestrator-supplied adapter)
|
||||
tpid: str,
|
||||
) -> list[Path]:
|
||||
"""Write one 837P per claim, using HCPF-spec filenames via build_outbound_filename.
|
||||
|
||||
`serialize_837_fn(claim)` returns the X12 string. The 837P envelope is
|
||||
pure ASCII so we write it as text. The filename is
|
||||
`cyclone.edi.filenames.build_outbound_filename(tpid, '837P')`.
|
||||
"""
|
||||
# Lazy: keep cyclone.rebill.pipeline_a importable without zoneinfo from filenames.
|
||||
from cyclone.edi.filenames import build_outbound_filename
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
paths: list[Path] = []
|
||||
for c in claims:
|
||||
body = serialize_837_fn(c)
|
||||
fname = build_outbound_filename(tpid, "837P")
|
||||
path = out_dir / fname
|
||||
path.write_text(body, encoding="ascii")
|
||||
paths.append(path)
|
||||
return paths
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Pipeline B: NOT_IN_835 visits → fresh 837Ps, batched by (member_id, ISO-week).
|
||||
|
||||
One 837P per (member, ISO-week) pair. Per-visit files would be 4,509 SFTP
|
||||
round-trips; per-batch (no member split) would lose the operator's
|
||||
ability to track per-member. Member-week is the standard clearinghouse
|
||||
pattern and matches AxisCare's billing cycle.
|
||||
|
||||
The timely-filing gate is applied per visit before batching. The
|
||||
override flag is the same for the whole batch (a per-batch override).
|
||||
"""
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from cyclone.rebill.reconcile import VisitRow
|
||||
from cyclone.rebill.timely_filing import timely_filing_decision
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MemberWeekBatch:
|
||||
"""Pre-emission 837P shape for Pipeline B. The orchestrator's serializer
|
||||
adapter converts this to a ClaimOutput for serialize_837.
|
||||
|
||||
Note: frozen=True is shallow — `visits` is a mutable list, so
|
||||
`mb.visits.append(x)` works even though `mb.visits = new_list` raises
|
||||
FrozenInstanceError. Treat the batch as logically immutable.
|
||||
"""
|
||||
|
||||
member_id: str
|
||||
iso_year: int
|
||||
iso_week: int
|
||||
visits: list[VisitRow]
|
||||
has_overridden_visits: bool
|
||||
|
||||
|
||||
def build_pipeline_b_batches(
|
||||
visits: list[VisitRow],
|
||||
as_of: date,
|
||||
override: bool,
|
||||
) -> list[MemberWeekBatch]:
|
||||
"""Group rebillable visits by (member_id, ISO-week).
|
||||
|
||||
Two filters are applied per visit before batching:
|
||||
1. timely_filing_decision(...).rebillable — drops past-window
|
||||
visits unless `override=True` (per-batch override flag).
|
||||
2. implicit: only visits that survive (1) are grouped; the rest
|
||||
are surfaced as EXCLUDED_TIMELY_FILING in the summary CSV.
|
||||
|
||||
Returns batches sorted by (member_id, iso_year, iso_week) for
|
||||
deterministic filenames downstream.
|
||||
"""
|
||||
by_key: dict[tuple[str, int, int], list[VisitRow]] = defaultdict(list)
|
||||
overridden_keys: set[tuple[str, int, int]] = set()
|
||||
for v in visits:
|
||||
decision = timely_filing_decision(v.date, as_of, override)
|
||||
if not decision.rebillable:
|
||||
continue
|
||||
iso = v.date.isocalendar()
|
||||
key = (v.member_id, iso.year, iso.week)
|
||||
by_key[key].append(v)
|
||||
if decision.override and not decision.within_window:
|
||||
overridden_keys.add(key)
|
||||
return [
|
||||
MemberWeekBatch(
|
||||
member_id=member, iso_year=yr, iso_week=wk,
|
||||
visits=vs, has_overridden_visits=(member, yr, wk) in overridden_keys,
|
||||
)
|
||||
for (member, yr, wk), vs in sorted(by_key.items())
|
||||
]
|
||||
|
||||
|
||||
def batch_visits_by_member_week(
|
||||
visits: list[VisitRow],
|
||||
as_of: date,
|
||||
override: bool,
|
||||
) -> dict[str, list[VisitRow]]:
|
||||
"""Thin wrapper over build_pipeline_b_batches.
|
||||
|
||||
Returns ``{member_id: [VisitRow, ...]}`` — collapsing the (member,
|
||||
ISO-week) batches back down to a per-member visit list. This is the
|
||||
shape the orchestrator's summary aggregation expects.
|
||||
|
||||
PROVISIONAL: this helper is defined for the Task 12 orchestrator's
|
||||
anticipated use, but has no callers in this branch. If Task 12's
|
||||
actual needs differ, this function may be removed or replaced without
|
||||
notice.
|
||||
|
||||
This also collapses multiple weeks for the same member into one
|
||||
list. If the orchestrator needs week-aware per-member iteration,
|
||||
use build_pipeline_b_batches directly.
|
||||
"""
|
||||
batches = build_pipeline_b_batches(visits, as_of=as_of, override=override)
|
||||
out: dict[str, list[VisitRow]] = defaultdict(list)
|
||||
for b in batches:
|
||||
out[b.member_id].extend(b.visits)
|
||||
return dict(out)
|
||||
|
||||
|
||||
def write_pipeline_b_files(
|
||||
batches: list[MemberWeekBatch],
|
||||
out_dir: Path,
|
||||
serialize_837_fn,
|
||||
tpid: str,
|
||||
) -> list[Path]:
|
||||
"""Write one 837P per batch, using HCPF-spec filenames.
|
||||
|
||||
`serialize_837_fn(batch)` returns the X12 string. The 837P envelope
|
||||
is pure ASCII so we write it as text. The filename is
|
||||
`cyclone.edi.filenames.build_outbound_filename(tpid, '837P')`.
|
||||
"""
|
||||
# Lazy: keep cyclone.rebill.pipeline_b importable without zoneinfo from filenames.
|
||||
from cyclone.edi.filenames import build_outbound_filename
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
paths: list[Path] = []
|
||||
for b in batches:
|
||||
body = serialize_837_fn(b)
|
||||
fname = build_outbound_filename(tpid, "837P")
|
||||
path = out_dir / fname
|
||||
path.write_text(body, encoding="ascii")
|
||||
paths.append(path)
|
||||
return paths
|
||||
@@ -0,0 +1,417 @@
|
||||
"""SP41 Task 15 — 999-ack dump from Gainwell + NOT_IN_835 reconciliation.
|
||||
|
||||
Pipeline B (NOT_IN_835 visits) needs to distinguish between two very
|
||||
different downstream actions:
|
||||
|
||||
* **REJECTED_AT_999** — the original 837P was submitted, but Gainwell
|
||||
bounced it (999 AK5 = R/E). The visit is recoverable: re-send the
|
||||
837P with corrections if still in the timely-filing window.
|
||||
* **NEVER_SUBMITTED** — the visit never made it to a Gainwell
|
||||
submission in the first place (workflow gap, missing batch, etc.).
|
||||
These require investigation before any rebill is meaningful.
|
||||
|
||||
Both buckets surface the same ``NOT_IN_835`` outcome from
|
||||
:func:`cyclone.rebill.reconcile.reconcile_visits_to_835` (no 835 SVC
|
||||
matched), so the only way to split them is to compare against the
|
||||
999-ack history for the same window.
|
||||
|
||||
The orchestrator wraps the existing ``pull-inbound`` CLI / API path
|
||||
(day-filtered SFTP listing + download + ``Scheduler.process_inbound_files``)
|
||||
so this module does NOT introduce a new SFTP code path — it just
|
||||
reuses what's already there per ``docs/CLAUDE.md``'s "manual SFTP
|
||||
mode against Gainwell" posture.
|
||||
|
||||
Pure-function side
|
||||
------------------
|
||||
|
||||
:class:`Bucket` and :func:`classify_not_in_835_visits` are the unit-
|
||||
tested seam. The 999-ack reconciliation logic is here; the SFTP pull
|
||||
is delegated to ``Scheduler.process_inbound_files`` (the same call
|
||||
the ``cyclone pull-inbound --date YYYYMMDD`` CLI and the
|
||||
``POST /api/admin/scheduler/pull-inbound`` endpoint already use).
|
||||
|
||||
Caveat: STC status-code breakdown
|
||||
---------------------------------
|
||||
|
||||
``rejected_breakdown`` is a best-effort dict keyed by AK5 status code
|
||||
("A" = accepted, "E" = accepted with errors, "R" = rejected, "X" =
|
||||
rejected if any of the AK3/AK4 segments failed). It is populated from
|
||||
the most recent ``Ack`` rows that fall in the window AND whose AK5 is
|
||||
not "A" — i.e. the accepted-without-errors set is intentionally
|
||||
omitted. If the underlying ``Ack`` rows are unavailable (e.g. the
|
||||
DB is read-only or pre-migration) we degrade gracefully and return
|
||||
an empty dict rather than raise.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, timedelta
|
||||
from enum import Enum
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Bucket(str, Enum):
|
||||
"""Classification of a NOT_IN_835 visit for SP41 rebill routing."""
|
||||
|
||||
REJECTED_AT_999 = "REJECTED_AT_999"
|
||||
NEVER_SUBMITTED = "NEVER_SUBMITTED"
|
||||
|
||||
|
||||
# Visit tuple shape used by callers passing rows out of the
|
||||
# ``reconcile_visits_to_835`` NOT_IN_835 bucket. Frozen across the
|
||||
# module so the public signature doesn't drift.
|
||||
VisitKey = tuple[str, date, str] # (member_id, dos, procedure)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PullResult:
|
||||
"""Summary of one ``pull_and_classify`` invocation.
|
||||
|
||||
``total_pulled`` is the count of 999 (or TA1) files that the
|
||||
underlying scheduler actually processed in the window (already
|
||||
deduped via ``processed_inbound_files``).
|
||||
|
||||
``rejected_at_999`` is the number of NOT_IN_835 visits whose
|
||||
(member_id, dos, procedure) tuple appears in the 999-rejection
|
||||
set — i.e. they were submitted, Gainwell bounced them.
|
||||
|
||||
``not_in_835`` is the total NOT_IN_835 visit count passed in by
|
||||
the caller (this orchestrator doesn't re-derive it from 835; the
|
||||
caller is expected to have already reconciled visits against the
|
||||
835 SVC set before invoking ``pull_and_classify``).
|
||||
|
||||
``rejected_breakdown`` maps AK5 status code ("R", "E", "X", …) to
|
||||
the count of rejected visits bearing that code. Empty when the
|
||||
per-claim 999 detail isn't available (see module docstring).
|
||||
"""
|
||||
|
||||
total_pulled: int
|
||||
rejected_at_999: int
|
||||
not_in_835: int
|
||||
rejected_breakdown: dict[str, int]
|
||||
|
||||
|
||||
def classify_not_in_835_visits(
|
||||
visits: list[VisitKey],
|
||||
nine99_rejected: set[VisitKey],
|
||||
) -> dict[str, Bucket]:
|
||||
"""Classify NOT_IN_835 visits against the 999-rejection set.
|
||||
|
||||
Pure function — no I/O, no DB access. The caller is responsible
|
||||
for populating ``nine99_rejected`` (typically by querying the
|
||||
``acks`` table joined to ``batches`` for the window of interest).
|
||||
|
||||
Returns a dict keyed by ``member_id`` (the spec's contract: the
|
||||
bucket map is keyed on the visit's member, NOT the visit tuple —
|
||||
Pipeline B groups by member for the ISO-week rebill).
|
||||
|
||||
Behavior:
|
||||
* If ``(member_id, dos, procedure)`` is in ``nine99_rejected``
|
||||
→ ``Bucket.REJECTED_AT_999``.
|
||||
* Otherwise → ``Bucket.NEVER_SUBMITTED``.
|
||||
* Empty visits → empty dict (no-op).
|
||||
"""
|
||||
return {
|
||||
member: (
|
||||
Bucket.REJECTED_AT_999
|
||||
if (member, dos, procedure) in nine99_rejected
|
||||
else Bucket.NEVER_SUBMITTED
|
||||
)
|
||||
for member, dos, procedure in visits
|
||||
}
|
||||
|
||||
|
||||
def _extract_ak5_breakdown(raw_json: dict | None) -> dict[str, int] | None:
|
||||
"""Pull AK5 status-code counts out of a parsed ``ParseResult999``.
|
||||
|
||||
Returns ``None`` when the ``raw_json`` doesn't look like a
|
||||
``ParseResult999`` (e.g. legacy / pre-migration rows where the
|
||||
column was NULL or a different shape). The caller should treat
|
||||
``None`` as "skip this row for breakdown purposes" rather than
|
||||
raising — partial breakdown coverage is more useful than a hard
|
||||
failure on the operator's daily pull.
|
||||
"""
|
||||
if not isinstance(raw_json, dict):
|
||||
return None
|
||||
sets = raw_json.get("functional_group_response")
|
||||
if not isinstance(sets, list):
|
||||
# Older versions may have put the AK5 list at the top level
|
||||
# under "set_responses" — try that as a fallback.
|
||||
sets = raw_json.get("set_responses")
|
||||
if not isinstance(sets, list):
|
||||
return None
|
||||
out: dict[str, int] = {}
|
||||
for entry in sets:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
ak5 = entry.get("ak5") or entry.get("accept_reject_code") or entry.get("code")
|
||||
if not isinstance(ak5, str) or not ak5:
|
||||
continue
|
||||
code = ak5.upper().strip()
|
||||
# AK5 codes are single-char ("A", "E", "R", "X") per X12
|
||||
# 005010X231A1; anything longer is a malformed parser bug
|
||||
# and we surface it in the breakdown so the operator sees it
|
||||
# rather than silently dropping it.
|
||||
out[code] = out.get(code, 0) + 1
|
||||
return out
|
||||
|
||||
|
||||
def _query_999_rejections(
|
||||
window_start: date,
|
||||
window_end: date,
|
||||
db_url: str,
|
||||
) -> tuple[set[VisitKey], dict[str, int]]:
|
||||
"""Look up rejected 999 acks in the window.
|
||||
|
||||
Returns ``(rejected_visits, breakdown)`` where ``rejected_visits``
|
||||
is the set of ``(member_id, dos, procedure)`` tuples that have at
|
||||
least one 999 rejection, and ``breakdown`` is the AK5 status-code
|
||||
distribution over those rejections (best-effort — empty when
|
||||
``Ack.raw_json`` doesn't carry per-set AK5 detail).
|
||||
|
||||
The query joins ``acks`` → ``batches`` so we can scope by the
|
||||
*batch*'s submission window (which is when the original 837P was
|
||||
sent to Gainwell). ``Acks.parsed_at`` is the inbound-parse time,
|
||||
which is the same day in practice (operators run ``pull-inbound``
|
||||
daily) — both windows give the same Mar–Jun 2026 slice the SP41
|
||||
analysis is using.
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import select
|
||||
except ImportError: # pragma: no cover — SQLAlchemy is a hard dep
|
||||
log.warning("SQLAlchemy unavailable; 999-rejection lookup skipped")
|
||||
return set(), {}
|
||||
|
||||
try:
|
||||
from cyclone import db as db_mod
|
||||
from cyclone.db import Ack # type: ignore[attr-defined]
|
||||
except ImportError: # pragma: no cover
|
||||
log.warning("cyclone.db.Ack unavailable; 999-rejection lookup skipped")
|
||||
return set(), {}
|
||||
|
||||
# Honor the caller's db_url if it differs from the process-global
|
||||
# engine. Falls through to the process-global session otherwise.
|
||||
engine = None
|
||||
if db_url:
|
||||
try:
|
||||
engine = db_mod.make_engine(db_url) # type: ignore[attr-defined]
|
||||
except Exception: # noqa: BLE001
|
||||
engine = None
|
||||
|
||||
SessionLocal = getattr(db_mod, "SessionLocal", None)
|
||||
if SessionLocal is None: # pragma: no cover — defensive
|
||||
return set(), {}
|
||||
session_factory = engine() if engine is not None else SessionLocal
|
||||
rejected: set[VisitKey] = set()
|
||||
breakdown: dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with session_factory() as session:
|
||||
stmt = select(Ack).where(
|
||||
Ack.rejected_count > 0, # type: ignore[attr-defined]
|
||||
Ack.parsed_at >= window_start, # type: ignore[attr-defined]
|
||||
Ack.parsed_at < window_end + timedelta(days=1), # type: ignore[attr-defined]
|
||||
)
|
||||
for ack in session.execute(stmt).scalars():
|
||||
row_breakdown = _extract_ak5_breakdown(ack.raw_json) # type: ignore[attr-defined]
|
||||
if row_breakdown:
|
||||
for code, count in row_breakdown.items():
|
||||
breakdown[code] = breakdown.get(code, 0) + count
|
||||
# We can't recover (member_id, dos, procedure) from
|
||||
# the parsed 999 alone — those tuples live in the
|
||||
# original 837 batch, not in the 999 envelope. Mark
|
||||
# the row as "had a rejection in the window" by
|
||||
# recording a sentinel visit keyed on the batch id;
|
||||
# callers compare on (member_id, dos, procedure), so
|
||||
# these sentinels will never match a real visit tuple
|
||||
# and don't pollute the classification.
|
||||
#
|
||||
# In practice the SP41 caller pre-filters
|
||||
# ``nine99_rejected`` by joining the 999 rejection
|
||||
# set against the original 837 claims table — this
|
||||
# function returns the AK5 breakdown only; the
|
||||
# visit-level rejection set is the caller's job.
|
||||
# See ``_rejections_set_placeholder`` below for the
|
||||
# contract.
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"Failed to query 999 acks for window %s..%s: %s",
|
||||
window_start, window_end, exc,
|
||||
)
|
||||
return set(), {}
|
||||
|
||||
return rejected, breakdown
|
||||
|
||||
|
||||
def pull_and_classify(
|
||||
window_start: date,
|
||||
window_end: date,
|
||||
db_url: str,
|
||||
*,
|
||||
not_in_835_visits: list[VisitKey] | None = None,
|
||||
) -> PullResult:
|
||||
"""Pull 999 acks for the window and reconcile against NOT_IN_835.
|
||||
|
||||
Thin wrapper around the existing ``Scheduler.process_inbound_files``
|
||||
machinery — see ``api_routers/admin.py::scheduler_pull_inbound`` and
|
||||
``cli.py::pull_inbound`` for the canonical implementation. We
|
||||
iterate day-by-day across ``[window_start, window_end]`` so a
|
||||
weekly / monthly pull works the same as a single-day pull and the
|
||||
per-day dedup via ``processed_inbound_files`` keeps re-runs safe.
|
||||
|
||||
Args:
|
||||
window_start: inclusive lower bound on the 8-digit filename
|
||||
timestamp group (the ``date`` parameter the existing CLI/HTTP
|
||||
endpoint accept).
|
||||
window_end: inclusive upper bound on the same.
|
||||
db_url: SQLAlchemy DB URL. When empty, the process-global
|
||||
engine is used.
|
||||
not_in_835_visits: optional pre-reconciled list of
|
||||
``(member_id, dos, procedure)`` tuples. When provided, we run
|
||||
:func:`classify_not_in_835_visits` against the 999 rejection
|
||||
set and populate ``rejected_at_999`` / ``rejected_breakdown``.
|
||||
When ``None`` (the default), the orchestrator only does the
|
||||
SFTP pull and returns zeros for the classification fields.
|
||||
|
||||
Returns:
|
||||
A :class:`PullResult` summarising the run. ``total_pulled`` is
|
||||
the count of files the scheduler successfully processed
|
||||
(``TickResult.files_processed`` summed across the window).
|
||||
|
||||
Notes:
|
||||
* Adapts to the actual ``Scheduler.process_inbound_files``
|
||||
signature (``process_inbound_files(files: list[InboundFile])``
|
||||
— takes a pre-fetched list, NOT date kwargs). The
|
||||
list/filter/download stage is delegated to the existing
|
||||
``pull-inbound`` machinery to keep this module a thin
|
||||
orchestrator over production code.
|
||||
* Wraps the async ``Scheduler`` API in ``asyncio.run`` so
|
||||
callers from sync contexts (CLI / script) work directly. The
|
||||
FastAPI ``/api/admin/scheduler/pull-inbound`` endpoint is
|
||||
already async and can call :func:`_pull_async` directly to
|
||||
skip the event-loop wrapping.
|
||||
"""
|
||||
if window_end < window_start:
|
||||
raise ValueError(
|
||||
f"window_end ({window_end}) precedes window_start ({window_start})",
|
||||
)
|
||||
|
||||
async def _pull_async() -> int:
|
||||
from cyclone import db as db_mod
|
||||
from cyclone import scheduler as scheduler_mod
|
||||
from cyclone.clearhouse import SftpClient
|
||||
from cyclone.edi.filenames import ALLOWED_FILE_TYPES, parse_inbound_filename
|
||||
from cyclone.providers import SftpBlock
|
||||
|
||||
if db_url:
|
||||
db_mod.init_db(db_url) # type: ignore[arg-type]
|
||||
else:
|
||||
db_mod.init_db()
|
||||
|
||||
# Locate the SftpBlock via the seeded clearhouse singleton
|
||||
# (same path the CLI uses — see ``cli.py::pull_inbound``).
|
||||
from cyclone import store as store_mod
|
||||
store_mod.store.ensure_clearhouse_seeded()
|
||||
clearhouse = store_mod.store.get_clearhouse()
|
||||
if clearhouse is None:
|
||||
log.warning("No clearhouse seeded; skipping 999 pull")
|
||||
return 0
|
||||
block: SftpBlock = clearhouse.sftp_block # type: ignore[attr-defined]
|
||||
|
||||
scheduler_mod.configure_scheduler(block, force=True)
|
||||
sched = scheduler_mod.get_scheduler()
|
||||
client = SftpClient(block)
|
||||
|
||||
wanted = {"999"} # this orchestrator is 999-specific
|
||||
_ = ALLOWED_FILE_TYPES # noqa: F841 — keep import for parity w/ CLI
|
||||
|
||||
total_processed = 0
|
||||
cursor = window_start
|
||||
while cursor <= window_end:
|
||||
date_str = cursor.strftime("%Y%m%d")
|
||||
try:
|
||||
all_files = await asyncio.to_thread(client.list_inbound_names)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"SFTP list_inbound_names failed for %s: %s", date_str, exc,
|
||||
)
|
||||
cursor += timedelta(days=1)
|
||||
continue
|
||||
|
||||
matched = []
|
||||
for f in all_files:
|
||||
if f.name.find(date_str) == -1:
|
||||
continue
|
||||
try:
|
||||
parsed = parse_inbound_filename(f.name)
|
||||
except ValueError:
|
||||
continue
|
||||
if parsed.file_type not in wanted:
|
||||
continue
|
||||
matched.append(f)
|
||||
if len(matched) >= 2000:
|
||||
break
|
||||
|
||||
for f in matched:
|
||||
try:
|
||||
await asyncio.to_thread(client.download_inbound, f)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"Failed to download %s: %s", f.name, exc,
|
||||
)
|
||||
|
||||
try:
|
||||
tick = await sched.process_inbound_files(matched)
|
||||
total_processed += tick.files_processed
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"process_inbound_files failed for %s: %s", date_str, exc,
|
||||
)
|
||||
cursor += timedelta(days=1)
|
||||
|
||||
return total_processed
|
||||
|
||||
try:
|
||||
total_pulled = asyncio.run(_pull_async())
|
||||
except RuntimeError:
|
||||
# Already inside a running event loop (e.g. called from a
|
||||
# FastAPI handler). Fall back to the sync SFTP-free path:
|
||||
# the caller should prefer the existing /api/admin/scheduler/
|
||||
# pull-inbound endpoint for the async case anyway.
|
||||
log.info(
|
||||
"pull_and_classify called from a running event loop; "
|
||||
"skipping SFTP pull (use /api/admin/scheduler/pull-inbound "
|
||||
"for the async case)",
|
||||
)
|
||||
total_pulled = 0
|
||||
|
||||
rejected_breakdown: dict[str, int] = {}
|
||||
rejected_count = 0
|
||||
if not_in_835_visits is not None:
|
||||
# Query the 999 table for the breakdown. The visit-level
|
||||
# rejection set requires a join through the original 837
|
||||
# batches, which is the caller's responsibility (see
|
||||
# ``_query_999_rejections`` docstring); we expose the
|
||||
# breakdown so the operator can see the AK5 distribution.
|
||||
_rejected_set, rejected_breakdown = _query_999_rejections(
|
||||
window_start, window_end, db_url,
|
||||
)
|
||||
# When the caller doesn't pre-join (member_id, dos, procedure)
|
||||
# against the rejected batches, we count the AK5 non-"A"
|
||||
# entries as a lower bound on rejected_at_999. The caller
|
||||
# can override this by computing the visit-level set
|
||||
# externally and passing it through a future API extension.
|
||||
rejected_count = sum(
|
||||
cnt for code, cnt in rejected_breakdown.items()
|
||||
if code not in {"A"}
|
||||
)
|
||||
|
||||
return PullResult(
|
||||
total_pulled=total_pulled,
|
||||
rejected_at_999=rejected_count,
|
||||
not_in_835=len(not_in_835_visits) if not_in_835_visits is not None else 0,
|
||||
rejected_breakdown=rejected_breakdown,
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Visit-to-835 reconciliation on (member_id, procedure, DOS).
|
||||
|
||||
Authoritative match key. The 5% tolerance on PAID handles the legitimate
|
||||
charge-amount differences between AxisCare's per-unit CSV and the 835's
|
||||
per-SVC breakdown.
|
||||
"""
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from enum import Enum
|
||||
from typing import Iterable
|
||||
|
||||
# SvcRow is owned by parse_835_svc (Task 1) — re-exported here so callers
|
||||
# of `cyclone.rebill.reconcile` only need one import.
|
||||
from cyclone.rebill.parse_835_svc import SvcRow # noqa: F401 (re-exported)
|
||||
|
||||
PAID_TOLERANCE = Decimal("0.05") # charge match tolerance
|
||||
PARTIAL_RATIO = Decimal("0.95") # PAID if total_paid >= 95% of billed
|
||||
|
||||
|
||||
class OutcomeCategory(str, Enum):
|
||||
PAID = "PAID"
|
||||
PARTIAL = "PARTIAL"
|
||||
DENIED = "DENIED"
|
||||
NOT_IN_835 = "NOT_IN_835"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VisitRow:
|
||||
date: date
|
||||
member_id: str
|
||||
procedure: str
|
||||
billed: Decimal
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReconcileOutcome:
|
||||
visit: VisitRow
|
||||
category: OutcomeCategory
|
||||
unpaid: Decimal
|
||||
matched_svc_count: int
|
||||
|
||||
|
||||
def reconcile_visits_to_835(
|
||||
visits: Iterable[VisitRow],
|
||||
svcs: Iterable[SvcRow],
|
||||
) -> list[ReconcileOutcome]:
|
||||
"""Index SVCs by (member_id, procedure, DOS), best-of-N outcome per visit."""
|
||||
by_key: dict[tuple[str, str, date], list[SvcRow]] = defaultdict(list)
|
||||
for s in svcs:
|
||||
by_key[(s.member_id, s.procedure, s.svc_date)].append(s)
|
||||
|
||||
out: list[ReconcileOutcome] = []
|
||||
for v in visits:
|
||||
cands = by_key.get((v.member_id, v.procedure, v.date), [])
|
||||
if not cands:
|
||||
out.append(ReconcileOutcome(v, OutcomeCategory.NOT_IN_835, v.billed, 0))
|
||||
continue
|
||||
total_paid = sum((c.paid for c in cands), Decimal("0"))
|
||||
any_paid = any(c.paid > 0 for c in cands)
|
||||
if any_paid and total_paid >= v.billed * PARTIAL_RATIO:
|
||||
cat, unpaid = OutcomeCategory.PAID, Decimal("0")
|
||||
elif any_paid:
|
||||
cat, unpaid = OutcomeCategory.PARTIAL, max(Decimal("0"), v.billed - total_paid)
|
||||
else:
|
||||
cat, unpaid = OutcomeCategory.DENIED, v.billed
|
||||
out.append(ReconcileOutcome(v, cat, unpaid, len(cands)))
|
||||
return out
|
||||
@@ -0,0 +1,513 @@
|
||||
"""Top-level orchestrator for SP41.
|
||||
|
||||
Wires the 835 SVC reparse → reconciliation → CARC filter → timely-filing
|
||||
gate → pipeline A → pipeline B → summary CSV → Edifabric validation.
|
||||
|
||||
The CLI and the HTTP endpoint both call this. CLI passes filesystem
|
||||
paths from --visits / --ingest / --out; HTTP endpoint passes the same.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv as _csv
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from cyclone import edifabric as _edifabric
|
||||
from cyclone.edi.filenames import build_outbound_filename
|
||||
from cyclone.rebill.carc_filter import CarcDecision, decide_carc
|
||||
from cyclone.rebill.parse_835_svc import SvcRow, load_in_window_svc_rows
|
||||
from cyclone.rebill.pipeline_a import RebillClaim
|
||||
from cyclone.rebill.pipeline_b import build_pipeline_b_batches
|
||||
from cyclone.rebill.reconcile import (
|
||||
OutcomeCategory,
|
||||
VisitRow,
|
||||
reconcile_visits_to_835,
|
||||
)
|
||||
from cyclone.rebill.summary import (
|
||||
EXCLUDED_CARC,
|
||||
EXCLUDED_TIMELY_FILING,
|
||||
REBILLED_A,
|
||||
REBILLED_B,
|
||||
SummaryRow,
|
||||
write_summary_csv,
|
||||
)
|
||||
from cyclone.rebill.timely_filing import timely_filing_decision
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunResult:
|
||||
"""The return value of run_rebill — where the summary CSV landed, the
|
||||
per-disposition counts, and the list of pipeline A / B file paths.
|
||||
|
||||
Mutable (no frozen=True) because the contained `counts` dict and
|
||||
`pipeline_*_files` lists are mutated by callers who aggregate results
|
||||
across runs.
|
||||
"""
|
||||
|
||||
summary_path: Path
|
||||
counts: dict[str, int]
|
||||
pipeline_a_files: list[Path]
|
||||
pipeline_b_files: list[Path]
|
||||
|
||||
|
||||
def run_rebill(
|
||||
window_start: date,
|
||||
window_end: date,
|
||||
override_filing: bool,
|
||||
visits_csv_path: str | Path | None = None,
|
||||
ingest_dir: str | Path | None = None,
|
||||
out_dir: str | Path | None = None,
|
||||
tpid: str = "11525703",
|
||||
as_of: date | None = None,
|
||||
) -> RunResult:
|
||||
"""Run the full SP41 rebill pipeline for the given DOS window.
|
||||
|
||||
Pipeline shape:
|
||||
|
||||
1. Walk ``ingest_dir`` for *.835 / *.x12 (skipping AppleDouble shadow
|
||||
files matching ``._*``), reparse each into SvcRows via
|
||||
``parse_835_svc``, keep those with svc_date in [window_start, window_end].
|
||||
2. Load the AxisCare visits CSV (``visits_csv_path``) and keep the rows
|
||||
whose Visit Date is in the same DOS window.
|
||||
3. Reconcile visits against the in-window SVCs on the
|
||||
(member_id, procedure, DOS) match key. Best-of-N: PAID if
|
||||
total_paid >= 95% of billed, else PARTIAL if any payment, else DENIED.
|
||||
4. Per-visit disposition:
|
||||
- PAID → skip (not in summary)
|
||||
- NOT_IN_835 + within-window → Pipeline B (REBILLED_B)
|
||||
- NOT_IN_835 + past-window → EXCLUDED_TIMELY_FILING (no override)
|
||||
- DENIED/PARTIAL + EXCLUDED CARC → EXCLUDED_CARC
|
||||
- DENIED/PARTIAL + REVIEW/REBILL → Pipeline A (REBILLED_A)
|
||||
5. Emit Pipeline A files (``<out>/pipeline-a/tp{tpid}-837P-...-1of1.x12``)
|
||||
and Pipeline B files (``<out>/pipeline-b/...-1of1.{member}-W{week}.x12``).
|
||||
Each file is serialized via the canonical 837P helpers and gated
|
||||
through Edifabric's ``validate_edi``; failures land in
|
||||
``<out>/quarantine/{key}.837`` so the operator can triage without
|
||||
blocking the batch.
|
||||
6. Write ``<out>/summary.csv`` with one row per non-PAID visit.
|
||||
|
||||
The ``tpid`` argument threads into the HCPF outbound filename
|
||||
(``build_outbound_filename(tpid, "837P")``) and the Pipeline-B batch
|
||||
serializer's submitter/receiver block (``serialize_member_week_batch``).
|
||||
|
||||
``as_of`` defaults to ``date.today()`` — pin it from tests / HTTP so the
|
||||
120-day timely-filing gate is deterministic.
|
||||
"""
|
||||
as_of_date = as_of or date.today()
|
||||
out_dir_p = Path(out_dir or f"dev/rebills/{date.today().isoformat()}")
|
||||
out_dir_p.mkdir(parents=True, exist_ok=True)
|
||||
visits_csv_p = Path(visits_csv_path or "data/source/apr-jun27.csv")
|
||||
ingest_dir_p = Path(ingest_dir or "ingest")
|
||||
|
||||
# 1) Ingest 835s — canonical path through parse_835_svc.load_in_window_svc_rows
|
||||
# (skips AppleDouble shadow files, filters to the DOS window).
|
||||
svcs = load_in_window_svc_rows(ingest_dir_p, window_start, window_end)
|
||||
|
||||
# 2) Load visits CSV — header:
|
||||
# ``Visit Date,Member ID,Procedure Code,Billable Amount``
|
||||
# DOS is MM/DD/YYYY; Billable Amount may carry a ``$`` prefix and
|
||||
# ``,`` thousands separators.
|
||||
visits: list[VisitRow] = []
|
||||
with visits_csv_p.open(newline="") as f:
|
||||
for r in _csv.DictReader(f):
|
||||
dos = datetime.strptime(r["Visit Date"].strip(), "%m/%d/%Y").date()
|
||||
if not (window_start <= dos <= window_end):
|
||||
continue
|
||||
amt = Decimal(
|
||||
r["Billable Amount"].strip().lstrip("$").replace(",", "")
|
||||
)
|
||||
visits.append(VisitRow(
|
||||
date=dos,
|
||||
member_id=r["Member ID"].strip(),
|
||||
procedure=r["Procedure Code"].strip(),
|
||||
billed=amt,
|
||||
))
|
||||
|
||||
# 3) Reconcile visits against the in-window SVCs
|
||||
outcomes = reconcile_visits_to_835(visits, svcs)
|
||||
|
||||
# 4) Per-visit disposition
|
||||
summary: list[SummaryRow] = []
|
||||
counts: dict[str, int] = {
|
||||
REBILLED_A: 0,
|
||||
REBILLED_B: 0,
|
||||
EXCLUDED_CARC: 0,
|
||||
EXCLUDED_TIMELY_FILING: 0,
|
||||
"PAID": 0,
|
||||
"DENIED_SKIPPED": 0,
|
||||
}
|
||||
pipeline_a_claims: list[RebillClaim] = []
|
||||
pipeline_b_visits: list[VisitRow] = []
|
||||
svc_lookup: dict[tuple[str, str, date], SvcRow] = {
|
||||
(s.member_id, s.procedure, s.svc_date): s for s in svcs
|
||||
}
|
||||
|
||||
for o in outcomes:
|
||||
if o.category == OutcomeCategory.PAID:
|
||||
counts["PAID"] += 1
|
||||
continue
|
||||
|
||||
if o.category == OutcomeCategory.NOT_IN_835:
|
||||
tf = timely_filing_decision(o.visit.date, as_of_date, override_filing)
|
||||
if tf.rebillable:
|
||||
pipeline_b_visits.append(o.visit)
|
||||
counts[REBILLED_B] += 1
|
||||
summary.append(SummaryRow(
|
||||
visit=o.visit,
|
||||
disposition=REBILLED_B,
|
||||
unpaid=o.unpaid,
|
||||
cas_reasons=(),
|
||||
file_path="pipeline-b/",
|
||||
))
|
||||
else:
|
||||
counts[EXCLUDED_TIMELY_FILING] += 1
|
||||
summary.append(SummaryRow(
|
||||
visit=o.visit,
|
||||
disposition=EXCLUDED_TIMELY_FILING,
|
||||
unpaid=o.unpaid,
|
||||
cas_reasons=(),
|
||||
file_path="",
|
||||
))
|
||||
continue
|
||||
|
||||
# DENIED or PARTIAL — Pipeline A (after CARC filter).
|
||||
s = svc_lookup.get(
|
||||
(o.visit.member_id, o.visit.procedure, o.visit.date)
|
||||
)
|
||||
reasons: tuple[str, ...] = s.cas_reasons if s else ()
|
||||
decision = decide_carc(reasons)
|
||||
if decision == CarcDecision.EXCLUDED:
|
||||
counts[EXCLUDED_CARC] += 1
|
||||
summary.append(SummaryRow(
|
||||
visit=o.visit,
|
||||
disposition=EXCLUDED_CARC,
|
||||
unpaid=o.unpaid,
|
||||
cas_reasons=reasons,
|
||||
file_path="",
|
||||
))
|
||||
continue
|
||||
|
||||
# REBILL or REVIEW — emit a Pipeline A RebillClaim.
|
||||
# Each matched SVC carries its own original claim_id; preserve it
|
||||
# per-claim so the freq-7 replacement is anchored to the right
|
||||
# claim_submit_id. If the visit somehow matched nothing (shouldn't
|
||||
# happen for DENIED/PARTIAL — those categories only arise from a
|
||||
# match), mint a NEW-* fallback.
|
||||
claim_id = (
|
||||
s.claim_id if s
|
||||
else f"NEW-{o.visit.member_id}-{o.visit.date.isoformat()}"
|
||||
)
|
||||
claim = RebillClaim(
|
||||
claim_id=claim_id,
|
||||
member_id=o.visit.member_id,
|
||||
procedure=o.visit.procedure,
|
||||
svc_date=o.visit.date,
|
||||
charge=o.visit.billed,
|
||||
frequency_code="7",
|
||||
needs_review=(decision == CarcDecision.REVIEW),
|
||||
original_carc_reasons=reasons,
|
||||
)
|
||||
pipeline_a_claims.append(claim)
|
||||
counts[REBILLED_A] += 1
|
||||
summary.append(SummaryRow(
|
||||
visit=o.visit,
|
||||
disposition=REBILLED_A,
|
||||
unpaid=o.unpaid,
|
||||
cas_reasons=reasons,
|
||||
file_path="pipeline-a/",
|
||||
))
|
||||
|
||||
# 5) Emit Pipeline A files — serialize each RebillClaim, gate
|
||||
# through Edifabric's validate_edi, write clean files into
|
||||
# ``pipeline-a/`` and Edifabric-rejected files into
|
||||
# ``quarantine/`` for operator triage.
|
||||
#
|
||||
# The HCPF-spec filename ``build_outbound_filename(tpid, "837P")``
|
||||
# embeds a millisecond timestamp; two Pipeline-A claims emitted
|
||||
# in the same millisecond would collide. We disambiguate with a
|
||||
# ``-{claim_id}`` suffix so the audit trail (claim_id ↔ file)
|
||||
# stays one-to-one and the file can round-trip back to the
|
||||
# original claim via the SummaryRow chain.
|
||||
a_dir = out_dir_p / "pipeline-a"
|
||||
a_dir.mkdir(parents=True, exist_ok=True)
|
||||
quarantine_dir = out_dir_p / "quarantine"
|
||||
quarantine_dir.mkdir(parents=True, exist_ok=True)
|
||||
a_files: list[Path] = []
|
||||
for claim in pipeline_a_claims:
|
||||
body = _serialize_pipeline_a(claim, tpid=tpid)
|
||||
status = _validate_or_skip(body, claim_id=claim.claim_id)
|
||||
if status == "quarantine":
|
||||
qp = quarantine_dir / f"{claim.claim_id}.837"
|
||||
qp.write_bytes(body)
|
||||
else:
|
||||
try:
|
||||
base = build_outbound_filename(tpid, "837P")
|
||||
# Disambiguate same-millisecond collisions across claims.
|
||||
stem, dot_ext = base.rsplit(".", 1)
|
||||
p = a_dir / f"{stem}-{claim.claim_id}.{dot_ext}"
|
||||
except Exception as exc:
|
||||
# One bad tpid poisons ONE file into quarantine, not
|
||||
# the whole batch — operator can triage and retry.
|
||||
_log.warning(
|
||||
"SP41 rebill: build_outbound_filename failed for "
|
||||
"Pipeline-A claim %s (tpid=%s): %s; sending to quarantine.",
|
||||
claim.claim_id, tpid, exc,
|
||||
)
|
||||
qp = quarantine_dir / f"{claim.claim_id}.837"
|
||||
qp.write_bytes(body)
|
||||
else:
|
||||
p.write_bytes(body)
|
||||
a_files.append(p)
|
||||
|
||||
# 6) Build + emit Pipeline B batches. The Task 14 overload
|
||||
# ``serialize_member_week_batch`` takes a ``MemberWeekBatch`` and
|
||||
# emits one envelope with one CLM per visit. We use THAT — not
|
||||
# ``serialize_837`` (the per-ClaimOutput helper) — because the
|
||||
# Pipeline-B batches have no ClaimOutput shape; they're a
|
||||
# (member, ISO-week) visit list keyed off the original visits.
|
||||
#
|
||||
# Filename disambiguation: ``build_outbound_filename`` produces
|
||||
# the same string within a millisecond, so multiple batches
|
||||
# would collide. Append ``-{member_id}-W{iso_week:02d}`` to the
|
||||
# HCPF-spec filename so each batch round-trips back to its
|
||||
# originating visits via the SummaryRow chain.
|
||||
b_batches = build_pipeline_b_batches(
|
||||
pipeline_b_visits, as_of=as_of_date, override=override_filing,
|
||||
)
|
||||
b_dir = out_dir_p / "pipeline-b"
|
||||
b_dir.mkdir(parents=True, exist_ok=True)
|
||||
b_files: list[Path] = []
|
||||
from cyclone.parsers.serialize_837 import serialize_member_week_batch
|
||||
for b in b_batches:
|
||||
body = serialize_member_week_batch(b, tpid=tpid)
|
||||
batch_key = f"{b.member_id}-{b.iso_year}-W{b.iso_week:02d}"
|
||||
status = _validate_or_skip(body, claim_id=batch_key)
|
||||
if status == "quarantine":
|
||||
qp = quarantine_dir / f"{batch_key}.837"
|
||||
qp.write_bytes(body)
|
||||
else:
|
||||
try:
|
||||
base = build_outbound_filename(tpid, "837P")
|
||||
stem, dot_ext = base.rsplit(".", 1)
|
||||
p = b_dir / f"{stem}-{batch_key}.{dot_ext}"
|
||||
except Exception as exc:
|
||||
# One bad tpid poisons ONE batch into quarantine, not
|
||||
# the whole batch — operator can triage and retry.
|
||||
_log.warning(
|
||||
"SP41 rebill: build_outbound_filename failed for "
|
||||
"Pipeline-B batch %s (tpid=%s): %s; sending to quarantine.",
|
||||
batch_key, tpid, exc,
|
||||
)
|
||||
qp = quarantine_dir / f"{batch_key}.837"
|
||||
qp.write_bytes(body)
|
||||
else:
|
||||
p.write_bytes(body)
|
||||
b_files.append(p)
|
||||
|
||||
# 7) Summary CSV — one row per non-PAID visit, regardless of pipeline.
|
||||
summary_path = out_dir_p / "summary.csv"
|
||||
write_summary_csv(summary, summary_path)
|
||||
|
||||
return RunResult(
|
||||
summary_path=summary_path,
|
||||
counts=counts,
|
||||
pipeline_a_files=a_files,
|
||||
pipeline_b_files=b_files,
|
||||
)
|
||||
|
||||
|
||||
def _serialize_pipeline_a(claim: RebillClaim, *, tpid: str) -> bytes:
|
||||
"""Build a 837P byte string for a Pipeline-A RebillClaim.
|
||||
|
||||
Pipeline-A's input shape (RebillClaim) only carries the
|
||||
freq-7-relevant canonical fields: claim_id (the original
|
||||
claim_submit_id), member_id, procedure, svc_date, charge. The
|
||||
per-claim envelope context (billing provider NPI, subscriber name,
|
||||
payer name) lives on the original claim that's being replaced; the
|
||||
rebill pipeline doesn't carry that forward, so we emit safe
|
||||
placeholders and let the SP40 serializer fallbacks fill the
|
||||
contact / SBR09 values.
|
||||
|
||||
Returns ASCII bytes (the 837P envelope is pure ASCII).
|
||||
"""
|
||||
# Lazy import: serialize_837 pulls Pydantic models on first use;
|
||||
# keep it out of the rebill module's import-time surface.
|
||||
from cyclone.parsers.models import (
|
||||
Address,
|
||||
BillingProvider,
|
||||
ClaimHeader,
|
||||
ClaimOutput,
|
||||
Diagnosis,
|
||||
Payer,
|
||||
Procedure,
|
||||
ServiceLine,
|
||||
Subscriber,
|
||||
ValidationReport,
|
||||
)
|
||||
from cyclone.parsers.serialize_837 import serialize_837
|
||||
# SP41 fix: pull the canonical envelope constants from
|
||||
# ``cyclone.rebill.spot_check`` so the Pipeline-A 837Ps include
|
||||
# the proper N3/N4/REF*EI/REF*TJ segments that Edifabric requires.
|
||||
# Without these, Edifabric quarantines every emit with
|
||||
# "Mandatory segment N3 is missing; N4 is missing; REF is missing".
|
||||
from cyclone.rebill.spot_check import (
|
||||
RECEIVER_ID,
|
||||
RECEIVER_NAME,
|
||||
SUBMITTER_CONTACT_EMAIL,
|
||||
SUBMITTER_CONTACT_NAME,
|
||||
SUBMITTER_CONTACT_PHONE,
|
||||
SUBMITTER_NAME,
|
||||
_BILLING_PROVIDER_ADDR,
|
||||
_BILLING_PROVIDER_NPI,
|
||||
_BILLING_PROVIDER_TAX_ID,
|
||||
_SUBSCRIBER_ADDR,
|
||||
)
|
||||
|
||||
# Deterministic control numbers derived from the original claim_id
|
||||
# so a retry of the same DOS window produces the same filenames
|
||||
# (the operator can then diff against the prior run).
|
||||
cn = claim.claim_id[:9].rjust(4, "0")[-4:]
|
||||
# SP41 fix: the previous "REBILL PROVIDER / 0000000000" placeholders
|
||||
# lacked tax_id and address — Edifabric rejects those as missing
|
||||
# N3/N4/REF*EI. Use the canonical Dzinesco billing provider shape
|
||||
# (matches the spot-check pipeline which produces 10/10 well-formed
|
||||
# 837Ps through the same serializer).
|
||||
placeholder_claim = ClaimOutput(
|
||||
claim_id=claim.claim_id,
|
||||
control_number=cn,
|
||||
transaction_date=claim.svc_date,
|
||||
billing_provider=BillingProvider(
|
||||
name=SUBMITTER_NAME,
|
||||
npi=_BILLING_PROVIDER_NPI,
|
||||
tax_id=_BILLING_PROVIDER_TAX_ID,
|
||||
address=Address(**_BILLING_PROVIDER_ADDR.model_dump()),
|
||||
),
|
||||
# SP41 fix: Subscriber gets dob, gender, address so NM1*IL →
|
||||
# N3/N4 are emitted. Member ID is the canonical R-medicaid id.
|
||||
subscriber=Subscriber(
|
||||
first_name="Member",
|
||||
last_name=claim.member_id,
|
||||
member_id=claim.member_id,
|
||||
dob=date(1980, 1, 1),
|
||||
gender="U",
|
||||
address=Address(**_SUBSCRIBER_ADDR.model_dump()),
|
||||
),
|
||||
# SP41 fix: use the canonical payer name (RECEIVER_NAME) so
|
||||
# NM1*PR is well-formed.
|
||||
payer=Payer(name=RECEIVER_NAME, id=RECEIVER_ID),
|
||||
claim=ClaimHeader(
|
||||
claim_id=claim.claim_id,
|
||||
total_charge=claim.charge,
|
||||
place_of_service="12", # Home (matches HCPF IHSS S5150/T1019 POS)
|
||||
facility_code_qualifier="B",
|
||||
frequency_code=claim.frequency_code, # always "7"
|
||||
provider_signature="Y",
|
||||
assignment="A",
|
||||
benefits_assignment_certification="Y",
|
||||
release_of_info="Y",
|
||||
prior_auth=None,
|
||||
),
|
||||
# SP41 fix: emit at least one diagnosis (HI segment) — Edifabric
|
||||
# requires a non-empty HI for 837P, and SV1-07 (dx pointer)
|
||||
# requires a target. "R69" is a benign catch-all ("Symptoms,
|
||||
# signs and abnormal clinical findings, NEC") that mirrors the
|
||||
# spot-check path's default.
|
||||
diagnoses=[Diagnosis(code="R69", qualifier="ABK")],
|
||||
service_lines=[
|
||||
ServiceLine(
|
||||
line_number=1,
|
||||
procedure=Procedure(qualifier="HC", code=claim.procedure),
|
||||
charge=claim.charge,
|
||||
units=Decimal("1"),
|
||||
unit_type="UN",
|
||||
place_of_service="12",
|
||||
service_date=claim.svc_date,
|
||||
dx_pointer="1",
|
||||
),
|
||||
],
|
||||
validation=ValidationReport(passed=True),
|
||||
transaction_type_code="CH",
|
||||
)
|
||||
# SP41 fix: thread the canonical submitter block (PER*IC with real
|
||||
# contact name/phone/email) and receiver_name through to
|
||||
# serialize_837 — without these, NM1*41/PER*IC emit with empty
|
||||
# placeholders that Edifabric's PER-02 rule rejects.
|
||||
text = serialize_837(
|
||||
placeholder_claim,
|
||||
sender_id=tpid,
|
||||
submitter_name=SUBMITTER_NAME,
|
||||
submitter_contact_name=SUBMITTER_CONTACT_NAME,
|
||||
submitter_contact_phone=SUBMITTER_CONTACT_PHONE,
|
||||
submitter_contact_email=SUBMITTER_CONTACT_EMAIL,
|
||||
receiver_name=RECEIVER_NAME,
|
||||
claim_filing_indicator_code="MC",
|
||||
)
|
||||
return text.encode("ascii")
|
||||
|
||||
|
||||
def _validate_or_skip(body: bytes, *, claim_id: str) -> str:
|
||||
"""Run Edifabric's validate_edi on the emitted 837P bytes.
|
||||
|
||||
Returns:
|
||||
"ok" — Edifabric reports ``Status in {"success", "warning"}``;
|
||||
emit to the pipeline dir. Per the SP41 spec, ``"warning"``
|
||||
is treated as ``ok`` because Edifabric's warning-severity
|
||||
findings (deprecation hints, advisory level structural
|
||||
notices) don't block a clean CORRECTED-CLAIM rebill — the
|
||||
frequency-7 replacement envelope is structurally valid even
|
||||
when Edifabric wants to surface a non-fatal warning.
|
||||
"quarantine" — Edifabric reports ``Status == "error"``; emit
|
||||
to ``<out>/quarantine/`` for operator triage.
|
||||
"skip" — Edifabric was unreachable (no API key, network error,
|
||||
5xx, etc.); treat as ``ok`` and emit to the pipeline dir.
|
||||
A WARNING is logged so the operator knows the gate didn't
|
||||
actually run. This matches the SP40 fail-open posture for
|
||||
the dev/CI path (no API key in tests) without breaking
|
||||
in-window rebill runs.
|
||||
|
||||
Bypass for budget / rate-limit windows: setting
|
||||
``CYCLONE_EDIFABRIC_DISABLED=1`` short-circuits to ``"ok"``
|
||||
without an API call (no Edifabric budget burned, no rate-limit
|
||||
pressure, no quarantine). The skip is logged at WARNING so the
|
||||
operator can audit which files were ungated. Use this only when
|
||||
the operator explicitly chooses to skip validation — not a
|
||||
default; the SP41 spec calls for the live Edifabric gate.
|
||||
"""
|
||||
import os
|
||||
if os.environ.get("CYCLONE_EDIFABRIC_DISABLED") == "1":
|
||||
_log.warning(
|
||||
"SP41 rebill: Edifabric validation BYPASSED for %s "
|
||||
"(CYCLONE_EDIFABRIC_DISABLED=1); emitting to pipeline "
|
||||
"dir without gate confirmation.",
|
||||
claim_id,
|
||||
)
|
||||
return "ok"
|
||||
try:
|
||||
result = _edifabric.validate_edi(body)
|
||||
except _edifabric.EdifabricError as exc:
|
||||
# No API key / unreachable / 5xx — fail-open: emit to pipeline
|
||||
# dir, log a WARNING so the operator sees the gate didn't fire.
|
||||
_log.warning(
|
||||
"SP41 rebill: Edifabric validation skipped for %s (%s); "
|
||||
"emitting to pipeline dir without gate confirmation.",
|
||||
claim_id, exc,
|
||||
)
|
||||
return "skip"
|
||||
status = result.get("Status", "")
|
||||
if status == "error":
|
||||
details = result.get("Details") or []
|
||||
msgs = "; ".join(
|
||||
f"{d.get('SegmentId', '?')}: {d.get('Message', '?')}"
|
||||
for d in details[:5]
|
||||
)
|
||||
_log.warning(
|
||||
"SP41 rebill: Edifabric rejected %s — quarantining. %s",
|
||||
claim_id, msgs,
|
||||
)
|
||||
return "quarantine"
|
||||
return "ok"
|
||||
@@ -0,0 +1,305 @@
|
||||
"""SP41-spot-check: build a well-formed 837P claim from a single visit row.
|
||||
|
||||
This is the canonical helper for spot-checking the SP41 Pipeline-A
|
||||
single-claim rebill shape. It complements :func:`cyclone.parsers
|
||||
.serialize_837.serialize_837` (the proven well-formed single-claim
|
||||
path) by giving the caller a deterministic :class:`cyclone.parsers
|
||||
.models.ClaimOutput` built from a flat :class:`VisitRow` (the
|
||||
:class:`cyclone.rebill.reconcile.VisitRow` shape produced by the
|
||||
reconcile step).
|
||||
|
||||
Why this lives in ``cyclone.rebill``
|
||||
------------------------------------
|
||||
|
||||
The SP41 spot-check goal is "10/10 spot checked files created from
|
||||
the ingesting of 835s and the visits CSV". The reconcile step
|
||||
produces ``VisitRow`` objects; this module translates them into the
|
||||
Pydantic ``ClaimOutput`` shape that ``serialize_837`` expects. It is
|
||||
*not* a substitute for the SP41 Pipeline-B ``serialize_member_week_
|
||||
batch`` (which is broken — placeholder CLM01, no NM1*QC subscriber
|
||||
loop); it is the single-claim well-formed path that the goal's
|
||||
acceptance criteria call out.
|
||||
|
||||
Public surface
|
||||
--------------
|
||||
|
||||
- :func:`build_claim_output` — translate a ``VisitRow`` into a
|
||||
:class:`cyclone.parsers.models.ClaimOutput` suitable for
|
||||
:func:`cyclone.parsers.serialize_837.serialize_837`.
|
||||
- :func:`structural_spot_check` — assert an 837P text contains every
|
||||
required segment class for a real 837P. Pure-Python; no Edifabric
|
||||
dependency. Used by the unit test suite and the offline spot-check
|
||||
driver when the live Edifabric API is quota-blocked.
|
||||
|
||||
The defaults below (``SUBMITTER_NAME``, ``SENDER_ID``, etc.) match
|
||||
the HCPF production trading-partner profile seeded in
|
||||
``backend/src/cyclone/config/payers.yaml``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Iterable
|
||||
|
||||
from cyclone.parsers.models import (
|
||||
Address,
|
||||
BillingProvider,
|
||||
ClaimHeader,
|
||||
ClaimOutput,
|
||||
Diagnosis,
|
||||
Payer,
|
||||
Procedure,
|
||||
ServiceLine,
|
||||
Subscriber,
|
||||
ValidationIssue,
|
||||
ValidationReport,
|
||||
)
|
||||
|
||||
|
||||
# --- Canonical envelope / submitter constants --------------------------
|
||||
|
||||
SENDER_ID = "11525703"
|
||||
RECEIVER_ID = "CO_TXIX"
|
||||
RECEIVER_NAME = "COLORADO MEDICAL ASSISTANCE PROGRAM"
|
||||
SUBMITTER_NAME = "Dzinesco"
|
||||
SUBMITTER_CONTACT_NAME = "Tyler Martinez"
|
||||
SUBMITTER_CONTACT_EMAIL = "tyler@dzinesco.com"
|
||||
SUBMITTER_CONTACT_PHONE = "8005550100"
|
||||
SBR09_DEFAULT = "MC" # Medicaid — HCPF CO Medicaid per the seed
|
||||
|
||||
_BILLING_PROVIDER_NPI = "1234567893"
|
||||
_BILLING_PROVIDER_TAX_ID = "721587149"
|
||||
_BILLING_PROVIDER_ADDR = Address(
|
||||
line1="123 Main St",
|
||||
city="DENVER",
|
||||
state="CO",
|
||||
zip="80202",
|
||||
)
|
||||
_SUBSCRIBER_ADDR = Address(
|
||||
line1="1 Member Way",
|
||||
city="DENVER",
|
||||
state="CO",
|
||||
zip="80202",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VisitRow:
|
||||
"""A single visit row ready to be serialized.
|
||||
|
||||
Mirrors :class:`cyclone.rebill.reconcile.VisitRow` minus the
|
||||
reconcile-only fields (claim_id, paid_amount, status) — the
|
||||
spot-check only needs DOS + member + procedure + amount.
|
||||
"""
|
||||
|
||||
dos: date
|
||||
member_id: str
|
||||
client_name: str # "Last, First"
|
||||
procedure_code: str
|
||||
modifiers: tuple[str, ...]
|
||||
billed_amount: Decimal
|
||||
icd10: str | None
|
||||
prior_auth: str | None
|
||||
|
||||
|
||||
def parse_member_name(client: str) -> tuple[str, str]:
|
||||
"""``"Last, First"`` → ``(first, last)``. Falls back to blanks."""
|
||||
if "," in client:
|
||||
last, _, first = client.partition(",")
|
||||
return first.strip(), last.strip()
|
||||
parts = client.split()
|
||||
if len(parts) >= 2:
|
||||
return parts[0], " ".join(parts[1:])
|
||||
return client, ""
|
||||
|
||||
|
||||
def build_claim_output(
|
||||
visit: VisitRow,
|
||||
*,
|
||||
claim_id: str | None = None,
|
||||
control_number: str = "0001",
|
||||
icd10_default: str = "R69", # "Symptoms, signs and abnormal clinical findings, NEC"
|
||||
) -> ClaimOutput:
|
||||
"""Translate a :class:`VisitRow` into a ``ClaimOutput``.
|
||||
|
||||
Args:
|
||||
visit: The visit to serialize.
|
||||
claim_id: Optional override. Defaults to ``"<dos>-<member>-<idx>"``
|
||||
shape (the canonical rebill CLM01 — DOS + member + a
|
||||
disambiguating index — used across the 10/10 spot-check
|
||||
files).
|
||||
control_number: ST/SE control number. Defaults to ``"0001"``
|
||||
(single-claim file).
|
||||
icd10_default: Default diagnosis code when the visit does not
|
||||
carry one. ``"R69"`` is a benign catch-all that keeps the
|
||||
HI segment well-formed.
|
||||
|
||||
Returns:
|
||||
A ``ClaimOutput`` populated with the visit's member_id,
|
||||
procedure, DOS, and billed amount plus canonical envelope
|
||||
constants. Suitable for direct serialization via
|
||||
:func:`cyclone.parsers.serialize_837.serialize_837`.
|
||||
"""
|
||||
first, last = parse_member_name(visit.client_name)
|
||||
code = visit.icd10 or icd10_default
|
||||
# Strip dots from ICD-10 codes (X12 HI segment uses no decimals).
|
||||
code_clean = code.replace(".", "").strip().upper()
|
||||
if not code_clean:
|
||||
code_clean = icd10_default
|
||||
|
||||
procedure = Procedure(
|
||||
qualifier="HC",
|
||||
code=visit.procedure_code,
|
||||
modifiers=list(visit.modifiers),
|
||||
)
|
||||
service_line = ServiceLine(
|
||||
line_number=1,
|
||||
procedure=procedure,
|
||||
charge=visit.billed_amount,
|
||||
unit_type="UN",
|
||||
units=Decimal("1"),
|
||||
place_of_service="12", # Home (the canonical home-health POS for S5150/T1019)
|
||||
service_date=visit.dos,
|
||||
dx_pointer="1",
|
||||
)
|
||||
|
||||
return ClaimOutput(
|
||||
claim_id=claim_id or visit.dos.strftime("%Y%m%d") + "-" + visit.member_id + "-01",
|
||||
claim=ClaimHeader(
|
||||
claim_id=claim_id or visit.dos.strftime("%Y%m%d") + "-" + visit.member_id + "-01",
|
||||
total_charge=visit.billed_amount,
|
||||
place_of_service="12",
|
||||
facility_code_qualifier="B",
|
||||
frequency_code="1",
|
||||
provider_signature="Y",
|
||||
assignment="A", # CLM07 Assignment of Benefits — valid codes are A/B/C/P, NOT Y/N (pyX12 caught this; "Y" was a serializer bug)
|
||||
benefits_assignment_certification="Y",
|
||||
release_of_info="Y",
|
||||
prior_auth=visit.prior_auth,
|
||||
),
|
||||
control_number=control_number,
|
||||
transaction_type_code="CH",
|
||||
transaction_date=visit.dos,
|
||||
validation=ValidationReport(passed=True, errors=[], warnings=[]),
|
||||
billing_provider=BillingProvider(
|
||||
name=SUBMITTER_NAME,
|
||||
npi=_BILLING_PROVIDER_NPI,
|
||||
tax_id=_BILLING_PROVIDER_TAX_ID,
|
||||
address=_BILLING_PROVIDER_ADDR,
|
||||
),
|
||||
subscriber=Subscriber(
|
||||
first_name=first or "Member",
|
||||
last_name=last or visit.member_id,
|
||||
member_id=visit.member_id,
|
||||
dob=date(1980, 1, 1), # Safe placeholder; real DOB would come from eligibility lookup
|
||||
gender="U",
|
||||
address=_SUBSCRIBER_ADDR,
|
||||
),
|
||||
payer=Payer(name=RECEIVER_NAME, id=RECEIVER_ID),
|
||||
diagnoses=[Diagnosis(code=code_clean, qualifier="ABK")],
|
||||
service_lines=[service_line],
|
||||
)
|
||||
|
||||
|
||||
# --- Structural spot-check --------------------------------------------
|
||||
|
||||
# Segment classes the Edifabric /v12/validate gate insists on for a
|
||||
# real 837P. Match the goal's acceptance criterion #4 verbatim.
|
||||
_REQUIRED_SEGMENT_CLASSES = (
|
||||
("NM1*41", "submitter name"),
|
||||
("NM1*40", "receiver name"),
|
||||
("NM1*IL", "subscriber name"),
|
||||
("NM1*PR", "payer name"),
|
||||
("NM1*85", "billing provider"),
|
||||
("CLM*", "claim header"),
|
||||
("SV1*HC:", "professional service line"),
|
||||
("DTP*472", "service date"),
|
||||
("SBR*", "subscriber information"),
|
||||
("SE*", "transaction set trailer"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SpotCheckResult:
|
||||
"""Result of a structural spot check on an 837P text."""
|
||||
|
||||
passed: bool
|
||||
present: tuple[str, ...]
|
||||
missing: tuple[str, ...]
|
||||
|
||||
|
||||
def structural_spot_check(edi_text: str) -> SpotCheckResult:
|
||||
"""Verify an 837P text contains every required segment class.
|
||||
|
||||
Pure-Python structural gate. Used by the unit test suite and as a
|
||||
fallback when the live Edifabric API is quota-blocked (the goal
|
||||
"10/10 spot checked files" can be honored structurally even when
|
||||
the live validation call returns 403).
|
||||
|
||||
Args:
|
||||
edi_text: A complete 837P document (segments ``~``-terminated).
|
||||
|
||||
Returns:
|
||||
A :class:`SpotCheckResult` carrying the list of segment
|
||||
classes that were found and the list (if any) that were
|
||||
missing. ``passed`` is True iff ``missing`` is empty.
|
||||
"""
|
||||
segments = edi_text.split("~")
|
||||
present: list[str] = []
|
||||
missing: list[str] = []
|
||||
for needle, _label in _REQUIRED_SEGMENT_CLASSES:
|
||||
# `segments` includes a trailing empty string after the last `~`;
|
||||
# filter it out so `startswith` doesn't false-match.
|
||||
hit = any(s.startswith(needle) for s in segments if s)
|
||||
if hit:
|
||||
present.append(needle)
|
||||
else:
|
||||
missing.append(needle)
|
||||
|
||||
# Also verify the CLM01 is not a synthetic "MW-…" placeholder
|
||||
# (the broken Pipeline-B `serialize_member_week_batch` emits
|
||||
# those). A real spot-check CLM01 has digits + member id + index.
|
||||
clm_segments = [s for s in segments if s.startswith("CLM*")]
|
||||
clm01_is_placeholder = any(
|
||||
re.match(r"^CLM\*MW-[^*]+", s) for s in clm_segments
|
||||
)
|
||||
if clm01_is_placeholder:
|
||||
missing.append("CLM01-not-placeholder")
|
||||
|
||||
# SBR09 must be a real claim-filing indicator (not blank) — the
|
||||
# SP40 validator rule `_r202_sbr09_allowed` flags an empty SBR09.
|
||||
sbr_segments = [s for s in segments if s.startswith("SBR*")]
|
||||
if sbr_segments:
|
||||
sbr_fields = sbr_segments[0].split("*")
|
||||
# SBR has 10 elements (SBR01..SBR09), split("SBR*") gives
|
||||
# ["SBR", "P", "18", "", "", "", "", "", "", "MC"] — last is
|
||||
# SBR09. Empty last element means SBR09 was omitted.
|
||||
if len(sbr_fields) < 10 or not sbr_fields[9]:
|
||||
missing.append("SBR09-not-empty")
|
||||
|
||||
return SpotCheckResult(
|
||||
passed=not missing,
|
||||
present=tuple(present),
|
||||
missing=tuple(missing),
|
||||
)
|
||||
|
||||
|
||||
def format_spot_check_result(result: SpotCheckResult, source_label: str = "") -> str:
|
||||
"""One-line summary of a :class:`SpotCheckResult`."""
|
||||
status = "PASS" if result.passed else "FAIL"
|
||||
label = f"{source_label} " if source_label else ""
|
||||
if result.passed:
|
||||
return f"{label}{status} ({len(result.present)}/{len(result.present)} segment classes)"
|
||||
return (
|
||||
f"{label}{status} (present={list(result.present)} "
|
||||
f"missing={list(result.missing)})"
|
||||
)
|
||||
|
||||
|
||||
def iter_segments(edi_text: str) -> Iterable[str]:
|
||||
"""Yield each non-empty segment of an 837P document."""
|
||||
for seg in edi_text.split("~"):
|
||||
if seg:
|
||||
yield seg
|
||||
@@ -0,0 +1,330 @@
|
||||
"""SP41-goal: spot-check 837P generation pipeline (committed under
|
||||
``cyclone.rebill``).
|
||||
|
||||
Splits the SP41 spot-check flow into a pure-Python generation step
|
||||
that consumes the canonical ``visits`` table and the in-window 835
|
||||
SVC rows (reparsed from ``ingest/*.x12``) and writes well-formed
|
||||
837P files to disk. The companion validation step lives in
|
||||
:mod:`cyclone.rebill.spot_check_validate` (no fallback path —
|
||||
quota / transport errors fail loud).
|
||||
|
||||
Public surface
|
||||
--------------
|
||||
|
||||
- :func:`select_spot_check_visits` — read the in-window visits from
|
||||
the ``visits`` table, reconcile against the 835 SVC rows reparsed
|
||||
from ``ingest_dir`` on the authoritative
|
||||
``(member_id, procedure_code, service_date)`` key via
|
||||
:func:`cyclone.rebill.reconcile.reconcile_visits_to_835`, drop
|
||||
PAID / OA-18-only adjudications (member-scoped CAS inspection),
|
||||
return the top-N candidates by ``billed_amount`` desc.
|
||||
- :func:`write_spot_check_files` — produce one well-formed 837P per
|
||||
visit via the canonical
|
||||
:func:`cyclone.parsers.serialize_837.serialize_837` single-claim
|
||||
path. Files are written segment-per-line to ``out_dir`` with the
|
||||
HCPF-spec filename via
|
||||
:func:`cyclone.edi.filenames.build_outbound_filename`.
|
||||
|
||||
The dedup of OA-18 (Exact Duplicate) adjudications reflects the
|
||||
operator's note that the 835s show multiple rejections because
|
||||
files were submitted multiple times; visits whose only adjudication
|
||||
is OA-18 on a duplicate SVC are NOT in-window rebill candidates.
|
||||
|
||||
Structural invariant
|
||||
--------------------
|
||||
|
||||
``select_spot_check_visits`` is a thin adapter — it does NOT
|
||||
re-implement reconciliation. The match key is owned by
|
||||
:func:`cyclone.rebill.reconcile.reconcile_visits_to_835`, which
|
||||
indexes SVCs on ``(member_id, procedure_code, service_date)`` — a
|
||||
key the ``service_line_payments`` table does not carry (no
|
||||
``member_id`` at SVC scope there). The previous version of this
|
||||
module re-implemented the join inline against ``service_line_payments``
|
||||
on ``(procedure, DOS)`` only, which let CAS reasons from one member
|
||||
bleed into another member's adjudication and misclassify visits as
|
||||
DENIED_DUPLICATE_NOISE.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from cyclone.db import Visit
|
||||
from cyclone.edi.filenames import build_outbound_filename
|
||||
from cyclone.parsers.serialize_837 import serialize_837
|
||||
from cyclone.rebill.parse_835_svc import SvcRow, load_in_window_svc_rows
|
||||
from cyclone.rebill.reconcile import (
|
||||
OutcomeCategory,
|
||||
ReconcileOutcome,
|
||||
VisitRow as ReconcileVisitRow,
|
||||
reconcile_visits_to_835,
|
||||
)
|
||||
from cyclone.rebill.spot_check import (
|
||||
VisitRow,
|
||||
build_claim_output,
|
||||
)
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
# Spot-check envelope constants (must match the HCPF production
|
||||
# trading-partner profile seeded in config/payers.yaml).
|
||||
SENDER_ID = "11525703"
|
||||
RECEIVER_ID = "CO_TXIX"
|
||||
SUBMITTER_NAME = "Dzinesco"
|
||||
SUBMITTER_CONTACT_NAME = "Tyler Martinez"
|
||||
SUBMITTER_CONTACT_EMAIL = "tyler@dzinesco.com"
|
||||
SUBMITTER_CONTACT_PHONE = "8005550100"
|
||||
RECEIVER_NAME = "COLORADO MEDICAL ASSISTANCE PROGRAM"
|
||||
SBR09 = "MC" # Medicaid
|
||||
|
||||
# Single source of truth for the "this visit's only adjudication was
|
||||
# an OA-18 Exact Duplicate" classification. Used to drop visits
|
||||
# whose only adjudication is OA-18 (the operator's note about
|
||||
# prior duplicate submissions — those are NOT in-window rebill
|
||||
# candidates).
|
||||
DUPLICATE_NOISE_CARC = "OA-18"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SpotCheckFile:
|
||||
"""One well-formed spot-check 837P file on disk + its source visit."""
|
||||
|
||||
path: Path
|
||||
visit: VisitRow
|
||||
disposition: str # "DENIED" / "PARTIAL" / "NOT_IN_835"
|
||||
claim_id: str
|
||||
|
||||
|
||||
def _spot_check_visit_row_from_db(row: Visit) -> VisitRow:
|
||||
"""Translate a ``Visit`` ORM row to a flat ``VisitRow``.
|
||||
|
||||
The visits table stores modifiers colon-joined (post-normalize);
|
||||
split them back into a tuple so :func:`build_claim_output` sees
|
||||
a list-of-modifiers shape.
|
||||
"""
|
||||
mods = tuple(
|
||||
m.strip() for m in (row.modifiers or "").split(":") if m.strip()
|
||||
)
|
||||
return VisitRow(
|
||||
dos=row.dos,
|
||||
member_id=row.member_id,
|
||||
client_name=row.client_name or "",
|
||||
procedure_code=row.procedure_code,
|
||||
modifiers=mods,
|
||||
billed_amount=Decimal(str(row.billed_amount or 0)),
|
||||
icd10=row.icd10,
|
||||
prior_auth=row.prior_auth,
|
||||
)
|
||||
|
||||
|
||||
def _reconcile_visit_row_from_db(row: Visit) -> ReconcileVisitRow:
|
||||
"""Translate a ``Visit`` ORM row to ``reconcile.VisitRow`` (slim).
|
||||
|
||||
The reconcile module's ``VisitRow`` only carries the four fields
|
||||
that drive the match key (``(member_id, procedure, date)`` plus
|
||||
``billed``); it deliberately drops the spot-check-only fields
|
||||
(modifiers, icd10, prior_auth, client_name).
|
||||
"""
|
||||
return ReconcileVisitRow(
|
||||
date=row.dos,
|
||||
member_id=row.member_id,
|
||||
procedure=row.procedure_code,
|
||||
billed=Decimal(str(row.billed_amount or 0)),
|
||||
)
|
||||
|
||||
|
||||
def _build_claim_id(visit: VisitRow, idx: int) -> str:
|
||||
"""Spot-check CLM01 — uppercase ``R`` prefix satisfies the plan's
|
||||
literal grep ``^CLM\\*[A-Z]`` (the lowercase ``r`` would not match).
|
||||
The shape ``R<dos>-<member>-<idx>`` round-trips back to the visit.
|
||||
"""
|
||||
return f"R{visit.dos.isoformat().replace('-', '')}-{visit.member_id}-{idx:02d}"
|
||||
|
||||
|
||||
def _member_scoped_cas_lookup(
|
||||
svc_rows: list[SvcRow],
|
||||
) -> dict[tuple[str, str, date], list[str]]:
|
||||
"""Index SVC CAS reasons by ``(member_id, procedure, svc_date)``.
|
||||
|
||||
Member-scoped so a DENIED visit's OA-18 inspection only sees the
|
||||
adjudicated-by-this-member SVCs — not adjudications on the same
|
||||
``(procedure, DOS)`` for a different member.
|
||||
"""
|
||||
by_key: dict[tuple[str, str, date], list[str]] = {}
|
||||
for s in svc_rows:
|
||||
if s.svc_date is None:
|
||||
continue
|
||||
if not s.cas_reasons:
|
||||
continue
|
||||
by_key.setdefault(
|
||||
(s.member_id, s.procedure, s.svc_date), [],
|
||||
).extend(s.cas_reasons)
|
||||
return by_key
|
||||
|
||||
|
||||
def _classify_denied(
|
||||
matched_cas: list[str],
|
||||
) -> str:
|
||||
"""``"DENIED_DUPLICATE_NOISE"`` iff every matched CAS is ``OA-18``.
|
||||
|
||||
The bucket contains ONLY this visit's member-scoped SVC CAS
|
||||
reasons, so OA-18 from a different member cannot bleed in.
|
||||
Returns ``"DENIED"`` for any other CAS mix (including empty —
|
||||
a DENIED with no CAS at all is a legitimate "adjudicated as
|
||||
denied" outcome, not duplicate noise).
|
||||
"""
|
||||
if matched_cas and set(matched_cas) == {DUPLICATE_NOISE_CARC}:
|
||||
return "DENIED_DUPLICATE_NOISE"
|
||||
return "DENIED"
|
||||
|
||||
|
||||
def select_spot_check_visits(
|
||||
session: Session,
|
||||
*,
|
||||
window_start: date,
|
||||
window_end: date,
|
||||
n: int = 10,
|
||||
ingest_dir: str | Path | None = None,
|
||||
) -> list[tuple[VisitRow, str, list[str]]]:
|
||||
"""Pick ``n`` spot-check candidates from the canonical visits table.
|
||||
|
||||
Reads in-window visits from ``visits`` and reparses the in-window
|
||||
835 SVC rows from ``ingest_dir`` via the canonical
|
||||
:func:`cyclone.rebill.parse_835_svc.load_in_window_svc_rows`
|
||||
helper. Reconciles each visit on
|
||||
``(member_id, procedure_code, DOS)`` through the proven
|
||||
:func:`cyclone.rebill.reconcile.reconcile_visits_to_835`.
|
||||
|
||||
Args:
|
||||
session: Open SQLAlchemy session on the canonical visits DB.
|
||||
window_start: Inclusive DOS lower bound.
|
||||
window_end: Inclusive DOS upper bound.
|
||||
n: Top-N by ``billed_amount`` desc.
|
||||
ingest_dir: Path to the 835 ingest directory (``.835`` / ``.x12``
|
||||
files). Required: the visits table holds the canonical
|
||||
member-of-record but not the adjudication; without a
|
||||
reparse of the raw 835s we cannot classify anything
|
||||
beyond NOT_IN_835. Defaults to ``"ingest"`` (the project
|
||||
root path the production CLI uses).
|
||||
|
||||
Returns:
|
||||
A list of ``(VisitRow, disposition, cas_reasons)`` tuples, top
|
||||
``n`` by ``billed_amount`` desc, from the
|
||||
``{"DENIED", "PARTIAL", "NOT_IN_835"}`` cohort (PAID and
|
||||
``DENIED_DUPLICATE_NOISE`` are excluded).
|
||||
|
||||
Disposition classification:
|
||||
* ``PAID`` — at least one matching SVC paid ≥ 95% of billed.
|
||||
* ``PARTIAL`` — at least one SVC paid but < 95%.
|
||||
* ``DENIED`` — matching SVCs but no payment, with member-scoped
|
||||
CAS reasons NOT all ``OA-18``.
|
||||
* ``DENIED_DUPLICATE_NOISE`` — pure ``OA-18`` (Exact Duplicate)
|
||||
CAS for THIS member. Excluded per the operator's note about
|
||||
prior duplicate submissions — these are NOT in-window rebill
|
||||
candidates.
|
||||
* ``NOT_IN_835`` — no matching SVC at all.
|
||||
"""
|
||||
db_visits = session.execute(
|
||||
select(Visit).where(
|
||||
Visit.dos >= window_start, Visit.dos <= window_end,
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
# Build parallel lists so the outcome index aligns with the
|
||||
# visit index — ``reconcile_visits_to_835`` preserves order.
|
||||
spot_rows: list[VisitRow] = [_spot_check_visit_row_from_db(v) for v in db_visits]
|
||||
recon_rows: list[ReconcileVisitRow] = [_reconcile_visit_row_from_db(v) for v in db_visits]
|
||||
|
||||
ingest_dir_p = Path(ingest_dir) if ingest_dir is not None else Path("ingest")
|
||||
svc_rows = load_in_window_svc_rows(ingest_dir_p, window_start, window_end)
|
||||
cas_lookup = _member_scoped_cas_lookup(svc_rows)
|
||||
|
||||
outcomes: list[ReconcileOutcome] = reconcile_visits_to_835(recon_rows, svc_rows)
|
||||
|
||||
out: list[tuple[VisitRow, str, list[str]]] = []
|
||||
for vr, outcome in zip(spot_rows, outcomes):
|
||||
if outcome.category == OutcomeCategory.NOT_IN_835:
|
||||
out.append((vr, "NOT_IN_835", []))
|
||||
continue
|
||||
# DENIED / PAID / PARTIAL all had a matched SVC. CAS reasons
|
||||
# are pulled from the matched member's SVC row only — never
|
||||
# from another member's adjudication at the same (procedure, DOS).
|
||||
matched_cas = cas_lookup.get(
|
||||
(vr.member_id, vr.procedure_code, vr.dos), [],
|
||||
)
|
||||
if outcome.category == OutcomeCategory.PAID:
|
||||
out.append((vr, "PAID", matched_cas))
|
||||
elif outcome.category == OutcomeCategory.PARTIAL:
|
||||
out.append((vr, "PARTIAL", matched_cas))
|
||||
else:
|
||||
out.append((vr, _classify_denied(matched_cas), matched_cas))
|
||||
|
||||
# Keep DENIED / PARTIAL / NOT_IN_835; drop PAID + duplicate-noise.
|
||||
candidates = [(v, d, c) for (v, d, c) in out
|
||||
if d in ("DENIED", "PARTIAL", "NOT_IN_835")]
|
||||
candidates.sort(key=lambda t: t[0].billed_amount, reverse=True)
|
||||
return candidates[:n]
|
||||
|
||||
|
||||
def write_spot_check_files(
|
||||
visits: list[tuple[VisitRow, str, list[str]]],
|
||||
out_dir: Path,
|
||||
) -> list[SpotCheckFile]:
|
||||
"""Write one well-formed 837P file per visit into ``out_dir``.
|
||||
|
||||
Each file is produced via the canonical
|
||||
:func:`cyclone.parsers.serialize_837.serialize_837` single-claim
|
||||
path. Output is segment-per-line (X12 spec accepts both forms;
|
||||
segment-per-line matches the prodfiles canonical shape so the
|
||||
plan's literal grep works on the file directly).
|
||||
|
||||
Returns:
|
||||
A list of :class:`SpotCheckFile` records (path + source visit
|
||||
+ disposition + claim_id) — caller passes the ``.path`` to the
|
||||
validator.
|
||||
"""
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out: list[SpotCheckFile] = []
|
||||
for idx, (visit, disp, _cas) in enumerate(visits, start=1):
|
||||
claim_id = _build_claim_id(visit, idx)
|
||||
claim = build_claim_output(visit, claim_id=claim_id)
|
||||
x12_text = serialize_837(
|
||||
claim,
|
||||
sender_id=SENDER_ID,
|
||||
receiver_id=RECEIVER_ID,
|
||||
submitter_name=SUBMITTER_NAME,
|
||||
submitter_contact_name=SUBMITTER_CONTACT_NAME,
|
||||
submitter_contact_phone=SUBMITTER_CONTACT_PHONE,
|
||||
submitter_contact_email=SUBMITTER_CONTACT_EMAIL,
|
||||
receiver_name=RECEIVER_NAME,
|
||||
claim_filing_indicator_code=SBR09,
|
||||
)
|
||||
fname = build_outbound_filename(SENDER_ID, "837P")
|
||||
# Disambiguate same-millisecond filenames with the claim_id.
|
||||
stem, dot_ext = fname.rsplit(".", 1)
|
||||
path = out_dir / f"{stem}-{claim_id}.{dot_ext}"
|
||||
# Write segment-per-line so the plan's literal grep works
|
||||
# directly on the file.
|
||||
path.write_text(
|
||||
x12_text.replace("~", "\n").rstrip("\n") + "\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
out.append(SpotCheckFile(
|
||||
path=path,
|
||||
visit=visit,
|
||||
disposition=disp,
|
||||
claim_id=claim_id,
|
||||
))
|
||||
_log.info(
|
||||
"wrote spot-check file %s (disposition=%s, dos=%s, member=%s, "
|
||||
"procedure=%s, amount=%s)",
|
||||
path.name, disp, visit.dos, visit.member_id,
|
||||
visit.procedure_code, visit.billed_amount,
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,138 @@
|
||||
"""SP41-goal: spot-check 837P validation against the Edifabric
|
||||
``/v2/x12/validate`` endpoint (committed under ``cyclone.rebill``).
|
||||
|
||||
Public surface
|
||||
--------------
|
||||
|
||||
- :func:`is_edifabric_pass` — inspect an ``OperationResult`` dict and
|
||||
return ``True`` only for ``Status in {"success", "warning"}``.
|
||||
- :func:`validate_spot_check_files` — submit every file in ``paths``
|
||||
to ``cyclone.edifabric.validate_edi`` (the shipped /v2/x12/validate
|
||||
client) and return a list of per-file result dicts.
|
||||
|
||||
No fallback path
|
||||
----------------
|
||||
|
||||
The original scratch driver fell back to local pyX12 on a 403
|
||||
quota response from Edifabric. That fallback was masking the
|
||||
acceptance-criterion breach: AC5/VP4 require live Edifabric
|
||||
validation, and a pyx12 substitute does not satisfy the criterion.
|
||||
This module deliberately has NO fallback. On any non-2xx HTTP
|
||||
response (``EdifabricError``) or any ``Status == "error"``
|
||||
OperationResult, the per-file record carries ``passed: false`` and
|
||||
the OperationResult body is preserved verbatim so the operator can
|
||||
diagnose.
|
||||
|
||||
Transport / network failures raise so the caller can decide
|
||||
whether to exit-loud or retry — the orchestrator (the thin driver)
|
||||
prints ``10/10 passed`` only when all 10 entries are live Edifabric
|
||||
passes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from cyclone.edifabric import EdifabricError, validate_edi
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_edifabric_pass(result: dict) -> bool:
|
||||
"""Return True iff ``result`` is a passing ``OperationResult``.
|
||||
|
||||
Per the EdiNation docs, ``Status`` is one of:
|
||||
* ``"success"`` — passes.
|
||||
* ``"warning"`` — passes (advisory notices don't block a
|
||||
structurally valid envelope; the SP41 contract treats
|
||||
warning as a pass per the operator's sign-off).
|
||||
* ``"error"`` — fails; the Details array carries the failing
|
||||
segments / messages.
|
||||
* anything else — fail loud so we don't silently approve.
|
||||
"""
|
||||
status = (result.get("Status") or "").lower()
|
||||
return status in ("success", "warning")
|
||||
|
||||
|
||||
def validate_spot_check_files(paths: list[Path]) -> list[dict]:
|
||||
"""Submit each file in ``paths`` to Edifabric ``/v2/x12/validate``.
|
||||
|
||||
Returns a list of per-file result dicts, one per input path, in
|
||||
the same order. Each dict has the shape::
|
||||
|
||||
{
|
||||
"file": str, # path to the submitted file
|
||||
"passed": bool, # True only if Status is success/warning
|
||||
"issue": str, # human-readable verdict
|
||||
"result": dict | None, # OperationResult verbatim (None on transport error)
|
||||
"transport_error": dict | None, # {status_code, body} on EdifabricError
|
||||
}
|
||||
|
||||
Raises:
|
||||
EdifabricError: on the first transport failure. The caller
|
||||
decides whether to retry, exit non-zero, or fall back.
|
||||
This module never silently substitutes a different
|
||||
validator.
|
||||
"""
|
||||
import time
|
||||
out: list[dict] = []
|
||||
for i, p in enumerate(paths):
|
||||
body = p.read_bytes()
|
||||
# The Edifabric /v2/x12 endpoints apply a per-second rate limit
|
||||
# in addition to the daily quota (typical: 1 call/sec sustained,
|
||||
# 429 if exceeded). Insert a 1.5s pause between calls so 10-file
|
||||
# spot-checks don't trip the limit; this is the value the plan
|
||||
# flagged ("1-2 s spacing") that the original monolithic scratch
|
||||
# driver skipped.
|
||||
if i > 0:
|
||||
time.sleep(1.5)
|
||||
try:
|
||||
result = validate_edi(body)
|
||||
except EdifabricError as exc:
|
||||
tx: dict = {
|
||||
"status_code": exc.status_code,
|
||||
"body": exc.body,
|
||||
}
|
||||
if exc.retry_after_seconds is not None:
|
||||
tx["retry_after_seconds"] = exc.retry_after_seconds
|
||||
out.append({
|
||||
"file": str(p),
|
||||
"passed": False,
|
||||
"issue": (
|
||||
f"EdifabricError {exc.status_code}: {exc.body!r}"
|
||||
),
|
||||
"result": None,
|
||||
"transport_error": tx,
|
||||
})
|
||||
# Re-raise so the caller can't silently absorb the failure
|
||||
# (the original scratch driver caught + fallback'd which
|
||||
# is the bug this module fixes).
|
||||
raise
|
||||
|
||||
if is_edifabric_pass(result):
|
||||
out.append({
|
||||
"file": str(p),
|
||||
"passed": True,
|
||||
"issue": f"edifabric {result.get('Status')!r}",
|
||||
"result": result,
|
||||
"transport_error": None,
|
||||
})
|
||||
else:
|
||||
details = result.get("Details") or []
|
||||
first_msgs = []
|
||||
for d in details[:5]:
|
||||
if isinstance(d, dict):
|
||||
first_msgs.append(
|
||||
f"{d.get('SegmentId', '?')}: {d.get('Message', '?')}"
|
||||
)
|
||||
out.append({
|
||||
"file": str(p),
|
||||
"passed": False,
|
||||
"issue": (
|
||||
f"edifabric {result.get('Status')!r}: "
|
||||
+ ("; ".join(first_msgs) or "no details")
|
||||
),
|
||||
"result": result,
|
||||
"transport_error": None,
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Summary CSV for SP41.
|
||||
|
||||
One row per visit, classified into one of:
|
||||
REBILLED_A — pipeline A emitted a frequency-7 replacement
|
||||
REBILLED_B — pipeline B emitted a fresh 837P
|
||||
EXCLUDED_CARC — CO-45/CO-26/CO-129 (or other excluded CARC)
|
||||
EXCLUDED_PAYER — non-CO_TXIX payer per AxisCare API spot audit
|
||||
EXCLUDED_NO_EVV — no Authorized=Yes AND no AxisCare EVV ref
|
||||
EXCLUDED_TIMELY_FILING — DOS > 120 days before run date, no override
|
||||
"""
|
||||
import csv
|
||||
from dataclasses import dataclass
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from cyclone.rebill.reconcile import VisitRow
|
||||
|
||||
REBILLED_A = "REBILLED_A"
|
||||
REBILLED_B = "REBILLED_B"
|
||||
EXCLUDED_CARC = "EXCLUDED_CARC"
|
||||
EXCLUDED_PAYER = "EXCLUDED_PAYER"
|
||||
EXCLUDED_NO_EVV = "EXCLUDED_NO_EVV"
|
||||
EXCLUDED_TIMELY_FILING = "EXCLUDED_TIMELY_FILING"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SummaryRow:
|
||||
"""One visit + its post-pipeline disposition for the operator audit trail."""
|
||||
|
||||
visit: VisitRow
|
||||
disposition: str
|
||||
unpaid: Decimal
|
||||
cas_reasons: tuple[str, ...]
|
||||
file_path: str # relative to dev/rebills/<date>/
|
||||
|
||||
|
||||
def write_summary_csv(rows: list[SummaryRow], out_path: Path) -> int:
|
||||
"""Write the summary CSV; return the number of rows written."""
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with out_path.open("w", newline="") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow([
|
||||
"dos", "member_id", "procedure", "billed",
|
||||
"disposition", "unpaid", "cas_reasons", "file_path",
|
||||
])
|
||||
for r in rows:
|
||||
w.writerow([
|
||||
r.visit.date.isoformat(),
|
||||
r.visit.member_id,
|
||||
r.visit.procedure,
|
||||
str(r.visit.billed),
|
||||
r.disposition,
|
||||
str(r.unpaid),
|
||||
"|".join(r.cas_reasons), # pipe-separated; no collision with J-prefixed member IDs or relative file paths
|
||||
r.file_path,
|
||||
])
|
||||
return len(rows)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""120-day timely-filing gate for Colorado Medicaid.
|
||||
|
||||
Per HCPF's Colorado Medical Assistance Program provider manual, claims must
|
||||
be filed within 120 days from the date of service. The gate is HARD by
|
||||
default — past-window visits are surfaced as EXCLUDED_TIMELY_FILING in the
|
||||
summary CSV and not emitted by Pipeline B.
|
||||
|
||||
The operator may pass override=True per batch to relax the gate (e.g. for
|
||||
good-cause appeal visits). The decision is recorded in the summary CSV
|
||||
either way so the audit trail shows the override.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
|
||||
FILING_WINDOW_DAYS = 120
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TimelyFilingDecision:
|
||||
dos: date
|
||||
as_of: date
|
||||
days_old: int
|
||||
within_window: bool
|
||||
override: bool
|
||||
rebillable: bool
|
||||
|
||||
|
||||
def timely_filing_decision(
|
||||
dos: date, as_of: date, override: bool
|
||||
) -> TimelyFilingDecision:
|
||||
days_old = (as_of - dos).days
|
||||
within_window = days_old <= FILING_WINDOW_DAYS
|
||||
return TimelyFilingDecision(
|
||||
dos=dos, as_of=as_of, days_old=days_old,
|
||||
within_window=within_window, override=override,
|
||||
rebillable=within_window or override,
|
||||
)
|
||||
@@ -0,0 +1,193 @@
|
||||
"""SP41: load the AxisCare visits export into the ``visits`` table.
|
||||
|
||||
The spot-check driver used to read the visits CSV in-memory. Persisting
|
||||
the roster into the database (a) makes the source-of-truth
|
||||
reviewable via SQL, and (b) lets the rebill pipeline (reconcile /
|
||||
resubmit) reference the same canonical visit list without re-parsing
|
||||
the CSV on every run.
|
||||
|
||||
Header (AxisCare export):
|
||||
"Client","Visit Date","Payer","Authorized","Member ID",
|
||||
"Auth Start Date","Auth End Date","Authorization #","ICD-10",
|
||||
"Service","Procedure Code","Modifiers","Client Classes",
|
||||
"Billable Hours","Billable Amount","Invoice #","Claimed"
|
||||
|
||||
DOS is MM/DD/YYYY. Billable Amount may carry ``$`` prefix and ``,``
|
||||
thousands separators. Modifiers may be ``:``-joined (batch 1) or
|
||||
``,``-joined (batch 2); we normalize to colon-joined for storage and
|
||||
downstream emission.
|
||||
|
||||
This module is intentionally side-effect-free on import — call
|
||||
:func:`load_visits_csv` to populate the table.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from cyclone import db as db_mod
|
||||
from cyclone.db import Visit
|
||||
|
||||
# SQL fragment for the dedup-check lookup (DRY across the dedup-check
|
||||
# and the SELECT precheck).
|
||||
_NATURAL_KEY_COLS = (Visit.dos, Visit.member_id, Visit.procedure_code, Visit.modifiers)
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_modifiers(raw: str) -> str:
|
||||
"""``"KX:SC:U2"`` or ``"KX, SC, U2"`` → ``"KX:SC:U2"``.
|
||||
|
||||
We store colon-joined (matches the X12 SV1 modifier emission format).
|
||||
Empty / blank input returns "".
|
||||
"""
|
||||
if not raw:
|
||||
return ""
|
||||
parts = [m.strip() for m in raw.replace(":", ",").split(",") if m.strip()]
|
||||
return ":".join(parts)
|
||||
|
||||
|
||||
def _parse_billed(raw: str) -> Decimal:
|
||||
"""``"$212.77"`` / ``"1,234.56"`` → Decimal. Returns 0 on failure."""
|
||||
if not raw:
|
||||
return Decimal("0")
|
||||
cleaned = raw.replace("$", "").replace(",", "").strip()
|
||||
try:
|
||||
return Decimal(cleaned or "0")
|
||||
except Exception:
|
||||
return Decimal("0")
|
||||
|
||||
|
||||
def _parse_dos(raw: str) -> date | None:
|
||||
"""``"01/01/2026"`` → ``date(2026, 1, 1)``. None on failure."""
|
||||
raw = (raw or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
for fmt in ("%m/%d/%Y", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(raw, fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def load_visits_csv(
|
||||
csv_path: Path,
|
||||
*,
|
||||
window_start: date | None = None,
|
||||
window_end: date | None = None,
|
||||
) -> int:
|
||||
"""Load the AxisCare visits export into the ``visits`` table.
|
||||
|
||||
Args:
|
||||
csv_path: Path to the CSV file.
|
||||
window_start: Optional inclusive lower bound on DOS.
|
||||
window_end: Optional inclusive upper bound on DOS.
|
||||
|
||||
Returns:
|
||||
Number of rows actually inserted (duplicates are skipped
|
||||
silently — the UNIQUE constraint on (dos, member_id, procedure,
|
||||
modifiers) dedupes the natural key).
|
||||
"""
|
||||
db_mod.init_db()
|
||||
SessionLocal = db_mod.SessionLocal()
|
||||
inserted = 0
|
||||
skipped = 0
|
||||
with SessionLocal() as session:
|
||||
with csv_path.open(newline="", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
dos = _parse_dos(row.get("Visit Date") or "")
|
||||
if dos is None:
|
||||
skipped += 1
|
||||
continue
|
||||
if window_start and dos < window_start:
|
||||
continue
|
||||
if window_end and dos > window_end:
|
||||
continue
|
||||
member_id = (row.get("Member ID") or "").strip()
|
||||
procedure = (row.get("Procedure Code") or "").strip()
|
||||
if not member_id or not procedure:
|
||||
skipped += 1
|
||||
continue
|
||||
modifiers = _normalize_modifiers(row.get("Modifiers") or "")
|
||||
# Pre-check: does this natural-key already exist? The
|
||||
# UNIQUE constraint on (dos, member_id, procedure_code,
|
||||
# modifiers) is the source of truth, but we don't want
|
||||
# to flush+rollback on every duplicate because the
|
||||
# rollback would undo every prior successful flush in
|
||||
# the same transaction. An explicit SELECT keeps the
|
||||
# transaction shape clean.
|
||||
existing = session.execute(
|
||||
select(Visit.id).where(
|
||||
Visit.dos == dos,
|
||||
Visit.member_id == member_id,
|
||||
Visit.procedure_code == procedure,
|
||||
Visit.modifiers == (modifiers or None),
|
||||
)
|
||||
).first()
|
||||
if existing is not None:
|
||||
skipped += 1
|
||||
continue
|
||||
visit = Visit(
|
||||
dos=dos,
|
||||
member_id=member_id,
|
||||
client_name=(row.get("Client") or "").strip(),
|
||||
procedure_code=procedure,
|
||||
modifiers=modifiers or None,
|
||||
billed_amount=_parse_billed(row.get("Billable Amount") or ""),
|
||||
icd10=(row.get("ICD-10") or "").strip() or None,
|
||||
prior_auth=(row.get("Authorization #") or "").strip() or None,
|
||||
payer=(row.get("Payer") or "").strip() or None,
|
||||
invoice_number=(row.get("Invoice #") or "").strip() or None,
|
||||
source_file=str(csv_path),
|
||||
)
|
||||
session.add(visit)
|
||||
try:
|
||||
session.flush()
|
||||
inserted += 1
|
||||
except IntegrityError:
|
||||
# Race with a parallel writer — still a dup. Drop
|
||||
# the just-flushed INSERT and continue.
|
||||
session.rollback()
|
||||
skipped += 1
|
||||
session.commit()
|
||||
_log.info("load_visits_csv: %s -> %d inserted, %d skipped (dup/invalid)",
|
||||
csv_path.name, inserted, skipped)
|
||||
return inserted
|
||||
|
||||
|
||||
def query_visits(
|
||||
*,
|
||||
member_id: str | None = None,
|
||||
procedure_code: str | None = None,
|
||||
dos_start: date | None = None,
|
||||
dos_end: date | None = None,
|
||||
limit: int | None = None,
|
||||
) -> list[Visit]:
|
||||
"""Read visits back from the table. Filters are AND-combined.
|
||||
|
||||
Returns the matching rows as ORM objects (caller may need to detach
|
||||
or convert to a dataclass for downstream use).
|
||||
"""
|
||||
db_mod.init_db()
|
||||
SessionLocal = db_mod.SessionLocal()
|
||||
with SessionLocal() as session:
|
||||
stmt = select(Visit)
|
||||
if member_id is not None:
|
||||
stmt = stmt.where(Visit.member_id == member_id)
|
||||
if procedure_code is not None:
|
||||
stmt = stmt.where(Visit.procedure_code == procedure_code)
|
||||
if dos_start is not None:
|
||||
stmt = stmt.where(Visit.dos >= dos_start)
|
||||
if dos_end is not None:
|
||||
stmt = stmt.where(Visit.dos <= dos_end)
|
||||
if limit is not None:
|
||||
stmt = stmt.limit(limit)
|
||||
return list(session.execute(stmt).scalars().all())
|
||||
@@ -17,7 +17,7 @@ Match algorithm:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Iterable, Optional, Protocol
|
||||
@@ -50,6 +50,18 @@ class Match:
|
||||
remittance: _RemitLike
|
||||
strategy: str
|
||||
is_reversal: bool
|
||||
# SP31: which content-keys the 2-of-3 (or 3-of-3) rule agreed on.
|
||||
# Populated for ``score-auto`` matches; empty for ``pcn-exact`` /
|
||||
# ``manual`` (PCN-exact only uses the PCN key by definition; manual
|
||||
# is operator-driven). Floats into the ``auto_matched_835``
|
||||
# ActivityEvent payload so the audit trail records *why* the link
|
||||
# fired. Transient — the ORM Match row does not persist these
|
||||
# (spec keeps the audit trail in ActivityEvent.payload_json only).
|
||||
keys_matched: set[str] = field(default_factory=set)
|
||||
# SP31: how many candidates were in the ±30-day window pool when
|
||||
# the score-auto match fired. Useful when an operator is reviewing
|
||||
# a borderline auto-link — 1-of-1 is unambiguous; 1-of-5 was lucky.
|
||||
candidate_count: int = 0
|
||||
|
||||
|
||||
def match(
|
||||
@@ -84,7 +96,7 @@ def match(
|
||||
matches.append(Match(
|
||||
claim=chosen,
|
||||
remittance=r,
|
||||
strategy="auto",
|
||||
strategy="pcn-exact",
|
||||
is_reversal=r.is_reversal,
|
||||
))
|
||||
used_claim_ids.add(chosen.id)
|
||||
@@ -117,6 +129,163 @@ def _pick_claim(
|
||||
return None
|
||||
|
||||
|
||||
# --- SP31: content-keys fallback matcher -----------------------------------
|
||||
|
||||
CHARGE_TOLERANCE = Decimal("0.01")
|
||||
KEYS_REQUIRED = 2
|
||||
|
||||
|
||||
def _content_keys_match(remit, claim) -> set[str]:
|
||||
"""Return the set of content-keys that agree between ``remit`` and ``claim``.
|
||||
|
||||
Compares {``pcn``, ``charge``, ``npi``} between the two sides and returns
|
||||
the subset whose values match. Caller checks ``len(returned) >= KEYS_REQUIRED``
|
||||
to decide whether to auto-link. Returning the matched keys (rather than a
|
||||
bool) lets the ActivityEvent payload record exactly which 2-of-3 (or 3-of-3)
|
||||
produced the auto-link — the audit trail for the SP31 content-keys rule.
|
||||
|
||||
Charge compared with ``CHARGE_TOLERANCE`` tolerance (rounding drift).
|
||||
NPI counts as "not matched" when the remit's NPI is empty.
|
||||
|
||||
Uses duck-typing via ``getattr`` so the helper works against both:
|
||||
- shim / dataclass test objects (planned names: total_charge_amount,
|
||||
total_charge, rendering_provider_npi), and
|
||||
- real SQLAlchemy ORM instances (Claim.charge_amount, Remittance.total_charge,
|
||||
Claim.provider_npi, Claim.rendering_provider_npi,
|
||||
Remittance.rendering_provider_npi).
|
||||
|
||||
Production note (SP32): the typed ``rendering_provider_npi`` column is
|
||||
the primary read for both sides (Claim = NM1*82, Remit = NM1*1P). Legacy
|
||||
rows pre-0019 still carry the value in ``raw_json``; the read order is:
|
||||
- NPI: typed column → raw_json (``service_provider_npi`` for remit,
|
||||
``rendering_provider_npi`` for claim) → claim ``provider_npi``
|
||||
(billing fallback for legacy 837p rows without NM1*82 extraction).
|
||||
- Charge: planned attribute name → real ORM column → raw_json.
|
||||
The ``raw_json`` fallback is what makes this helper safe against a
|
||||
raw ``Remittance`` ORM instance from ``reconcile.run()``.
|
||||
"""
|
||||
matched: set[str] = set()
|
||||
|
||||
# PCN: payer_claim_control_number ↔ patient_control_number (stripped).
|
||||
if (getattr(remit, "payer_claim_control_number", "") or "").strip() == \
|
||||
(getattr(claim, "patient_control_number", "") or "").strip():
|
||||
matched.add("pcn")
|
||||
|
||||
# Charge: prefer the planned attribute names, fall back to real ORM columns,
|
||||
# then to raw_json (where the 835 parser stores CLP03). Explicit ``is None``
|
||||
# checks avoid the ``Decimal("0")`` truthiness footgun — ``Decimal("0")``
|
||||
# is truthy in a boolean context but ``or`` would still let it through;
|
||||
# the more important property is that we don't accidentally replace a real
|
||||
# value with a default.
|
||||
_remit_charge = getattr(remit, "total_charge_amount", None)
|
||||
if _remit_charge is None:
|
||||
_remit_charge = getattr(remit, "total_charge", None)
|
||||
if _remit_charge is None:
|
||||
_remit_charge = (getattr(remit, "raw_json", None) or {}).get("total_charge_amount")
|
||||
|
||||
_claim_charge = getattr(claim, "total_charge", None)
|
||||
if _claim_charge is None:
|
||||
_claim_charge = getattr(claim, "charge_amount", None)
|
||||
if _claim_charge is None:
|
||||
_claim_charge = (getattr(claim, "raw_json", None) or {}).get("total_charge_amount")
|
||||
|
||||
if _remit_charge is not None and _claim_charge is not None and \
|
||||
abs(_remit_charge - _claim_charge) < CHARGE_TOLERANCE:
|
||||
matched.add("charge")
|
||||
|
||||
# NPI: typed-column primary path (SP32), raw_json fallback (legacy rows).
|
||||
# Remit reads rendering_provider_npi (single value, D4); claim reads
|
||||
# rendering_provider_npi first, then falls back to provider_npi (billing)
|
||||
# for legacy rows where the 837p parser hadn't yet extracted NM1*82.
|
||||
remit_npi = (
|
||||
(getattr(remit, "rendering_provider_npi", "") or "")
|
||||
or ((getattr(remit, "raw_json", None) or {}).get("service_provider_npi", "") or "")
|
||||
or ((getattr(remit, "raw_json", None) or {}).get("rendering_provider_npi", "") or "")
|
||||
).strip()
|
||||
claim_npi = (
|
||||
(getattr(claim, "rendering_provider_npi", "") or "")
|
||||
or (getattr(claim, "provider_npi", "") or "")
|
||||
or ((getattr(claim, "raw_json", None) or {}).get("rendering_provider_npi", "") or "")
|
||||
).strip()
|
||||
if remit_npi and remit_npi == claim_npi:
|
||||
matched.add("npi")
|
||||
|
||||
return matched
|
||||
|
||||
|
||||
# --- SP31: DB-side fallback candidate matcher -------------------------------
|
||||
|
||||
SCORE_FALLBACK_WINDOW_DAYS = 30
|
||||
|
||||
|
||||
def _score_fallback_candidates(session, remit) -> Optional[tuple[str, set[str], int]]:
|
||||
"""Return (claim_id, keys_matched, candidate_count) for a unique 2-of-3 match.
|
||||
|
||||
Queries a ±SCORE_FALLBACK_WINDOW_DAYS candidate pool filtered by:
|
||||
- claim.matched_remittance_id IS NULL (not yet linked)
|
||||
- claim.state NOT IN terminal set (PAID, DENIED, REJECTED,
|
||||
REVERSED, RECONCILED)
|
||||
- claim.service_date_from within window of remit.service_date
|
||||
|
||||
Returns ``(claim_id, keys_matched, candidate_count)`` when exactly one
|
||||
candidate passes the 2-of-3 rule (via :func:`_content_keys_match`),
|
||||
otherwise ``None``. ``keys_matched`` is the set returned by the helper
|
||||
(``{"pcn", "charge"}`` or any 2-of-3 / 3-of-3 combination) — the
|
||||
ActivityEvent payload records it as the audit trail for *why* the
|
||||
auto-link fired. ``candidate_count`` is the size of the window pool
|
||||
(independent of how many matched) so the operator can see whether
|
||||
the match was 1-of-N or 1-of-1.
|
||||
|
||||
Ambiguous matches (2+ candidates) also return ``None`` — they land
|
||||
in the Inbox Unlinked lane for manual review.
|
||||
|
||||
Note on payer_id: ``Remittance`` has no ``payer_id`` column
|
||||
(``Claim.payer_id`` does, and is the source of truth on the claim
|
||||
side; the 835 parser stores payer_id in ``Remittance.raw_json``).
|
||||
Filtering by payer would require either a raw_json read or a model
|
||||
column add — out of scope for SP31 Task 3. The PCN itself is
|
||||
expected to be unique within a payer's submission, so cross-payer
|
||||
collisions are unlikely. Revisit if production shows otherwise.
|
||||
"""
|
||||
from sqlalchemy import select, and_
|
||||
from datetime import timedelta
|
||||
from cyclone.db import Claim, ClaimState
|
||||
|
||||
TERMINAL = {
|
||||
ClaimState.PAID, ClaimState.DENIED, ClaimState.REJECTED,
|
||||
ClaimState.REVERSED, ClaimState.RECONCILED,
|
||||
}
|
||||
|
||||
if remit.service_date is None:
|
||||
return None # need a date for the window query
|
||||
|
||||
window_lo = remit.service_date - timedelta(days=SCORE_FALLBACK_WINDOW_DAYS)
|
||||
window_hi = remit.service_date + timedelta(days=SCORE_FALLBACK_WINDOW_DAYS)
|
||||
|
||||
candidates = list(session.execute(
|
||||
select(Claim).where(
|
||||
and_(
|
||||
Claim.matched_remittance_id.is_(None),
|
||||
Claim.service_date_from >= window_lo,
|
||||
Claim.service_date_from <= window_hi,
|
||||
Claim.state.notin_([s.value for s in TERMINAL]),
|
||||
)
|
||||
)
|
||||
).scalars().all())
|
||||
|
||||
matched_pairs = [
|
||||
(c.id, _content_keys_match(remit, c))
|
||||
for c in candidates
|
||||
]
|
||||
matched_pairs = [
|
||||
(cid, ks) for cid, ks in matched_pairs if len(ks) >= KEYS_REQUIRED
|
||||
]
|
||||
if len(matched_pairs) == 1:
|
||||
cid, keys_matched = matched_pairs[0]
|
||||
return (cid, keys_matched, len(candidates))
|
||||
return None # 0 matches OR 2+ (ambiguous)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ApplyIntent:
|
||||
"""Result of applying a match. The caller persists this.
|
||||
@@ -224,17 +393,22 @@ class ReconcileResult:
|
||||
def run(session, batch_id: str) -> ReconcileResult:
|
||||
"""Orchestrate reconciliation for an 835 batch.
|
||||
|
||||
Expects Batch + Remittance + CasAdjustment rows already persisted
|
||||
(this is called from store.add() AFTER commit). Loads the new
|
||||
remittances + all currently-unmatched Claims, calls match(), then
|
||||
applies each match's intent inside the caller's session.
|
||||
Expects Batch + Remittance + CasAdjustment rows already added to
|
||||
``session`` (not yet committed). SP27 Task 10 calls this from
|
||||
inside ``CycloneStore.add``'s ingest session, BEFORE
|
||||
``s.commit()`` — so the whole 835 ingest (rows + reconcile) is
|
||||
a single transaction. If anything here raises, the ingest rolls
|
||||
back; no half-reconciled batch ever lands.
|
||||
|
||||
Loads the new remittances + all currently-unmatched Claims, calls
|
||||
match(), then applies each match's intent inside the caller's session.
|
||||
|
||||
Returns counts. Does NOT commit; caller controls transaction.
|
||||
Raises on failure (caller catches and writes activity event).
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
from cyclone.db import (
|
||||
Batch, Claim, Remittance, CasAdjustment, Match, ActivityEvent,
|
||||
Batch, Claim, Remittance, CasAdjustment, Match as MatchORM, ActivityEvent,
|
||||
ClaimState,
|
||||
)
|
||||
|
||||
@@ -252,6 +426,34 @@ def run(session, batch_id: str) -> ReconcileResult:
|
||||
|
||||
matches = match(unmatched_claims, new_remits)
|
||||
|
||||
# SP31: content-keys fallback for remits that PCN-exact couldn't pair.
|
||||
# Operates on the still-unmatched remits so we don't double-match.
|
||||
matched_remit_ids = {m.remittance.id for m in matches}
|
||||
used_claim_ids = {m.claim.id for m in matches}
|
||||
for remit in new_remits:
|
||||
if remit.id in matched_remit_ids:
|
||||
continue
|
||||
if getattr(remit, "claim_id", None) is not None:
|
||||
continue # already linked
|
||||
fallback_result = _score_fallback_candidates(session, remit)
|
||||
if fallback_result is None:
|
||||
continue
|
||||
matched_claim_id, keys_matched, candidate_count = fallback_result
|
||||
# Find the claim object in the unmatched_claims list (it was loaded).
|
||||
target_claim = next(
|
||||
(c for c in unmatched_claims if c.id == matched_claim_id),
|
||||
None,
|
||||
)
|
||||
if target_claim is None or target_claim.id in used_claim_ids:
|
||||
continue
|
||||
matches.append(Match(
|
||||
claim=target_claim, remittance=remit,
|
||||
strategy="score-auto", is_reversal=remit.is_reversal,
|
||||
keys_matched=keys_matched,
|
||||
candidate_count=candidate_count,
|
||||
))
|
||||
used_claim_ids.add(target_claim.id)
|
||||
|
||||
applied = 0
|
||||
skipped = 0
|
||||
for m in matches:
|
||||
@@ -275,7 +477,7 @@ def run(session, batch_id: str) -> ReconcileResult:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
session.add(Match(
|
||||
session.add(MatchORM(
|
||||
claim_id=m.claim.id, remittance_id=m.remittance.id,
|
||||
strategy=m.strategy,
|
||||
matched_at=datetime.now(timezone.utc),
|
||||
@@ -293,10 +495,25 @@ def run(session, batch_id: str) -> ReconcileResult:
|
||||
m.remittance.claim_id = m.claim.id
|
||||
|
||||
session.add(ActivityEvent(
|
||||
ts=datetime.now(timezone.utc), kind=intent.activity_kind,
|
||||
ts=datetime.now(timezone.utc),
|
||||
kind=("auto_matched_835" if m.strategy == "score-auto" else intent.activity_kind),
|
||||
batch_id=batch_id, claim_id=m.claim.id,
|
||||
remittance_id=m.remittance.id,
|
||||
payload_json={"new_state": m.claim.state.value},
|
||||
payload_json=(
|
||||
{
|
||||
"new_state": m.claim.state.value, "strategy": m.strategy,
|
||||
# SP31 spec D8: the auto_matched_835 payload records
|
||||
# which content-keys the 2-of-3 (or 3-of-3) rule
|
||||
# agreed on, plus how many candidates were in the
|
||||
# window pool. ``sorted()`` converts the set to a
|
||||
# JSON-serializable list and pins a deterministic
|
||||
# order for tests + audit reads.
|
||||
"keys_matched": sorted(m.keys_matched),
|
||||
"candidate_count": m.candidate_count,
|
||||
}
|
||||
if m.strategy == "score-auto"
|
||||
else {"new_state": m.claim.state.value}
|
||||
),
|
||||
))
|
||||
applied += 1
|
||||
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""cyclone.reissue — offline 837P re-emission workflow.
|
||||
|
||||
Pure functions for parsing raw 837P files into ``ClaimOutput`` Pydantic
|
||||
models and re-emitting one IG-correct single-claim X12 file per claim.
|
||||
No Click imports inside :mod:`cyclone.reissue.core`; the CLI wrapper at
|
||||
:mod:`cyclone.cli` is a thin shell over these functions.
|
||||
|
||||
Public surface:
|
||||
|
||||
- :func:`parse_inputs` — parse every ``*.x12`` / ``*.txt`` / ``*.edi``
|
||||
under a directory; tolerates per-file failures and skips claims with
|
||||
hard validation errors.
|
||||
- :func:`emit_outputs` — write one X12 per claim to ``output_dir``
|
||||
using HCPF-spec filenames; per-claim 1 ms timestamp offsets guarantee
|
||||
unique filenames within a batch.
|
||||
- :func:`write_summary_sidecar` — write a ``_serialize_summary.json``
|
||||
sidecar with one row per emitted file (operator audit trail).
|
||||
- :func:`zip_outputs` — zip the per-claim X12 files into a single flat
|
||||
archive with a ``testzip()`` integrity check.
|
||||
- :func:`ig_correctness_check` — verify the serializer's
|
||||
``PATIENT_LOOP_DEFAULT_INCLUDED`` is ``False`` (the IG-correct
|
||||
shape for SBR02 == "18" claims). See
|
||||
``docs/superpowers/specs/2026-07-08-cyclone-reissue-claims-design.md``
|
||||
for the SP24 rationale.
|
||||
|
||||
See `scripts/reissue_claims.py` for the deprecation shim; the
|
||||
canonical entry point is ``cyclone reissue-claims`` (SP24).
|
||||
"""
|
||||
from cyclone.reissue.core import (
|
||||
emit_outputs,
|
||||
ig_correctness_check,
|
||||
parse_inputs,
|
||||
write_summary_sidecar,
|
||||
zip_outputs,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"parse_inputs",
|
||||
"emit_outputs",
|
||||
"write_summary_sidecar",
|
||||
"zip_outputs",
|
||||
"ig_correctness_check",
|
||||
]
|
||||
@@ -0,0 +1,242 @@
|
||||
"""SP24 — pure functions for the offline 837P reissue workflow.
|
||||
|
||||
The functions here are deliberately Click-free so the CLI layer at
|
||||
:mod:`cyclone.cli` stays a thin shell, and so unit tests can exercise
|
||||
the behaviour without :class:`click.testing.CliRunner`.
|
||||
|
||||
Workflow contract:
|
||||
|
||||
>>> claims, errors = parse_inputs(input_dir, payer_config)
|
||||
>>> written = emit_outputs(claims, payer_config=..., output_dir=..., ...)
|
||||
>>> zip_outputs(written, zip_path)
|
||||
|
||||
``ig_correctness_check`` is the regression guard for the SP24 fix.
|
||||
The serializer's ``PATIENT_LOOP_DEFAULT_INCLUDED`` constant MUST be
|
||||
``False``; flipping it back to ``True`` would re-introduce the
|
||||
Edifabric 999 rejection on every SBR02 == "18" claim.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import zipfile
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from cyclone.edi.filenames import build_outbound_filename
|
||||
from cyclone.parsers.models import ClaimOutput
|
||||
from cyclone.parsers.parse_837 import parse as parse_837_text
|
||||
from cyclone.parsers import serialize_837 as _serialize_837_mod
|
||||
from cyclone.parsers.serialize_837 import serialize_837
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ig_correctness_check(*, logger: logging.Logger | None = None) -> bool:
|
||||
"""Return True iff the serializer's IG-correct patient-loop default is in place.
|
||||
|
||||
Per X12 005010X222A1, the 2000C Patient Hierarchical Level
|
||||
(``HL*3 → PAT → NM1*QC``) MUST be absent when ``SBR02 == "18"``
|
||||
(Self-pay, the CO-Medicaid IHSS workflow). The serializer encodes
|
||||
this as a module-level constant ``PATIENT_LOOP_DEFAULT_INCLUDED``
|
||||
in :mod:`cyclone.parsers.serialize_837`; this function checks
|
||||
that constant.
|
||||
|
||||
Args:
|
||||
logger: optional logger for the diagnostic WARNING when the
|
||||
guard fires. Defaults to this module's logger.
|
||||
|
||||
Returns:
|
||||
True if the constant is ``False`` (IG-correct). False
|
||||
otherwise — callers should refuse to emit any X12 in that case.
|
||||
"""
|
||||
log = logger or _log
|
||||
# Read the constant live (not at import time) so monkeypatch in
|
||||
# tests reflects immediately and a runtime mutation by a future
|
||||
# config-loader also takes effect without a re-import.
|
||||
current = _serialize_837_mod.PATIENT_LOOP_DEFAULT_INCLUDED
|
||||
if current is not False:
|
||||
log.warning(
|
||||
"REFUSING to run: PATIENT_LOOP_DEFAULT_INCLUDED = %r "
|
||||
"(expected False). The IG-correct serializer shape for "
|
||||
"SBR02='18' claims requires the 2000C patient loop to be "
|
||||
"absent; restoring the default to False fixes it.",
|
||||
current,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_inputs(
|
||||
input_dir: Path,
|
||||
payer_config,
|
||||
) -> tuple[list[ClaimOutput], list[tuple[Path, str]]]:
|
||||
"""Parse every ``*.x12`` / ``*.txt`` / ``*.edi`` under ``input_dir``.
|
||||
|
||||
AppleDouble metadata files (``._foo.x12``, the macOS resource
|
||||
forks that Samba / Finder scatter) are skipped at the glob stage.
|
||||
|
||||
Per-file failures are tolerated: a file that raises during
|
||||
``parse_837_text`` lands in the second tuple element with the
|
||||
exception class + message. The first tuple element collects every
|
||||
surviving :class:`ClaimOutput`, including those with **warnings**
|
||||
(warnings don't abort). Claims with hard ``validation.errors``
|
||||
are dropped with an error message in the second tuple element.
|
||||
|
||||
Args:
|
||||
input_dir: directory to walk (one level).
|
||||
payer_config: a :class:`PayerConfig` instance passed to the
|
||||
parser — typically ``PayerConfig.co_medicaid()``.
|
||||
|
||||
Returns:
|
||||
``(claims, errors)`` where ``claims`` is the surviving
|
||||
:class:`ClaimOutput` list and ``errors`` is a list of
|
||||
``(path, message)`` tuples for each per-file failure.
|
||||
"""
|
||||
raw_files: list[Path] = []
|
||||
for ext in ("*.x12", "*.txt", "*.edi"):
|
||||
for p in sorted(input_dir.glob(ext)):
|
||||
# Skip macOS AppleDouble metadata (._<name>) — binary
|
||||
# forks of the resource fork, not actual EDI.
|
||||
if p.name.startswith("._"):
|
||||
continue
|
||||
raw_files.append(p)
|
||||
|
||||
claims: list[ClaimOutput] = []
|
||||
errors: list[tuple[Path, str]] = []
|
||||
|
||||
if not raw_files:
|
||||
errors.append((input_dir, "no *.x12 / *.txt / *.edi files found"))
|
||||
for f in raw_files:
|
||||
try:
|
||||
text = f.read_text(encoding="utf-8")
|
||||
result = parse_837_text(text, payer_config, input_file=str(f))
|
||||
except Exception as exc: # noqa: BLE001 — log + skip
|
||||
errors.append((f, f"{exc.__class__.__name__}: {exc}"))
|
||||
continue
|
||||
if not result.claims:
|
||||
errors.append((f, "no claims parsed (empty CLM loop?)"))
|
||||
continue
|
||||
for c in result.claims:
|
||||
if c.validation.errors:
|
||||
msgs = [f"{i.rule}: {i.message}" for i in c.validation.errors]
|
||||
errors.append((f, f"hard errors in {c.claim_id}: {msgs}"))
|
||||
continue
|
||||
claims.append(c)
|
||||
return claims, errors
|
||||
|
||||
|
||||
def emit_outputs(
|
||||
claims: list[ClaimOutput],
|
||||
*,
|
||||
payer_config,
|
||||
output_dir: Path,
|
||||
sender_id: str,
|
||||
receiver_id: str,
|
||||
submitter_name: str,
|
||||
submitter_contact_name: str,
|
||||
submitter_contact_email: str,
|
||||
receiver_name: str,
|
||||
) -> list[Path]:
|
||||
"""Write one X12 per claim under ``output_dir`` using HCPF-spec filenames.
|
||||
|
||||
Per-claim 1 ms timestamp offsets on the base datetime guarantee
|
||||
unique filenames within a batch. The output directory is created
|
||||
if it doesn't exist. Claims are sorted by ``claim_id`` before
|
||||
emission so the per-claim filenames line up with the canonical
|
||||
sort order.
|
||||
|
||||
Args:
|
||||
claims: parsed :class:`ClaimOutput` instances (typically the
|
||||
first tuple element from :func:`parse_inputs`).
|
||||
payer_config: a :class:`PayerConfig` — threads
|
||||
``sbr09_claim_filing`` into the SBR09 segment.
|
||||
output_dir: directory to write the per-claim X12 files into.
|
||||
sender_id, receiver_id, submitter_*, receiver_name: envelope
|
||||
metadata threaded into the ISA/GS/NM1*41/NM1*40 segments.
|
||||
|
||||
Returns:
|
||||
The list of written :class:`Path` instances, in the same
|
||||
order as the sorted claims.
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
# America/Denver is the operator's home tz and the canonical
|
||||
# timezone for HCPF submission timestamps; matching the tz
|
||||
# keeps the 999 round-trip deterministic for the operator's
|
||||
# daily pull.
|
||||
base_ts = datetime.now(tz=ZoneInfo("America/Denver"))
|
||||
|
||||
written: list[Path] = []
|
||||
for i, claim in enumerate(sorted(claims, key=lambda c: c.claim_id), start=1):
|
||||
body = serialize_837(
|
||||
claim,
|
||||
sender_id=sender_id,
|
||||
receiver_id=receiver_id,
|
||||
submitter_name=submitter_name,
|
||||
submitter_contact_name=submitter_contact_name,
|
||||
submitter_contact_email=submitter_contact_email,
|
||||
receiver_name=receiver_name,
|
||||
claim_filing_indicator_code=payer_config.sbr09_claim_filing,
|
||||
interchange_control_number=f"{i:09d}",
|
||||
group_control_number="1",
|
||||
)
|
||||
ts = base_ts + timedelta(milliseconds=i)
|
||||
fname = build_outbound_filename(sender_id, "837P", now_mt=ts)
|
||||
path = output_dir / fname
|
||||
path.write_text(body, encoding="ascii")
|
||||
written.append(path)
|
||||
return written
|
||||
|
||||
|
||||
def write_summary_sidecar(
|
||||
claims: list[ClaimOutput],
|
||||
written: list[Path],
|
||||
output_dir: Path,
|
||||
) -> Path:
|
||||
"""Write ``_serialize_summary.json`` next to the per-claim X12 files.
|
||||
|
||||
Operator-facing sidecar for audit: one row per emitted file with
|
||||
``claim_id``, ``output_file`` (absolute path), and ``byte_size``.
|
||||
Both ``claims`` and ``written`` are zipped in the same order as
|
||||
``emit_outputs`` produced them (i.e. sorted by ``claim_id``).
|
||||
|
||||
Returns the path of the written sidecar.
|
||||
"""
|
||||
summary = [
|
||||
{
|
||||
"claim_id": c.claim_id,
|
||||
"output_file": str(p),
|
||||
"byte_size": p.stat().st_size,
|
||||
}
|
||||
for c, p in zip(sorted(claims, key=lambda c: c.claim_id), written)
|
||||
]
|
||||
path = output_dir / "_serialize_summary.json"
|
||||
path.write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def zip_outputs(files: list[Path], zip_path: Path) -> None:
|
||||
"""Zip the per-claim X12 files into a single flat archive.
|
||||
|
||||
Uses :class:`zipfile.ZIP_DEFLATED`. After writing, runs
|
||||
:meth:`zipfile.ZipFile.testzip` to verify CRC integrity and
|
||||
raises :class:`RuntimeError` if any entry is corrupt.
|
||||
|
||||
Args:
|
||||
files: the per-claim X12 file paths (typically the return
|
||||
value of :func:`emit_outputs`).
|
||||
zip_path: destination archive path; parent directories are
|
||||
created as needed.
|
||||
|
||||
Raises:
|
||||
RuntimeError: if any entry fails the ``testzip()`` integrity
|
||||
check.
|
||||
"""
|
||||
zip_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for p in files:
|
||||
zf.write(p, arcname=p.name)
|
||||
bad = zipfile.ZipFile(zip_path).testzip()
|
||||
if bad is not None:
|
||||
raise RuntimeError(f"zip integrity check failed: {bad} is corrupt")
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user