Files
cyclone/backend/src/cyclone/store/claim_detail.py
T
Tyler Martinez c8f8f5d3c6 refactor(sp21): split store.py into cyclone/store/ subpackage (14 modules)
Behaviour-preserving structural split of the 2,995-LOC store.py into
a 14-module subpackage with a thin facade. CycloneStore class keeps
its full method surface as 1-line delegations to module functions.

14 modules:
  exceptions, records, orm_builders, ui, write,
  batches, claim_detail, kpis, acks,
  backups, inbox, providers
  (+ __init__.py facade)

Facade re-exports 12 public symbols (CycloneStore, store, BatchRecord*,
BatchKind, AlreadyMatchedError/NotMatchedError/InvalidStateError, utcnow,
dashboard_kpis, check_matched_pair_drift) + 3 private helpers
(_claim_status_from_validation, _persist_835_remit, _remittance_835_row)
to preserve the 4 test files that import them.

Zero public API changes, zero test changes, zero importer changes.
CycloneStore._lock and CycloneStore._batches.clear() remain intact for
the 7 test files still using the cleanup idiom.

Verified: 1,176 / 1 (pre-existing isolation flake) / 10 tests pass —
identical to baseline.
2026-06-29 15:57:31 -06:00

711 lines
26 KiB
Python

"""Claim- and remittance-detail read APIs.
Includes the only large non-write query in the store:
``get_claim_detail`` which joins Claim + CasAdjustment + ActivityEvent
history for the right-drawer UI.
Also hosts ``check_matched_pair_drift()`` — the SP27 startup invariant
audit that returns the count of Claim↔Remit pairs where
``matched_remittance_id`` doesn't agree with the FK in ``remittances.claim_id``.
It lives here (not in its own module) because it reads the same pair of
tables as ``get_claim_detail``'s matched-remittance summary.
"""
from __future__ import annotations
from datetime import timezone
from decimal import Decimal
from cyclone import db
from cyclone.db import (
ActivityEvent,
CasAdjustment,
Claim,
ClaimState,
Remittance,
)
from .ui import (
CLAIM_DETAIL_HISTORY_LIMIT,
_date_in_bounds,
_iso_z,
_svc_to_wire_dict,
to_ui_claim_detail,
to_ui_claim_from_orm,
to_ui_provider,
to_ui_remittance_from_orm,
to_ui_remittance_with_adjustments,
)
def get_remittance(remittance_id: str) -> dict | None:
"""Return a UI-shaped remittance dict with ``adjustments`` array.
Joins the persisted ``CasAdjustment`` rows for ``remittance_id``
and labels each via :mod:`cyclone.parsers.cas_codes`. Returns
``None`` when the remittance is not found so the API layer can
map that to a 404.
SP7: also returns the per-line SVC composites
(``serviceLinePayments``) and the CLP-level (claim-level) CAS
bucket (``claimLevelAdjustments``) so the remit drawer can show
per-line payments + adjustments without a second fetch.
"""
with db.SessionLocal()() as s:
row = s.get(Remittance, remittance_id)
if row is None:
return None
cas_rows = (
s.query(CasAdjustment)
.filter(CasAdjustment.remittance_id == remittance_id)
.all()
)
parsed_at = (
row.batch.parsed_at if row.batch is not None else row.received_at
)
if parsed_at is not None and parsed_at.tzinfo is None:
parsed_at = parsed_at.replace(tzinfo=timezone.utc)
body = to_ui_remittance_with_adjustments(
row,
batch_id=row.batch_id,
parsed_at=parsed_at,
cas_rows=cas_rows,
)
# SP7: per-line SVC composites + claim-level CAS bucket.
from cyclone.db import ServiceLinePayment as SLP
slps = (
s.query(SLP)
.filter(SLP.remittance_id == remittance_id)
.order_by(SLP.line_number)
.all()
)
body["serviceLinePayments"] = [
_svc_to_wire_dict(svc) for svc in slps
]
body["claimLevelAdjustments"] = [
{
"id": c.id,
"group_code": c.group_code,
"reason_code": c.reason_code,
"amount": str(Decimal(str(c.amount))),
"quantity": (
str(Decimal(str(c.quantity)))
if c.quantity is not None
else None
),
}
for c in cas_rows
if c.service_line_payment_id is None
]
return body
def get_claim_detail(claim_id: str) -> dict | None:
"""Return the SP4 detail-drawer shape for one claim, or ``None``.
Drives ``GET /api/claims/{claim_id}``. Returns the spec-shaped
dict from :func:`to_ui_claim_detail` (header + state + parties +
validation + service lines + diagnoses + raw segments) stitched
with the claim's recent activity history and, if paired, a
matched-remittance summary.
Returns ``None`` when ``claim_id`` is not in the DB so the API
layer can map that to a 404 — the URL-driven drawer
distinguishes "claim doesn't exist" from "fetch failed" (the
spec §3.4 calls for a distinct 404 state in the drawer).
The history is capped at :data:`CLAIM_DETAIL_HISTORY_LIMIT`
(50, per the spec) and ordered ``ts DESC`` so the most recent
event is first. The status string in ``matchedRemittance``
follows the same ``reconciled``/``received`` mapping used by
:func:`to_ui_remittance_from_orm`.
"""
# Lazy import — same pattern used throughout this module to
# avoid a circular store ↔ db import on cold start.
from cyclone import db as _db
with _db.SessionLocal()() as s:
row = s.get(Claim, claim_id)
if row is None:
return None
history_rows = (
s.query(ActivityEvent)
.filter(ActivityEvent.claim_id == claim_id)
.order_by(ActivityEvent.ts.desc())
.limit(CLAIM_DETAIL_HISTORY_LIMIT)
.all()
)
# Claim.batch_id is FK NOT NULL with ON DELETE CASCADE, so
# ``row.batch`` is always populated in normal flow. Re-attach
# UTC only when SQLite drops the tzinfo on read.
parsed_at = row.batch.parsed_at
if parsed_at.tzinfo is None:
parsed_at = parsed_at.replace(tzinfo=timezone.utc)
detail = to_ui_claim_detail(
row,
batch_id=row.batch_id,
parsed_at=parsed_at,
)
detail["stateHistory"] = [
{
"kind": ev.kind,
# SQLite drops tzinfo on read; rows are stored UTC
# at write time (see ``add`` / ``manual_match``),
# so re-attach UTC if needed to keep the spec
# contract that ``ts`` ends in Z.
"ts": _iso_z(ev.ts),
"batchId": ev.batch_id,
"remittanceId": ev.remittance_id,
}
for ev in history_rows
]
if row.matched_remittance_id is not None:
remit = s.get(Remittance, row.matched_remittance_id)
if remit is not None:
status = (
"reconciled"
if remit.status_code in ("21", "22")
else "received"
)
detail["matchedRemittance"] = {
"id": remit.id,
"totalPaid": float(remit.total_paid or 0),
"status": status,
"receivedAt": _iso_z(remit.received_at),
}
# If the remittance was deleted out from under the FK
# (the FK is ``ON DELETE SET NULL`` so the column is
# already cleared in normal flow), the matched_remittance_id
# would be None here and we wouldn't enter this branch.
# If the FK is non-null but the row is gone (e.g. tests
# that bypass the cascade), fall through with the
# default ``None`` — the UI shows "no match" rather
# than crashing.
# SP7 §5.2: slim per-line projection so the ServiceLinesTable
# can show Paid + Adjustments columns without a second fetch.
# The 837 side is keyed by ``claim_service_line_number`` (the
# 1-based line number from raw_json) since 837 service lines
# are not a separate ORM table.
from cyclone.db import (
LineReconciliation, ServiceLinePayment, CasAdjustment,
)
slim_lrs = list(
s.query(LineReconciliation)
.filter(LineReconciliation.claim_id == claim_id)
.all()
)
svc_ids_for_cas = [
lr.service_line_payment_id
for lr in slim_lrs
if lr.service_line_payment_id is not None
]
cas_sums_by_svc: dict = {}
svc_by_id_slim: dict = {}
if svc_ids_for_cas:
cas_rows = (
s.query(CasAdjustment.service_line_payment_id, CasAdjustment.amount)
.filter(CasAdjustment.service_line_payment_id.in_(svc_ids_for_cas))
.all()
)
from collections import defaultdict
agg = defaultdict(lambda: Decimal("0"))
for svc_id, amount in cas_rows:
agg[svc_id] += Decimal(str(amount))
cas_sums_by_svc = {k: str(v) for k, v in agg.items()}
for svc in (
s.query(ServiceLinePayment)
.filter(ServiceLinePayment.id.in_(svc_ids_for_cas))
.all()
):
svc_by_id_slim[svc.id] = svc
slim_by_num: dict = {
lr.claim_service_line_number: lr
for lr in slim_lrs
if lr.claim_service_line_number is not None
}
line_reconciliation_slim: list = []
for sl in detail["serviceLines"]:
ln = sl.get("lineNumber")
lr = slim_by_num.get(ln)
if lr is None:
line_reconciliation_slim.append({
"lineNumber": ln,
"status": "unmatched_837_only",
"paid": None,
"adjustmentsSum": None,
})
continue
svc = (
svc_by_id_slim.get(lr.service_line_payment_id)
if lr.service_line_payment_id
else None
)
line_reconciliation_slim.append({
"lineNumber": ln,
"status": lr.status,
"paid": str(Decimal(str(svc.payment))) if svc else None,
"adjustmentsSum": (
cas_sums_by_svc.get(lr.service_line_payment_id)
if lr.service_line_payment_id
else None
),
})
detail["lineReconciliation"] = line_reconciliation_slim
return detail
def iter_claims(
*,
batch_id: str | None = None,
status: str | None = None,
provider_npi: str | None = None,
payer: str | None = None,
date_from: str | None = None,
date_to: str | None = None,
sort: str | None = None,
order: str = "desc",
limit: int = 100,
offset: int = 0,
) -> list[dict]:
"""Return UI-shaped claim dicts from the DB.
Filters mirror the in-memory version. The ``payer`` filter is
a case-insensitive substring on the payer's ``name``, recovered
from each claim's ``raw_json`` payload (the DB stores it there
because ``Claim`` itself only carries ``payer_id``).
"""
with db.SessionLocal()() as s:
q = s.query(Claim)
if batch_id is not None:
q = q.filter(Claim.batch_id == batch_id)
if status is not None:
q = q.filter(Claim.state == ClaimState(status))
if provider_npi is not None:
q = q.filter(Claim.provider_npi == provider_npi)
rows = q.all()
# Bulk-load matched-remittance totals so the UI's "Received"
# KPI + per-claim received_amount reflect real paid amounts
# rather than always-0. One SQL roundtrip for the whole page
# rather than per-claim lookups.
matched_ids = [
r.matched_remittance_id
for r in rows
if r.matched_remittance_id
]
received_by_remit: dict[str, float] = {}
if matched_ids:
for rid, total_paid in (
s.query(Remittance.id, Remittance.total_paid)
.filter(Remittance.id.in_(matched_ids))
.all()
):
received_by_remit[rid] = float(total_paid or 0)
out: list[dict] = []
for r in rows:
raw = r.raw_json or {}
bp = raw.get("billing_provider", {})
payer_obj = raw.get("payer", {})
sub = raw.get("subscriber", {})
claim_hdr = raw.get("claim", {})
service_lines = raw.get("service_lines", [])
parsed_at_iso = (
r.batch.parsed_at.isoformat().replace("+00:00", "Z")
if r.batch is not None
else ""
)
cpt = (
service_lines[0].get("procedure", {}).get("code", "")
if service_lines
else ""
)
out.append({
"id": r.id,
"patientName": (
f"{sub.get('first_name', '')} "
f"{sub.get('last_name', '')}".strip()
),
"providerNpi": bp.get("npi") or r.provider_npi or "",
"payerName": payer_obj.get("name") or "",
"cptCode": cpt,
"billedAmount": float(r.charge_amount or 0),
"receivedAmount": received_by_remit.get(
r.matched_remittance_id, 0.0
),
"status": r.state.value if hasattr(r.state, "value") else str(r.state),
"state": r.state.value if hasattr(r.state, "value") else str(r.state),
"denialReason": None,
"submissionDate": parsed_at_iso,
"batchId": r.batch_id,
"parsedAt": parsed_at_iso,
# Keep these so we can sort on them in-memory below.
"_sort_billedAmount": float(r.charge_amount or 0),
"_sort_submissionDate": parsed_at_iso,
})
if payer is not None:
needle = payer.casefold()
out = [
c for c in out
if needle in (c.get("payerName") or "").casefold()
]
out = [
c for c in out
if _date_in_bounds(c, "submissionDate", date_from, date_to)
]
if sort is not None:
out.sort(
key=lambda c: c.get(f"_sort_{sort}", 0) or 0,
reverse=(order == "desc"),
)
# Drop the private sort keys before returning.
for c in out:
c.pop("_sort_billedAmount", None)
c.pop("_sort_submissionDate", None)
return out[offset:offset + limit]
def iter_remittances(
*,
batch_id: str | None = None,
payer: str | None = None,
claim_id: str | None = None,
date_from: str | None = None,
date_to: str | None = None,
sort: str | None = None,
order: str = "desc",
limit: int = 100,
offset: int = 0,
) -> list[dict]:
"""Return UI-shaped remittance dicts from the DB."""
with db.SessionLocal()() as s:
q = s.query(Remittance)
if batch_id is not None:
q = q.filter(Remittance.batch_id == batch_id)
if claim_id is not None:
q = q.filter(Remittance.claim_id == claim_id)
rows = q.all()
# Bulk-fetch all CAS rows for these remittances in one query
# (SP3 P2 follow-up — fixes the list-view's empty adjustments
# expansion). N+1-free.
cas_by_remit: dict[str, list] = {}
if rows:
from cyclone.parsers.cas_codes import reason_label
cas_rows = (
s.query(CasAdjustment)
.filter(CasAdjustment.remittance_id.in_([r.id for r in rows]))
.all()
)
for c in cas_rows:
cas_by_remit.setdefault(c.remittance_id, []).append(c)
out: list[dict] = []
for r in rows:
raw = r.raw_json or {}
parsed_at_iso = (
r.batch.parsed_at.isoformat().replace("+00:00", "Z")
if r.batch is not None
else r.received_at.isoformat().replace("+00:00", "Z")
)
payer_name = ""
if r.batch is not None and r.batch.raw_result_json:
payer_name = (
r.batch.raw_result_json.get("payer", {}).get("name", "")
)
adjustments = [
{
"group": c.group_code,
"reason": c.reason_code,
"label": reason_label(c.group_code, c.reason_code),
"amount": float(c.amount),
"quantity": float(c.quantity) if c.quantity is not None else None,
}
for c in cas_by_remit.get(r.id, [])
]
out.append({
"id": r.id,
"claimId": r.claim_id or "",
"payerName": payer_name,
"paidAmount": float(r.total_paid or 0),
"adjustmentAmount": float(r.adjustment_amount or 0),
"status": (
"reconciled" if r.status_code in ("21", "22")
else "received"
),
"denialReason": None,
"validationWarnings": [],
"receivedDate": r.received_at.isoformat().replace("+00:00", "Z"),
"batchId": r.batch_id,
"parsedAt": parsed_at_iso,
"adjustments": adjustments,
"_sort_receivedDate": r.received_at.isoformat().replace("+00:00", "Z"),
})
if payer is not None:
out = [r for r in out if r.get("payerName") == payer]
out = [
r for r in out
if _date_in_bounds(r, "receivedDate", date_from, date_to)
]
if sort is not None:
out.sort(
key=lambda r: r.get(f"_sort_{sort}", 0) or 0,
reverse=(order == "desc"),
)
for r in out:
r.pop("_sort_receivedDate", None)
return out[offset:offset + limit]
def distinct_providers() -> list[dict]:
"""Group claims by NPI and return one row per provider."""
with db.SessionLocal()() as s:
rows = s.query(Claim).all()
by_npi: dict[str, dict] = {}
for r in rows:
npi = r.provider_npi or ""
if npi not in by_npi:
raw = r.raw_json or {}
bp = raw.get("billing_provider", {})
by_npi[npi] = to_ui_provider(
npi=npi,
name=bp.get("name") or "",
tax_id=bp.get("tax_id"),
address=None,
city=None,
state=None,
zip=None,
phone=None,
claim_count=0,
outstanding_ar=0.0,
)
by_npi[npi]["claimCount"] += 1
return list(by_npi.values())
def recent_activity(*, limit: int = 200) -> list[dict]:
"""Return recent activity events from the DB, newest first.
SP21 Task 2.5: each row also carries ``claimId`` and
``remittanceId`` (read from the ORM columns) so the Dashboard's
Recent-activity card can route clicks to the right entity
drawer via ``src/lib/event-routing.ts``. Both are nullable
strings; the wire shape uses camelCase keys to match the
existing ``npi`` / ``amount`` fields and the frontend
``Activity`` interface.
"""
with db.SessionLocal()() as s:
rows = (
s.query(ActivityEvent)
.order_by(ActivityEvent.ts.desc())
.limit(limit)
.all()
)
return [
{
"id": f"ae-{r.id}",
"kind": r.kind,
"message": (r.payload_json or {}).get("message", ""),
"timestamp": r.ts.isoformat().replace("+00:00", "Z"),
"npi": (r.payload_json or {}).get("npi"),
"amount": (r.payload_json or {}).get("amount"),
"claimId": r.claim_id,
"remittanceId": r.remittance_id,
}
for r in rows
]
def check_matched_pair_drift() -> int:
"""Audit the ``Claim.matched_remittance_id`` ↔ ``Remittance.claim_id``
FK pair at startup. Non-blocking (SP27 Task 11).
The matched pair is a denormalized FK pair maintained transactionally
by ``manual_match``, ``manual_unmatch``, and ``reconcile.run``. A
pre-existing mismatch (e.g. a row written before a state migration
that added one column but not the other) would otherwise stay
invisible until the next operator pair attempt fails confusingly.
This check logs the count + up to N examples so operators can
investigate without booting the system.
Returns the number of drifted rows (0 means clean). Does not
raise; bootstrap continues even if drift is detected.
Count semantics: this returns *drifted rows*, not *drifted pairs*.
A single broken pair (``Claim.matched_remittance_id = X`` AND
``Remittance.claim_id = Y != nil`` with neither pointing back)
can produce TWO drifted rows — one in case A (the claim) and
one in case B (the remit). In practice drift is almost always
asymmetric (one side NULL), so count == count(pairs); for the
fully-symmetric minority, divide by ~2 when alerting. Real drift
should be fixed by repairing the writer path, not by counting.
Cases:
A. Claim ``matched_remittance_id = X`` but the paired remit X's
``claim_id`` is either NULL or doesn't point back to the claim.
B. Remit ``claim_id = A`` but the paired claim A's
``matched_remittance_id`` is either NULL or doesn't point back.
"""
import logging
from sqlalchemy import select
log = logging.getLogger(__name__)
with db.SessionLocal()() as s:
# Case A: claim says X is paired, but X.claim_id doesn't point back.
case_a = list(
s.execute(
select(
Claim.id.label("claim_id"),
Claim.matched_remittance_id.label("claimed_remit_id"),
Remittance.claim_id.label("remit_points_to"),
)
.outerjoin(
Remittance,
Remittance.id == Claim.matched_remittance_id,
)
.where(Claim.matched_remittance_id.is_not(None))
.where(
(Remittance.claim_id.is_(None))
| (Remittance.claim_id != Claim.id)
)
).all()
)
# Case B: remit says A is paired, but A.matched_remittance_id
# doesn't point back.
case_b = list(
s.execute(
select(
Remittance.id.label("remit_id"),
Remittance.claim_id.label("claimed_claim_id"),
Claim.matched_remittance_id.label("claim_points_to"),
)
.outerjoin(
Claim,
Claim.id == Remittance.claim_id,
)
.where(Remittance.claim_id.is_not(None))
.where(
(Claim.matched_remittance_id.is_(None))
| (Claim.matched_remittance_id != Remittance.id)
)
).all()
)
total = len(case_a) + len(case_b)
if total == 0:
log.info("matched-pair drift check: 0 mismatches (clean)")
return 0
log.warning(
"matched-pair drift check: %d mismatched pair(s) (showing up to 5 "
"of each). Investigate via SELECT against claim / remittance; "
"manual re-pair via /api/claims/{id}/manual-match will repair.",
total,
)
for r in case_a[:5]:
log.warning(
" case A: claim %s -> remit %s, but remit.claim_id=%r",
r.claim_id,
r.claimed_remit_id,
r.remit_points_to,
)
for r in case_b[:5]:
log.warning(
" case B: remit %s -> claim %s, but claim.matched_remittance_id=%r",
r.remit_id,
r.claimed_claim_id,
r.claim_points_to,
)
return total
# ---------------------------------------------------------------------------
# Aggregate counters (SP25 / SP27): full-population counts and sums that
# the /api/* endpoints expose so page-local reductions (25 rows + live-tail
# delta) can never silently understate the true population.
# ---------------------------------------------------------------------------
_ITER_UNBOUNDED = 2**31 - 1
def count_claims(
*,
batch_id: str | None = None,
status: str | None = None,
provider_npi: str | None = None,
payer: str | None = None,
date_from: str | None = None,
date_to: str | None = None,
) -> int:
"""Count claims that would be returned by ``iter_claims``.
Same filter parameters as ``iter_claims`` (excluding
``sort``/``order``/``limit``/``offset``, which don't affect cardinality).
"""
rows = iter_claims(
batch_id=batch_id, status=status, provider_npi=provider_npi,
payer=payer, date_from=date_from, date_to=date_to,
sort=None, order="desc", limit=_ITER_UNBOUNDED, offset=0,
)
return len(rows)
def count_remittances(
*,
batch_id: str | None = None,
payer: str | None = None,
claim_id: str | None = None,
date_from: str | None = None,
date_to: str | None = None,
) -> int:
"""Count remittances that would be returned by ``iter_remittances``.
Same filter parameters as ``iter_remittances`` (excluding
``sort``/``order``/``limit``/``offset``).
"""
rows = iter_remittances(
batch_id=batch_id, payer=payer, claim_id=claim_id,
date_from=date_from, date_to=date_to,
sort=None, order="desc", limit=_ITER_UNBOUNDED, offset=0,
)
return len(rows)
def summarize_remittances(
*,
batch_id: str | None = None,
payer: str | None = None,
claim_id: str | None = None,
date_from: str | None = None,
date_to: str | None = None,
) -> dict:
"""Return ``{count, total_paid, total_adjustments}`` summed over the
remittance population that ``iter_remittances`` would return under
the same filters. Backs ``GET /api/remittances/summary`` — the
Remittances page's KPI tiles (added in SP27).
"""
rows = iter_remittances(
batch_id=batch_id, payer=payer, claim_id=claim_id,
date_from=date_from, date_to=date_to,
sort=None, order="desc", limit=_ITER_UNBOUNDED, offset=0,
)
total_paid = 0.0
total_adjustments = 0.0
for r in rows:
total_paid += float(r.get("paidAmount") or 0)
total_adjustments += float(r.get("adjustmentAmount") or 0)
return {
"count": len(rows),
"total_paid": total_paid,
"total_adjustments": total_adjustments,
}