feat(sp27): pin Claim↔Remit matched-pair invariant + startup drift audit

Add check_matched_pair_drift() at module level in store.py — read-only
audit of the Claim.matched_remittance_id ↔ Remittance.claim_id FK pair.
Logs WARNING on drift with up to 5 examples of each of two cases
(claim-side and remit-side), returns the count of drifted rows. Wired
into api.py::lifespan right after db.init_db() and ensure_clearhouse_seeded(),
wrapped in try/except so a query failure logs but doesn't crash boot.

The symmetric-write + symmetric-clear invariants on
manual_match / manual_unmatch were already correct; this commit pins
them with focused regression tests in test_store_match_invariant.py
(8 tests covering both directions, both happy-path, rollback pin, and
drift-check unit behavior). PCN asymmetry in the fixture prevents the
auto-match inside CycloneStore.add (Task 10) from pre-pairing, so
manual_match is the only writer.

Migration 0016 adds the missing ix_claims_matched_remittance_id index —
the drift check scans WHERE matched_remittance_id IS NOT NULL on every
boot and would become a full-table scan past ~10k claims. Symmetric
with ix_remittances_claim_id (added in 0007). Migration tests bumped
from head=15 to head=16.
This commit is contained in:
Nora
2026-06-29 12:21:34 -06:00
parent d5f95b4f3c
commit f5d119fbe7
7 changed files with 580 additions and 7 deletions
+10
View File
@@ -146,6 +146,16 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
except Exception as exc: # noqa: BLE001
log.exception("SP9 seed failed: %s", exc)
# SP27 Task 11: startup audit for the Claim.matched_remittance_id ↔
# Remittance.claim_id denormalized FK pair. Non-blocking — mismatches
# are logged at WARNING with up to 5 examples of each case so the
# operator can investigate without the system failing to boot.
try:
from cyclone.store import check_matched_pair_drift
check_matched_pair_drift()
except Exception as exc: # noqa: BLE001
log.exception("matched-pair drift check failed: %s", exc)
# SP16: configure the inbound MFT polling scheduler. The
# dzinesco clearhouse singleton (seeded by SP9) carries the SFTP
# block — multi-provider polling is out of scope for v1.
+7
View File
@@ -317,6 +317,13 @@ class Claim(Base):
Index("ix_claims_state", "state"),
Index("ix_claims_patient_control_number", "patient_control_number"),
Index("ix_claims_service_date_from", "service_date_from"),
# SP27 Task 11: matched-pair drift check (run at startup)
# scans ``WHERE matched_remittance_id IS NOT NULL``. Without
# this index it's a full claim scan. The reverse side
# (``ix_remittances_claim_id``) is added in 0007. Pure index
# (non-unique) — a claim without a match is fine, reversals
# leave the previous claim/claim match intact.
Index("ix_claims_matched_remittance_id", "matched_remittance_id"),
)
@@ -0,0 +1,15 @@
-- version: 16
-- Add the missing index on claims.matched_remittance_id.
--
-- SP27 Task 11 added ``check_matched_pair_drift`` at startup, which
-- scans ``WHERE Claim.matched_remittance_id IS NOT NULL``. Without an
-- index this is a full-table scan; becomes a noticeable boot
-- latency cost past ~10k claims. The companion index on the
-- reverse side (``remittances.claim_id``) was added in 0007.
--
-- A plain index is enough — neither side is unique (reversals
-- re-reference the original PCN, and a claim without a match is
-- fine).
CREATE INDEX ix_claims_matched_remittance_id
ON claims(matched_remittance_id);
+104
View File
@@ -878,6 +878,110 @@ class _BatchesShim:
s.commit()
def check_matched_pair_drift() -> int:
"""Audit the ``Claim.matched_remittance_id`` ↔ ``Remittance.claim_id``
FK pair at startup. Non-blocking (SP27 Task 11).
The matched pair is a denormalized FK pair maintained transactionally
by ``manual_match``, ``manual_unmatch``, and ``reconcile.run``. A
pre-existing mismatch (e.g. a row written before a state migration
that added one column but not the other) would otherwise stay
invisible until the next operator pair attempt fails confusingly.
This check logs the count + up to N examples so operators can
investigate without booting the system.
Returns the number of drifted rows (0 means clean). Does not
raise; bootstrap continues even if drift is detected.
Count semantics: this returns *drifted rows*, not *drifted pairs*.
A single broken pair (``Claim.matched_remittance_id = X`` AND
``Remittance.claim_id = Y != nil`` with neither pointing back)
can produce TWO drifted rows — one in case A (the claim) and
one in case B (the remit). In practice drift is almost always
asymmetric (one side NULL), so count == count(pairs); for the
fully-symmetric minority, divide by ~2 when alerting. Real drift
should be fixed by repairing the writer path, not by counting.
Cases:
A. Claim ``matched_remittance_id = X`` but the paired remit X's
``claim_id`` is either NULL or doesn't point back to the claim.
B. Remit ``claim_id = A`` but the paired claim A's
``matched_remittance_id`` is either NULL or doesn't point back.
"""
import logging
from sqlalchemy import select
log = logging.getLogger(__name__)
with db.SessionLocal()() as s:
# Case A: claim says X is paired, but X.claim_id doesn't point back.
case_a = list(
s.execute(
select(
Claim.id.label("claim_id"),
Claim.matched_remittance_id.label("claimed_remit_id"),
Remittance.claim_id.label("remit_points_to"),
)
.outerjoin(
Remittance,
Remittance.id == Claim.matched_remittance_id,
)
.where(Claim.matched_remittance_id.is_not(None))
.where(
(Remittance.claim_id.is_(None))
| (Remittance.claim_id != Claim.id)
)
).all()
)
# Case B: remit says A is paired, but A.matched_remittance_id
# doesn't point back.
case_b = list(
s.execute(
select(
Remittance.id.label("remit_id"),
Remittance.claim_id.label("claimed_claim_id"),
Claim.matched_remittance_id.label("claim_points_to"),
)
.outerjoin(
Claim,
Claim.id == Remittance.claim_id,
)
.where(Remittance.claim_id.is_not(None))
.where(
(Claim.matched_remittance_id.is_(None))
| (Claim.matched_remittance_id != Remittance.id)
)
).all()
)
total = len(case_a) + len(case_b)
if total == 0:
log.info("matched-pair drift check: 0 mismatches (clean)")
return 0
log.warning(
"matched-pair drift check: %d mismatched pair(s) (showing up to 5 "
"of each). Investigate via SELECT against claim / remittance; "
"manual re-pair via /api/claims/{id}/manual-match will repair.",
total,
)
for r in case_a[:5]:
log.warning(
" case A: claim %s -> remit %s, but remit.claim_id=%r",
r.claim_id,
r.claimed_remit_id,
r.remit_points_to,
)
for r in case_b[:5]:
log.warning(
" case B: remit %s -> claim %s, but claim.matched_remittance_id=%r",
r.remit_id,
r.claimed_claim_id,
r.claim_points_to,
)
return total
# ---------------------------------------------------------------------------
# CycloneStore: the SQLAlchemy-backed facade.
# ---------------------------------------------------------------------------