feat(sp27): pin Claim↔Remit matched-pair invariant + startup drift audit
Add check_matched_pair_drift() at module level in store.py — read-only audit of the Claim.matched_remittance_id ↔ Remittance.claim_id FK pair. Logs WARNING on drift with up to 5 examples of each of two cases (claim-side and remit-side), returns the count of drifted rows. Wired into api.py::lifespan right after db.init_db() and ensure_clearhouse_seeded(), wrapped in try/except so a query failure logs but doesn't crash boot. The symmetric-write + symmetric-clear invariants on manual_match / manual_unmatch were already correct; this commit pins them with focused regression tests in test_store_match_invariant.py (8 tests covering both directions, both happy-path, rollback pin, and drift-check unit behavior). PCN asymmetry in the fixture prevents the auto-match inside CycloneStore.add (Task 10) from pre-pairing, so manual_match is the only writer. Migration 0016 adds the missing ix_claims_matched_remittance_id index — the drift check scans WHERE matched_remittance_id IS NOT NULL on every boot and would become a full-table scan past ~10k claims. Symmetric with ix_remittances_claim_id (added in 0007). Migration tests bumped from head=15 to head=16.
This commit is contained in:
@@ -146,6 +146,16 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.exception("SP9 seed failed: %s", exc)
|
||||
|
||||
# SP27 Task 11: startup audit for the Claim.matched_remittance_id ↔
|
||||
# Remittance.claim_id denormalized FK pair. Non-blocking — mismatches
|
||||
# are logged at WARNING with up to 5 examples of each case so the
|
||||
# operator can investigate without the system failing to boot.
|
||||
try:
|
||||
from cyclone.store import check_matched_pair_drift
|
||||
check_matched_pair_drift()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.exception("matched-pair drift check failed: %s", exc)
|
||||
|
||||
# SP16: configure the inbound MFT polling scheduler. The
|
||||
# dzinesco clearhouse singleton (seeded by SP9) carries the SFTP
|
||||
# block — multi-provider polling is out of scope for v1.
|
||||
|
||||
@@ -317,6 +317,13 @@ class Claim(Base):
|
||||
Index("ix_claims_state", "state"),
|
||||
Index("ix_claims_patient_control_number", "patient_control_number"),
|
||||
Index("ix_claims_service_date_from", "service_date_from"),
|
||||
# SP27 Task 11: matched-pair drift check (run at startup)
|
||||
# scans ``WHERE matched_remittance_id IS NOT NULL``. Without
|
||||
# this index it's a full claim scan. The reverse side
|
||||
# (``ix_remittances_claim_id``) is added in 0007. Pure index
|
||||
# (non-unique) — a claim without a match is fine, reversals
|
||||
# leave the previous claim/claim match intact.
|
||||
Index("ix_claims_matched_remittance_id", "matched_remittance_id"),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
-- version: 16
|
||||
-- Add the missing index on claims.matched_remittance_id.
|
||||
--
|
||||
-- SP27 Task 11 added ``check_matched_pair_drift`` at startup, which
|
||||
-- scans ``WHERE Claim.matched_remittance_id IS NOT NULL``. Without an
|
||||
-- index this is a full-table scan; becomes a noticeable boot
|
||||
-- latency cost past ~10k claims. The companion index on the
|
||||
-- reverse side (``remittances.claim_id``) was added in 0007.
|
||||
--
|
||||
-- A plain index is enough — neither side is unique (reversals
|
||||
-- re-reference the original PCN, and a claim without a match is
|
||||
-- fine).
|
||||
|
||||
CREATE INDEX ix_claims_matched_remittance_id
|
||||
ON claims(matched_remittance_id);
|
||||
@@ -878,6 +878,110 @@ class _BatchesShim:
|
||||
s.commit()
|
||||
|
||||
|
||||
def check_matched_pair_drift() -> int:
|
||||
"""Audit the ``Claim.matched_remittance_id`` ↔ ``Remittance.claim_id``
|
||||
FK pair at startup. Non-blocking (SP27 Task 11).
|
||||
|
||||
The matched pair is a denormalized FK pair maintained transactionally
|
||||
by ``manual_match``, ``manual_unmatch``, and ``reconcile.run``. A
|
||||
pre-existing mismatch (e.g. a row written before a state migration
|
||||
that added one column but not the other) would otherwise stay
|
||||
invisible until the next operator pair attempt fails confusingly.
|
||||
This check logs the count + up to N examples so operators can
|
||||
investigate without booting the system.
|
||||
|
||||
Returns the number of drifted rows (0 means clean). Does not
|
||||
raise; bootstrap continues even if drift is detected.
|
||||
|
||||
Count semantics: this returns *drifted rows*, not *drifted pairs*.
|
||||
A single broken pair (``Claim.matched_remittance_id = X`` AND
|
||||
``Remittance.claim_id = Y != nil`` with neither pointing back)
|
||||
can produce TWO drifted rows — one in case A (the claim) and
|
||||
one in case B (the remit). In practice drift is almost always
|
||||
asymmetric (one side NULL), so count == count(pairs); for the
|
||||
fully-symmetric minority, divide by ~2 when alerting. Real drift
|
||||
should be fixed by repairing the writer path, not by counting.
|
||||
|
||||
Cases:
|
||||
A. Claim ``matched_remittance_id = X`` but the paired remit X's
|
||||
``claim_id`` is either NULL or doesn't point back to the claim.
|
||||
B. Remit ``claim_id = A`` but the paired claim A's
|
||||
``matched_remittance_id`` is either NULL or doesn't point back.
|
||||
"""
|
||||
import logging
|
||||
from sqlalchemy import select
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
with db.SessionLocal()() as s:
|
||||
# Case A: claim says X is paired, but X.claim_id doesn't point back.
|
||||
case_a = list(
|
||||
s.execute(
|
||||
select(
|
||||
Claim.id.label("claim_id"),
|
||||
Claim.matched_remittance_id.label("claimed_remit_id"),
|
||||
Remittance.claim_id.label("remit_points_to"),
|
||||
)
|
||||
.outerjoin(
|
||||
Remittance,
|
||||
Remittance.id == Claim.matched_remittance_id,
|
||||
)
|
||||
.where(Claim.matched_remittance_id.is_not(None))
|
||||
.where(
|
||||
(Remittance.claim_id.is_(None))
|
||||
| (Remittance.claim_id != Claim.id)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
# Case B: remit says A is paired, but A.matched_remittance_id
|
||||
# doesn't point back.
|
||||
case_b = list(
|
||||
s.execute(
|
||||
select(
|
||||
Remittance.id.label("remit_id"),
|
||||
Remittance.claim_id.label("claimed_claim_id"),
|
||||
Claim.matched_remittance_id.label("claim_points_to"),
|
||||
)
|
||||
.outerjoin(
|
||||
Claim,
|
||||
Claim.id == Remittance.claim_id,
|
||||
)
|
||||
.where(Remittance.claim_id.is_not(None))
|
||||
.where(
|
||||
(Claim.matched_remittance_id.is_(None))
|
||||
| (Claim.matched_remittance_id != Remittance.id)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
|
||||
total = len(case_a) + len(case_b)
|
||||
if total == 0:
|
||||
log.info("matched-pair drift check: 0 mismatches (clean)")
|
||||
return 0
|
||||
|
||||
log.warning(
|
||||
"matched-pair drift check: %d mismatched pair(s) (showing up to 5 "
|
||||
"of each). Investigate via SELECT against claim / remittance; "
|
||||
"manual re-pair via /api/claims/{id}/manual-match will repair.",
|
||||
total,
|
||||
)
|
||||
for r in case_a[:5]:
|
||||
log.warning(
|
||||
" case A: claim %s -> remit %s, but remit.claim_id=%r",
|
||||
r.claim_id,
|
||||
r.claimed_remit_id,
|
||||
r.remit_points_to,
|
||||
)
|
||||
for r in case_b[:5]:
|
||||
log.warning(
|
||||
" case B: remit %s -> claim %s, but claim.matched_remittance_id=%r",
|
||||
r.remit_id,
|
||||
r.claimed_claim_id,
|
||||
r.claim_points_to,
|
||||
)
|
||||
return total
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CycloneStore: the SQLAlchemy-backed facade.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user