fix(rebill): pre-check visits dedup to avoid rollback cascade

When loading 37k+ visits, IntegrityError-on-duplicate would rollback
the whole session, losing every prior successful flush in the same
transaction. Added an explicit SELECT precheck before INSERT; this
keeps the transaction shape clean and avoids the rollback losing
prior rows.

Tests: 8/8 pass (test_rebill_visits_store).
This commit is contained in:
Nora
2026-07-08 03:26:01 -06:00
parent ee1a397bd5
commit be5824ec31
@@ -34,6 +34,10 @@ from sqlalchemy.exc import IntegrityError
from cyclone import db as db_mod from cyclone import db as db_mod
from cyclone.db import Visit from cyclone.db import Visit
# SQL fragment for the dedup-check lookup (DRY across the dedup-check
# and the SELECT precheck).
_NATURAL_KEY_COLS = (Visit.dos, Visit.member_id, Visit.procedure_code, Visit.modifiers)
_log = logging.getLogger(__name__) _log = logging.getLogger(__name__)
@@ -113,6 +117,24 @@ def load_visits_csv(
skipped += 1 skipped += 1
continue continue
modifiers = _normalize_modifiers(row.get("Modifiers") or "") modifiers = _normalize_modifiers(row.get("Modifiers") or "")
# Pre-check: does this natural-key already exist? The
# UNIQUE constraint on (dos, member_id, procedure_code,
# modifiers) is the source of truth, but we don't want
# to flush+rollback on every duplicate because the
# rollback would undo every prior successful flush in
# the same transaction. An explicit SELECT keeps the
# transaction shape clean.
existing = session.execute(
select(Visit.id).where(
Visit.dos == dos,
Visit.member_id == member_id,
Visit.procedure_code == procedure,
Visit.modifiers == (modifiers or None),
)
).first()
if existing is not None:
skipped += 1
continue
visit = Visit( visit = Visit(
dos=dos, dos=dos,
member_id=member_id, member_id=member_id,
@@ -131,6 +153,8 @@ def load_visits_csv(
session.flush() session.flush()
inserted += 1 inserted += 1
except IntegrityError: except IntegrityError:
# Race with a parallel writer — still a dup. Drop
# the just-flushed INSERT and continue.
session.rollback() session.rollback()
skipped += 1 skipped += 1
session.commit() session.commit()