fix(rebill): pre-check visits dedup to avoid rollback cascade
When loading 37k+ visits, IntegrityError-on-duplicate would rollback the whole session, losing every prior successful flush in the same transaction. Added an explicit SELECT precheck before INSERT; this keeps the transaction shape clean and avoids the rollback losing prior rows. Tests: 8/8 pass (test_rebill_visits_store).
This commit is contained in:
@@ -34,6 +34,10 @@ from sqlalchemy.exc import IntegrityError
|
|||||||
from cyclone import db as db_mod
|
from cyclone import db as db_mod
|
||||||
from cyclone.db import Visit
|
from cyclone.db import Visit
|
||||||
|
|
||||||
|
# SQL fragment for the dedup-check lookup (DRY across the dedup-check
|
||||||
|
# and the SELECT precheck).
|
||||||
|
_NATURAL_KEY_COLS = (Visit.dos, Visit.member_id, Visit.procedure_code, Visit.modifiers)
|
||||||
|
|
||||||
_log = logging.getLogger(__name__)
|
_log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@@ -113,6 +117,24 @@ def load_visits_csv(
|
|||||||
skipped += 1
|
skipped += 1
|
||||||
continue
|
continue
|
||||||
modifiers = _normalize_modifiers(row.get("Modifiers") or "")
|
modifiers = _normalize_modifiers(row.get("Modifiers") or "")
|
||||||
|
# Pre-check: does this natural-key already exist? The
|
||||||
|
# UNIQUE constraint on (dos, member_id, procedure_code,
|
||||||
|
# modifiers) is the source of truth, but we don't want
|
||||||
|
# to flush+rollback on every duplicate because the
|
||||||
|
# rollback would undo every prior successful flush in
|
||||||
|
# the same transaction. An explicit SELECT keeps the
|
||||||
|
# transaction shape clean.
|
||||||
|
existing = session.execute(
|
||||||
|
select(Visit.id).where(
|
||||||
|
Visit.dos == dos,
|
||||||
|
Visit.member_id == member_id,
|
||||||
|
Visit.procedure_code == procedure,
|
||||||
|
Visit.modifiers == (modifiers or None),
|
||||||
|
)
|
||||||
|
).first()
|
||||||
|
if existing is not None:
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
visit = Visit(
|
visit = Visit(
|
||||||
dos=dos,
|
dos=dos,
|
||||||
member_id=member_id,
|
member_id=member_id,
|
||||||
@@ -131,6 +153,8 @@ def load_visits_csv(
|
|||||||
session.flush()
|
session.flush()
|
||||||
inserted += 1
|
inserted += 1
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
|
# Race with a parallel writer — still a dup. Drop
|
||||||
|
# the just-flushed INSERT and continue.
|
||||||
session.rollback()
|
session.rollback()
|
||||||
skipped += 1
|
skipped += 1
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|||||||
Reference in New Issue
Block a user