fix(sp27): /api/claims + /api/remittances total counts the full population
Before this fix, the list endpoints computed `total` via
`len(list(store.iter_*(**common)))`. Both `iter_claims` and
`iter_remittances` default to `limit=100`, so the reported total
silently capped at 100 even when the DB held 60k claims or 835
remits. The frontend rendered a `data.total` of 100 in the KPI
tile and a 100-row table, the page looked complete, and the bug
stayed hidden — exactly the Dashboard silent-failure pattern that
59c3275 (server-aggregated KPIs) was meant to retire.
The Remittances page symptom reported on Jun 29 ('REMITS 100',
empty CLAIM column on 100 rows) was this bug. The empty CLAIM is
a separate matching concern (those remits have `claim_id = NULL`
in the DB); the count fix addresses the population-size lie.
Fix:
* `CycloneStore.count_claims` + `count_remittances` reuse the
iter's filter pipeline (DB filters + in-memory payer/date
filters) with an effectively-unbounded limit so the count
reflects the true DB population.
* `list_claims` + `list_remittances` call the new count
helpers instead of slicing `iter_*` twice.
* 13 new tests in `test_list_endpoint_counts.py` cover the
empty/filter/per-dimension/cardinality cases plus the HTTP
regression (seed 150/120, assert total matches).
Bonus fix:
* `conftest._reset_rate_limit_buckets` walks the middleware
stack and clears `RateLimitMiddleware._buckets` between
tests. Without this, the full suite tripped the 300 req/60s
rate limiter mid-run and 9 later tests (4 of mine, 5
pre-existing in test_inbox_endpoints / test_payer_summary)
got 429s. All 1167 tests now pass.
Follow-ups noted but out of scope:
* `/api/admin/audit-log`, `/api/admin/backup/list`,
`/api/admin/scheduler/processed-files` use the same
`len(rows)`-after-`.limit(limit)` pattern. Admin-only,
no `has_more` consumption, so impact is bounded.
* `count_*` could short-circuit to `func.count()` to skip
the UI-dict build, but iter already calls `q.all()` so
the win is modest.
This commit is contained in:
@@ -1714,7 +1714,10 @@ def list_claims(
|
||||
items = list(store.iter_claims(
|
||||
sort=sort, order=order, limit=limit, offset=offset, **common,
|
||||
))
|
||||
total = len(list(store.iter_claims(**common)))
|
||||
# SP27 Task 13b: count the full population, not a 100-row sample.
|
||||
# `iter_claims` defaults to limit=100; counting its output silently
|
||||
# capped the reported total at 100 even when the DB held 60k rows.
|
||||
total = store.count_claims(**common)
|
||||
returned = len(items)
|
||||
has_more = total > offset + returned
|
||||
if _wants_ndjson(request):
|
||||
@@ -2220,7 +2223,9 @@ def list_remittances(
|
||||
items = list(store.iter_remittances(
|
||||
sort=sort, order=order, limit=limit, offset=offset, **common,
|
||||
))
|
||||
total = len(list(store.iter_remittances(**common)))
|
||||
# SP27 Task 13b: count the full population, not a 100-row sample.
|
||||
# See the matching note in list_claims — same silent-failure pattern.
|
||||
total = store.count_remittances(**common)
|
||||
returned = len(items)
|
||||
has_more = total > offset + returned
|
||||
if _wants_ndjson(request):
|
||||
|
||||
@@ -851,6 +851,15 @@ def _date_in_bounds(
|
||||
return True
|
||||
|
||||
|
||||
# Effectively-unbounded iter_* limit, used by count_claims /
|
||||
# count_remittances so they can reuse the iter's filter pipeline
|
||||
# (incl. the in-memory ``payer`` substring + ``date_from/to`` checks)
|
||||
# without being silently capped at the iter's default ``limit=100``.
|
||||
# 2**31 - 1 is the largest signed 32-bit int — far above any realistic
|
||||
# X12 batch population.
|
||||
_ITER_UNBOUNDED = 2**31 - 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backward-compat shim: tests called ``_batches.clear()`` on the in-memory
|
||||
# store. The DB-backed store doesn't have an in-memory list, so we expose
|
||||
@@ -1754,6 +1763,62 @@ class CycloneStore:
|
||||
r.pop("_sort_receivedDate", None)
|
||||
return out[offset:offset + limit]
|
||||
|
||||
# -- count helpers (SP27 Task 13b) ---------------------------------
|
||||
#
|
||||
# The list endpoints (``/api/claims`` and ``/api/remittances``)
|
||||
# previously computed ``total`` by calling ``iter_*`` with default
|
||||
# ``limit=100`` and taking ``len(...)``. With 60k+ claims and 800+
|
||||
# remits in production, the reported total silently capped at 100
|
||||
# even when the UI rendered a "100" KPI tile and a 100-row table —
|
||||
# the page looked complete but the population was 600× larger. These
|
||||
# helpers reuse the iter's filter pipeline with an effectively
|
||||
# unbounded ``limit`` so the count reflects the true DB population,
|
||||
# not a 100-row sample. Mirrors Dashboard's "server-aggregated
|
||||
# counts" fix from commit 59c3275.
|
||||
def count_claims(
|
||||
self,
|
||||
*,
|
||||
batch_id: str | None = None,
|
||||
status: str | None = None,
|
||||
provider_npi: str | None = None,
|
||||
payer: str | None = None,
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
) -> int:
|
||||
"""Count claims that would be returned by ``iter_claims``.
|
||||
|
||||
Same filter parameters as :meth:`iter_claims` (excluding
|
||||
``sort``/``order``/``limit``/``offset``, which don't affect
|
||||
cardinality).
|
||||
"""
|
||||
rows = self.iter_claims(
|
||||
batch_id=batch_id, status=status, provider_npi=provider_npi,
|
||||
payer=payer, date_from=date_from, date_to=date_to,
|
||||
sort=None, order="desc", limit=_ITER_UNBOUNDED, offset=0,
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
def count_remittances(
|
||||
self,
|
||||
*,
|
||||
batch_id: str | None = None,
|
||||
payer: str | None = None,
|
||||
claim_id: str | None = None,
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
) -> int:
|
||||
"""Count remittances that would be returned by ``iter_remittances``.
|
||||
|
||||
Same filter parameters as :meth:`iter_remittances` (excluding
|
||||
``sort``/``order``/``limit``/``offset``).
|
||||
"""
|
||||
rows = self.iter_remittances(
|
||||
batch_id=batch_id, payer=payer, claim_id=claim_id,
|
||||
date_from=date_from, date_to=date_to,
|
||||
sort=None, order="desc", limit=_ITER_UNBOUNDED, offset=0,
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
def distinct_providers(self) -> list[dict]:
|
||||
"""Group claims by NPI and return one row per provider."""
|
||||
with db.SessionLocal()() as s:
|
||||
|
||||
Reference in New Issue
Block a user