diff --git a/README.md b/README.md index 89a11af..0f04602 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,69 @@ npm run build npm test ``` +## Live updates + +The Claims, Remittances, and Activity pages stay current without +manual refresh. The backend publishes an internal event on every +store write, the page opens a streaming HTTP connection to the +matching `/api//stream` endpoint, and new rows are +appended to the table the moment they hit the database. + +### Wire format + +Each stream endpoint emits newline-delimited JSON. The first batch +is the **snapshot** of currently-known rows; after that comes +**`snapshot_end`** with the count, then the **live** events. + +```json +{"type":"item","data":{"id":"CLM-1", "...":"..."}} +{"type":"item","data":{"id":"CLM-2", "...":"..."}} +{"type":"snapshot_end","data":{"count":2}} +{"type":"item","data":{"id":"CLM-3", "...":"..."}} ← live +{"type":"heartbeat","data":{"ts":"2026-06-20T23:17:09Z"}} ← idle keep-alive +``` + +Lines are `{"type": ..., "data": ...}`; known types are `item`, +`snapshot_end`, `heartbeat`, and (rare) `item_dropped` / +`error`. Heartbeats keep the connection alive when nothing is +happening — clients flip to `stalled` after 30s of total silence +(heartbeat or otherwise) and surface a **↻ Reconnect** button. + +### Endpoints + +| Method | Path | Subscribes to | Default sort | +| ------ | -------------------------- | ------------------- | ------------------- | +| GET | `/api/claims/stream` | `claim_written` | `-submission_date` | +| GET | `/api/remittances/stream` | `remittance_written`| `-received_date` | +| GET | `/api/activity/stream` | `activity_recorded` | `-timestamp` (limit 50) | + +All three accept the same query params as their non-streaming +counterparts (`status`, `payer`, `date_from`, …) so a frontend can +swap a one-shot fetch for a tail with no URL surgery. Responses +are `Content-Type: application/x-ndjson`. + +### Status pill + +The pill in each toolbar reflects the current connection state: + +| Status | Badge variant | What it means | +| ------------- | ------------- | ------------------------------------------------------------------ | +| `live` | success | snapshot received, listening for new events | +| `connecting` | warning | opening the stream (initial mount) | +| `reconnecting`| warning | previous attempt failed, backing off before retry | +| `stalled` | destructive | no event (including heartbeat) for 30s — click **↻ Reconnect** | +| `error` | destructive | stream errored — click **↻ Reconnect** | +| `closed` | destructive | page unmounted | + +The reconnect button is only shown on `stalled` and `error`. The +backoff schedule on error is `1s → 2s → 4s → 8s → 16s → 30s` capped. + +### Knobs + +| Env var | Default | Effect | +| ---------------------------- | ------- | ------------------------------------------------------- | +| `CYCLONE_TAIL_HEARTBEAT_S` | `15` | Idle heartbeat interval (seconds). Lower it for tests. | + ## Persistence Parsed batches, claims, remittances, matches, and activity events are @@ -94,8 +157,9 @@ backup API). . ├── backend/ │ ├── src/cyclone/ -│ │ ├── api.py # FastAPI app, GET + parse routes -│ │ ├── store.py # InMemoryStore, mappers +│ │ ├── api.py # FastAPI app, GET + parse routes, /api/{resource}/stream +│ │ ├── pubsub.py # in-process EventBus (drop-oldest, per-kind fan-out) +│ │ ├── store.py # InMemoryStore, mappers, publish-on-write │ │ ├── __main__.py # `python -m cyclone serve` │ │ ├── cli.py # click CLI │ │ └── parsers/ # X12 tokenizer, models, validator, writers @@ -105,14 +169,20 @@ backup API). │ ├── test_api_gets.py # 6 GET endpoints │ ├── test_store.py # store + mappers + iterators │ ├── test_api_streaming.py +│ ├── test_api_stream_live.py # 3 live-tail endpoints + disconnect cleanup +│ ├── test_pubsub.py # EventBus + subscribe/unsubscribe │ └── test_api_parse_persists.py ├── src/ # React + Vite + TypeScript UI │ ├── components/ -│ │ └── ui/ # Skeleton, EmptyState, ErrorState, FilterChips, Pagination, … +│ │ ├── ui/ # Skeleton, EmptyState, ErrorState, FilterChips, Pagination, … +│ │ └── TailStatusPill.tsx # live-tail status badge + reconnect button │ ├── pages/ # Claims, Remittances, Providers, Activity, Upload │ ├── hooks/ # useBatches, useClaims, useRemittances, useProviders, useActivity, useParse +│ │ # + useTailStream, useMergedTail (live tail) │ ├── lib/ # api.ts (6 GET + parse837/parse835/health), format.ts, utils.ts +│ │ # + tail-stream.ts (NDJSON parser) │ ├── store/ # zustand sample-data + parsed-batches store +│ │ # + tail-store.ts (FIFO-capped live tail slices) │ └── types/ # shared TS types ├── docs/ │ ├── reference/ # condensed 837P/835/X12/CO Medicaid notes @@ -123,7 +193,7 @@ backup API). ## Roadmap -Sub-projects 2, 3, and the first half of 4 are **shipped**. Next up: +Sub-projects 2, 3, 4, and 5 are **shipped**. Next up: - **Sub-project 3 (shipped) — More 837P/835 features.** - **837P validation rules:** R034 enforces `REF*G1` on frequency-code @@ -154,9 +224,23 @@ Sub-projects 2, 3, and the first half of 4 are **shipped**. Next up: (404) states are all distinct. Powered by a new backend endpoint `GET /api/claims/{claim_id}` that returns the full drawer payload in a single round-trip. - - **Remaining SP4:** batch diff view, real-time streaming of NDJSON - GET responses, advanced filters (date range, multi-status, saved - filter sets). + - **Remaining SP4:** batch diff view, advanced filters (date range, + multi-status, saved filter sets). +- **Sub-project 5 (shipped) — Live updates.** + - The Claims, Remittances, and Activity pages stream new rows in + real time over Server-Sent Events encoded as newline-delimited + JSON. The backend publishes a `claim_written` / + `remittance_written` / `activity_recorded` event on every store + write; the frontend opens an HTTP stream and dispatches each + event into a per-resource Zustand store that the page reads on + top of its base query results. + - A small status pill in each toolbar surfaces the connection + state — `live` / `connecting` / `reconnecting` / `stalled` / + `error` / `closed` — and offers a manual **↻ Reconnect** button + on `stalled` or `error`. Stale connections (no event for 30s, + heartbeat included) flip to `stalled` automatically; the + back-off ladder on errors is 1s → 2s → 4s → 8s → 16s → 30s + capped. See the "Live updates" section below for details. ### SP3 endpoints @@ -184,6 +268,17 @@ ACKs and lets you download the regenerated 999 text. when the claim doesn't exist — distinct from a transient fetch failure so the UI can render a dedicated not-found state. +### SP5 endpoints (live updates) + +- `GET /api/claims/stream` — NDJSON stream of claim events. Emits a + snapshot (filtered by the same query params as `GET /api/claims`), + then `snapshot_end`, then live `claim_written` events as they + arrive, with a 15s idle heartbeat. +- `GET /api/remittances/stream` — same shape for remittances + (subscribes to `remittance_written`, default sort `-received_date`). +- `GET /api/activity/stream` — same shape for activity events + (subscribes to `activity_recorded`, default `limit=50`). + ## License No license file yet; this is internal-use software. Add a `LICENSE` file diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 107dde3..a946363 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ dev = [ "pytest>=8.0", "pytest-cov>=4.1", + "pytest-asyncio>=0.23,<1", "httpx>=0.27,<1", ] @@ -32,3 +33,4 @@ where = ["src"] [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-ra --strict-markers" +asyncio_mode = "auto" diff --git a/backend/src/cyclone/__main__.py b/backend/src/cyclone/__main__.py index 7632e8c..153b5f3 100644 --- a/backend/src/cyclone/__main__.py +++ b/backend/src/cyclone/__main__.py @@ -27,10 +27,8 @@ def main() -> None: ] if reload: sys.argv.append("--reload") - # Initialize the DB (engine, migrations, schema) before the app - # boots so the first request hits a populated database. - from cyclone import db - db.init_db() + # DB + EventBus are initialized by the FastAPI lifespan handler + # (cyclone.api:lifespan); no explicit pre-init is needed here. import uvicorn uvicorn.main() diff --git a/backend/src/cyclone/api.py b/backend/src/cyclone/api.py index f8cf591..65fbb58 100644 --- a/backend/src/cyclone/api.py +++ b/backend/src/cyclone/api.py @@ -15,10 +15,13 @@ plus GET/POST with any header. from __future__ import annotations +import asyncio import json import logging +import os import uuid -from typing import Any, Iterator +from contextlib import asynccontextmanager +from typing import Any, AsyncIterator from fastapi import FastAPI, File, HTTPException, Query, Request, UploadFile from fastapi.middleware.cors import CORSMiddleware @@ -47,6 +50,7 @@ from cyclone.parsers.serialize_270 import serialize_270 from cyclone.parsers.serialize_999 import serialize_999 from cyclone.parsers.batch_ack_builder import build_ack_for_batch from cyclone.parsers.validator_835 import validate as validate_835 +from cyclone.pubsub import EventBus from cyclone.store import ( AlreadyMatchedError, BatchRecord, @@ -58,6 +62,29 @@ from cyclone.store import ( log = logging.getLogger(__name__) +def _ndjson_line(event: dict) -> bytes: + """Serialize one event dict as a single NDJSON line (UTF-8, trailing ``\\n``). + + Used by the live-tail streaming endpoints to emit a uniform wire format + that the frontend ``tail-stream.ts`` parser can split on newlines. + Compact separators keep each line small and avoid ambiguity with embedded + whitespace. + """ + return (json.dumps(event, separators=(",", ":")) + "\n").encode("utf-8") + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator[None]: + """Initialize per-process resources (DB + EventBus) before the app + serves requests. No teardown needed for Cyclone's local-only posture. + """ + from cyclone import db + + db.init_db() + app.state.event_bus = EventBus() + yield + + # Mirror cli._PAYER_FACTORIES. Kept local so the API doesn't depend on the # Click-based CLI module. PAYER_FACTORIES: dict[str, Any] = { @@ -76,6 +103,7 @@ app = FastAPI( title="Cyclone 837P / 835 Parser API", version=__version__, description="Upload X12 837P and 835 files and receive parsed records as JSON or NDJSON.", + lifespan=lifespan, ) app.add_middleware( @@ -264,7 +292,7 @@ async def parse_837( parsed_at=utcnow(), result=result, ) - store.add(rec) + store.add(rec, event_bus=request.app.state.event_bus) if _client_wants_json(request): body = json.loads(result.model_dump_json()) @@ -472,7 +500,7 @@ async def parse_835_endpoint( parsed_at=utcnow(), result=result, ) - store.add(rec) + store.add(rec, event_bus=request.app.state.event_bus) if _client_wants_json(request): body = json.loads(result.model_dump_json()) @@ -712,6 +740,123 @@ def list_claims( } +# --------------------------------------------------------------------------- # +# Live-tail NDJSON streaming endpoints (Phase 3 — SP5) +# --------------------------------------------------------------------------- # + + +def _heartbeat_seconds() -> float: + """Return the configured tail heartbeat interval. + + Read from ``CYCLONE_TAIL_HEARTBEAT_S`` at call time so tests can + monkeypatch the env var without reloading the module. Defaults to + 15s (the production cadence); tests override to a small value (e.g. + 0.2s) to keep their runtime bounded. + """ + raw = os.environ.get("CYCLONE_TAIL_HEARTBEAT_S", "15") + try: + v = float(raw) + except ValueError: + return 15.0 + return v if v > 0 else 15.0 + + +async def _tail_events( + request: Request, bus: EventBus, kinds: list[str] +) -> AsyncIterator[bytes]: + """Forward subscribed events as ``item`` lines with periodic heartbeats. + + Polls the underlying ``asyncio.Queue`` directly (via + :meth:`EventBus.subscribe_raw`) instead of awaiting the bus's + async-iterator wrapper. ``asyncio.wait_for`` cancels the inner + future on timeout, which would otherwise terminate the bus + iterator at its ``await`` point and break subsequent + ``__anext__`` calls with ``StopAsyncIteration``. Polling + ``queue.get()`` is idempotent under cancellation, so heartbeats + don't poison the subscription. + + A ``try/finally`` unsubscribes the queue from the bus when the + caller disconnects or the generator is garbage collected — + otherwise the bus would leak one queue per open stream. + """ + hb_s = _heartbeat_seconds() + queue, _sub = bus.subscribe_raw(kinds) + try: + while True: + if await request.is_disconnected(): + return + get_task = asyncio.ensure_future(queue.get()) + sleep_task = asyncio.ensure_future(asyncio.sleep(hb_s)) + try: + done, pending = await asyncio.wait( + {get_task, sleep_task}, + return_when=asyncio.FIRST_COMPLETED, + ) + except BaseException: + get_task.cancel() + sleep_task.cancel() + raise + for t in pending: + t.cancel() + if get_task in done: + event = get_task.result() + yield _ndjson_line({"type": "item", "data": event}) + else: + yield _ndjson_line({ + "type": "heartbeat", + "data": {"ts": utcnow().isoformat().replace("+00:00", "Z")}, + }) + finally: + bus.unsubscribe(queue, kinds) + + +@app.get("/api/claims/stream") +async def claims_stream( + request: Request, + status: str | None = Query(None), + provider_npi: str | None = Query(None), + payer: str | None = Query(None), + date_from: str | None = Query(None), + date_to: str | None = Query(None), + sort: str | None = Query(None), + order: str = Query("desc"), + limit: int = Query(100, ge=1, le=1000), +) -> StreamingResponse: + """Stream Claims as NDJSON: snapshot first, then live events. + + Wire format: + * ``{"type":"item","data":}`` per snapshot row, then per + new ``claim_written`` event + * ``{"type":"snapshot_end","data":{"count":N}}`` after the snapshot + * ``{"type":"heartbeat","data":{"ts":}}`` every + ``CYCLONE_TAIL_HEARTBEAT_S`` seconds when idle + + Query params mirror :func:`list_claims` so a frontend can swap a + one-shot fetch for a tail with no URL surgery. + + NOTE: registered before ``/api/claims/{claim_id}`` so the literal + ``stream`` path segment doesn't get matched as a claim id. + """ + bus: EventBus = request.app.state.event_bus + + async def gen() -> AsyncIterator[bytes]: + # 1. Snapshot (eager — iter_claims returns a list already). + rows = store.iter_claims( + status=status, provider_npi=provider_npi, payer=payer, + date_from=date_from, date_to=date_to, + sort=sort or "-submission_date", order=order, limit=limit, + ) + for row in rows: + yield _ndjson_line({"type": "item", "data": row}) + yield _ndjson_line({"type": "snapshot_end", "data": {"count": len(rows)}}) + + # 2. Subscribe + heartbeats. + async for chunk in _tail_events(request, bus, ["claim_written"]): + yield chunk + + return StreamingResponse(gen(), media_type="application/x-ndjson") + + @app.get("/api/claims/{claim_id}") def get_claim_detail_endpoint(claim_id: str) -> dict: """Return one claim with full drawer context (SP4). @@ -865,6 +1010,45 @@ def list_remittances( } +@app.get("/api/remittances/stream") +async def remittances_stream( + request: Request, + payer: str | None = Query(None), + claim_id: str | None = Query(None), + date_from: str | None = Query(None), + date_to: str | None = Query(None), + sort: str | None = Query(None), + order: str = Query("desc"), + limit: int = Query(100, ge=1, le=1000), +) -> StreamingResponse: + """Stream Remittances as NDJSON: snapshot first, then live events. + + Subscribes to ``remittance_written``. Default sort is + ``-received_date`` (newest-first), matching the list endpoint's + most common sort. + + NOTE: registered before ``/api/remittances/{remittance_id}`` so + the literal ``stream`` path segment doesn't get matched as a + remittance id. + """ + bus: EventBus = request.app.state.event_bus + + async def gen() -> AsyncIterator[bytes]: + rows = store.iter_remittances( + payer=payer, claim_id=claim_id, + date_from=date_from, date_to=date_to, + sort=sort or "-received_date", order=order, limit=limit, + ) + for row in rows: + yield _ndjson_line({"type": "item", "data": row}) + yield _ndjson_line({"type": "snapshot_end", "data": {"count": len(rows)}}) + + async for chunk in _tail_events(request, bus, ["remittance_written"]): + yield chunk + + return StreamingResponse(gen(), media_type="application/x-ndjson") + + @app.get("/api/remittances/{remittance_id}") def get_remittance(remittance_id: str) -> dict: """Return one remittance with its labeled CAS ``adjustments`` array. @@ -939,6 +1123,43 @@ def list_activity( } +@app.get("/api/activity/stream") +async def activity_stream( + request: Request, + kind: str | None = Query(None), + since: str | None = Query(None), + limit: int = Query(50, ge=1, le=500), +) -> StreamingResponse: + """Stream Activity events as NDJSON: snapshot first, then live events. + + Subscribes to ``activity_recorded``. Default ``limit`` is 50 + (smaller than the list endpoint's 200) because activity is + high-volume — callers usually want the most recent handful, not a + full replay. + """ + bus: EventBus = request.app.state.event_bus + + async def gen() -> AsyncIterator[bytes]: + # Snapshot reuses the same in-memory filter as ``list_activity`` + # so the two endpoints are interchangeable for the snapshot + # half. + events = store.recent_activity(limit=limit) + if kind is not None: + events = [e for e in events if e["kind"] == kind] + if since is not None: + events = [e for e in events if e["timestamp"] >= since] + for ev in events: + yield _ndjson_line({"type": "item", "data": ev}) + yield _ndjson_line({ + "type": "snapshot_end", "data": {"count": len(events)}, + }) + + async for chunk in _tail_events(request, bus, ["activity_recorded"]): + yield chunk + + return StreamingResponse(gen(), media_type="application/x-ndjson") + + # --------------------------------------------------------------------------- # # 999 ACKs (read views) # --------------------------------------------------------------------------- # diff --git a/backend/src/cyclone/pubsub.py b/backend/src/cyclone/pubsub.py new file mode 100644 index 0000000..9b0e7b3 --- /dev/null +++ b/backend/src/cyclone/pubsub.py @@ -0,0 +1,113 @@ +"""In-process pub/sub for live-tail event streaming. + +``EventBus`` is a tiny, async-first fan-out broker. Publishers call +``publish(kind, payload)`` and never block; if a subscriber's per-kind +queue is full, the oldest event is dropped so the slow consumer cannot +stall the producer. Subscribers receive events through an async +iterator that yields ``{**payload, "_kind": kind}``. + +The bus is intentionally not thread-safe: it is designed to run on a +single asyncio event loop, which matches the FastAPI/uvicorn +deployment model used elsewhere in this project. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator + + +class EventBus: + """Per-kind fan-out bus with drop-oldest overflow per subscriber.""" + + def __init__(self, max_queue_size: int = 256) -> None: + self._max_queue_size = max_queue_size + self._subscribers: dict[str, list[asyncio.Queue[dict]]] = {} + + async def publish(self, kind: str, payload: dict) -> None: + """Fan ``payload`` out to every subscriber of ``kind``. + + Never blocks. If a subscriber queue is full, the oldest queued + event is discarded to make room. + """ + event = {**payload, "_kind": kind} + for queue in list(self._subscribers.get(kind, ())): + self._enqueue_or_drop_oldest(queue, event) + + def _enqueue_or_drop_oldest( + self, queue: asyncio.Queue[dict], event: dict + ) -> None: + try: + queue.put_nowait(event) + except asyncio.QueueFull: + # Drop the oldest event to make room, then enqueue. + queue.get_nowait() + queue.task_done() + queue.put_nowait(event) + + def subscribe(self, kinds: list[str]) -> AsyncIterator[dict]: + """Return an async iterator delivering events for any of ``kinds``.""" + queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=self._max_queue_size) + for kind in kinds: + self._subscribers.setdefault(kind, []).append(queue) + return self._iterator(queue) + + def subscribe_raw( + self, kinds: list[str] + ) -> tuple[asyncio.Queue[dict], AsyncIterator[dict]]: + """Like :meth:`subscribe` but also returns the underlying queue. + + Most consumers should use :meth:`subscribe`. The live-tail + endpoints use ``subscribe_raw`` so they can race + ``queue.get()`` against a heartbeat sleep without poisoning + the subscription iterator: ``asyncio.wait_for`` cancels the + inner future on timeout, which terminates an async generator + at its ``await`` point and breaks all subsequent + ``__anext__`` calls. Polling ``queue.get()`` directly avoids + that because ``Queue.get`` is idempotent under cancellation. + + Callers MUST pair this with :meth:`unsubscribe` in a + ``try/finally`` to release the queue when the consumer + disconnects — otherwise the bus leaks a queue per stream. + """ + queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=self._max_queue_size) + for kind in kinds: + self._subscribers.setdefault(kind, []).append(queue) + return queue, self._iterator(queue) + + def unsubscribe( + self, queue: asyncio.Queue[dict], kinds: list[str] + ) -> None: + """Remove ``queue`` from each kind's subscriber list. + + Idempotent: missing entries are silently ignored so callers + can run the cleanup in a ``finally`` without worrying about + double-unsubscribe on disconnect-then-GeneratorExit. + """ + for kind in kinds: + subs = self._subscribers.get(kind) + if not subs: + continue + try: + subs.remove(queue) + except ValueError: + pass + if not subs: + self._subscribers.pop(kind, None) + + async def _iterator(self, queue: asyncio.Queue[dict]) -> AsyncIterator[dict]: + while True: + yield await queue.get() + queue.task_done() + + +def get_event_bus() -> EventBus: + """Return the process-wide EventBus attached to the FastAPI app state. + + Late-imports ``cyclone.api`` to avoid a circular import at module + load time (the API module imports pubsub indirectly via lifespan). + Tests stub this via monkeypatch. + """ + from cyclone.api import app # late import to avoid circular + + return app.state.event_bus diff --git a/backend/src/cyclone/store.py b/backend/src/cyclone/store.py index 9f11286..7c3c247 100644 --- a/backend/src/cyclone/store.py +++ b/backend/src/cyclone/store.py @@ -809,7 +809,12 @@ class CycloneStore: # -- write path ----------------------------------------------------- - def add(self, record: BatchRecord) -> None: + def add( + self, + record: BatchRecord, + *, + event_bus: "EventBus | None" = None, + ) -> None: """Persist a parsed batch (837P or 835) to the DB. For 837P batches: inserts the Batch row, one Claim row per @@ -830,10 +835,22 @@ class CycloneStore: has a fresh ``uuid4`` id from the API). O(n) per row, but acceptable for the small fixture sizes — production load is one batch at a time via the API, not bulk inserts. + + When ``event_bus`` is provided, publishes one ``claim_written`` + or ``remittance_written`` event per newly-inserted row plus an + ``activity_recorded`` event per activity row, after commit. The + publish calls are best-effort — failures are logged but do not + roll back the persisted batch. """ + from cyclone.pubsub import EventBus + import logging log = logging.getLogger(__name__) + # Track rows we actually inserted so we can publish events for them. + inserted_claim_ids: list[str] = [] + inserted_remit_ids: list[str] = [] + with db.SessionLocal()() as s: batch_row = Batch( id=record.id, @@ -870,6 +887,7 @@ class CycloneStore: "amount": float(claim.claim.total_charge or 0.0), }, )) + inserted_claim_ids.append(claim.claim_id) elif isinstance(record, BatchRecord835): result835: ParseResult835 = record.result payer_name = result835.payer.name @@ -903,6 +921,7 @@ class CycloneStore: "amount": float(cp.total_paid or 0.0), }, )) + inserted_remit_ids.append(cp.payer_claim_control_number) else: raise TypeError( f"Unsupported BatchRecord subclass: {type(record).__name__}" @@ -922,6 +941,83 @@ class CycloneStore: "reconcile.run failed for batch %s", record.id, ) + # Publish live-tail events synchronously. EventBus.publish is async + # but its body is purely synchronous ``put_nowait`` enqueues; we + # bypass the async wrapper and call the internal enqueue directly + # so callers (sync FastAPI endpoints, sync test harnesses) don't + # need to await. + if event_bus is not None and (inserted_claim_ids or inserted_remit_ids): + self._publish_events_sync( + event_bus, record, inserted_claim_ids, inserted_remit_ids, + ) + + def _publish_events_sync( + self, + event_bus: "EventBus", + record: BatchRecord, + claim_ids: list[str], + remit_ids: list[str], + ) -> None: + """Build UI-shaped payloads for newly-inserted rows and publish. + + Runs after commit so subscribers can immediately re-fetch from + the API and see consistent data. Each ``claim_written`` / + ``remittance_written`` payload is identical to what the matching + list endpoint would return for that row. + + This is sync because EventBus's enqueue path is sync; we don't + need a coroutine for ``put_nowait``. + """ + import logging + log = logging.getLogger(__name__) + + try: + with db.SessionLocal()() as s: + for cid in claim_ids: + row = s.get(Claim, cid) + if row is None: + continue + ui = to_ui_claim_from_orm( + row, batch_id=row.batch_id or record.id, + parsed_at=record.parsed_at, + ) + self._sync_publish(event_bus, "claim_written", ui) + for rid in remit_ids: + row = s.get(Remittance, rid) + if row is None: + continue + ui = to_ui_remittance_from_orm( + row, batch_id=row.batch_id or record.id, + parsed_at=record.parsed_at, + ) + self._sync_publish(event_bus, "remittance_written", ui) + # Activity events for this batch. + from sqlalchemy import select + activity_rows = s.execute( + select(ActivityEvent).where(ActivityEvent.batch_id == record.id) + ).scalars().all() + for arow in activity_rows: + ui = { + "kind": arow.kind, + "ts": arow.ts.isoformat().replace("+00:00", "Z"), + "batchId": arow.batch_id, + "claimId": arow.claim_id, + "remittanceId": arow.remittance_id, + "payload": arow.payload_json, + } + self._sync_publish(event_bus, "activity_recorded", ui) + except Exception: + log.exception("add: event publish failed for batch %s", record.id) + + @staticmethod + def _sync_publish(event_bus: "EventBus", kind: str, payload: dict) -> None: + """Synchronous fan-out helper. Mirrors ``EventBus.publish`` but + bypasses the async wrapper so callers don't need an event loop. + """ + event = {**payload, "_kind": kind} + for queue in list(event_bus._subscribers.get(kind, ())): + event_bus._enqueue_or_drop_oldest(queue, event) + def _run_reconcile(self, batch_id: str) -> None: """T10 stub: invoke the reconcile orchestrator for a batch. diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index bc23067..0ee2b75 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -19,10 +19,22 @@ import pytest @pytest.fixture(autouse=True) def _auto_init_db(tmp_path, monkeypatch): - """Point CYCLONE_DB_URL at a per-test SQLite file and init the schema.""" + """Point CYCLONE_DB_URL at a per-test SQLite file and init the schema. + + Also wires a fresh ``EventBus`` onto ``app.state`` because ``TestClient`` + does not invoke the FastAPI lifespan handler unless used as a context + manager. The bus is reset between tests so subscribers don't leak. + """ monkeypatch.setenv("CYCLONE_DB_URL", f"sqlite:///{tmp_path}/test.db") from cyclone import db + from cyclone.api import app + from cyclone.pubsub import EventBus + db._reset_for_tests() db.init_db() - yield - db._reset_for_tests() \ No newline at end of file + app.state.event_bus = EventBus() + try: + yield + finally: + app.state.event_bus = None + db._reset_for_tests() \ No newline at end of file diff --git a/backend/tests/test_api_stream_live.py b/backend/tests/test_api_stream_live.py new file mode 100644 index 0000000..e825324 --- /dev/null +++ b/backend/tests/test_api_stream_live.py @@ -0,0 +1,607 @@ +"""Live-tail NDJSON streaming endpoint tests (Phase 3). + +Each ``/api/.../stream`` endpoint: + 1. yields the current snapshot as ``{"type":"item","data":}`` lines + 2. terminates the snapshot with ``{"type":"snapshot_end","data":{"count":N}}`` + 3. subscribes to the relevant EventBus kind and forwards new events + 4. emits a ``{"type":"heartbeat","data":{"ts":}}`` line every + ``CYCLONE_TAIL_HEARTBEAT_S`` seconds (default 15) when idle + 5. returns cleanly when the client disconnects + 6. responds with ``application/x-ndjson`` + +Implementation note (deviation from plan) +----------------------------------------- +The plan suggested using ``httpx.AsyncClient(transport=ASGITransport(app))`` +and ``client.stream()``. That doesn't work for true long-lived streams: + +* ``httpx.ASGITransport``'s response is buffered (``ASGIResponseStream`` + joins every body chunk at the end), so ``client.stream()`` blocks until + the app finishes — a live tail never finishes. +* ``starlette.requests.Request.is_disconnected()`` relies on a + ``http.disconnect`` message in the ASGI receive channel, which + ``ASGITransport`` only delivers *after* the response body completes + (the receive coroutine awaits ``response_complete``). So even if the + caller closes the client side, the endpoint never observes a + disconnect. + +Instead, we call the endpoint coroutine directly, build a synthetic +``Request`` and ``StreamingResponse``, and iterate ``body_iterator`` +ourselves. That gives us byte-by-byte streaming, a real disconnect +signal via ``body_iterator.aclose()`` (which throws ``CancelledError`` +into the generator), and works for arbitrarily long-lived tails. + +The public surface (snapshot + subscribe + heartbeat + disconnect +cleanup) is verified exactly as the plan requires; only the transport +differs. +""" + +from __future__ import annotations + +import asyncio +import json +from datetime import date, datetime, timezone +from decimal import Decimal + +import pytest +from starlette.requests import Request + +from cyclone.api import app, claims_stream, remittances_stream, activity_stream +from cyclone.parsers.models import ( + BatchSummary, + BillingProvider, + ClaimHeader, + ClaimOutput, + Envelope, + Payer, + ParseResult, + Subscriber, + ValidationReport, +) +from cyclone.parsers.models_835 import ( + ClaimAdjustment, + ClaimPayment, + Envelope as Envelope835, + FinancialInfo, + ParseResult835, + Payer835, + Payee835, + ReassociationTrace, + ServicePayment, + BatchSummary as BatchSummary835, +) +from cyclone.pubsub import EventBus +from cyclone.store import ( + BatchRecord, + BatchRecord835, + CycloneStore, +) + + +@pytest.fixture(autouse=True) +def _short_heartbeat(monkeypatch): + """Set a 0.2s heartbeat for all stream tests in this module. + + The production default is 15s. Tests use a tight value so that + generators sitting idle after the test reads its lines exit + promptly — ``aclose()`` on the body iterator cancels the + generator and the heartbeat timeout is the de-facto idle + signal in these tests. + """ + monkeypatch.setenv("CYCLONE_TAIL_HEARTBEAT_S", "0.2") + + +# --------------------------------------------------------------------------- # +# Test helpers +# --------------------------------------------------------------------------- # + + +def _make_claim_837(claim_id: str = "CLM-1", charge: str = "124.00") -> ClaimOutput: + return ClaimOutput( + claim_id=claim_id, + control_number="0001", + transaction_date=date(2026, 6, 19), + billing_provider=BillingProvider(name="Test", npi="1234567890"), + subscriber=Subscriber( + first_name="Jane", last_name="Doe", member_id=f"M-{claim_id}", + ), + payer=Payer(name="Test Payer", id="P1"), + claim=ClaimHeader( + claim_id=claim_id, total_charge=Decimal(charge), + frequency_code="1", place_of_service="11", + ), + diagnoses=[], + service_lines=[], + validation=ValidationReport(passed=True, errors=[], warnings=[]), + raw_segments=[], + ) + + +def _make_result_837(n: int = 3, batch_id: str = "b-uuid-1") -> ParseResult: + claims = [ + _make_claim_837(f"CLM-{i}", charge=str(100 + i)) + for i in range(1, n + 1) + ] + return ParseResult( + envelope=Envelope( + sender_id="S", receiver_id="R", control_number="0001", + transaction_date=date(2026, 6, 19), + ), + claims=claims, + summary=BatchSummary( + input_file="test.txt", control_number="0001", + transaction_date=date(2026, 6, 19), + total_claims=n, passed=n, failed=0, + ), + ) + + +def _make_batch_record(kind: str = "837p", result=None, batch_id: str = "b-uuid-1", + n: int = 3): + return BatchRecord( + id=batch_id, kind=kind, input_filename="test.txt", + parsed_at=datetime(2026, 6, 19, 12, 0, tzinfo=timezone.utc), + result=result if result is not None else _make_result_837(n=n), + ) + + +def _make_remit_with_cas(remit_id="CLP-1", status="1", charge="124.00", + paid="62.00", cas_amount="62.00", pcn=None): + cp = ClaimPayment( + payer_claim_control_number=pcn if pcn is not None else remit_id, + status_code=status, + status_label="Primary", + total_charge=charge, total_paid=paid, + service_payments=[ + ServicePayment( + line_number=1, procedure_qualifier="HC", procedure_code="99213", + charge=charge, payment=paid, + adjustments=[ClaimAdjustment(group_code="CO", reason_code="45", + amount=cas_amount)], + ), + ], + ) + return cp + + +def _make_835_result(claims): + return ParseResult835( + envelope=Envelope835( + sender_id="S", receiver_id="R", control_number="0001", + transaction_date=date(2026, 6, 19), + ), + financial_info=FinancialInfo( + handling_code="C", paid_amount=Decimal("0"), + credit_debit_flag="C", payment_method=None, + ), + trace=ReassociationTrace( + trace_type_code="1", trace_number="0001", + originating_company_id="S", + ), + payer=Payer835(name="X", id="SKCO0"), + payee=Payee835(name="Y", npi="1234567890"), + claims=claims, + summary=BatchSummary835( + input_file="era.txt", control_number="0001", + transaction_date=date(2026, 6, 19), + total_claims=len(claims), passed=len(claims), failed=0, + ), + ) + + +def _make_request(path: str = "/api/claims/stream", + query_string: bytes = b"") -> Request: + """Build a Starlette ``Request`` wired to ``app.state`` with a no-op + receive channel so the endpoint's ``is_disconnected()`` polling + works without raising ``RuntimeError: Receive channel has not + been made available``. + """ + # ``receive`` blocks until a message is delivered. We never deliver + # one, so the cancel scope inside ``is_disconnected()`` always + # returns ``False`` — exactly what we want for tests that close + # via ``body_iterator.aclose()`` rather than the disconnect path. + receive_started = asyncio.Event() + + async def receive(): + receive_started.set() + # Block forever; is_disconnected's cancel scope will cancel + # this await and we'll return None (treated as "no message"). + await asyncio.Event().wait() + + scope = { + "type": "http", + "method": "GET", + "headers": [], + "query_string": query_string, + "path": path, + "app": app, + "scheme": "http", + "server": ("testserver", 80), + "client": ("testclient", 50000), + } + return Request(scope, receive=receive) + + +def _resolve_query_defaults(endpoint) -> dict: + """Extract the actual default values from a FastAPI endpoint signature. + + ``Query(None)`` etc. are marker objects — when called directly we + need to substitute their ``.default`` so the endpoint function + sees the same values it would receive over HTTP. + """ + import inspect + sig = inspect.signature(endpoint) + resolved: dict = {} + for name, param in sig.parameters.items(): + default = param.default + if hasattr(default, "default"): + resolved[name] = default.default + else: + resolved[name] = default + return resolved + + +async def _call_endpoint(endpoint, path: str): + """Invoke ``endpoint(request, **defaults)`` with resolved defaults.""" + request = _make_request(path) + defaults = _resolve_query_defaults(endpoint) + # Drop ``request`` — it's the positional we pass explicitly. + defaults.pop("request", None) + return await endpoint(request, **defaults) + + +async def _read_lines(body_iter, n: int, timeout: float = 5.0) -> list[str]: + """Read up to ``n`` lines from a StreamingResponse body iterator.""" + buffer = b"" + lines: list[str] = [] + async with asyncio.timeout(timeout): + while len(lines) < n: + try: + chunk = await body_iter.__anext__() + except StopAsyncIteration: + break + buffer += chunk + while b"\n" in buffer and len(lines) < n: + raw, buffer = buffer.split(b"\n", 1) + if raw: + lines.append(raw.decode("utf-8")) + return lines + + +async def _read_one_line(body_iter, timeout: float = 5.0) -> str | None: + """Read a single NDJSON line from the body iterator, or None on timeout.""" + buffer = b"" + async with asyncio.timeout(timeout): + while True: + try: + chunk = await body_iter.__anext__() + except StopAsyncIteration: + return None + buffer += chunk + if b"\n" in buffer: + raw, _ = buffer.split(b"\n", 1) + if raw: + return raw.decode("utf-8") + + +async def _drain_until_disconnect(body_iter, max_chunks: int = 20) -> None: + """Close the body iterator and swallow the cancellation.""" + try: + await body_iter.aclose() + except (asyncio.CancelledError, GeneratorExit): + pass + + +async def _read_until_type( + body_iter, target: str, *, timeout: float = 5.0, max_lines: int = 50 +) -> dict | None: + """Read lines from the body iterator until one has ``type == target``. + + Used by tests that need to skip past heartbeats or multi-item + snapshot prefixes to assert on a specific event. Returns the first + matching parsed object, or ``None`` if ``max_lines`` is reached + without a match. + """ + buffer = b"" + seen = 0 + async with asyncio.timeout(timeout): + while seen < max_lines: + try: + chunk = await body_iter.__anext__() + except StopAsyncIteration: + return None + buffer += chunk + while b"\n" in buffer: + raw, buffer = buffer.split(b"\n", 1) + if not raw: + continue + seen += 1 + obj = json.loads(raw.decode("utf-8")) + if obj.get("type") == target: + return obj + return None + + +# --------------------------------------------------------------------------- # +# Task 11 — claims stream +# --------------------------------------------------------------------------- # + + +async def test_claims_stream_yields_snapshot_then_snapshot_end(): + """3 claims pre-loaded → 3 item lines + 1 snapshot_end line.""" + s = CycloneStore() + s.add(_make_batch_record(n=3)) + + response = await _call_endpoint(claims_stream, "/api/claims/stream") + assert response.media_type.startswith("application/x-ndjson") + + lines = await _read_lines(response.body_iterator, n=4) + items = [json.loads(line) for line in lines] + assert len(items) == 4 + # First three are items; the order isn't guaranteed without an + # explicit sort, so check shape not exact ordering. + for item in items[:3]: + assert item["type"] == "item" + assert "id" in item["data"] + # Fourth is the snapshot_end terminator. + assert items[3] == {"type": "snapshot_end", "data": {"count": 3}} + await _drain_until_disconnect(response.body_iterator) + + +async def test_claims_stream_emits_new_item_after_publish(): + """After snapshot_end, a bus.publish('claim_written', …) → new item line.""" + s = CycloneStore() + s.add(_make_batch_record(n=1)) + + response = await _call_endpoint(claims_stream, "/api/claims/stream") + # Drain snapshot phase: 1 item + 1 snapshot_end. + lines = await _read_lines(response.body_iterator, n=2) + assert json.loads(lines[1]) == { + "type": "snapshot_end", "data": {"count": 1}, + } + # Advance the generator one more step so ``subscribe_raw`` runs + # and registers the subscriber queue. The next yield will be a + # heartbeat (with the 0.2s env var set in some tests); we don't + # care about its content here, just that we've moved past the + # sync subscription code. + _ = await _read_one_line(response.body_iterator, timeout=2.0) + + # Now publish a claim_written event and assert a new item line + # streams out. + payload = { + "id": "NEW-CLM-1", + "patientName": "Live Tail", + "providerNpi": "1234567890", + "payerName": "Live Payer", + "cptCode": "99213", + "billedAmount": 999.99, + "receivedAmount": 0.0, + "status": "submitted", + "state": "submitted", + "denialReason": None, + "submissionDate": "2026-06-19T12:00:00Z", + "batchId": "b-uuid-1", + "parsedAt": "2026-06-19T12:00:00Z", + } + await app.state.event_bus.publish("claim_written", payload) + + new_line = await _read_one_line(response.body_iterator, timeout=1.0) + assert new_line is not None, "no item line arrived after publish" + obj = json.loads(new_line) + assert obj["type"] == "item" + assert obj["data"]["id"] == "NEW-CLM-1" + await _drain_until_disconnect(response.body_iterator) + + +async def test_claims_stream_heartbeat_after_idle(monkeypatch): + """With heartbeat=0.2s, a heartbeat line appears within ~1s of idle.""" + monkeypatch.setenv("CYCLONE_TAIL_HEARTBEAT_S", "0.2") + s = CycloneStore() + s.add(_make_batch_record(n=1)) + + response = await _call_endpoint(claims_stream, "/api/claims/stream") + # Drain snapshot (2 lines). + lines = await _read_lines(response.body_iterator, n=2) + assert json.loads(lines[1]) == { + "type": "snapshot_end", "data": {"count": 1}, + } + + # Read subsequent lines until we see a heartbeat or time out. + deadline = asyncio.get_event_loop().time() + 1.5 + saw_heartbeat = False + while asyncio.get_event_loop().time() < deadline: + new_line = await _read_one_line( + response.body_iterator, timeout=0.5, + ) + if new_line is None: + continue + obj = json.loads(new_line) + if obj["type"] == "heartbeat": + assert "ts" in obj["data"] + saw_heartbeat = True + break + assert saw_heartbeat, "no heartbeat line arrived within 1.5s" + await _drain_until_disconnect(response.body_iterator) + + +# --------------------------------------------------------------------------- # +# Task 12 — remittances stream +# --------------------------------------------------------------------------- # + + +async def test_remittances_stream_yields_snapshot_then_subscribes(): + """Pre-loaded 835 batch → snapshot (1 item + snapshot_end).""" + s = CycloneStore() + cp = _make_remit_with_cas(remit_id="CLP-1", pcn="PCN-1") + rec = BatchRecord835( + id="b-835-1", kind="835", input_filename="era.txt", + parsed_at=datetime(2026, 6, 19, 12, 0, tzinfo=timezone.utc), + result=_make_835_result([cp]), + ) + s.add(rec) + + response = await _call_endpoint( + remittances_stream, "/api/remittances/stream", + ) + assert response.media_type.startswith("application/x-ndjson") + # Read the full snapshot (1 item + snapshot_end), skipping nothing. + snap_end = await _read_until_type( + response.body_iterator, "snapshot_end", timeout=2.0, + ) + assert snap_end == {"type": "snapshot_end", "data": {"count": 1}} + await _drain_until_disconnect(response.body_iterator) + + +async def test_remittances_stream_emits_after_upsert_remittance(): + """A bus.publish('remittance_written', …) → new item line on stream.""" + s = CycloneStore() + cp = _make_remit_with_cas(remit_id="CLP-0", pcn="PCN-0") + rec = BatchRecord835( + id="b-835-0", kind="835", input_filename="era0.txt", + parsed_at=datetime(2026, 6, 19, 12, 0, tzinfo=timezone.utc), + result=_make_835_result([cp]), + ) + s.add(rec) + + response = await _call_endpoint( + remittances_stream, "/api/remittances/stream", + ) + # Drain snapshot to the terminator. + snap_end = await _read_until_type( + response.body_iterator, "snapshot_end", timeout=2.0, + ) + assert snap_end == {"type": "snapshot_end", "data": {"count": 1}} + # Advance one more step so the subscription is registered before + # we publish. The next yielded line may be a heartbeat (with the + # 0.2s heartbeat fixture) or a real event — either is fine, we + # just need ``subscribe_raw`` to have run. + _ = await _read_one_line(response.body_iterator, timeout=2.0) + + payload = { + "id": "PCN-NEW", + "claimId": "", + "payerName": "Tail Payer", + "paidAmount": 12.34, + "adjustmentAmount": 0.0, + "status": "received", + "denialReason": None, + "validationWarnings": [], + "receivedDate": "2026-06-19T12:00:00Z", + "batchId": "b-835-1", + "parsedAt": "2026-06-19T12:00:00Z", + "adjustments": [], + } + await app.state.event_bus.publish("remittance_written", payload) + + # Read until the new item line appears (skip any heartbeats that + # race in before the published event). + new_obj = await _read_until_type( + response.body_iterator, "item", timeout=2.0, + ) + assert new_obj is not None, "no item line arrived after publish" + assert new_obj["data"]["id"] == "PCN-NEW" + await _drain_until_disconnect(response.body_iterator) + + +# --------------------------------------------------------------------------- # +# Task 13 — activity stream +# --------------------------------------------------------------------------- # + + +async def test_activity_stream_yields_snapshot_then_subscribes(): + """An 837 batch seeds activity rows → snapshot has at least 1 item.""" + s = CycloneStore() + s.add(_make_batch_record(n=2)) + + response = await _call_endpoint( + activity_stream, "/api/activity/stream", + ) + assert response.media_type.startswith("application/x-ndjson") + # Read to snapshot_end — activity may have multiple items. + snap_end = await _read_until_type( + response.body_iterator, "snapshot_end", timeout=2.0, + ) + assert snap_end is not None + assert snap_end["data"]["count"] >= 1 + await _drain_until_disconnect(response.body_iterator) + + +async def test_activity_stream_emits_after_record_activity(): + """A bus.publish('activity_recorded', …) → new item line on stream.""" + s = CycloneStore() + s.add(_make_batch_record(n=1)) + + response = await _call_endpoint( + activity_stream, "/api/activity/stream", + ) + # Drain snapshot. + snap_end = await _read_until_type( + response.body_iterator, "snapshot_end", timeout=2.0, + ) + assert snap_end is not None + # Advance one step so subscription is registered. + _ = await _read_one_line(response.body_iterator, timeout=2.0) + + payload = { + "kind": "claim_submitted", + "ts": "2026-06-19T12:00:00Z", + "batchId": "b-uuid-1", + "claimId": "LIVE-CLM", + "remittanceId": None, + "payload": {"message": "live tail event"}, + } + await app.state.event_bus.publish("activity_recorded", payload) + + new_obj = await _read_until_type( + response.body_iterator, "item", timeout=2.0, + ) + assert new_obj is not None, "no item line arrived after publish" + assert new_obj["data"]["kind"] == "claim_submitted" + assert new_obj["data"]["claimId"] == "LIVE-CLM" + await _drain_until_disconnect(response.body_iterator) + + +# --------------------------------------------------------------------------- # +# Task 14 — disconnect cancels subscription +# --------------------------------------------------------------------------- # + + +async def test_client_disconnect_cancels_subscription(monkeypatch): + """After closing the body iterator, no claim_written subscribers remain. + + Closes the body iterator (the test-side disconnect signal — see + the module docstring). The generator must propagate the + ``CancelledError`` and tear down its bus subscription. + """ + monkeypatch.setenv("CYCLONE_TAIL_HEARTBEAT_S", "0.1") + s = CycloneStore() + s.add(_make_batch_record(n=1)) + + bus: EventBus = app.state.event_bus + response = await _call_endpoint(claims_stream, "/api/claims/stream") + # Read the full snapshot to the terminator. + snap_end = await _read_until_type( + response.body_iterator, "snapshot_end", timeout=2.0, + ) + assert snap_end == {"type": "snapshot_end", "data": {"count": 1}} + # Advance one step into the subscription phase so subscribe_raw + # has run and registered the queue. + _ = await _read_one_line(response.body_iterator, timeout=2.0) + + # Pre-condition: at least one subscriber registered while the + # stream was open. + assert len(bus._subscribers.get("claim_written", [])) >= 1 + + # Client disconnect: close the body iterator. The endpoint + # generator receives ``CancelledError`` via Starlette's + # cancellation path and returns; its `bus.subscribe_raw` queue is + # released. Wait briefly for cleanup. + await _drain_until_disconnect(response.body_iterator) + for _ in range(50): + if not bus._subscribers.get("claim_written"): + break + await asyncio.sleep(0.02) + + assert bus._subscribers.get("claim_written", []) == [], ( + f"subscribers leaked after disconnect: " + f"{bus._subscribers.get('claim_written')!r}" + ) \ No newline at end of file diff --git a/backend/tests/test_pubsub.py b/backend/tests/test_pubsub.py new file mode 100644 index 0000000..d1a4906 --- /dev/null +++ b/backend/tests/test_pubsub.py @@ -0,0 +1,156 @@ +"""Tests for the in-process EventBus. + +These exercise the drop-oldest overflow strategy, per-kind fan-out, and +the non-blocking publish contract. They are async because the bus +exposes an async iterator for subscribers. +""" + +from __future__ import annotations + +import asyncio +import time + +import pytest + +from cyclone.pubsub import EventBus, get_event_bus + + +@pytest.mark.asyncio +async def test_publish_no_subscribers_is_noop(): + bus = EventBus() + # No subscribers at all — must not raise. + await bus.publish("x", {"a": 1}) + + +@pytest.mark.asyncio +async def test_publish_to_n_subscribers_delivers_to_all(): + bus = EventBus() + queues = [bus.subscribe(["claim"]) for _ in range(3)] + + await bus.publish("claim", {"id": "C1"}) + + received = [await asyncio.wait_for(q.__anext__(), timeout=0.5) for q in queues] + for event in received: + assert event == {"id": "C1", "_kind": "claim"} + + +@pytest.mark.asyncio +async def test_subscriber_only_receives_matching_kinds(): + bus = EventBus() + queue = bus.subscribe(["a"]) + + await bus.publish("b", {"n": 1}) + await bus.publish("a", {"n": 2}) + + event = await asyncio.wait_for(queue.__anext__(), timeout=0.5) + assert event == {"n": 2, "_kind": "a"} + + +@pytest.mark.asyncio +async def test_slow_subscriber_does_not_block_publish(): + bus = EventBus(max_queue_size=2) + queue = bus.subscribe(["e"]) + + async def emit(): + for i in range(3): + await bus.publish("e", {"i": i}) + + start = time.monotonic() + await emit() + elapsed = time.monotonic() - start + + # Publishing must not block even though the subscriber queue is full + # after the second event; the drop-oldest policy keeps publishes non-blocking. + assert elapsed < 0.05, f"publish blocked for {elapsed:.3f}s" + + # The slow subscriber retains only the last two events. + seen = [] + for _ in range(2): + seen.append(await asyncio.wait_for(queue.__anext__(), timeout=0.5)) + assert [e["i"] for e in seen] == [1, 2] + + +@pytest.mark.asyncio +async def test_queue_overflow_drops_oldest(): + bus = EventBus(max_queue_size=2) + queue = bus.subscribe(["e"]) + + for i in range(3): + await bus.publish("e", {"i": i}) + + seen = [] + for _ in range(2): + seen.append(await asyncio.wait_for(queue.__anext__(), timeout=0.5)) + assert [e["i"] for e in seen] == [1, 2] + + +@pytest.mark.asyncio +async def test_multiple_concurrent_subscribers_each_get_all_events(): + bus = EventBus() + queues = [bus.subscribe(["claim"]) for _ in range(2)] + + for i in range(5): + await bus.publish("claim", {"i": i}) + + for q in queues: + seen = [] + for _ in range(5): + seen.append(await asyncio.wait_for(q.__anext__(), timeout=0.5)) + assert [e["i"] for e in seen] == [0, 1, 2, 3, 4] + + +def test_get_event_bus_raises_when_app_state_uninitialized(): + """get_event_bus() late-imports cyclone.api; if the lifespan handler + hasn't set ``app.state.event_bus`` (or it's been cleared), the + AttributeError must surface so callers can fail loudly.""" + from cyclone.api import app + + # The autouse conftest fixture sets app.state.event_bus for every test + # so that endpoint code can call request.app.state.event_bus. We pop + # the key entirely here so __getattr__ raises AttributeError rather + # than returning None for an explicitly-None slot. + state_dict = app.state._state + saved = state_dict.pop("event_bus", None) + try: + with pytest.raises((AttributeError, RuntimeError)): + get_event_bus() + finally: + if saved is not None: + state_dict["event_bus"] = saved + + +def test_get_event_bus_returns_app_state_bus(): + """When the lifespan handler has initialised the bus, get_event_bus() + returns it. Validates the happy path under the same late-import.""" + from cyclone.api import app + from cyclone.pubsub import EventBus + + saved = getattr(app.state, "event_bus", None) + bus = EventBus() + app.state.event_bus = bus + try: + assert get_event_bus() is bus + finally: + app.state.event_bus = saved + + +def test_unsubscribe_removes_queue_from_each_kind(): + bus = EventBus() + q1, _ = bus.subscribe_raw(["a", "b"]) + q2, _ = bus.subscribe_raw(["a"]) + + bus.unsubscribe(q1, ["a", "b"]) + + assert q1 not in bus._subscribers.get("a", []) + assert q1 not in bus._subscribers.get("b", []) + assert q2 in bus._subscribers["a"] + + +def test_unsubscribe_is_idempotent(): + bus = EventBus() + q, _ = bus.subscribe_raw(["a"]) + bus.unsubscribe(q, ["a"]) + # Second call must not raise. + bus.unsubscribe(q, ["a"]) + bus.unsubscribe(q, ["b"]) # wrong kind — also fine + assert "a" not in bus._subscribers diff --git a/backend/tests/test_store.py b/backend/tests/test_store.py index 86bfe91..17bbbc9 100644 --- a/backend/tests/test_store.py +++ b/backend/tests/test_store.py @@ -90,6 +90,85 @@ def test_module_singleton(): assert isinstance(store, CycloneStore) +@pytest.mark.asyncio +async def test_add_837_publishes_claim_written_and_activity_recorded_events(): + """store.add() with event_bus= should publish one claim_written event + per inserted claim and one activity_recorded event per activity row.""" + from cyclone.pubsub import EventBus + + bus = EventBus() + claim_q = bus.subscribe(["claim_written"]) + activity_q = bus.subscribe(["activity_recorded"]) + + s = CycloneStore() + rec = _make_batch_record() + s.add(rec, event_bus=bus) + + # Two claims + two claim_submitted activity events expected. + import asyncio + claims = [] + for _ in range(2): + claims.append(await asyncio.wait_for(claim_q.__anext__(), timeout=0.5)) + activities = [] + for _ in range(2): + activities.append(await asyncio.wait_for(activity_q.__anext__(), timeout=0.5)) + + # Each event has _kind and a payload with the expected shape. + for ev in claims: + assert ev["_kind"] == "claim_written" + assert "id" in ev + for ev in activities: + assert ev["_kind"] == "activity_recorded" + assert ev["kind"] == "claim_submitted" + assert "claimId" in ev + + +def test_add_835_publishes_remittance_written_and_activity_recorded_events(): + """835 batches publish remittance_written events (one per remit) and + activity_recorded events (one per activity row).""" + import asyncio + from cyclone.pubsub import EventBus + from cyclone.store import BatchRecord835 + + # Reuse the 835 result helper from test_store_reconcile.py. + from test_store_reconcile import _make_835_result, _make_remit_with_cas + + bus = EventBus() + remit_q = bus.subscribe(["remittance_written"]) + activity_q = bus.subscribe(["activity_recorded"]) + + cp = _make_remit_with_cas(remit_id="CLP-PUB", pcn="PCN-PUB") + rec = BatchRecord835( + id="b-835-1", kind="835", input_filename="test835.txt", + parsed_at=datetime(2026, 6, 19, 12, 0, tzinfo=timezone.utc), + result=_make_835_result([cp]), + ) + CycloneStore().add(rec, event_bus=bus) + + loop = asyncio.new_event_loop() + try: + remits = [] + for _ in range(1): + remits.append(loop.run_until_complete( + asyncio.wait_for(remit_q.__anext__(), timeout=0.5) + )) + activities = [] + for _ in range(1): + activities.append(loop.run_until_complete( + asyncio.wait_for(activity_q.__anext__(), timeout=0.5) + )) + finally: + loop.close() + + for ev in remits: + assert ev["_kind"] == "remittance_written" + assert ev["id"] == "PCN-PUB" + for ev in activities: + assert ev["_kind"] == "activity_recorded" + assert ev["kind"] == "remit_received" + assert ev["remittanceId"] == "PCN-PUB" + + def test_add_837_persists_batch_and_claims(): s = CycloneStore() rec = _make_batch_record() diff --git a/docs/superpowers/plans/2026-06-20-cyclone-live-tail.md b/docs/superpowers/plans/2026-06-20-cyclone-live-tail.md new file mode 100644 index 0000000..82da356 --- /dev/null +++ b/docs/superpowers/plans/2026-06-20-cyclone-live-tail.md @@ -0,0 +1,754 @@ +# Sub-project 5 — Live Tail (NDJSON Pub/Sub): Implementation Plan + +**Date:** 2026-06-20 +**Status:** Approved +**Branch:** `live-tail` +**Spec:** `docs/superpowers/specs/2026-06-20-cyclone-live-tail-design.md` + +26 tasks across 5 phases. Follows the same task pattern as the SP3/SP4 plans (`- [ ] Step 1 → Step N → commit`). TDD throughout. + +Adds real-time "tail -f" updates to `Claims`, `Remittances`, and `ActivityLog`: when a new claim/remittance/activity event lands in the DB, it appears on the relevant page without a manual refresh. Mechanism is **long-lived NDJSON GET endpoints** backed by an **in-process asyncio pub/sub bus** — no SSE, no polling, no new infra deps. + +## Task 0: Add `pytest-asyncio` to dev deps + +**Files:** +- Modify: `backend/pyproject.toml` (append `"pytest-asyncio>=0.23,<1"` to `dev` list) +- Modify: `backend/pyproject.toml` (under `[tool.pytest.ini_options]`, set `asyncio_mode = "auto"`) + +```bash +.venv/bin/pip install -e '.[dev]' +.venv/bin/pytest -q # smoke: full suite still green +``` + +- [ ] **Step 1: Add the dep + config** +- [ ] **Step 2: Install + run full suite, confirm green** +- [ ] **Step 3: Commit** + +```bash +git add backend/pyproject.toml backend/uv.lock +git commit -m "build(deps): pytest-asyncio for SP5 stream tests" +``` + +## File inventory + +**Backend (2 new, 3 modified):** +- `backend/src/cyclone/pubsub.py` — `EventBus` (NEW) +- `backend/src/cyclone/api.py` — +3 streaming endpoints; lifespan handler init `app.state.event_bus` (MODIFIED) +- `backend/src/cyclone/store.py` — `upsert_claim/upsert_remittance/record_activity` publish via injected bus (MODIFIED) +- `backend/src/cyclone/__main__.py` — drop the `db.init_db()` call (now lives in lifespan) (MODIFIED) +- `backend/tests/test_pubsub.py` — ~6 tests (NEW) +- `backend/tests/test_api_stream_live.py` — ~8 tests (NEW) + +**Frontend (5 new, 4 modified):** +- `src/lib/tail-stream.ts` — NDJSON parser over `ReadableStream` (NEW) +- `src/store/tail-store.ts` — Zustand append-only, FIFO-capped slices (NEW) +- `src/hooks/useTailStream.ts` — connection lifecycle hook (NEW) +- `src/hooks/useMergedTail.ts` — base + tail merge with filter (NEW) +- `src/components/TailStatusPill.tsx` — status badge + reconnect button (NEW) +- `src/lib/tail-stream.test.ts` — ~5 tests (NEW) +- `src/store/tail-store.test.ts` — ~5 tests (NEW) +- `src/hooks/useTailStream.test.ts` — ~6 tests (NEW) +- `src/hooks/useMergedTail.test.ts` — ~4 tests (NEW) +- `src/pages/Claims.tsx` — swap `data.items` → `useMergedTail` + toolbar pill (MODIFIED) +- `src/pages/Remittances.tsx` — same (MODIFIED) +- `src/pages/ActivityLog.tsx` — same (MODIFIED) +- `src/pages/Claims.test.tsx` — +2 live-tail integration tests (MODIFIED) + +Total: **9 new files, 6 modified, ~36 new tests.** + +--- + +## Phase 1 — Backend: `EventBus` + +### Task 1: `cyclone.pubsub.EventBus` skeleton + `publish` + +**Files:** +- Create: `backend/src/cyclone/pubsub.py` +- Create: `backend/tests/test_pubsub.py` + +API per spec §3.1: +```python +class EventBus: + def __init__(self, max_queue_size: int = 256) -> None: ... + async def publish(self, kind: str, payload: dict) -> None: ... + def subscribe(self, kinds: list[str]) -> "AsyncIterator[dict]": ... +``` + +Internals: +- `self._subscribers: dict[str, list[asyncio.Queue[dict]]]` +- `publish`: iterate `_subscribers[kind]`, call `queue.put_nowait(event)`. On `QueueFull`, drop oldest via `get_nowait() + task_done`, then enqueue. Never blocks. +- `subscribe`: returns an async generator; each yielded value is `{**payload, "_kind": kind}`. +- The subscriber queue size equals `max_queue_size` (default 256). + +Tests (`backend/tests/test_pubsub.py`, ~6): +- `test_publish_no_subscribers_is_noop` — `publish("x", {"a": 1})` on a fresh bus doesn't raise. +- `test_publish_to_n_subscribers_delivers_to_all` +- `test_subscriber_only_receives_matching_kinds` — subscribe to `["a"]`, publish `b` + `a`, only `a` arrives. +- `test_slow_subscriber_does_not_block_publish` — one subscriber queue full, publish from another task completes within 10ms. +- `test_queue_overflow_drops_oldest` — fill a queue to capacity, publish another, the oldest is gone. +- `test_multiple_concurrent_subscribers_each_get_all_events` + +- [ ] **Step 1: Add failing tests** in `backend/tests/test_pubsub.py` (`@pytest.mark.asyncio` on each) +- [ ] **Step 2: Run tests, confirm red** +- [ ] **Step 3: Implement `EventBus`** in `backend/src/cyclone/pubsub.py` +- [ ] **Step 4: Run tests, confirm green** +- [ ] **Step 5: Commit** + +```bash +git add backend/src/cyclone/pubsub.py backend/tests/test_pubsub.py +git commit -m "feat(pubsub): EventBus with drop-oldest overflow, per-kind fan-out" +``` + +### Task 2: Module-level `get_event_bus()` accessor + +**Files:** +- Modify: `backend/src/cyclone/pubsub.py` +- Modify: `backend/tests/test_pubsub.py` + +Add (per spec §3.3): +```python +def get_event_bus() -> EventBus: + """Module-level accessor; tests stub via monkeypatch.""" + from cyclone.api import app # late import to avoid circular + return app.state.event_bus +``` + +Test: +- `test_get_event_bus_returns_attribute_error_when_app_not_initialized` — calling before lifespan raises; verifies the late-import contract. + +- [ ] **Step 1: Add failing test** +- [ ] **Step 2: Implement the accessor** +- [ ] **Step 3: Run tests, confirm green** +- [ ] **Step 4: Commit** + +```bash +git add backend/src/cyclone/pubsub.py backend/tests/test_pubsub.py +git commit -m "feat(pubsub): get_event_bus() late-import accessor" +``` + +### Task 3: Phase 1 commit + full suite + +```bash +.venv/bin/pytest -q +``` + +Expected: ~417 + 1 skip + ~7 new = ~424 pass. All green. + +- [ ] **Step 1: Run full backend suite, confirm green** +- [ ] **Step 2: Tag phase complete (no commit unless changes)** + +--- + +## Phase 2 — Backend: lifespan + store publishing + +### Task 4: Add FastAPI lifespan handler that initializes `event_bus` + +**Files:** +- Modify: `backend/src/cyclone/api.py` + +Add at module top: +```python +from contextlib import asynccontextmanager +from cyclone.pubsub import EventBus + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Initialize DB schema/migrations inside the lifespan so the bus has + # access to a fully-populated store before any request lands. + from cyclone import db + db.init_db() + app.state.event_bus = EventBus() + yield + # No teardown needed — single-process local-only. + +app = FastAPI(..., lifespan=lifespan) +``` + +Important: keep the existing `@app.on_event("startup")` if present, OR remove it and rely solely on the lifespan. (Verify which is currently in use; if `@app.on_event` is absent, this is a pure add.) + +- [ ] **Step 1: Add the lifespan handler + wire it to `app`** +- [ ] **Step 2: Smoke: import `cyclone.api` succeeds; `app.router.lifespan_context` is set** +- [ ] **Step 3: Commit** + +```bash +git add backend/src/cyclone/api.py +git commit -m "feat(api): lifespan handler initializes EventBus + db" +``` + +### Task 5: Drop `db.init_db()` from `__main__.py` + +**Files:** +- Modify: `backend/src/cyclone/__main__.py` + +Remove the explicit `db.init_db()` call before `uvicorn.main()` — lifespan handles it now. + +- [ ] **Step 1: Remove the line** +- [ ] **Step 2: Smoke: `python -m cyclone serve &; curl /api/health; kill %1`** +- [ ] **Step 3: Commit** + +```bash +git add backend/src/cyclone/__main__.py +git commit -m "refactor(main): rely on lifespan for db init (SP5)" +``` + +### Task 6: `store.upsert_claim` publishes `claim_written` + +**Files:** +- Modify: `backend/src/cyclone/store.py` +- Modify: `backend/tests/test_store.py` + +Change signature to accept an optional `event_bus: EventBus | None = None`. After the `s.commit()` for an upsert: +```python +if event_bus is not None: + await event_bus.publish("claim_written", ) +``` + +Test: +- `test_upsert_claim_publishes_event_with_ui_shape` — instantiate an `EventBus`, subscribe to `["claim_written"]`, call `upsert_claim(bus=bus, ...)`, drain the subscriber, assert event payload has same keys as `iter_claims` returns. + +- [ ] **Step 1: Add failing test** +- [ ] **Step 2: Implement the publish call** +- [ ] **Step 3: Run test, confirm green** +- [ ] **Step 4: Commit** + +```bash +git add backend/src/cyclone/store.py backend/tests/test_store.py +git commit -m "feat(store): upsert_claim publishes claim_written to EventBus" +``` + +### Task 7: `store.upsert_remittance` + `record_activity` publish + +**Files:** +- Modify: `backend/src/cyclone/store.py` +- Modify: `backend/tests/test_store.py` + +Same pattern as Task 6 for the other two write paths: +- `upsert_remittance` → `"remittance_written"` (payload = `to_ui_remittance` shape) +- `record_activity` → `"activity_recorded"` (payload = `to_activity_event` shape) + +Tests: +- `test_upsert_remittance_publishes_remittance_written` +- `test_record_activity_publishes_activity_recorded` + +- [ ] **Step 1: Add 2 failing tests** +- [ ] **Step 2: Implement both publish calls** +- [ ] **Step 3: Run tests, confirm green** +- [ ] **Step 4: Commit** + +```bash +git add backend/src/cyclone/store.py backend/tests/test_store.py +git commit -m "feat(store): upsert_remittance + record_activity publish events" +``` + +### Task 8: Wire `event_bus` into parse-837 / parse-835 endpoints + +**Files:** +- Modify: `backend/src/cyclone/api.py` + +Update the two parse endpoints to pass `event_bus=app.state.event_bus` into the store calls. Test impact: existing parse-837/parse-835 tests should continue to pass — the bus is wired but no subscribers are attached in those tests, so `publish` is a no-op. + +- [ ] **Step 1: Wire the bus into both endpoints** +- [ ] **Step 2: Run full backend suite, confirm green** +- [ ] **Step 3: Commit** + +```bash +git add backend/src/cyclone/api.py +git commit -m "feat(api): parse endpoints pass EventBus into store writes" +``` + +### Task 9: Phase 2 commit + full suite + +```bash +.venv/bin/pytest -q +``` + +Expected: ~424 + ~3 new = ~427 pass. All green. + +- [ ] **Step 1: Run full suite, confirm green** +- [ ] **Step 2: Tag phase complete** + +--- + +## Phase 3 — Backend: streaming endpoints + +### Task 10: `_stream_ndjson` helper + +**Files:** +- Modify: `backend/src/cyclone/api.py` + +Helper that wraps a generator of dict events into NDJSON lines: +```python +def _ndjson_line(event: dict) -> bytes: + return (json.dumps(event, separators=(",", ":")) + "\n").encode("utf-8") +``` + +- [ ] **Step 1: Add the helper** +- [ ] **Step 2: Commit** + +```bash +git add backend/src/cyclone/api.py +git commit -m "refactor(api): _ndjson_line helper for streaming responses" +``` + +### Task 11: `GET /api/claims/stream` + +**Files:** +- Modify: `backend/src/cyclone/api.py` +- Create: `backend/tests/test_api_stream_live.py` + +Endpoint signature mirrors `list_claims` query params (forwarded): +```python +@app.get("/api/claims/stream") +async def claims_stream( + request: Request, + status: str | None = None, + provider_npi: str | None = None, + payer: str | None = None, + date_from: str | None = None, + date_to: str | None = None, + sort: str | None = None, + order: str = "desc", + limit: int = 100, +): + bus: EventBus = request.app.state.event_bus + async def gen(): + # 1. Snapshot + rows = store.iter_claims( + status=status, provider_npi=provider_npi, payer=payer, + date_from=date_from, date_to=date_to, + sort=sort or "-submission_date", order=order, limit=limit, + ) + for row in rows: + yield _ndjson_line({"type": "item", "data": row}) + yield _ndjson_line({"type": "snapshot_end", "data": {"count": len(rows)}}) + # 2. Subscribe + async for event in bus.subscribe(["claim_written"]): + if await request.is_disconnected(): + return + yield _ndjson_line({"type": "item", "data": event}) + return StreamingResponse(gen(), media_type="application/x-ndjson") +``` + +Tests (with `httpx.AsyncClient` + `pytest-asyncio`): +- `test_claims_stream_yields_snapshot_then_snapshot_end` — pre-populate 3 claims, hit stream, assert first 3 lines are `{"type":"item"}`, 4th is `{"type":"snapshot_end","data":{"count":3}}`. +- `test_claims_stream_emits_new_item_after_publish` — open stream in one task, call `bus.publish("claim_written", payload)` in another, assert the new item appears within 100ms. +- `test_claims_stream_heartbeat_after_15s_idle` — override heartbeat to 1s via env var or test fixture; assert heartbeat line arrives within 1.5s. + +- [ ] **Step 1: Add 3 failing tests** +- [ ] **Step 2: Implement the endpoint (without heartbeat first)** +- [ ] **Step 3: Run tests, confirm green** +- [ ] **Step 4: Add the 15s heartbeat via `asyncio.wait_for` loop** (overridable for tests) +- [ ] **Step 5: Run tests, confirm green** +- [ ] **Step 6: Commit** + +```bash +git add backend/src/cyclone/api.py backend/tests/test_api_stream_live.py +git commit -m "feat(api): GET /api/claims/stream — snapshot + subscribe + heartbeat" +``` + +### Task 12: `GET /api/remittances/stream` + +**Files:** +- Modify: `backend/src/cyclone/api.py` +- Modify: `backend/tests/test_api_stream_live.py` + +Same shape; subscribes to `["remittance_written"]`. Default sort `-received_date`. + +Tests: +- `test_remittances_stream_yields_snapshot_then_subscribes` +- `test_remittances_stream_emits_after_upsert_remittance` + +- [ ] **Step 1: Add 2 failing tests** +- [ ] **Step 2: Implement the endpoint** +- [ ] **Step 3: Run tests, confirm green** +- [ ] **Step 4: Commit** + +```bash +git add backend/src/cyclone/api.py backend/tests/test_api_stream_live.py +git commit -m "feat(api): GET /api/remittances/stream" +``` + +### Task 13: `GET /api/activity/stream` + +**Files:** +- Modify: `backend/src/cyclone/api.py` +- Modify: `backend/tests/test_api_stream_live.py` + +Same shape; subscribes to `["activity_recorded"]`. Default sort `-timestamp`, default `limit=50` (smaller — activity is high-volume). + +Tests: +- `test_activity_stream_yields_snapshot_then_subscribes` +- `test_activity_stream_emits_after_record_activity` + +- [ ] **Step 1: Add 2 failing tests** +- [ ] **Step 2: Implement the endpoint** +- [ ] **Step 3: Run tests, confirm green** +- [ ] **Step 4: Commit** + +```bash +git add backend/src/cyclone/api.py backend/tests/test_api_stream_live.py +git commit -m "feat(api): GET /api/activity/stream (limit=50)" +``` + +### Task 14: Client disconnect cancels subscription + +**Files:** +- Modify: `backend/tests/test_api_stream_live.py` + +Test: +- `test_client_disconnect_cancels_subscription` — open stream, read `snapshot_end`, close the response, then assert `bus._subscribers["claim_written"]` is empty (use a fresh bus per test). + +- [ ] **Step 1: Add failing test** +- [ ] **Step 2: If not already implemented (the `request.is_disconnected()` check in Task 11 covers it), confirm** +- [ ] **Step 3: Run test, confirm green** +- [ ] **Step 4: Commit if any change** + +### Task 15: Phase 3 commit + full suite + +```bash +.venv/bin/pytest -q +``` + +Expected: ~427 + ~10 new = ~437 pass. All green. + +- [ ] **Step 1: Run full suite, confirm green** +- [ ] **Step 2: Tag phase complete** + +--- + +## Phase 4 — Frontend: NDJSON parser + tail store + +### Task 16: `src/lib/tail-stream.ts` — NDJSON parser + +**Files:** +- Create: `src/lib/tail-stream.ts` +- Create: `src/lib/tail-stream.test.ts` + +API per spec §3.4: +```ts +export type TailEvent = + | { type: "item"; data: unknown } + | { type: "snapshot_end"; data: { count: number } } + | { type: "heartbeat"; data: { ts: string } } + | { type: "item_dropped"; data: { id: string } } + | { type: "error"; data: { message: string } }; + +export type TailResource = "claims" | "remittances" | "activity"; + +export async function* streamTail( + resource: TailResource, + opts?: { signal?: AbortSignal; baseUrl?: string }, +): AsyncIterableIterator; +``` + +Implementation: `fetch(${baseUrl}/api/${resource}/stream, { signal })` with `Accept: application/x-ndjson` header; pipe `response.body!` through `TextDecoderStream` then an NDJSON-line splitter that yields `JSON.parse`d typed events. Malformed lines are skipped with `console.warn`. + +Tests (`src/lib/tail-stream.test.ts`, ~5): +- `test_parses_well_formed_ndjson_into_typed_events` — feed a `ReadableStream` of well-formed lines (use a `TransformStream` helper in tests). +- `test_yields_typed_error_on_error_line` — `{"type":"error","data":{"message":"x"}}` yields as `{ type: "error", data: { message: "x" } }`. +- `test_handles_heartbeat_silently` — heartbeat line yields correctly typed. +- `test_abort_signal_cancels_mid_stream` — pass an `AbortController`, abort after first event, generator throws/returns without further yield. +- `test_malformed_line_skipped_next_valid_still_emitted` — bad JSON in middle, surrounding good lines still parse. + +- [ ] **Step 1: Add failing tests** +- [ ] **Step 2: Implement `streamTail`** +- [ ] **Step 3: Run tests, confirm green** +- [ ] **Step 4: Commit** + +```bash +git add src/lib/tail-stream.ts src/lib/tail-stream.test.ts +git commit -m "feat(frontend): NDJSON streamTail parser for /api/{resource}/stream" +``` + +### Task 17: `src/store/tail-store.ts` — Zustand append-only with FIFO cap + +**Files:** +- Create: `src/store/tail-store.ts` +- Create: `src/store/tail-store.test.ts` + +API per spec §3.4: +```ts +export const TAIL_CAP = 10_000; + +interface TailStore { + claims: Record; + remittances: Record; + activity: ActivityEvent[]; + addClaim: (c: ClaimListItem) => void; + addRemittance: (r: RemitListItem) => void; + addActivity: (a: ActivityEvent) => void; + reset: (resource: TailResource) => void; +} + +export const useTailStore = create((set) => ({ ... })); +``` + +Dedup: `addClaim(c)` is a no-op when `c.id` already present. FIFO cap: when `Object.keys(claims).length > TAIL_CAP`, drop the oldest by id (sort keys by insertion order via a small parallel `OrderMap`, or just evict the first 100 oldest when the cap is exceeded). + +Tests (~5): +- `test_add_claim_adds_new_claim_keyed_by_id` +- `test_add_claim_with_duplicate_id_is_noop` +- `test_reset_claims_clears_only_claims_slice` +- `test_add_activity_appends` (no stable id) +- `test_fifo_cap_evicts_oldest_when_over_10000` + +- [ ] **Step 1: Add failing tests** +- [ ] **Step 2: Implement the store** +- [ ] **Step 3: Run tests, confirm green** +- [ ] **Step 4: Commit** + +```bash +git add src/store/tail-store.ts src/store/tail-store.test.ts +git commit -m "feat(frontend): tail-store Zustand with FIFO cap 10k per slice" +``` + +### Task 18: Phase 4 commit + frontend check + +```bash +npm run typecheck +npm test -- --run +``` + +Expected: 124 + ~10 new = ~134 frontend pass. + +- [ ] **Step 1: Run typecheck + tests, confirm green** +- [ ] **Step 2: Commit if any change** + +--- + +## Phase 5 — Frontend: hooks + page integration + +### Task 19: `src/hooks/useTailStream.ts` — connection lifecycle + +**Files:** +- Create: `src/hooks/useTailStream.ts` +- Create: `src/hooks/useTailStream.test.ts` + +API per spec §3.4: +```ts +export type TailStatus = "connecting" | "live" | "reconnecting" | "closed" | "stalled" | "error"; + +export function useTailStream(resource: TailResource): { + status: TailStatus; + lastEventAt: Date | null; + error: Error | null; + forceReconnect: () => void; +}; +``` + +Behavior (per spec): +- On mount: opens stream; status = `connecting`. +- On `snapshot_end`: status = `live`. +- On `item`: dispatches to `useTailStore`; updates `lastEventAt`. +- On error: status = `reconnecting`; backoff 1s → 2s → 4s → 8s → 16s → 30s cap. +- If no event (incl. heartbeat) for 30s: status = `stalled`. +- On unmount: abort; status = `closed`. +- `forceReconnect()`: cancels and re-opens immediately. + +Tests (~6) — mock `streamTail` with a vitest module mock: +- `test_on_mount_status_connecting_then_live_after_snapshot_end` +- `test_on_stream_error_status_error` +- `test_on_abort_status_closed_no_reconnect` +- `test_on_event_dispatches_to_tail_store` — assert `useTailStore.getState().addClaim` is called. +- `test_reconnect_status_cycles_reconnecting_to_connecting_to_live` +- `test_reconnect_dedup_duplicate_items_appear_once_in_store` + +- [ ] **Step 1: Add failing tests** +- [ ] **Step 2: Implement the hook** (use `useEffect` + `useRef` for backoff state; `AbortController` per attempt) +- [ ] **Step 3: Run tests, confirm green** +- [ ] **Step 4: Commit** + +```bash +git add src/hooks/useTailStream.ts src/hooks/useTailStream.test.ts +git commit -m "feat(frontend): useTailStream lifecycle hook with backoff + stall detection" +``` + +### Task 20: `src/hooks/useMergedTail.ts` — merge base + tail with filter + +**Files:** +- Create: `src/hooks/useMergedTail.ts` +- Create: `src/hooks/useMergedTail.test.ts` + +API per spec §3.4: +```ts +export function useMergedTail( + resource: TailResource, + baseItems: T[], + filterFn?: (item: T) => boolean, +): T[]; +``` + +Behavior: +- Reads the matching slice from `useTailStore`. +- Applies `filterFn` to tail items if provided. +- Dedups by `id` against `baseItems`. +- Returns `baseItems` followed by tail items in arrival order. + +Tests (~4): +- `test_merges_base_and_tail_by_id_base_first` +- `test_filter_predicate_drops_tail_items_that_dont_match` +- `test_empty_tail_store_returns_base_unchanged` +- `test_new_tail_item_appears_at_top` + +- [ ] **Step 1: Add failing tests** +- [ ] **Step 2: Implement the hook** +- [ ] **Step 3: Run tests, confirm green** +- [ ] **Step 4: Commit** + +```bash +git add src/hooks/useMergedTail.ts src/hooks/useMergedTail.test.ts +git commit -m "feat(frontend): useMergedTail merges base + tail with filter" +``` + +### Task 21: `src/components/TailStatusPill.tsx` — status badge + reconnect button + +**Files:** +- Create: `src/components/TailStatusPill.tsx` + +Small component (no test needed beyond a smoke render check inside Claims.test.tsx): +- Props: `{ status: TailStatus; lastEventAt: Date | null; onReconnect: () => void }` +- Renders the existing `` with variant per status (live=success, connecting/reconnecting=warning, closed/stalled/error=error/destructive), plus "Last event: 12s ago" subtitle (computed via `useEffect` ticking every second), plus a "↻ Reconnect" button shown only when `status === "stalled" || status === "error"`. + +- [ ] **Step 1: Implement the component** +- [ ] **Step 2: Commit** + +```bash +git add src/components/TailStatusPill.tsx +git commit -m "feat(frontend): TailStatusPill with reconnect button" +``` + +### Task 22: `Claims.tsx` — swap to `useMergedTail` + toolbar pill + +**Files:** +- Modify: `src/pages/Claims.tsx` +- Modify: `src/pages/Claims.test.tsx` + +Wire-up: +- Replace `data?.items ?? []` with `useMergedTail("claims", data?.items ?? [], filterFn)`. +- Call `useTailStream("claims")` for status. +- Render `` in the toolbar with status + lastEventAt + onReconnect = `forceReconnect`. +- Existing tests must still pass; add 2 new tests: + - `test_live_tail_arrival_triggers_row_in_table` — mock `streamTail` to emit an `item` for a new claim, assert row count + row text appears. + - `test_status_pill_shows_live_after_stream_connects` — mock `streamTail` to emit `snapshot_end`, assert pill text contains `live`. + +- [ ] **Step 1: Add 2 failing tests** +- [ ] **Step 2: Wire up the page** +- [ ] **Step 3: Run tests, confirm green** +- [ ] **Step 4: Commit** + +```bash +git add src/pages/Claims.tsx src/pages/Claims.test.tsx +git commit -m "feat(frontend): Claims page wires live tail + TailStatusPill" +``` + +### Task 23: `Remittances.tsx` + `ActivityLog.tsx` — same pattern + +**Files:** +- Modify: `src/pages/Remittances.tsx` +- Modify: `src/pages/ActivityLog.tsx` + +Mirror Task 22: swap to `useMergedTail("remittances", ...)` / `useMergedTail("activity", ...)`, render `` in the toolbar. Use the same `filterFn` the page already passes to its list query. + +- [ ] **Step 1: Wire both pages** +- [ ] **Step 2: Run typecheck + tests** +- [ ] **Step 3: Commit** + +```bash +git add src/pages/Remittances.tsx src/pages/ActivityLog.tsx +git commit -m "feat(frontend): Remittances + ActivityLog wire live tail" +``` + +### Task 24: Phase 5 commit + full suite + +```bash +npm run typecheck +npm test -- --run +.venv/bin/pytest -q +``` + +Expected: ~134 + ~10 new = ~144 frontend; ~437 backend. All green. + +- [ ] **Step 1: Run all checks, confirm green** +- [ ] **Step 2: Commit if any change** + +--- + +## Phase 6 — Docs + smoke + merge + +### Task 25: End-to-end smoke + +Per spec §8 smoke: + +1. Start backend (`backend/.venv/bin/python -m cyclone serve &`). +2. Start frontend (`npm run dev &`). +3. Open `http://localhost:5173/claims` in browser. +4. In another tab (or via curl), POST `co_medicaid_837p.txt` to `/api/parse-837` (`curl -F file=@...`). +5. Verify the first `/claims` tab shows the new rows appearing without manual refresh. +6. Verify the status pill reads `live`. +7. Verify the backend logs show no errors. +8. Kill both servers. + +- [ ] **Step 1: Run the smoke** +- [ ] **Step 2: Commit empty** + +```bash +git commit --allow-empty -m "smoke: end-to-end SP5 (live tail) flow passes" +``` + +### Task 26: README + merge to main + +**Files:** +- Modify: `README.md` — under "Roadmap" or new "Live updates" section, document the tail behavior + status pill + reconnect UX. + +- [ ] **Step 1: Update README** +- [ ] **Step 2: Commit** + +```bash +git add README.md +git commit -m "docs: document live-tail behavior in README" +``` + +### Task 27: Cleanup + merge to main + +1. `git status` — clean. +2. `git merge --ff-only live-tail`. +3. From `main`: `cd backend && .venv/bin/pytest -q` → must show ~437 pass + 1 skip; `cd .. && npm test -- --run` → ~144 pass. +4. `git worktree remove .worktrees/live-tail` (if applicable). +5. `git branch -d live-tail`. +6. Update SP4 plan checkboxes (if any drift) — not applicable here; this is the SP5 plan itself. Tick all `[ ]` to `[x]` in this file before committing. + +- [ ] **Step 1: Verify clean** +- [ ] **Step 2: Discard stale uncommitted changes (if any)** +- [ ] **Step 3: Fast-forward merge to main** +- [ ] **Step 4: Remove worktree + delete branch** +- [ ] **Step 5: Tick all `[ ]` to `[x]` in this file; commit as `docs: tick SP5 plan checkboxes`** + +--- + +## Test count targets + +| Phase | New backend | New frontend | Running total | +|-------|-------------|--------------|---------------| +| Start (post-SP4) | 417 + 1 skip | 124 | 417 / 124 | +| 0 (deps) | 0 | 0 | 417 / 124 | +| P1 end | +7 | 0 | 424 / 124 | +| P2 end | +3 | 0 | 427 / 124 | +| P3 end | +10 | 0 | 437 / 124 | +| P4 end | 0 | +10 | 437 / 134 | +| P5 end | 0 | +10 (incl. 2 page tests) | 437 / 144 | +| **Final target** | **~437 + 1 skip** | **~144** | — | + +## Risk reminders + +- **`pytest-asyncio` config** — set `asyncio_mode = "auto"` so `@pytest.mark.asyncio` isn't required per test. Verify no other test file gets accidentally parallelized. +- **Lifespan replaces on_event** — if any existing startup hook is wired via `@app.on_event("startup")`, either fold it into the lifespan or leave both (idempotent). Check before deleting the existing handler in `__main__`. +- **Tail-store growth** — cap at 10k per slice (FIFO). Operators who never refresh the page won't OOM the tab. +- **Heartbeat timing in tests** — make the 15s heartbeat overridable via env var (`CYCLONE_HEARTBEAT_INTERVAL_S`) so tests can use 0.1s. +- **Stall threshold vs reconnect** — 30s stall threshold; the hook should NOT auto-reconnect on stall (user confirms). Distinguish from `reconnecting` (auto-backoff in progress). +- **NDJSON partial lines** — the splitter must buffer trailing partial chunks across stream chunks; verify with a test that streams a line across two `ReadableStream` pulls. +- **Filter drift on the client** — tail-store contains items that may not match the current page filter; `useMergedTail` filters them out. Verify with a test that flips the filter mid-stream. +- **ActivityLog already streams from parse-837** — the new tail stream is **additive**, not a replacement. The page already has its own `parse837` NDJSON consumer; the tail stream survives parse completion. + +## Worktree note + +Use `superpowers:using-git-worktrees` to create `.worktrees/live-tail` from `main` before Task 1. All commits land on the `live-tail` branch until the final ff-merge to `main`. \ No newline at end of file diff --git a/src/components/TailStatusPill.test.tsx b/src/components/TailStatusPill.test.tsx new file mode 100644 index 0000000..a7aaa8f --- /dev/null +++ b/src/components/TailStatusPill.test.tsx @@ -0,0 +1,184 @@ +// @vitest-environment happy-dom +// TailStatusPill is a leaf component with no async state and no hooks +// beyond a single setInterval — happy-dom is the right env so we can +// inspect the rendered DOM. +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { TailStatusPill } from "./TailStatusPill"; +import type { TailStatus } from "@/hooks/useTailStream"; + +/** + * Minimal render helper — the project doesn't ship + * `@testing-library/react`, so we render straight into a real + * `createRoot` and inspect the DOM via `container.innerHTML` and + * `container.querySelector`. + * + * Note: React 18's `createRoot` defers DOM mutation until the calling + * microtask flushes unless we wrap the render in `act`. The pill's only + * effect is a `setInterval`, so we use `act(() => { root.render(node) })` + * — sync act is fine because no state updates fire during render. + */ +function renderInto(node: React.ReactElement): { + container: HTMLDivElement; + root: Root; + unmount: () => void; +} { + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + act(() => { + root.render(node); + }); + return { + container, + root, + unmount: () => { + act(() => root.unmount()); + container.remove(); + }, + }; +} + +const ALL_STATUSES: TailStatus[] = [ + "connecting", + "live", + "reconnecting", + "closed", + "stalled", + "error", +]; + +describe("TailStatusPill", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-06-20T12:00:30Z")); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("test_renders_badge_for_each_status", () => { + // Iterate every TailStatus and assert no crash, plus a sensible label. + // The plan's smoke spec calls for "renders all 5 statuses"; we test + // all 6 (closed is also a status even if rare in practice). + for (const status of ALL_STATUSES) { + const { container, root, unmount } = renderInto( + {}} + />, + ); + + // The pill root should be present. + const pill = container.querySelector('[data-testid="tail-status-pill"]'); + expect(pill).not.toBeNull(); + + // The Badge child should carry the human label — match by text + // content (the badge text is the only handoff the user reads). + const expectedLabels: Record = { + connecting: "Connecting", + live: "Live", + reconnecting: "Reconnecting", + closed: "Closed", + stalled: "Stalled", + error: "Error", + }; + expect(pill?.textContent).toContain(expectedLabels[status]); + + unmount(); + // Silence "unmount was not wrapped in act" warnings; renderInto's + // unmount is a sync root.unmount() and React tolerates this for + // leaf components with no async state. + void root; + } + }); + + it("test_shows_reconnect_button_only_for_stalled_and_error", () => { + const onReconnect = vi.fn(); + + for (const status of ALL_STATUSES) { + const { container, unmount } = renderInto( + , + ); + + const button = container.querySelector( + '[data-testid="tail-status-pill-reconnect"]', + ); + + if (status === "stalled" || status === "error") { + expect(button).not.toBeNull(); + expect(button?.textContent).toContain("Reconnect"); + } else { + expect(button).toBeNull(); + } + + unmount(); + } + }); + + it("test_subtitle_says_never_when_lastEventAt_is_null", () => { + const { container, unmount } = renderInto( + {}} + />, + ); + + const subtitle = container.querySelector( + '[data-testid="tail-status-pill-subtitle"]', + ); + expect(subtitle).not.toBeNull(); + expect(subtitle?.textContent).toBe("Last event: never"); + + unmount(); + }); + + it("test_subtitle_shows_age_in_seconds_when_lastEventAt_recent", () => { + // 12 seconds ago — fake-timer driven so the assertion is stable. + const eventAt = new Date("2026-06-20T12:00:18Z"); + const { container, unmount } = renderInto( + {}} + />, + ); + + const subtitle = container.querySelector( + '[data-testid="tail-status-pill-subtitle"]', + ); + expect(subtitle?.textContent).toBe("Last event: 12s ago"); + + unmount(); + }); + + it("test_reconnect_button_invokes_onReconnect_when_clicked", () => { + const onReconnect = vi.fn(); + const { container, unmount } = renderInto( + , + ); + + const button = container.querySelector( + '[data-testid="tail-status-pill-reconnect"]', + ) as HTMLButtonElement | null; + expect(button).not.toBeNull(); + button?.click(); + expect(onReconnect).toHaveBeenCalledTimes(1); + + unmount(); + }); +}); diff --git a/src/components/TailStatusPill.tsx b/src/components/TailStatusPill.tsx new file mode 100644 index 0000000..7251745 --- /dev/null +++ b/src/components/TailStatusPill.tsx @@ -0,0 +1,122 @@ +// --------------------------------------------------------------------------- +// Live-tail status pill (sub-project 5, Phase 5 Task 21). +// +// Tiny presentational component used in the toolbar of `Claims`, +// `Remittances`, and `ActivityLog` to surface the connection status of +// `useTailStream`. Renders the existing `` (variants from +// `src/components/ui/badge.tsx`) plus a "Last event: 12s ago" subtitle +// that re-ticks every second, plus a "↻ Reconnect" button that only +// appears when the connection has visibly failed (`stalled` or `error`). +// +// Variant mapping (per spec): +// live → success +// connecting → warning +// reconnecting → warning +// closed → destructive +// stalled → destructive +// error → destructive +// --------------------------------------------------------------------------- + +import { useEffect, useState } from "react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import type { TailStatus } from "@/hooks/useTailStream"; + +export interface TailStatusPillProps { + status: TailStatus; + lastEventAt: Date | null; + onReconnect: () => void; +} + +const STATUS_LABEL: Record = { + connecting: "Connecting…", + live: "Live", + reconnecting: "Reconnecting…", + closed: "Closed", + stalled: "Stalled", + error: "Error", +}; + +type BadgeVariant = "success" | "warning" | "destructive"; + +const STATUS_VARIANT: Record = { + live: "success", + connecting: "warning", + reconnecting: "warning", + closed: "destructive", + stalled: "destructive", + error: "destructive", +}; + +/** + * Render a duration in seconds as a compact human label. We deliberately + * keep the units small — under a minute is the common case for a healthy + * tail (the stall detector fires at 30s, so anything beyond is a real + * outage the user will want to investigate). + */ +function formatAge(seconds: number): string { + if (seconds < 0) return "0s"; + if (seconds < 60) return `${seconds}s`; + const m = Math.floor(seconds / 60); + const s = seconds % 60; + if (m < 60) return s === 0 ? `${m}m` : `${m}m ${s}s`; + const h = Math.floor(m / 60); + return `${h}h ${m % 60}m`; +} + +export function TailStatusPill({ + status, + lastEventAt, + onReconnect, +}: TailStatusPillProps): React.JSX.Element { + // Re-tick every second so the "Last event: 12s ago" subtitle stays + // current without the parent having to re-render. We mount the + // interval only once per pill instance. + const [now, setNow] = useState(() => Date.now()); + useEffect(() => { + const id = setInterval(() => setNow(Date.now()), 1000); + return () => clearInterval(id); + }, []); + + const ageSeconds = lastEventAt + ? Math.max(0, Math.floor((now - lastEventAt.getTime()) / 1000)) + : null; + + const subtitle = + ageSeconds === null + ? "Last event: never" + : `Last event: ${formatAge(ageSeconds)} ago`; + + // Only show the manual reconnect affordance when the connection has + // visibly failed. `closed` is reserved for the post-unmount state and + // is intentionally not actionable — the component is about to vanish. + const showReconnect = status === "stalled" || status === "error"; + + return ( +
+ + {STATUS_LABEL[status]} + + + {subtitle} + + {showReconnect && ( + + )} +
+ ); +} diff --git a/src/hooks/useMergedTail.test.ts b/src/hooks/useMergedTail.test.ts new file mode 100644 index 0000000..82d025b --- /dev/null +++ b/src/hooks/useMergedTail.test.ts @@ -0,0 +1,231 @@ +// @vitest-environment happy-dom +// React's act-aware env — mirror the other hook tests. +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { beforeEach, describe, expect, it } from "vitest"; +import { useMergedTail } from "./useMergedTail"; +import { useTailStore } from "@/store/tail-store"; +import type { Claim, Remittance, Activity } from "@/types"; + +/** + * Same `renderHook` shim used in `useClaimDetail`, `useDrawerUrlState`, + * etc. — the project doesn't ship `@testing-library/react`, so we wire a + * Probe component into a real `createRoot` and read the hook's return + * value through a shared `result` object. `act` flushes React's state + * updates between micro-tasks. + */ +function renderHook(setup: () => TResult): { + result: { current: TResult | undefined }; + unmount: () => void; +} { + const result: { current: TResult | undefined } = { current: undefined }; + const container = document.createElement("div"); + document.body.appendChild(container); + + function Probe() { + result.current = setup(); + return null; + } + + const root: Root = createRoot(container); + act(() => { + root.render(React.createElement(Probe)); + }); + + return { + result, + unmount: () => { + act(() => root.unmount()); + container.remove(); + }, + }; +} + +// --------------------------------------------------------------------------- +// Sample factories. The hook is generic over `T extends { id: string }`, +// so for `claims` we use Claim, for `remittances` we use Remittance, etc. +// Tests populate the store via `useTailStore.getState().addX(...)` — the +// same path the live-tail hook uses — so the merge is exercised against +// the real store shape (including the `claimOrder`/`remitOrder` indexing +// the store maintains for the keyed-by-id slices). +// --------------------------------------------------------------------------- + +function claim(id: string, patientName: string): Claim { + return { + id, + patientName, + providerNpi: "1234567890", + payerName: "Medicaid", + cptCode: "99213", + billedAmount: 100, + receivedAmount: 0, + status: "submitted", + submissionDate: "2026-06-20T00:00:00Z", + }; +} + +function remit(id: string, claimId: string): Remittance { + return { + id, + claimId, + payerName: "Medicaid", + paidAmount: 100, + adjustmentAmount: 0, + receivedDate: "2026-06-20", + checkNumber: id, + status: "received", + }; +} + +function activity(id: string, message: string): Activity { + return { + id, + kind: "claim_submitted", + message, + timestamp: "2026-06-20T00:00:00Z", + }; +} + +describe("useMergedTail", () => { + beforeEach(() => { + // Singleton store — clear each slice between tests so cases are + // independent. Mirrors the pattern in `tail-store.test.ts`. + useTailStore.getState().reset("claims"); + useTailStore.getState().reset("remittances"); + useTailStore.getState().reset("activity"); + }); + + it("test_merges_base_and_tail_by_id_base_first", () => { + // Base: [A, B]. Tail store adds [B, C, D] — note B is a duplicate + // of a base item (mirrors what a snapshot replay on reconnect would + // produce). After dedup, tail should contribute only [C, D]. + const base = [claim("A", "Alice"), claim("B", "Bob")]; + const { addClaim } = useTailStore.getState(); + addClaim(claim("B", "Bob-Updated")); // deduped — first write wins + addClaim(claim("C", "Carol")); + addClaim(claim("D", "Dave")); + + const { result, unmount } = renderHook(() => + useMergedTail("claims", base), + ); + + const merged = result.current ?? []; + // Base items first, in their original order; then tail in arrival + // order, with duplicates of base ids dropped. + expect(merged.map((c) => c.id)).toEqual(["A", "B", "C", "D"]); + // The base version of B must be preserved (the store's first-write- + // wins rule keeps "Bob", not "Bob-Updated"). + expect(merged[1]?.patientName).toBe("Bob"); + + unmount(); + }); + + it("test_filter_predicate_drops_tail_items_that_dont_match", () => { + // Base: [A]. Tail store: [B, C, D]. Predicate `id !== "D"` drops + // the trailing item — the filter must run AFTER dedup and only + // affect the tail contribution. + const base = [claim("A", "Alice")]; + const { addClaim } = useTailStore.getState(); + addClaim(claim("B", "Bob")); + addClaim(claim("C", "Carol")); + addClaim(claim("D", "Dave")); + + const { result, unmount } = renderHook(() => + useMergedTail("claims", base, (c) => c.id !== "D"), + ); + + const merged = result.current ?? []; + expect(merged.map((c) => c.id)).toEqual(["A", "B", "C"]); + + unmount(); + }); + + it("test_empty_tail_store_returns_base_unchanged", () => { + const base = [claim("A", "Alice"), claim("B", "Bob")]; + + const { result, unmount } = renderHook(() => + useMergedTail("claims", base), + ); + + const merged = result.current ?? []; + expect(merged).toHaveLength(2); + expect(merged.map((c) => c.id)).toEqual(["A", "B"]); + // The hook returns base verbatim when the tail slice is empty — + // no copy churn, so `toBe` reference equality holds. + expect(merged[0]).toBe(base[0]); + expect(merged[1]).toBe(base[1]); + + unmount(); + }); + + it("test_new_tail_item_appears_after_base", async () => { + // Base: [A, B]. Tail store empty initially. After the component + // mounts, we push a new claim to the store; the merged list must + // include the new tail item, surfaced via re-render. (Plan calls + // this "appears_at_top" — in this hook's contract the new item + // comes AFTER base items, not before, so the assertion is that it + // appears at all and is positioned correctly.) + const base = [claim("A", "Alice"), claim("B", "Bob")]; + + const { result, unmount } = renderHook(() => + useMergedTail("claims", base), + ); + + expect(result.current?.map((c) => c.id)).toEqual(["A", "B"]); + + // New item lands in the store → zustand notifies subscribers → the + // hook re-renders with the merged result. + await act(async () => { + useTailStore.getState().addClaim(claim("C", "Carol")); + }); + + const merged = result.current ?? []; + // Base first, then tail in arrival order. + expect(merged.map((c) => c.id)).toEqual(["A", "B", "C"]); + expect(merged[2]?.patientName).toBe("Carol"); + + unmount(); + }); + + it("test_activity_resource_uses_array_slice_and_dedupes_by_id", () => { + // Activity doesn't have a keyed-by-id map — it's a plain array. + // The hook must still dedup against baseItems by id (Activity has + // an id, even though the store's addActivity doesn't dedup). + const base = [activity("A", "alpha"), activity("B", "bravo")]; + const { addActivity } = useTailStore.getState(); + addActivity(activity("B", "bravo-dup")); // duplicate id + addActivity(activity("C", "charlie")); + addActivity(activity("D", "delta")); + + const { result, unmount } = renderHook(() => + useMergedTail("activity", base), + ); + + const merged = result.current ?? []; + // Base first, then tail in arrival order. Tail's "B" is dropped + // because it's a duplicate of base's "B". + expect(merged.map((a) => a.id)).toEqual(["A", "B", "C", "D"]); + + unmount(); + }); + + it("test_remittance_resource_orders_by_remitOrder", () => { + // Smoke test for the remittances slice — exercise the second keyed + // slice so we know the claim/remit branches are symmetric. + const base = [remit("R-1", "CLM-1")]; + const { addRemittance } = useTailStore.getState(); + addRemittance(remit("R-2", "CLM-2")); + addRemittance(remit("R-3", "CLM-3")); + + const { result, unmount } = renderHook(() => + useMergedTail("remittances", base), + ); + + const merged = result.current ?? []; + expect(merged.map((r) => r.id)).toEqual(["R-1", "R-2", "R-3"]); + + unmount(); + }); +}); diff --git a/src/hooks/useMergedTail.ts b/src/hooks/useMergedTail.ts new file mode 100644 index 0000000..b1dca07 --- /dev/null +++ b/src/hooks/useMergedTail.ts @@ -0,0 +1,77 @@ +// --------------------------------------------------------------------------- +// Live-tail merge hook (sub-project 5, Phase 5 Task 20). +// +// Reads the matching slice of `useTailStore` and returns a single array: +// `baseItems` first (in the order the caller supplied them), then any new +// tail items in arrival order, with duplicates of base ids removed. +// +// Design notes: +// - The dedup runs against `baseItems` (not the other way around) because +// the base list is the authoritative snapshot from the page's JSON +// fetch — the tail slice is an opportunistic delta, so the base wins +// when an id appears in both. +// - The optional `filterFn` is applied to the tail slice AFTER dedup so a +// page-specific filter (e.g. "only show me submitted claims") doesn't +// accidentally include items that were already filtered out of base. +// - We don't `useMemo` here: `baseItems` is a new array reference on +// every render of the caller, so memo deps would invalidate anyway. +// The merge is cheap (one Set build + two filters) and zustand's +// selector ensures we only re-render when the slice actually changes. +// --------------------------------------------------------------------------- + +import { useTailStore } from "@/store/tail-store"; +import type { TailResource } from "@/lib/tail-stream"; + +export function useMergedTail( + resource: TailResource, + baseItems: T[], + filterFn?: (item: T) => boolean, +): T[] { + // Select the slice that matches the resource. We return a fresh array + // each time so the consumer gets a stable iteration order even when + // zustand hands us a new container (Record or array) reference. + const tailSlice = useTailStore((s) => { + switch (resource) { + case "claims": { + // The store keys claims by id in a Record for O(1) updates, but + // also keeps an `claimOrder` array so we can iterate in arrival + // order. Filter out any holes (defensive — shouldn't happen but + // type-narrows the result to Claim[]). + const out: unknown[] = []; + for (const id of s.claimOrder) { + const v = s.claims[id]; + if (v !== undefined) out.push(v); + } + return out; + } + case "remittances": { + const out: unknown[] = []; + for (const id of s.remitOrder) { + const v = s.remittances[id]; + if (v !== undefined) out.push(v); + } + return out; + } + case "activity": + // Activity is already an array — the store appends in arrival + // order, so iteration order is exactly what we want. + return s.activity as unknown[]; + } + }); + + const baseIds = new Set(); + for (const b of baseItems) baseIds.add(b.id); + + const tailAfterDedup: unknown[] = []; + for (const t of tailSlice) { + const id = (t as { id: string }).id; + if (baseIds.has(id)) continue; + tailAfterDedup.push(t); + } + + const tailAfterFilter = filterFn + ? tailAfterDedup.filter((t) => filterFn(t as T)) + : tailAfterDedup; + + return [...baseItems, ...(tailAfterFilter as T[])]; +} diff --git a/src/hooks/useTailStream.test.ts b/src/hooks/useTailStream.test.ts new file mode 100644 index 0000000..5247e4b --- /dev/null +++ b/src/hooks/useTailStream.test.ts @@ -0,0 +1,403 @@ +// @vitest-environment happy-dom +// React's `act` warnings need an act-aware environment; mirror the other +// hook tests in this repo (`useClaimDetail`, `useReconciliation`, ...). +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { + afterEach, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; +import { useTailStream } from "./useTailStream"; +import { useTailStore } from "@/store/tail-store"; +import type { Claim } from "@/types"; +import type { TailEvent } from "@/lib/tail-stream"; + +// --------------------------------------------------------------------------- +// Mock `streamTail` so we can drive the async iterator from the test. The +// hook calls `streamTail(resource, { signal })` once per connection attempt; +// each call must return an async iterator we control. We replace it with a +// factory that records every generator we hand out so the test can push +// events / errors / close on demand. +// --------------------------------------------------------------------------- + +vi.mock("@/lib/tail-stream", () => ({ + streamTail: vi.fn(), +})); + +import { streamTail } from "@/lib/tail-stream"; +const mockStreamTail = vi.mocked(streamTail); + +/** + * One controllable async iterator. `push(ev)` resolves the next pending + * `next()` with `ev`. `failWith(err)` rejects the next pending `next()`. + * `close()` resolves any pending `next()` with `done: true` so the + * `for await` loop exits cleanly (this is how a server-side EOF looks to + * the hook — the spec says that triggers a reconnect). + * + * If `next()` is called when no event is queued and the iterator isn't + * closed, we suspend on a promise — exactly mirroring `ReadableStream`'s + * behaviour and letting the test decide when each event arrives. + */ +function makeCtrl() { + const queue: TailEvent[] = []; + let resolveNext: ((v: IteratorResult) => void) | null = null; + let rejectNext: ((e: unknown) => void) | null = null; + let done = false; + let pendingError: unknown = null; + + const next = (): Promise> => { + if (pendingError !== null) { + const e = pendingError; + pendingError = null; + return Promise.reject(e); + } + if (queue.length > 0) { + return Promise.resolve({ value: queue.shift() as TailEvent, done: false }); + } + if (done) { + return Promise.resolve({ value: undefined as unknown as TailEvent, done: true }); + } + return new Promise>((resolve, reject) => { + resolveNext = resolve; + rejectNext = reject; + }); + }; + + const iter: AsyncIterableIterator = { + next, + return: () => { + done = true; + if (resolveNext) { + const r = resolveNext; + resolveNext = null; + rejectNext = null; + r({ value: undefined as unknown as TailEvent, done: true }); + } + return Promise.resolve({ value: undefined as unknown as TailEvent, done: true }); + }, + throw: (err: unknown) => { + done = true; + if (rejectNext) { + const r = rejectNext; + resolveNext = null; + rejectNext = null; + r(err); + } + return Promise.reject(err); + }, + [Symbol.asyncIterator]() { + return this; + }, + }; + + return { + iter, + push(ev: TailEvent): void { + if (resolveNext) { + const r = resolveNext; + resolveNext = null; + rejectNext = null; + r({ value: ev, done: false }); + } else { + queue.push(ev); + } + }, + close(): void { + done = true; + if (resolveNext) { + const r = resolveNext; + resolveNext = null; + rejectNext = null; + r({ value: undefined as unknown as TailEvent, done: true }); + } + }, + failWith(err: unknown): void { + if (rejectNext) { + const r = rejectNext; + resolveNext = null; + rejectNext = null; + r(err); + } else { + pendingError = err; + } + }, + }; +} + +type Ctrl = ReturnType; + +/** Configure `mockStreamTail` so every call returns a fresh controllable iterator. */ +function trackCalls(): { ctrls: Ctrl[] } { + const ctrls: Ctrl[] = []; + mockStreamTail.mockImplementation(() => { + const c = makeCtrl(); + ctrls.push(c); + return c.iter; + }); + return { ctrls }; +} + +/** Same `renderHook` shim used by `useClaimDetail`, `useReconciliation`, etc. */ +function renderHook(setup: () => TResult): { + result: { current: TResult | undefined }; + unmount: () => void; +} { + const result: { current: TResult | undefined } = { current: undefined }; + const container = document.createElement("div"); + document.body.appendChild(container); + + function Probe() { + result.current = setup(); + return null; + } + + const root: Root = createRoot(container); + act(() => { + root.render(React.createElement(Probe)); + }); + + return { + result, + unmount: () => { + act(() => root.unmount()); + container.remove(); + }, + }; +} + +/** Flush microtasks + React state until `predicate()` holds (or we time out). */ +async function waitFor( + predicate: () => boolean, + timeoutMs = 1000, +): Promise { + const start = Date.now(); + while (!predicate()) { + if (Date.now() - start > timeoutMs) { + throw new Error( + `waitFor: predicate did not hold within ${timeoutMs}ms`, + ); + } + await act(async () => { + await Promise.resolve(); + }); + } +} + +/** Build a valid Claim shape for the dispatch test. */ +function makeClaim(id: string, patientName: string): Claim { + return { + id, + patientName, + providerNpi: "1234567890", + payerName: "Medicaid", + cptCode: "99213", + billedAmount: 100, + receivedAmount: 0, + status: "submitted", + submissionDate: "2026-06-20T00:00:00Z", + }; +} + +describe("useTailStream", () => { + beforeEach(() => { + mockStreamTail.mockReset(); + // Reset the singleton tail-store between tests so each case sees a + // clean claims/remittances/activity slate. + useTailStore.getState().reset("claims"); + useTailStore.getState().reset("remittances"); + useTailStore.getState().reset("activity"); + }); + + afterEach(() => { + // Restore real timers so a fake-timers-using test doesn't leak into + // the next case. + vi.useRealTimers(); + }); + + it("test_on_mount_status_connecting_then_live_after_snapshot_end", async () => { + const { ctrls } = trackCalls(); + const { result, unmount } = renderHook(() => useTailStream("claims")); + + // Wait for the effect to call streamTail once. + await waitFor(() => ctrls.length === 1); + // Initial status: connecting. + expect(result.current?.status).toBe("connecting"); + + // Server finishes replaying the snapshot — status flips to live. + ctrls[0].push({ type: "snapshot_end", data: { count: 0 } }); + await waitFor(() => result.current?.status === "live"); + expect(result.current?.status).toBe("live"); + + unmount(); + }); + + it("test_on_stream_error_status_error", async () => { + const { ctrls } = trackCalls(); + const { result, unmount } = renderHook(() => useTailStream("claims")); + + await waitFor(() => ctrls.length === 1); + + // Simulate a stream-level error event (yielded, not thrown). The hook + // should turn this into an Error and surface `status === "error"`. + ctrls[0].failWith(new Error("boom")); + + await waitFor(() => result.current?.status === "error"); + expect(result.current?.status).toBe("error"); + expect(result.current?.error).toBeInstanceOf(Error); + expect((result.current?.error as Error).message).toBe("boom"); + + unmount(); + }); + + it("test_on_abort_status_closed_no_reconnect", async () => { + const { ctrls } = trackCalls(); + const { unmount } = renderHook(() => useTailStream("claims")); + + await waitFor(() => ctrls.length === 1); + + // Capture the AbortSignal the hook passed to streamTail — the contract + // is that the hook aborts it on unmount. + const call = mockStreamTail.mock.calls[0]; + expect(call).toBeDefined(); + const opts = call[1] as { signal?: AbortSignal } | undefined; + expect(opts?.signal).toBeDefined(); + expect(opts?.signal?.aborted).toBe(false); + + unmount(); + + // After unmount the signal MUST be aborted — that's how `streamTail` + // tears down its fetch loop, and it's the externally-observable + // evidence that we cleaned up. (`status === "closed"` is set on the + // hook's state, but a renderHook shim can't observe post-unmount + // re-renders, so we assert on the abort instead.) + expect(opts?.signal?.aborted).toBe(true); + + // No reconnect must be scheduled. Advance past the entire backoff + // window and confirm `streamTail` is still at exactly one call. + vi.useFakeTimers(); + try { + await act(async () => { + vi.advanceTimersByTime(60_000); + }); + expect(mockStreamTail).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); + + it("test_on_event_dispatches_to_tail_store", async () => { + const { ctrls } = trackCalls(); + const { unmount } = renderHook(() => useTailStream("claims")); + + await waitFor(() => ctrls.length === 1); + + // Bring the stream to "live" so the dispatcher is in the happy path. + ctrls[0].push({ type: "snapshot_end", data: { count: 0 } }); + await waitFor(() => useTailStore.getState().claims !== undefined); + + // Push an item — it must end up in the claims slice. + ctrls[0].push({ + type: "item", + data: makeClaim("CLM-1", "Patient One"), + }); + await waitFor( + () => useTailStore.getState().claims["CLM-1"] !== undefined, + ); + + const stored = useTailStore.getState().claims["CLM-1"]; + expect(stored).toBeDefined(); + expect(stored?.patientName).toBe("Patient One"); + + unmount(); + }); + + it("test_reconnect_status_cycles_reconnecting_to_connecting_to_live", async () => { + const { ctrls } = trackCalls(); + + // Compress the backoff schedule so the test doesn't sit on a real + // 1-second timer. We override the global setTimeout / clearTimeout to + // schedule timers via `setImmediate`-style micro-tasks instead of + // wall-clock — but the simpler path is just `vi.useFakeTimers` and + // manually advance. We go with the simpler path. + vi.useFakeTimers(); + try { + const { result, unmount } = renderHook(() => useTailStream("claims")); + await act(async () => { + // Yield so the initial useEffect runs and calls streamTail. + await Promise.resolve(); + }); + expect(ctrls.length).toBe(1); + + // First attempt fails — hook sets status=error and schedules a + // reconnect after the first backoff step (1000ms in the real + // schedule, but we advance just past it). + ctrls[0].failWith(new Error("first failed")); + await act(async () => { + await Promise.resolve(); + }); + expect(result.current?.status).toBe("error"); + + // Advance past the first backoff step. The hook should reopen — + // status becomes "connecting" (attempt counter resets on success + // but only after snapshot_end; while retrying it's still + // "reconnecting" until the new attempt actually opens). + await act(async () => { + vi.advanceTimersByTime(1500); + await Promise.resolve(); + }); + expect(ctrls.length).toBe(2); + + // Second attempt completes the snapshot — status flips to live. + ctrls[1].push({ type: "snapshot_end", data: { count: 0 } }); + await waitFor(() => result.current?.status === "live"); + expect(result.current?.status).toBe("live"); + + unmount(); + } finally { + vi.useRealTimers(); + } + }); + + it("test_reconnect_dedup_duplicate_items_appear_once_in_store", async () => { + const { ctrls } = trackCalls(); + const { unmount } = renderHook(() => useTailStream("claims")); + + await waitFor(() => ctrls.length === 1); + ctrls[0].push({ type: "snapshot_end", data: { count: 0 } }); + await waitFor(() => useTailStore.getState().claims !== undefined); + + // Push the same claim twice (mimicking a snapshot replay on + // reconnect). The store must keep exactly one copy. + ctrls[0].push({ + type: "item", + data: makeClaim("CLM-DUP", "Original"), + }); + ctrls[0].push({ + type: "item", + data: makeClaim("CLM-DUP", "Updated"), + }); + + await waitFor( + () => useTailStore.getState().claims["CLM-DUP"] !== undefined, + ); + // Give the second push time to be processed — we want to assert it + // was rejected by the store's first-write-wins rule. + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + + expect(Object.keys(useTailStore.getState().claims)).toHaveLength(1); + expect(useTailStore.getState().claims["CLM-DUP"]?.patientName).toBe( + "Original", + ); + + unmount(); + }); +}); diff --git a/src/hooks/useTailStream.ts b/src/hooks/useTailStream.ts new file mode 100644 index 0000000..5394c22 --- /dev/null +++ b/src/hooks/useTailStream.ts @@ -0,0 +1,237 @@ +// --------------------------------------------------------------------------- +// Live-tail connection lifecycle hook (sub-project 5, Phase 5 Task 19). +// +// Opens a `streamTail(resource)` connection, dispatches `item` events into +// the matching `useTailStore` slice, exposes the connection status to the +// page (for ``), and survives transient backend failures +// with an exponential-backoff retry (1s → 2s → 4s → 8s → 16s, capped at +// 30s — spec §3.4). A stall detector flips status to `stalled` after 30s +// of silence (no event, including heartbeats) so the UI can show a stale +// connection without polling. +// +// Design notes: +// - One effect per `(resource, reconnectNonce)` pair. Bumping +// `reconnectNonce` from `forceReconnect()` re-runs the effect, which +// aborts the in-flight stream (cleanup) and opens a fresh one. +// - The AbortController is stored in a ref so `forceReconnect` can abort +// even from outside React's render cycle. +// - Backoff is indexed by an `attempt` counter that resets to 0 once the +// server completes a snapshot (i.e. we're "live"). A clean server-side +// EOF is treated like an error — we reconnect with backoff. +// - `setStatus("closed")` fires from the effect cleanup; the consumer +// (e.g. ``) typically unmounts at the same time, so +// this update is rarely observed — but it's there for any parent that +// keeps the consumer rendered after the hook is detached. +// --------------------------------------------------------------------------- + +import { useCallback, useEffect, useRef, useState } from "react"; +import { streamTail, type TailResource } from "@/lib/tail-stream"; +import { useTailStore } from "@/store/tail-store"; +import type { Activity, Claim, Remittance } from "@/types"; + +export type TailStatus = + | "connecting" + | "live" + | "reconnecting" + | "closed" + | "stalled" + | "error"; + +export interface UseTailStreamResult { + status: TailStatus; + lastEventAt: Date | null; + error: Error | null; + forceReconnect: () => void; +} + +/** Backoff schedule per spec §3.4: 1s, 2s, 4s, 8s, 16s, then cap at 30s. */ +const BACKOFF_STEPS_MS: readonly number[] = [ + 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, +]; + +/** No event (including heartbeat) for this long → flip to `stalled`. */ +const STALL_TIMEOUT_MS = 30_000; + +function backoffDelayMs(attempt: number): number { + const i = Math.min(Math.max(attempt, 0), BACKOFF_STEPS_MS.length - 1); + return BACKOFF_STEPS_MS[i] as number; +} + +/** + * Dispatch a single item event into the matching slice of the tail store. + * The store slices are typed with the canonical shapes (`Claim`, + * `Remittance`, `Activity`); the stream yields `unknown` so we cast here. + */ +function dispatch(resource: TailResource, data: unknown): void { + const store = useTailStore.getState(); + switch (resource) { + case "claims": + store.addClaim(data as Claim); + break; + case "remittances": + store.addRemittance(data as Remittance); + break; + case "activity": + store.addActivity(data as Activity); + break; + } +} + +export function useTailStream(resource: TailResource): UseTailStreamResult { + const [status, setStatus] = useState("connecting"); + const [lastEventAt, setLastEventAt] = useState(null); + const [error, setError] = useState(null); + + /** + * Bumping this state causes the effect to re-run, aborting the current + * stream and starting a fresh one. `forceReconnect` is the only place + * we mutate it from outside the effect itself. + */ + const [reconnectNonce, setReconnectNonce] = useState(0); + + /** + * Hold the in-flight AbortController so `forceReconnect` can tear it + * down even when called from a stale render. The effect cleanup also + * aborts via this ref. + */ + const abortRef = useRef(null); + + const forceReconnect = useCallback(() => { + setReconnectNonce((n) => n + 1); + }, []); + + useEffect(() => { + let cancelled = false; + let attempt = 0; + let stallTimer: ReturnType | null = null; + let backoffTimer: ReturnType | null = null; + + const clearStall = (): void => { + if (stallTimer) { + clearTimeout(stallTimer); + stallTimer = null; + } + }; + + /** + * Re-arm the stall detector. Called on every event (including + * heartbeats and `item_dropped` notices) so a quiet backend that + * still pings every <30s doesn't get flagged as stalled. The stall + * timer only flips status if we were already in a "still trying" + * state — once we're `closed` or `error`, a missed heartbeat + * shouldn't override that. + */ + const armStall = (): void => { + clearStall(); + stallTimer = setTimeout(() => { + if (cancelled) return; + setStatus((prev) => { + if ( + prev === "closed" || + prev === "reconnecting" || + prev === "error" || + prev === "stalled" + ) { + return prev; + } + return "stalled"; + }); + }, STALL_TIMEOUT_MS); + }; + + const scheduleReconnect = (): void => { + if (cancelled) return; + if (backoffTimer) { + clearTimeout(backoffTimer); + backoffTimer = null; + } + const delay = backoffDelayMs(attempt); + backoffTimer = setTimeout(() => { + if (cancelled) return; + backoffTimer = null; + attempt += 1; + void openOnce(); + }, delay); + }; + + const openOnce = async (): Promise => { + if (cancelled) return; + const controller = new AbortController(); + abortRef.current = controller; + // First attempt → "connecting". Subsequent retries → "reconnecting" + // so the user can see that we've lost the prior connection. + setStatus(attempt === 0 ? "connecting" : "reconnecting"); + + try { + const iter = streamTail(resource, { signal: controller.signal }); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const ev of iter) { + if (cancelled) return; + setLastEventAt(new Date()); + armStall(); + + switch (ev.type) { + case "snapshot_end": + // Server finished replaying; we are now live. Reset the + // backoff counter so a transient blip doesn't penalize the + // next outage. + attempt = 0; + setStatus("live"); + break; + case "item": + dispatch(resource, ev.data); + break; + case "heartbeat": + case "item_dropped": + // No state change — the stall timer has already been + // re-armed. These exist purely so the hook knows the + // connection is alive. + break; + case "error": + // Yielded (not thrown) error event. Promote to a thrown + // Error so the catch block below runs the reconnect + // machinery. + throw new Error(ev.data.message); + } + } + // Stream finished cleanly (server-side EOF). Treat as a + // reconnect-trigger: if we're still mounted, schedule a retry. + if (!cancelled) { + setStatus("reconnecting"); + scheduleReconnect(); + } + } catch (err) { + if (cancelled) return; + // An abort during teardown isn't really an error — the stream + // is being torn down on purpose. Skip the reconnect. + if (controller.signal.aborted) return; + const e = err instanceof Error ? err : new Error(String(err)); + setError(e); + setStatus("error"); + scheduleReconnect(); + } + }; + + void openOnce(); + armStall(); + + return () => { + cancelled = true; + if (abortRef.current) { + abortRef.current.abort(); + abortRef.current = null; + } + if (backoffTimer) { + clearTimeout(backoffTimer); + backoffTimer = null; + } + clearStall(); + // Per spec: on unmount, status = closed. This is the only way a + // parent that keeps the consumer mounted (e.g. for a transition) + // can observe the closed state. + setStatus("closed"); + }; + }, [resource, reconnectNonce]); + + return { status, lastEventAt, error, forceReconnect }; +} diff --git a/src/lib/tail-stream.test.ts b/src/lib/tail-stream.test.ts new file mode 100644 index 0000000..654b572 --- /dev/null +++ b/src/lib/tail-stream.test.ts @@ -0,0 +1,152 @@ +// @vitest-environment node +// `streamTail` pipes a `fetch()` response body through a `TextDecoderStream` +// and a manual NDJSON-line splitter. We mock `fetch` with `vi.stubGlobal` +// and drive a fake `ReadableStream` from the test, so no DOM/happy-dom +// is needed — Node 22's globals (Response, ReadableStream, TextDecoderStream) +// are sufficient. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { streamTail, type TailEvent } from "./tail-stream"; + +/** + * Build a controllable ReadableStream and a tiny driver. The + * stream's controller is captured at construction so the test can decide + * exactly when each NDJSON line is enqueued and when to close. The body + * is wrapped in a `Response` (matching what `fetch()` would return). + */ +function makeStream() { + const encoder = new TextEncoder(); + let controllerRef: ReadableStreamDefaultController | undefined; + const stream = new ReadableStream({ + start(c) { + controllerRef = c; + }, + }); + return { + response: new Response(stream, { + status: 200, + headers: { "content-type": "application/x-ndjson" }, + }), + push: (line: string) => { + controllerRef?.enqueue(encoder.encode(line + "\n")); + }, + close: () => controllerRef?.close(), + }; +} + +/** + * Drain an async iterable into an array. Useful for asserting the full + * emission list without manually awaiting each `.next()`. + */ +async function collect(iter: AsyncIterableIterator): Promise { + const out: T[] = []; + for await (const v of iter) out.push(v); + return out; +} + +describe("streamTail", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("test_parses_well_formed_ndjson_into_typed_events", async () => { + const driver = makeStream(); + const fetchMock = vi.fn().mockResolvedValue(driver.response); + vi.stubGlobal("fetch", fetchMock); + + const gen = streamTail("claims"); + driver.push('{"type":"item","data":{"id":"CLM-1"}}'); + driver.push('{"type":"item","data":{"id":"CLM-2"}}'); + driver.push('{"type":"item","data":{"id":"CLM-3"}}'); + driver.close(); + + const first = await gen.next(); + const second = await gen.next(); + const third = await gen.next(); + const tail = await gen.next(); + + expect(first.done).toBe(false); + expect(second.done).toBe(false); + expect(third.done).toBe(false); + expect(tail.done).toBe(true); + expect([first.value, second.value, third.value]).toEqual([ + { type: "item", data: { id: "CLM-1" } }, + { type: "item", data: { id: "CLM-2" } }, + { type: "item", data: { id: "CLM-3" } }, + ]); + + // The fetch call itself must target /api/claims/stream with the + // NDJSON Accept header. + expect(fetchMock).toHaveBeenCalledTimes(1); + const [calledUrl, calledInit] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(calledUrl).toBe("/api/claims/stream"); + expect(calledInit.headers).toMatchObject({ Accept: "application/x-ndjson" }); + }); + + it("test_yields_typed_error_on_error_line", async () => { + const driver = makeStream(); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(driver.response)); + const gen = streamTail("remittances"); + driver.push('{"type":"error","data":{"message":"x"}}'); + driver.close(); + const events = await collect(gen); + expect(events).toEqual([ + { type: "error", data: { message: "x" } }, + ]); + }); + + it("test_handles_heartbeat_silently", async () => { + // "Silently" here means: still yields the correctly-typed heartbeat + // (no special filtering) — the consumer (useTailStream) decides what + // to do with it. The lib just has to forward it. + const driver = makeStream(); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(driver.response)); + const gen = streamTail("activity"); + driver.push('{"type":"heartbeat","data":{"ts":"2026-06-20T12:00:00Z"}}'); + driver.close(); + const events = await collect(gen); + expect(events).toEqual([ + { type: "heartbeat", data: { ts: "2026-06-20T12:00:00Z" } }, + ]); + }); + + it("test_abort_signal_cancels_mid_stream", async () => { + const driver = makeStream(); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(driver.response)); + const ac = new AbortController(); + const gen = streamTail("claims", { signal: ac.signal }); + + // First event arrives normally. + driver.push('{"type":"item","data":{"id":"CLM-1"}}'); + const first = await gen.next(); + expect(first.done).toBe(false); + + // Abort before any more events arrive, then close the upstream so + // any pending read completes. The generator must exit cleanly + // (no throw) and not yield further. + ac.abort(); + driver.close(); + const second = await gen.next(); + expect(second.done).toBe(true); + }); + + it("test_malformed_line_skipped_next_valid_still_emitted", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const driver = makeStream(); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(driver.response)); + const gen = streamTail("claims"); + driver.push('{"type":"item","data":{"id":"CLM-1"}}'); + driver.push("this is not json"); + driver.push('{"type":"item","data":{"id":"CLM-2"}}'); + driver.close(); + const events = await collect(gen); + expect(events).toEqual([ + { type: "item", data: { id: "CLM-1" } }, + { type: "item", data: { id: "CLM-2" } }, + ]); + // The malformed line must produce a console.warn so operators can spot + // a buggy backend without crashing the whole tail. + expect(warnSpy).toHaveBeenCalled(); + warnSpy.mockRestore(); + }); +}); diff --git a/src/lib/tail-stream.ts b/src/lib/tail-stream.ts new file mode 100644 index 0000000..6a8b3fc --- /dev/null +++ b/src/lib/tail-stream.ts @@ -0,0 +1,189 @@ +// --------------------------------------------------------------------------- +// Live-tail NDJSON stream parser (sub-project 5, Phase 4 Task 16). +// +// Opens `GET /api/{resource}/stream` with `Accept: application/x-ndjson`, +// pipes the response body through a `TextDecoderStream` + a manual +// newline-splitter, and yields one typed `TailEvent` per line. The +// higher-level `useTailStream` hook (Phase 5) is what subscribes to this +// generator and dispatches into `tail-store`; this module is intentionally +// pure parsing — no React, no Zustand, no fetch options other than the +// ones documented in the spec. +// --------------------------------------------------------------------------- + +/** + * One event yielded by `streamTail`. The data shapes mirror the backend's + * `GET /api/{resource}/stream` NDJSON contract (see `backend/api.py`). + * + * `item.data` is intentionally `unknown` — the consumer casts to the + * page-specific shape (Claim / Remittance / Activity). The other variants + * carry the data shape the spec mandates, so consumers get a friendly + * type for the non-item events. + */ +export type TailEvent = + | { type: "item"; data: unknown } + | { type: "snapshot_end"; data: { count: number } } + | { type: "heartbeat"; data: { ts: string } } + | { type: "item_dropped"; data: { id: string } } + | { type: "error"; data: { message: string } }; + +export type TailResource = "claims" | "remittances" | "activity"; + +export interface StreamTailOptions { + /** Forwarded to `fetch`; aborting the controller cleanly exits the generator. */ + signal?: AbortSignal; + /** Override the base URL (defaults to "" — i.e. same-origin). Used in tests. */ + baseUrl?: string; +} + +const KNOWN_TYPES: ReadonlySet = new Set([ + "item", + "snapshot_end", + "heartbeat", + "item_dropped", + "error", +]); + +/** + * Open a live-tail NDJSON stream and yield one typed event per line. + * + * The generator is single-use: call `gen.return()` (or break out of a + * `for await` loop) to close the underlying fetch. When `opts.signal` + * aborts, the generator exits cleanly without throwing. + * + * @example + * for await (const ev of streamTail("claims")) { + * if (ev.type === "item") console.log(ev.data); + * } + */ +export async function* streamTail( + resource: TailResource, + opts?: StreamTailOptions, +): AsyncIterableIterator { + const base = opts?.baseUrl ?? ""; + const url = `${base}/api/${resource}/stream`; + + let res: Response; + try { + res = await fetch(url, { + headers: { Accept: "application/x-ndjson" }, + signal: opts?.signal, + }); + } catch (err) { + // Aborting the signal makes fetch reject with a DOMException; the spec + // says "if signal aborts, exit cleanly (don't throw)", so we just + // return — the consumer sees a completed iterator. + if (opts?.signal?.aborted) return; + throw err; + } + + if (!res.ok) { + throw new Error(`stream open failed: ${res.status}`); + } + if (!res.body) { + throw new Error("stream has no body"); + } + + // TextDecoderStream handles the chunked UTF-8 boundary problem for us; + // after this, we have a stream of decoded strings. We then split each + // chunk on \n and JSON.parse each non-empty line. + const stringStream = res.body.pipeThrough(new TextDecoderStream()); + const reader = stringStream.getReader(); + + let buffer = ""; + try { + // eslint-disable-next-line no-constant-condition + while (true) { + // Check the abort signal between reads so a quick abort doesn't + // require us to wait for a network read to complete. + if (opts?.signal?.aborted) return; + + let chunk: string | undefined; + let done: boolean; + try { + const result = await reader.read(); + chunk = result.value; + done = result.done; + } catch (err) { + if (opts?.signal?.aborted) return; + throw err; + } + + if (done) break; + if (chunk) buffer += chunk; + + let nl = buffer.indexOf("\n"); + while (nl !== -1) { + const line = buffer.slice(0, nl).replace(/\r$/, ""); + buffer = buffer.slice(nl + 1); + if (line.length > 0) { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch (err) { + // Malformed lines are the backend's bug, not ours. Log and + // continue so a single bad frame doesn't kill the stream. + console.warn( + "tail-stream: malformed NDJSON line, skipping", + { line, err }, + ); + nl = buffer.indexOf("\n"); + continue; + } + + if ( + typeof parsed === "object" && + parsed !== null && + "type" in parsed && + typeof (parsed as { type: unknown }).type === "string" && + KNOWN_TYPES.has((parsed as { type: string }).type as TailEvent["type"]) + ) { + yield parsed as TailEvent; + } else { + // Unknown / wrong-shape line — log and skip. A future + // backend event type shouldn't crash the page. + console.warn( + "tail-stream: unknown event shape, skipping", + { line }, + ); + } + } + nl = buffer.indexOf("\n"); + } + } + + // Flush any trailing partial line (no terminating newline). + const tail = buffer.replace(/\r$/, ""); + if (tail.length > 0) { + try { + const parsed = JSON.parse(tail) as unknown; + if ( + typeof parsed === "object" && + parsed !== null && + "type" in parsed && + typeof (parsed as { type: unknown }).type === "string" && + KNOWN_TYPES.has((parsed as { type: string }).type as TailEvent["type"]) + ) { + yield parsed as TailEvent; + } else { + console.warn( + "tail-stream: unknown trailing event shape, skipping", + { line: tail }, + ); + } + } catch (err) { + console.warn( + "tail-stream: malformed trailing NDJSON line, skipping", + { line: tail, err }, + ); + } + } + } finally { + // Release the reader lock so the underlying connection can close + // when the test / consumer drops the iterator. + try { + reader.releaseLock(); + } catch { + // already released — ignore + } + } +} diff --git a/src/pages/ActivityLog.tsx b/src/pages/ActivityLog.tsx index 5eaca51..73850cb 100644 --- a/src/pages/ActivityLog.tsx +++ b/src/pages/ActivityLog.tsx @@ -1,6 +1,9 @@ import { useCallback, useMemo } from "react"; import { useSearchParams } from "react-router-dom"; import { useActivity } from "@/hooks/useActivity"; +import { useTailStream } from "@/hooks/useTailStream"; +import { useMergedTail } from "@/hooks/useMergedTail"; +import { TailStatusPill } from "@/components/TailStatusPill"; import { ActivityFeed } from "@/components/ActivityFeed"; import { ActivityFilters, type SinceValue } from "@/components/ActivityFilters"; import { Skeleton } from "@/components/ui/skeleton"; @@ -69,12 +72,17 @@ export function ActivityLog() { }); const allItems = data?.items ?? []; + + // SP5: live-tail wiring. `useMergedTail` appends tail-arriving events + // to the base query result, then we apply the same client-side + // multi-kind filter so live events respect the active filter set. + const { status: tailStatus, lastEventAt: tailLastEventAt, forceReconnect } = + useTailStream("activity"); + const merged = useMergedTail("activity", allItems); const items = useMemo(() => { - // Server already returned a single-kind slice when `apiKind` is set; - // otherwise apply the (multi-)kind filter locally. - if (selectedKinds.length <= 1) return allItems; - return allItems.filter((a) => selectedKinds.includes(a.kind)); - }, [allItems, selectedKinds]); + if (selectedKinds.length <= 1) return merged; + return merged.filter((a) => selectedKinds.includes(a.kind)); + }, [merged, selectedKinds]); const writeParams = useCallback( (mutate: (next: URLSearchParams) => void) => { @@ -152,6 +160,16 @@ export function ActivityLog() { ) : null}
+
+ {/* SP5: live-tail status pill, right-aligned in the toolbar. */} +
+ +
+
{isLoading ? (
{Array.from({ length: 5 }).map((_, i) => ( diff --git a/src/pages/Claims.test.tsx b/src/pages/Claims.test.tsx index b870ffb..32ac777 100644 --- a/src/pages/Claims.test.tsx +++ b/src/pages/Claims.test.tsx @@ -10,6 +10,7 @@ import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { Claims } from "./Claims"; import { api } from "@/lib/api"; +import { useTailStore } from "@/store/tail-store"; import type { Claim, ClaimDetail } from "@/types"; // Module-level mock — vitest hoists vi.mock above imports. We mock the @@ -38,6 +39,30 @@ vi.mock("@/lib/api", () => { }; }); +// --------------------------------------------------------------------------- +// Live-tail hook mock (sub-project 5, Phase 5 Task 22 page-level tests). +// +// We mock `useTailStream` directly (per the plan's "latter is simpler" +// guidance) so the page-level tests don't have to drive the real +// `streamTail` parser or the real AbortController-based reconnect loop. +// `useTailStream` is exercised exhaustively in its own test file +// (`useTailStream.test.ts`); the page only needs to observe that the +// returned status flows through to `` and that items +// pushed into the tail store surface as rows via `useMergedTail`. +// +// The mock returns `status: "live"` by default — that matches what the +// real hook settles to after a `snapshot_end` event arrives, which is +// the state the page sees in production ~all of the time. +// --------------------------------------------------------------------------- +vi.mock("@/hooks/useTailStream", () => ({ + useTailStream: vi.fn(() => ({ + status: "live", + lastEventAt: null, + error: null, + forceReconnect: vi.fn(), + })), +})); + const SAMPLE_CLAIMS: Claim[] = [ { id: "CLM-1", @@ -190,6 +215,10 @@ describe("Claims page drawer wiring", () => { (api.getClaimDetail as unknown as ReturnType).mockResolvedValue( SAMPLE_DETAIL ); + // Live-tail slice is a singleton zustand store — clear it between + // tests so the live-arrival test doesn't see rows from a previous + // case. + useTailStore.getState().reset("claims"); }); afterEach(() => { @@ -420,4 +449,86 @@ describe("Claims page drawer wiring", () => { document.body.querySelector('[data-testid="keyboard-cheatsheet"]') ).toBeNull(); }); + + // ------------------------------------------------------------------- + // Live-tail integration (sub-project 5, Phase 5 Task 22 page tests). + // + // We mock `useTailStream` at the module level (see the top of this + // file) so the page sees a settled `status: "live"` without driving + // the real streamTail parser. To exercise the "tail arrival triggers a + // row" path we push a new claim directly into `useTailStore` — this + // is exactly what the real `useTailStream` hook does on `item` events, + // so we cover the same render path (zustand notify → useMergedTail + // re-derive → `` mount). + // ------------------------------------------------------------------- + + it("test_live_tail_arrival_triggers_row_in_table", async () => { + const { unmount } = renderClaims(); + + // Base list (SAMPLE_CLAIMS, set in beforeEach) must be visible first. + await settle( + () => document.body.textContent?.includes("CLM-1") ?? false, + ); + + // The new claim id is brand new (not in base, not in tail store yet). + const TAIL_ID = "CLM-TAIL-1"; + expect( + document.body.textContent?.includes(TAIL_ID) ?? false, + ).toBe(false); + + // Simulate a live-arriving claim — this is what `useTailStream` + // does when it dispatches an `item` event. Wrap in `act` because + // zustand subscribers re-render synchronously. + await act(async () => { + useTailStore.getState().addClaim({ + id: TAIL_ID, + patientName: "Live Arrival", + providerNpi: "1234567890", + payerName: "Colorado Medicaid", + cptCode: "99215", + billedAmount: 999, + receivedAmount: 0, + status: "submitted", + submissionDate: "2026-06-20", + }); + }); + + // The new row must appear — proves the end-to-end path + // zustand.addClaim → useTailStore notify → useMergedTail re-derive + // → mounts the row with the new id. + await settle( + () => document.body.textContent?.includes(TAIL_ID) ?? false, + ); + expect(document.body.textContent).toContain(TAIL_ID); + // The row's secondary cells should also be present so we know the + // row fully rendered, not just that the id appeared somewhere + // (e.g. in the search-input placeholder or similar). + expect(document.body.textContent).toContain("Live Arrival"); + expect(document.body.textContent).toContain("99215"); + + unmount(); + }); + + it("test_status_pill_shows_live_after_stream_connects", async () => { + const { unmount } = renderClaims(); + + // Wait for the base table to mount — once it's there, the toolbar + // (which contains the pill) has also mounted. + await settle( + () => document.body.textContent?.includes("CLM-1") ?? false, + ); + + // The mocked `useTailStream` returns `status: "live"`, so the + // should render with the human label "Live" + // (per `STATUS_LABEL` in `TailStatusPill.tsx`). We assert on the + // pill's textContent so this stays robust against future styling + // changes (the test doesn't depend on Tailwind classes). + const pill = document.body.querySelector( + '[data-testid="tail-status-pill"]', + ); + expect(pill).not.toBeNull(); + expect(pill?.textContent).toContain("Live"); + + unmount(); + }); }); diff --git a/src/pages/Claims.tsx b/src/pages/Claims.tsx index 5b4dae8..2286b70 100644 --- a/src/pages/Claims.tsx +++ b/src/pages/Claims.tsx @@ -27,9 +27,12 @@ import { FilterChips, type FilterChipOption } from "@/components/ui/filter-chips import { Pagination } from "@/components/ui/pagination"; import { useClaims } from "@/hooks/useClaims"; import { useDrawerUrlState } from "@/hooks/useDrawerUrlState"; +import { useTailStream } from "@/hooks/useTailStream"; +import { useMergedTail } from "@/hooks/useMergedTail"; +import { TailStatusPill } from "@/components/TailStatusPill"; import { useAppStore } from "@/store"; import { fmt } from "@/lib/format"; -import type { ClaimStatus } from "@/types"; +import type { Claim, ClaimStatus } from "@/types"; import { cn } from "@/lib/utils"; const ALL = "all" as const; @@ -74,7 +77,28 @@ export function Claims() { }; const { data, isLoading, isError, error, refetch, dataUpdatedAt } = useClaims(params); - const items = data?.items ?? []; + + // SP5 live-tail wiring (sub-project 5, Phase 5 Task 22). + // + // The tail stream emits every claim that lands in the DB — including + // ones that don't match the user's current `status` / `provider_npi` + // filter. Mirror the server-side filter the page already applies via + // `useClaims` so a newly-arrived item that wouldn't pass the same + // predicate is dropped before it can show up in the table. `useMergedTail` + // applies this to the tail slice only; base items are already filtered + // server-side. + const tailFilterFn = useMemo( + () => (c: Claim) => { + if (status !== ALL && c.status !== status) return false; + if (npi !== ALL && c.providerNpi !== npi) return false; + return true; + }, + [status, npi], + ); + + const { status: tailStatus, lastEventAt: tailLastEventAt, forceReconnect } = + useTailStream("claims"); + const items = useMergedTail("claims", data?.items ?? [], tailFilterFn); const totals = useMemo( () => ({ @@ -190,6 +214,16 @@ export function Claims() { ))} + + {/* SP5: live-tail status pill. `ml-auto` pins it to the right + edge of the toolbar so it doesn't crowd the search input. */} +
+ +
diff --git a/src/pages/Remittances.test.tsx b/src/pages/Remittances.test.tsx index b6105fe..1d3e9bf 100644 --- a/src/pages/Remittances.test.tsx +++ b/src/pages/Remittances.test.tsx @@ -10,6 +10,7 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { Remittances } from "./Remittances"; import { api } from "@/lib/api"; +import { useTailStore } from "@/store/tail-store"; // Module-level mock: vitest hoists `vi.mock` calls above imports. We only // stub the method this page touches via the `useRemittances` hook. @@ -23,6 +24,18 @@ vi.mock("@/lib/api", () => ({ }, })); +// Mock the live-tail hook so the page renders the pill in the settled +// `live` state without driving the real streamTail parser. The hook's +// own test file (`useTailStream.test.ts`) covers the lifecycle in depth. +vi.mock("@/hooks/useTailStream", () => ({ + useTailStream: vi.fn(() => ({ + status: "live", + lastEventAt: null, + error: null, + forceReconnect: vi.fn(), + })), +})); + /** * Minimal `render` helper using react-dom/client + act(). Mirrors the * `renderIntoContainer` helper in `Reconciliation.test.tsx` — see that @@ -178,6 +191,10 @@ const SAMPLE_REMITS = [ describe("Remittances", () => { beforeEach(() => { vi.clearAllMocks(); + // Singleton tail-store: clear the remittances slice between tests + // so a tail-arrival case (if added later) doesn't see rows from a + // previous test. + useTailStore.getState().reset("remittances"); ( api.listRemittances as unknown as ReturnType ).mockResolvedValue({ @@ -389,4 +406,36 @@ describe("Remittances", () => { unmount(); }); + + // ------------------------------------------------------------------- + // Live-tail status pill (sub-project 5, Phase 5 Task 23, optional + // page-level test). Cheap smoke check: the toolbar renders the pill + // with the `live` label whenever the mocked stream is settled. The + // hook's lifecycle is covered exhaustively in its own test file. + // ------------------------------------------------------------------- + + it("test_status_pill_shows_live_in_toolbar", async () => { + ( + api.listRemittances as unknown as ReturnType + ).mockResolvedValue({ + items: [], + total: 0, + returned: 0, + has_more: false, + }); + + const { unmount } = renderIntoContainer(React.createElement(Remittances)); + + // Wait for the empty-state to mount — once the page body is rendered, + // the toolbar (and thus the pill) is in the DOM. + await waitForText("Remittances · awaiting first 835"); + + const pill = document.body.querySelector( + '[data-testid="tail-status-pill"]', + ); + expect(pill).not.toBeNull(); + expect(pill?.textContent).toContain("Live"); + + unmount(); + }); }); \ No newline at end of file diff --git a/src/pages/Remittances.tsx b/src/pages/Remittances.tsx index 880dbad..1d13d2e 100644 --- a/src/pages/Remittances.tsx +++ b/src/pages/Remittances.tsx @@ -1,4 +1,4 @@ -import { Fragment, useCallback, useState } from "react"; +import { Fragment, useCallback, useMemo, useState } from "react"; import { ChevronDown, ChevronRight, Receipt } from "lucide-react"; import { Table, @@ -17,9 +17,12 @@ import { Pagination } from "@/components/ui/pagination"; import { KeyboardCheatsheet } from "@/components/KeyboardCheatsheet"; import { useRemittances } from "@/hooks/useRemittances"; import { useRowKeyboard } from "@/hooks/useRowKeyboard"; +import { useTailStream } from "@/hooks/useTailStream"; +import { useMergedTail } from "@/hooks/useMergedTail"; +import { TailStatusPill } from "@/components/TailStatusPill"; import { fmt } from "@/lib/format"; import { cn } from "@/lib/utils"; -import type { CasAdjustment, RemittanceStatus } from "@/types"; +import type { CasAdjustment, Remittance, RemittanceStatus } from "@/types"; const PAGE_SIZE = 25; @@ -71,7 +74,25 @@ export function Remittances() { limit: PAGE_SIZE, offset: (page - 1) * PAGE_SIZE, }); - const items = data?.items ?? []; + + // SP5 live-tail wiring (sub-project 5, Phase 5 Task 23). The tail + // stream emits every remittance that lands — including ones that + // don't match the user's current `status` filter chip. Drop those + // tail items so a newly-arrived remit that wouldn't match the page's + // intent doesn't flash into the table. Base items are NOT filtered + // here (they reflect whatever the list query returned, matching the + // existing page behavior). + const tailFilterFn = useMemo( + () => (r: Remittance) => { + if (status && r.status !== status) return false; + return true; + }, + [status], + ); + + const { status: tailStatus, lastEventAt: tailLastEventAt, forceReconnect } = + useTailStream("remittances"); + const items = useMergedTail("remittances", data?.items ?? [], tailFilterFn); const total = items.reduce( (acc, r) => ({ @@ -167,15 +188,25 @@ export function Remittances() {
- { - setStatus((v as RemittanceStatus | null) ?? null); - setPage(1); - }} - eyebrow="Status" - /> +
+ { + setStatus((v as RemittanceStatus | null) ?? null); + setPage(1); + }} + eyebrow="Status" + /> + {/* SP5: live-tail status pill, right-aligned in the toolbar. */} +
+ +
+
diff --git a/src/store/tail-store.test.ts b/src/store/tail-store.test.ts new file mode 100644 index 0000000..ca61566 --- /dev/null +++ b/src/store/tail-store.test.ts @@ -0,0 +1,131 @@ +// @vitest-environment node +// `useTailStore` is a pure zustand store with no DOM / React dependencies, +// so a node environment is fine. We import `getState()` and `setState` +// from the store directly (no React render) and assert on the resulting +// state shape. + +import { beforeEach, describe, expect, it } from "vitest"; +import type { Activity, Claim, Remittance } from "@/types"; +import { TAIL_CAP, useTailStore } from "./tail-store"; + +function makeClaim(id: string, overrides: Partial = {}): Claim { + return { + id, + patientName: `Patient ${id}`, + providerNpi: "1234567890", + payerName: "Medicaid", + cptCode: "99213", + billedAmount: 100, + receivedAmount: 0, + status: "submitted", + submissionDate: "2026-06-20T00:00:00Z", + ...overrides, + }; +} + +function makeRemittance(id: string, overrides: Partial = {}): Remittance { + return { + id, + claimId: `CLM-${id}`, + payerName: "Medicaid", + paidAmount: 100, + adjustmentAmount: 0, + receivedDate: "2026-06-20", + checkNumber: id, + status: "received", + ...overrides, + }; +} + +function makeActivity(id: string, overrides: Partial = {}): Activity { + return { + id, + kind: "claim_submitted", + message: `Event ${id}`, + timestamp: "2026-06-20T00:00:00Z", + ...overrides, + }; +} + +describe("useTailStore", () => { + // Each test starts from a known-empty state. The store is module-level + // (singleton), so we use `reset()` to clear between cases — that's also + // what production code calls on stream open. + beforeEach(() => { + useTailStore.getState().reset("claims"); + useTailStore.getState().reset("remittances"); + useTailStore.getState().reset("activity"); + }); + + it("test_add_claim_adds_new_claim_keyed_by_id", () => { + const { addClaim } = useTailStore.getState(); + addClaim(makeClaim("CLM-1")); + addClaim(makeClaim("CLM-2")); + const { claims } = useTailStore.getState(); + expect(Object.keys(claims)).toHaveLength(2); + expect(claims["CLM-1"]?.id).toBe("CLM-1"); + expect(claims["CLM-2"]?.id).toBe("CLM-2"); + }); + + it("test_add_claim_with_duplicate_id_is_noop", () => { + const { addClaim } = useTailStore.getState(); + addClaim(makeClaim("CLM-1", { patientName: "Original" })); + addClaim(makeClaim("CLM-1", { patientName: "Updated" })); + const { claims } = useTailStore.getState(); + expect(Object.keys(claims)).toHaveLength(1); + // First write wins; duplicates are silently dropped so the snapshot + // replay on reconnect doesn't trample the canonical row. + expect(claims["CLM-1"]?.patientName).toBe("Original"); + }); + + it("test_reset_claims_clears_only_claims_slice", () => { + const { addClaim, addRemittance, addActivity, reset } = useTailStore.getState(); + addClaim(makeClaim("CLM-1")); + addRemittance(makeRemittance("RMT-1")); + addActivity(makeActivity("ACT-1")); + + reset("claims"); + const s = useTailStore.getState(); + expect(Object.keys(s.claims)).toHaveLength(0); + expect(Object.keys(s.remittances)).toHaveLength(1); + expect(s.activity).toHaveLength(1); + }); + + it("test_add_activity_appends", () => { + const { addActivity } = useTailStore.getState(); + addActivity(makeActivity("ACT-1", { message: "first" })); + addActivity(makeActivity("ACT-2", { message: "second" })); + addActivity(makeActivity("ACT-3", { message: "third" })); + const { activity } = useTailStore.getState(); + expect(activity).toHaveLength(3); + // Activity has no stable id; it lives in an array. Insertion order is + // the only ordering signal. + expect(activity.map((a) => a.message)).toEqual(["first", "second", "third"]); + }); + + it("test_fifo_cap_evicts_oldest_when_over_10000", () => { + // Insert 5 over the cap so the eviction policy must actually run. + // The plan calls for evicting the oldest 100 (or enough to be back at + // the cap) — we just assert: size == TAIL_CAP and the very-first + // items are gone. + // + // 10 005 individual `addClaim` calls is intentionally a stress test + // of the eviction path: each call rebuilds the dict + order array + // so the wall-clock cost is O(N^2) in the number of items ever + // inserted. Bump the per-test timeout from the default 5s to 60s + // so this case can complete in CI on slow runners. + const N = TAIL_CAP + 5; + const { addClaim } = useTailStore.getState(); + for (let i = 0; i < N; i++) { + addClaim(makeClaim(`CLM-${i.toString().padStart(6, "0")}`)); + } + const { claims } = useTailStore.getState(); + expect(Object.keys(claims)).toHaveLength(TAIL_CAP); + // The five oldest (CLM-000000 .. CLM-000004) must be gone. + expect(claims["CLM-000000"]).toBeUndefined(); + expect(claims["CLM-000004"]).toBeUndefined(); + // The most recent (CLM-0010004) must be present. + const lastKey = `CLM-${(N - 1).toString().padStart(6, "0")}`; + expect(claims[lastKey]).toBeDefined(); + }, 60_000); +}); diff --git a/src/store/tail-store.ts b/src/store/tail-store.ts new file mode 100644 index 0000000..29de8f8 --- /dev/null +++ b/src/store/tail-store.ts @@ -0,0 +1,161 @@ +// --------------------------------------------------------------------------- +// Live-tail append-only store (sub-project 5, Phase 4 Task 17). +// +// Three independent slices (`claims` / `remittances` / `activity`) that +// `useTailStream` (Phase 5) writes into as `item` events arrive. Reads +// happen via `useMergedTail` (also Phase 5), which merges each slice with +// the page's JSON fetch results and applies the page's filter. +// +// Design notes: +// - `claims` and `remittances` are key-by-id maps (id is stable), so +// duplicate snapshots are dedup'd by `addClaim`/`addRemittance` (first +// write wins — the snapshot replay on reconnect must not trample the +// canonical row). +// - `activity` is an append-only array (event ids aren't guaranteed to +// be unique across snapshots; the ActivityLog renders in arrival +// order). +// - Each slice is FIFO-capped at `TAIL_CAP` (10 000) so a runaway tail +// can't grow the heap unbounded. Oldest entries are evicted in the +// add function itself — `reset` is what production calls when a new +// stream opens. +// --------------------------------------------------------------------------- + +import { create } from "zustand"; +import type { Activity, Claim, Remittance } from "@/types"; +import type { TailResource } from "@/lib/tail-stream"; + +/** Maximum number of items retained per slice before FIFO eviction kicks in. */ +export const TAIL_CAP = 10_000; + +/** + * Per-slice eviction batch. When the cap is exceeded, drop the oldest + * `EVICT_BATCH` entries in a single `set()` call. Picking a batch > 1 + * amortizes the cost of eviction when a high-volume stream pushes many + * items per frame — and 100 is small enough that a 10 005-item insert + * still lands within one `set()` call. + * + * The store does NOT require this batch to be exactly the overflow + * amount — it always drains down to the cap, so a +5 overflow evicts 5 + * even when EVICT_BATCH is 100. The batch is just an upper bound on how + * many we delete in a single pass. + */ +const EVICT_BATCH = 100; + +interface TailStore { + // --- Slices (keyed by id for the two that have stable ids) ----------- + claims: Record; + remittances: Record; + activity: Activity[]; + + // --- Insertion-order trackers (kept in sync with the dicts above) ---- + // These are private to the store; consumers only read the dicts. We + // store them as part of the state so they reactively update with the + // same `set()` call (zustand shallow-merges, so the new array ref is + // what triggers a re-render in subscribers that select `claims`). + claimOrder: string[]; + remitOrder: string[]; + + // --- Setters --------------------------------------------------------- + addClaim: (c: Claim) => void; + addRemittance: (r: RemitListItem) => void; + addActivity: (a: Activity) => void; + reset: (resource: TailResource) => void; +} + +/** + * The spec's sketch uses the placeholder name `RemitListItem` for the + * remittance shape; locally we just use the existing `Remittance` type + * from `@/types` (same value, no need to add a duplicate). + */ +type RemitListItem = Remittance; + +function evictOldest( + order: string[], + dict: Record, + batch: number, +): { order: string[]; dict: Record } { + if (order.length <= TAIL_CAP) return { order, dict }; + // Drain down to the cap; never evict more than `batch` per call so a + // 5-item overflow evicts 5, but a 1 000-item overflow evicts 100 in + // this pass and the remaining 900 in subsequent add() calls. + const toDrop = Math.min(batch, order.length - TAIL_CAP); + const dropped = order.slice(0, toDrop); + const nextOrder = order.slice(toDrop); + // Object.assign is consistently faster than `{ ...dict }` in V8 for + // large dicts; we follow up with `delete` for each evicted id. + const nextDict: Record = Object.assign({}, dict); + for (const id of dropped) delete nextDict[id]; + return { order: nextOrder, dict: nextDict }; +} + +export const useTailStore = create((set) => ({ + claims: {}, + remittances: {}, + activity: [], + claimOrder: [], + remitOrder: [], + + addClaim: (c) => + set((s) => { + // Dedup: first write wins. The snapshot replay on reconnect + // produces the same id repeatedly; we want the first occurrence + // to stick so the canonical row isn't overwritten by an older + // version that happened to be in the snapshot. + if (s.claims[c.id]) return s; + // Object.assign is faster than `{ ...s.claims, [c.id]: c }` for + // large dicts; this hot path is called once per `item` event. + const nextClaims: Record = Object.assign({}, s.claims, { + [c.id]: c, + }); + const nextOrder = s.claimOrder.concat(c.id); + if (nextOrder.length > TAIL_CAP) { + const { order, dict } = evictOldest(nextOrder, nextClaims, EVICT_BATCH); + return { claims: dict as Record, claimOrder: order }; + } + return { claims: nextClaims, claimOrder: nextOrder }; + }), + + addRemittance: (r) => + set((s) => { + if (s.remittances[r.id]) return s; + const nextRemits: Record = Object.assign( + {}, + s.remittances, + { [r.id]: r }, + ); + const nextOrder = s.remitOrder.concat(r.id); + if (nextOrder.length > TAIL_CAP) { + const { order, dict } = evictOldest(nextOrder, nextRemits, EVICT_BATCH); + return { + remittances: dict as Record, + remitOrder: order, + }; + } + return { remittances: nextRemits, remitOrder: nextOrder }; + }), + + addActivity: (a) => + set((s) => { + // Activity has no stable id, so it's a plain append. FIFO cap + // evicts the oldest with a single `slice` (the typical case is + // +1, +1, +1; an extreme burst falls back to multiple slice + // passes). + let next = [...s.activity, a]; + if (next.length > TAIL_CAP) { + next = next.slice(next.length - TAIL_CAP); + } + return { activity: next }; + }), + + reset: (resource) => + set(() => { + switch (resource) { + case "claims": + return { claims: {}, claimOrder: [] }; + case "remittances": + return { remittances: {}, remitOrder: [] }; + case "activity": + return { activity: [] }; + } + }), +}));