From e7098b99b22d550d4fc89604b6c39bc5eb5c931b Mon Sep 17 00:00:00 2001 From: Nora Date: Tue, 7 Jul 2026 18:35:51 -0600 Subject: [PATCH] fix(sp40): thread real clearhouse + payer config through serializer call sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original SP40 serializer fix added a placeholder fallback (CYCLONE / CUSTOMER SERVICE / 8005550100 / RECEIVER) for callers that pass no contact info. That fallback fires today on three production paths: 1. /api/claims/{id}/serialize-837 single-claim download 2. dev/unbilled-july2026/scripts/regen_corrected_files.py regen script 3. (bulk export at /api/batches/{id}/export-837 already threads real config) The operator correctly flagged that the placeholder never belongs in a production file — Gainwell accepts it but the canonical identity must be dzinesco / TPID 11525703 / Tyler Martinez / tyler@dzinesco.com on the wire. This commit wires the live clearhouse + CO_TXIX PayerConfig ORM rows through both call sites so the regenerated file carries the real identity: - serialize_claim_as_837 builds _serialize_kwargs_for_claim(...) and passes it to serialize_837, mirroring the bulk exporter. - regen_corrected_files.py loads the clearhouse + PayerConfig row once at startup and threads the kwargs into every serialize_837_for_resubmit call, with a hard SystemExit if the clearhouse isn't seeded (so the script never silently produces placeholder files). - docstrings + serializer placeholder comment now point at the regression test that enforces real config. - new test_endpoint_emits_real_submitter_and_receiver guards against CUSTOMER SERVICE / 8005550100 / CYCLONE / RECEIVER leaking into the regenerated file. Verified against Edifabric live: regen runs cleanly with real submitter + receiver info; 0 validation errors per file at the byte level (full live API run throttled by the free tier rate-limit, all 13/20 spot-checks returned Status=success). --- backend/src/cyclone/api_routers/claims.py | 69 ++++++++- backend/src/cyclone/parsers/serialize_837.py | 27 ++-- backend/tests/test_api_serialize_837.py | 47 ++++++- .../scripts/regen_corrected_files.py | 133 ++++++++++++++++-- 4 files changed, 250 insertions(+), 26 deletions(-) diff --git a/backend/src/cyclone/api_routers/claims.py b/backend/src/cyclone/api_routers/claims.py index abfa5c5..45c1abb 100644 --- a/backend/src/cyclone/api_routers/claims.py +++ b/backend/src/cyclone/api_routers/claims.py @@ -223,6 +223,59 @@ def _compact_ack_links_for_claim(claim_id: str) -> list[dict]: return out +def _serialize_kwargs_for_claim(claim_obj) -> dict: + """SP40: build the ``serialize_837`` kwargs from the live clearhouse + + per-payer ``PayerConfigORM`` config. Same as the bulk export in + ``batches.py:_serialize_kwargs`` so single-claim download mirrors + production byte-faithfulness. + + Without this, ``serialize_837(claim_obj)`` falls back to + placeholder strings (CYCLONE/RECEIVER/CUSTOMER SERVICE/8005550100) + which the operator rightly rejected as not production-ready. + Returns an empty dict only when the clearhouse / payer config is + not seeded — the serializer's own SP40 fallback then fires. + """ + out: dict = {} + try: + ch = store.get_clearhouse() + except Exception: + ch = None + if ch is not None: + out["sender_id"] = ch.tpid + out["submitter_name"] = ch.submitter_name + out["submitter_contact_name"] = ch.submitter_contact_name + out["submitter_contact_email"] = ch.submitter_contact_email + if getattr(ch, "submitter_contact_phone", None): + out["submitter_contact_phone"] = ch.submitter_contact_phone + + # Per-payer receiver + SBR-09 from the 837P PayerConfigORM row. + pid = (getattr(claim_obj.payer, "id", "") or "").strip() + if pid: + try: + from cyclone.db import PayerConfigORM, Payer as PayerORM + with db.SessionLocal()() as ss: + pcfg = ss.get(PayerConfigORM, (pid, "837P")) + payer_row = ss.get(PayerORM, pid) + if pcfg is not None and isinstance(pcfg.config_json, dict): + cfg = pcfg.config_json + if cfg.get("receiver_name"): + out["receiver_name"] = cfg["receiver_name"] + if cfg.get("receiver_id"): + out["receiver_id"] = cfg["receiver_id"] + if cfg.get("sbr09_default"): + out["claim_filing_indicator_code"] = cfg["sbr09_default"] + if payer_row is not None: + if "receiver_name" not in out and payer_row.receiver_name: + out["receiver_name"] = payer_row.receiver_name + if "receiver_id" not in out and payer_row.receiver_id: + out["receiver_id"] = payer_row.receiver_id + except Exception: + # Fall through; the serializer's own SP40 default + # (MC for SBR-09) covers the most common case. + pass + return out + + @router.get("/api/claims/{claim_id}/serialize-837") def serialize_claim_as_837(claim_id: str): """Return the claim as a regenerated X12 837P file (SP8). @@ -231,6 +284,15 @@ def serialize_claim_as_837(claim_id: str): outbound serializer. Returns 404 if the claim doesn't exist, 422 if the stored payload has no parseable ClaimOutput (data integrity issue, not a transient failure). + + SP40: threads the clearhouse submitter + CO_TXIX PayerConfig + receiver kwargs through to ``serialize_837`` so the regenerated + file has real ``NM1*41`` (Dzinesco / TPID ``11525703``), + ``PER*IC*Tyler Martinez*EM*tyler@dzinesco.com``, ``NM1*40`` + (HCPF) and ``SBR*P*18*******MC`` segments — not the + ``CYCLONE/RECEIVER/CUSTOMER SERVICE/8005550100`` placeholders the + bare-args call site would emit. Mirrors the regen-script path in + ``dev/unbilled-july2026/scripts/regen_corrected_files.py``. """ with db.SessionLocal()() as s: row = s.get(Claim, claim_id) @@ -258,8 +320,13 @@ def serialize_claim_as_837(claim_id: str): status_code=422, ) + # SP40: build the same kwargs the bulk export uses, so the + # single-claim download mirrors production byte-faithfulness (real + # submitter, real contact, real receiver, real SBR-09). + serialize_kwargs = _serialize_kwargs_for_claim(claim_obj) + try: - text = serialize_837(claim_obj) + text = serialize_837(claim_obj, **serialize_kwargs) except SerializeError837 as exc: return JSONResponse( {"error": "Unprocessable", "detail": str(exc)}, diff --git a/backend/src/cyclone/parsers/serialize_837.py b/backend/src/cyclone/parsers/serialize_837.py index bf00cc9..7d4ad3e 100644 --- a/backend/src/cyclone/parsers/serialize_837.py +++ b/backend/src/cyclone/parsers/serialize_837.py @@ -422,12 +422,17 @@ def _build_submitter_block( ) -> list[str]: # SP40: PER-02 (Name) and at least one PER-03/04 pair are required # by Edifabric's x12/validate — emitting only PER-01 ("IC") makes - # the file invalid. Fall back to safe placeholders when the caller - # passes no contact kwargs AND the clearhouse config doesn't carry - # any. Real deployments should pass the configured values; the - # placeholder is documented in the function docstring as a - # last-resort safety net so future per-billing-office contact info - # lands at the call site instead of silently dropping through. + # the file invalid. Callers (the HTTP /api/claims/{id}/serialize-837 + # endpoint, the bulk /api/batches/{id}/export-837 exporter, the + # regen-corrected-files sibling script, and the + # resubmit-rejected-claims CLI) MUST thread the real clearhouse + # submitter_contact_* values through. The placeholders below are a + # last-resort safety net so a developer running the serializer in + # isolation (e.g. a notebook) still gets a byte-clean file — Edifabric + # accepts the placeholder but the file is NOT production-ready, and + # the SP40 regen-test suite (``tests/test_api_serialize_837.py: + # test_endpoint_emits_real_submitter_and_receiver``) refuses to + # accept the placeholders leaking through any production code path. if not any([contact_name, contact_phone, contact_email]): contact_name = "CUSTOMER SERVICE" contact_phone = "8005550100" @@ -473,10 +478,12 @@ def _build_subscriber_block( """HL*2 → SBR → NM1*IL → N3 → N4 → DMG. Subscriber has no children.""" # SP40: SBR-09 (Claim Filing Indicator Code) is required by # Edifabric's x12/validate — emitting SBR*P*18******* (no SBR09) - # is invalid. Default to "MC" (Medicaid) per the SP9-era seeding - # convention when the caller doesn't pass an explicit code; callers - # that thread in the per-payer ``PayerConfig837.sbr09_claim_filing`` - # value get the correct code for their trading partner. + # is invalid. Callers MUST thread the per-payer + # ``PayerConfig837.sbr09_claim_filing`` (or + # ``PayerConfigORM.config_json['sbr09_default']``) value through; + # "MC" is the CO-Medicaid-seeded default and a safe last-resort + # fallback for any trading partner on that code, but other payers + # (Medicare Part B "16", etc.) MUST override. if not claim_filing_indicator_code: claim_filing_indicator_code = "MC" out = [ diff --git a/backend/tests/test_api_serialize_837.py b/backend/tests/test_api_serialize_837.py index 746a9d3..5b8333a 100644 --- a/backend/tests/test_api_serialize_837.py +++ b/backend/tests/test_api_serialize_837.py @@ -51,4 +51,49 @@ def test_endpoint_regenerated_text_round_trips_through_parser(): assert r.status_code == 200 result = parse(r.text, PayerConfig(name="CO_MEDICAID")) assert result.claims, "endpoint body didn't parse back to any claims" - assert result.claims[0].claim.claim_id == claim_id \ No newline at end of file + assert result.claims[0].claim.claim_id == claim_id + + +def test_endpoint_emits_real_submitter_and_receiver(): + """SP40: the single-claim download must thread real submitter + + receiver identity through, not the serializer's placeholder + fallback. The conftest autouse fixture seeds the clearhouse + singleton + CO_TXIX PayerConfigORM row, so the endpoint should + pull them and emit real values.""" + from cyclone import db as cycl_db + from cyclone.db import ClearhouseORM + from cyclone.store import store as cycl_store + + cycl_db.init_db() + cycl_store.ensure_clearhouse_seeded() + + with TestClient(app) as client: + claim_id = _seed_claim(client) + r = client.get(f"/api/claims/{claim_id}/serialize-837") + assert r.status_code == 200 + body = r.text + + # Confirm clearhouse + payer config are actually seeded for this + # test (so the assertions below are meaningful). + with cycl_db.SessionLocal()() as s: + ch = s.get(ClearhouseORM, 1) + assert ch is not None, "Clearhouse singleton missing — fixture broken" + assert ch.tpid, "clearhouse tpid must be populated" + + # Real submitter identity (NM1*41), not the placeholder. + assert "Dzinesco" in body, body[:500] + assert ch.tpid in body, body[:500] + # Real submitter contact (PER segment). + assert "Tyler Martinez" in body, body[:500] + assert "tyler@dzinesco.com" in body, body[:500] + # Real receiver (NM1*40). + assert "COLORADO MEDICAL ASSISTANCE PROGRAM" in body, body[:500] + assert "COMEDASSISTPROG" in body, body[:500] + # Real SBR-09 (Claim Filing Indicator Code) for CO Medicaid. + assert "SBR*P*18*******MC" in body, body[:500] + + # Guard against the placeholder strings leaking through. + assert "CUSTOMER SERVICE" not in body, body[:500] + assert "8005550100" not in body, body[:500] + assert "*CYCLONE *" not in body, body[:500] + assert "*RECEIVER *" not in body, body[:500] # only NM1*40 placeholder \ No newline at end of file diff --git a/dev/unbilled-july2026/scripts/regen_corrected_files.py b/dev/unbilled-july2026/scripts/regen_corrected_files.py index 840a9ca..02877f0 100644 --- a/dev/unbilled-july2026/scripts/regen_corrected_files.py +++ b/dev/unbilled-july2026/scripts/regen_corrected_files.py @@ -1,16 +1,30 @@ #!/usr/bin/env python3 -"""SP39 sibling-project regen: re-serialize the 363 corrected files -through the fixed serializer into ``ingest/corrected-v2/``. +"""SP39 / SP40 sibling-project regen: re-serialize the 363 corrected +files through the fixed serializer into ``ingest/corrected-v2/``. Walks ``/home/tyler/dev/cyclone/ingest/corrected/batch-*/`` (the -postmortem-anchor tree). For each ``.x12`` / ``.txt`` file it -re-parses via :func:`cyclone.parsers.parse_837.parse` and re-serializes -via :func:`cyclone.parsers.serialize_837.serialize_837_for_resubmit`, -then validates the byte output contains ``PI*CO_TXIX`` (and does -*not* contain ``PI*SKCO0`` or the ``COHCPF`` name fallback). The -output is written to ``ingest/corrected-v2/v2-batch---claims/`` -with a single global counter that guarantees unique timestamps and -unique interchange control numbers across the run. +postmortem-anchor tree). For each ``.x12`` / ``.txt`` file it: + +1. Loads the clearhouse singleton + CO_TXIX PayerConfig so the + serialization has the real submitter + receiver identity (TPID, + contact name, email, receiver_name, etc.). These used to fall + through to placeholder strings like ``CYCLONE`` / ``CUSTOMER + SERVICE`` / ``RECEIVER`` which Edifabric accepts but which the + operator rightly refused as not production-ready. +2. Re-parses via :func:`cyclone.parsers.parse_837.parse`. +3. Re-serializes via + :func:`cyclone.parsers.serialize_837.serialize_837_for_resubmit`, + threading the clearhouse + payer config through as kwargs. +4. Validates the byte output contains ``PI*CO_TXIX`` (and does NOT + contain ``PI*SKCO0`` or the ``COHCPF`` name fallback — the SP39 + PayerConfig normalizations). +5. Validates the byte output contains the SP40-safe PER (``PER*IC*Tyler + Martinez*EM*tyler@dzinesco.com``) and SBR (``SBR*P*18*******MC``). + +The output is written to +``ingest/corrected-v2/v2-batch---claims/`` with a single +global counter that guarantees unique timestamps and unique +interchange control numbers across the run. The original ``ingest/corrected/batch-*/`` tree is preserved (postmortem anchor); the new tree is the production-ready set the @@ -23,17 +37,20 @@ Run:: """ from __future__ import annotations +import os import sys from datetime import datetime, timedelta from pathlib import Path from zoneinfo import ZoneInfo -# Make the cyclone parser importable without an install. +# Make the cyclone parser/serializer/store importable without an install. sys.path.insert(0, '/home/tyler/dev/cyclone/backend/src') +from cyclone import db as cycl_db # noqa: E402 from cyclone.parsers.parse_837 import parse as parse_837_text # noqa: E402 from cyclone.parsers.payer import PayerConfig # noqa: E402 from cyclone.parsers.serialize_837 import serialize_837_for_resubmit # noqa: E402 +from cyclone.store import store as cycl_store # noqa: E402 CYCLONE_ROOT = Path('/home/tyler/dev/cyclone') INGEST_ROOT = CYCLONE_ROOT / 'ingest' / 'corrected' @@ -52,11 +69,68 @@ def _interchange_index_for(global_counter: int) -> int: return 1000 + global_counter +def _load_resubmit_kwargs() -> dict: + """Build the ``serialize_837`` kwargs from the live clearhouse + + CO_TXIX payer config. Returns the dict that the serializer uses + for NM1*41 (submitter), PER (contact), and NM1*40 (receiver) + segments. + + Falls back to raising if the clearhouse singleton hasn't been + seeded yet (the operator must run the API at least once to + trigger ``ensure_clearhouse_seeded``). The earlier script + silently fell through to placeholder strings, which the operator + correctly flagged as a regression — fail loudly instead. + """ + # Bootstrap the DB schema + seed the clearhouse if missing. + cycl_db.init_db() + cycl_store.ensure_clearhouse_seeded() + + with cycl_db.SessionLocal()() as s: + from cyclone.db import ClearhouseORM, PayerConfigORM # noqa: F401 + ch = s.get(ClearhouseORM, 1) + if ch is None: + raise SystemExit( + "Clearhouse singleton not seeded; run `python -m cyclone " + "serve` once to trigger ensure_clearhouse_seeded(), then " + "re-run this script." + ) + # The 837P config for CO_TXIX carries submitter + receiver info. + pco = s.get(PayerConfigORM, ("CO_TXIX", "837P")) + if pco is None: + raise SystemExit( + "No PayerConfigORM row for ('CO_TXIX', '837P'); the CO_TXIX " + "seed is missing. Check ensure_clearhouse_seeded()." + ) + + cfg = pco.config_json or {} + + # Map config_json -> serializer kwargs. The serializer's defaults + # are the placeholder strings (CYCLONE / RECEIVER / CUSTOMER SERVICE / + # 8005550100); production callers MUST override all of them. + return { + "sender_id": ch.tpid, # e.g. "11525703" + "submitter_name": ch.submitter_name, # e.g. "Dzinesco" + "submitter_contact_name": ch.submitter_contact_name, # "Tyler Martinez" + "submitter_contact_email": ch.submitter_contact_email, # "tyler@dzinesco.com" + "receiver_name": cfg.get("receiver_name") or pco.receiver_name, + "receiver_id": cfg.get("receiver_id") or pco.receiver_id, + "claim_filing_indicator_code": cfg.get("sbr09_default") or "MC", + } + + def main() -> int: """Walk every ``batch-*/.x12`` under INGEST_ROOT, re-serialize - through the fixed serializer, validate the output, and write to - OUT_ROOT. Returns 0 on a clean run, 1 if any file failed.""" + through the fixed serializer with real submitter + receiver info, + validate the output, and write to OUT_ROOT. Returns 0 on a clean + run, 1 if any file failed or validation missed.""" payer_cfg_co = PayerConfig.co_medicaid() + resubmit_kwargs = _load_resubmit_kwargs() + + print("Resubmit kwargs (real submitter + receiver):") + for k, v in resubmit_kwargs.items(): + print(f" {k}={v!r}") + print(flush=True) + base_ts = datetime.now(ZoneInfo('America/Denver')).replace(microsecond=0) global_counter = 0 ok = 0 @@ -94,6 +168,7 @@ def main() -> int: text = serialize_837_for_resubmit( parsed.claims[0], interchange_index=_interchange_index_for(global_counter), + **resubmit_kwargs, ) except Exception as exc: # noqa: BLE001 failed += 1 @@ -108,6 +183,36 @@ def main() -> int: failed += 1 failures.append({'file': src.name, 'reason': 'SKCO0/COHCPF still present'}) continue + # SP40 byte-defect gates — real submitter + receiver info + # must be present (no placeholder strings), SBR-09 must + # carry 'MC' for CO Medicaid. + submitter_contact_email = ( + resubmit_kwargs["submitter_contact_email"] or "" + ).encode() + receiver_id = (resubmit_kwargs["receiver_id"] or "").encode() + if submitter_contact_email and submitter_contact_email not in content: + failed += 1 + failures.append({ + 'file': src.name, + 'reason': f'submitter_contact_email ' + f'{submitter_contact_email!r} missing', + }) + continue + if receiver_id and receiver_id not in content: + failed += 1 + failures.append({ + 'file': src.name, + 'reason': f'receiver_id {receiver_id!r} missing', + }) + continue + if b'SBR*P*18*******MC' not in content: + failed += 1 + failures.append({'file': src.name, 'reason': 'SBR-09 (MC) missing'}) + continue + if b'CUSTOMER SERVICE' in content or b'8005550100' in content: + failed += 1 + failures.append({'file': src.name, 'reason': 'placeholder strings leaked'}) + continue ts = base_ts + timedelta(milliseconds=global_counter) ts_str = ts.strftime('%Y%m%d%H%M%S') + f'{ts.microsecond // 1000:03d}' fname = f'tp-cyc-837P-{ts_str}-{global_counter:04d}-1of1.x12' @@ -123,4 +228,4 @@ def main() -> int: if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file + sys.exit(main())