#!/usr/bin/env python3 """SP39 / SP40 sibling-project regen: re-serialize the 363 corrected files through the fixed serializer into ``ingest/corrected-v2/``. Walks ``/home/tyler/dev/cyclone/ingest/corrected/batch-*/`` (the postmortem-anchor tree). For each ``.x12`` / ``.txt`` file it: 1. Loads the clearhouse singleton + CO_TXIX PayerConfig so the serialization has the real submitter + receiver identity (TPID, contact name, email, receiver_name, etc.). These used to fall through to placeholder strings like ``CYCLONE`` / ``CUSTOMER SERVICE`` / ``RECEIVER`` which Edifabric accepts but which the operator rightly refused as not production-ready. 2. Re-parses via :func:`cyclone.parsers.parse_837.parse`. 3. Re-serializes via :func:`cyclone.parsers.serialize_837.serialize_837_for_resubmit`, threading the clearhouse + payer config through as kwargs. 4. Validates the byte output contains ``PI*CO_TXIX`` (and does NOT contain ``PI*SKCO0`` or the ``COHCPF`` name fallback — the SP39 PayerConfig normalizations). 5. Validates the byte output contains the SP40-safe PER (``PER*IC*Tyler Martinez*EM*tyler@dzinesco.com``) and SBR (``SBR*P*18*******MC``). **Filename format** — HCPF X12 File Naming Standards Quick Guide: ``tp{tpid}-{transaction_type}-yyyymmddhhmmssSSS-1of1.x12`` (example: ``tp11525703-837P-20260708132243505-1of1.x12``). Built via :func:`cyclone.edi.filenames.build_outbound_filename` (the single canonical HCPF outbound builder) so the 17-digit millisecond-precision timestamp + "1of1" sequence + ".x12" extension are guaranteed. Each file gets a unique millisecond slot by spreading the batch 1ms apart over the run, so 363 files per regen land in ~363ms with millisecond-level uniqueness — well within the single-second "1of1" window the spec allows. The output is written to ``ingest/corrected-v2/v2-batch---claims/``. The original ``ingest/corrected/batch-*/`` tree is preserved (postmortem anchor); the new tree is the production-ready set the operator pushes to Gainwell. Run:: cd /home/tyler/dev/unbilled-july2026 /home/tyler/dev/cyclone/backend/.venv/bin/python scripts/regen_corrected_files.py """ from __future__ import annotations import sys from pathlib import Path # Make the cyclone parser/serializer/store importable without an install. sys.path.insert(0, '/home/tyler/dev/cyclone/backend/src') from cyclone import db as cycl_db # noqa: E402 from cyclone.edi.filenames import build_outbound_filename # noqa: E402 from cyclone.parsers.parse_837 import parse as parse_837_text # noqa: E402 from cyclone.parsers.payer import PayerConfig # noqa: E402 from cyclone.parsers.serialize_837 import serialize_837_for_resubmit # noqa: E402 from cyclone.store import store as cycl_store # noqa: E402 CYCLONE_ROOT = Path('/home/tyler/dev/cyclone') INGEST_ROOT = CYCLONE_ROOT / 'ingest' / 'corrected' OUT_ROOT = CYCLONE_ROOT / 'ingest' / 'corrected-v2' def _batch_id_from_dirname(d: Path) -> str: """``batch---claims`` -> ```` (the first hyphen-delimited chunk).""" return d.name.split('-')[1] def _interchange_index_for(global_counter: int) -> int: """The serialize helper needs a unique ISA13 per file; offset by 1000 so the resulting ISA13s stay well above the seeded fixture range (0-999).""" return 1000 + global_counter def _load_resubmit_kwargs() -> dict: """Build the ``serialize_837`` kwargs from the live clearhouse + CO_TXIX payer config. Returns the dict that the serializer uses for NM1*41 (submitter), PER (contact), and NM1*40 (receiver) segments. Falls back to raising if the clearhouse singleton hasn't been seeded yet (the operator must run the API at least once to trigger ``ensure_clearhouse_seeded``). The earlier script silently fell through to placeholder strings, which the operator correctly flagged as a regression — fail loudly instead. """ # Bootstrap the DB schema + seed the clearhouse if missing. cycl_db.init_db() cycl_store.ensure_clearhouse_seeded() with cycl_db.SessionLocal()() as s: from cyclone.db import ClearhouseORM, PayerConfigORM # noqa: F401 ch = s.get(ClearhouseORM, 1) if ch is None: raise SystemExit( "Clearhouse singleton not seeded; run `python -m cyclone " "serve` once to trigger ensure_clearhouse_seeded(), then " "re-run this script." ) # The 837P config for CO_TXIX carries submitter + receiver info. pco = s.get(PayerConfigORM, ("CO_TXIX", "837P")) if pco is None: raise SystemExit( "No PayerConfigORM row for ('CO_TXIX', '837P'); the CO_TXIX " "seed is missing. Check ensure_clearhouse_seeded()." ) cfg = pco.config_json or {} # Map config_json -> serializer kwargs. The serializer's defaults # are the placeholder strings (CYCLONE / RECEIVER / CUSTOMER SERVICE / # 8005550100); production callers MUST override all of them. return { "sender_id": ch.tpid, # e.g. "11525703" "submitter_name": ch.submitter_name, # e.g. "Dzinesco" "submitter_contact_name": ch.submitter_contact_name, # "Tyler Martinez" "submitter_contact_email": ch.submitter_contact_email, # "tyler@dzinesco.com" "receiver_name": cfg.get("receiver_name") or pco.receiver_name, "receiver_id": cfg.get("receiver_id") or pco.receiver_id, "claim_filing_indicator_code": cfg.get("sbr09_default") or "MC", } def main() -> int: """Walk every ``batch-*/.x12`` under INGEST_ROOT, re-serialize through the fixed serializer with real submitter + receiver info, validate the output, and write to OUT_ROOT. Returns 0 on a clean run, 1 if any file failed or validation missed. Filenames follow the HCPF X12 File Naming Standards Quick Guide: ``tp{tpid}-837P-yyyymmddhhmmssSSS-1of1.x12``. The 17-digit millisecond-precision timestamp comes from :func:`build_outbound_filename` so the spec format is guaranteed (no ``-cyc-`` segment, no trailing ``-{counter}-`` suffix, no abbreviation). Each file's timestamp is offset by 1ms so all 363 files land in distinct millisecond slots. """ payer_cfg_co = PayerConfig.co_medicaid() resubmit_kwargs = _load_resubmit_kwargs() tpid = resubmit_kwargs["sender_id"] # ISA06 — used in the filename assert tpid.isdigit(), f"TPID must be digits, got {tpid!r}" print("Resubmit kwargs (real submitter + receiver):") for k, v in resubmit_kwargs.items(): print(f" {k}={v!r}") print(f"Filename template: tp{tpid}-837P-yyyymmddhhmmssSSS-1of1.x12") print(f" (built via cyclone.edi.filenames.build_outbound_filename)", flush=True) global_counter = 0 ok = 0 failed = 0 failures: list[dict] = [] batch_dirs = sorted(p for p in INGEST_ROOT.iterdir() if p.is_dir() and p.name.startswith('batch-')) print(f'Found {len(batch_dirs)} batch dirs under {INGEST_ROOT}', flush=True) for batch_dir in batch_dirs: batch_id = _batch_id_from_dirname(batch_dir) files = sorted(batch_dir.glob('*.x12')) + sorted(batch_dir.glob('*.txt')) if not files: continue n = len(files) out_dir = OUT_ROOT / f'v2-batch-{batch_id}-{n}-claims' out_dir.mkdir(parents=True, exist_ok=True) print(f' {batch_id[:12]}... ({n} files) -> {out_dir}', flush=True) for src in files: global_counter += 1 # Each file's filename = real HCPF format with a unique # millisecond slot (1ms stride from now onward). fname = build_outbound_filename( tpid, "837P", now_mt=_now_mt_plus_ms(global_counter), ) raw = src.read_text() try: parsed = parse_837_text(raw, payer_cfg_co) except Exception as exc: # noqa: BLE001 failed += 1 failures.append({'file': src.name, 'reason': f'parse: {exc}'}) continue if not parsed.claims: failed += 1 failures.append({'file': src.name, 'reason': 'no claims parsed'}) continue try: text = serialize_837_for_resubmit( parsed.claims[0], interchange_index=_interchange_index_for(global_counter), **resubmit_kwargs, ) except Exception as exc: # noqa: BLE001 failed += 1 failures.append({'file': src.name, 'reason': f'serialize: {exc}'}) continue content = text.encode() if b'PI*CO_TXIX' not in content: failed += 1 failures.append({'file': src.name, 'reason': 'PI*CO_TXIX missing'}) continue if b'PI*SKCO0' in content or b'*COHCPF****' in content: failed += 1 failures.append({'file': src.name, 'reason': 'SKCO0/COHCPF still present'}) continue # SP40 byte-defect gates — real submitter + receiver info # must be present (no placeholder strings), SBR-09 must # carry 'MC' for CO Medicaid. submitter_contact_email = ( resubmit_kwargs["submitter_contact_email"] or "" ).encode() receiver_id = (resubmit_kwargs["receiver_id"] or "").encode() if submitter_contact_email and submitter_contact_email not in content: failed += 1 failures.append({ 'file': src.name, 'reason': f'submitter_contact_email ' f'{submitter_contact_email!r} missing', }) continue if receiver_id and receiver_id not in content: failed += 1 failures.append({ 'file': src.name, 'reason': f'receiver_id {receiver_id!r} missing', }) continue if b'SBR*P*18*******MC' not in content: failed += 1 failures.append({'file': src.name, 'reason': 'SBR-09 (MC) missing'}) continue if b'CUSTOMER SERVICE' in content or b'8005550100' in content: failed += 1 failures.append({'file': src.name, 'reason': 'placeholder strings leaked'}) continue # Final HCPF-spec filename validation guard (cheaper to # re-check the filename than to discover a malformed # filename on upload to Gainwell). from cyclone.edi.filenames import is_outbound_filename if not is_outbound_filename(fname): failed += 1 failures.append({'file': src.name, 'reason': f'filename {fname!r} does not match ' f'HCPF OUTBOUND_RE (SPEC VIOLATION)'}) continue (out_dir / fname).write_text(text) ok += 1 print(f'\nDONE: ok={ok} failed={failed}', flush=True) if failures: print('First failures:') for f in failures[:5]: print(f' {f}') return 0 if failed == 0 else 1 def _now_mt_plus_ms(ms_offset: int): """Return the current Mountain Time as a tz-aware datetime, offset by ``ms_offset`` milliseconds. Used to spread the regen run across distinct millisecond slots so each file's filename is unique within the spec's 17-digit yyyymmddhhmmssSSS window.""" from datetime import datetime, timedelta from zoneinfo import ZoneInfo return datetime.now(ZoneInfo("America/Denver")) + timedelta(milliseconds=ms_offset) if __name__ == '__main__': sys.exit(main())