cyclone/dev/unbilled-july2026/scripts/regen_corrected_files.py

#!/usr/bin/env python3
"""SP39 / SP40 sibling-project regen: re-serialize the 363 corrected
files through the fixed serializer into ``ingest/corrected-v2/``.

Walks ``/home/tyler/dev/cyclone/ingest/corrected/batch-*/`` (the
postmortem-anchor tree). For each ``.x12`` / ``.txt`` file it:

1. Loads the clearhouse singleton + CO_TXIX PayerConfig so the
   serialization has the real submitter + receiver identity (TPID,
   contact name, email, receiver_name, etc.). These used to fall
   through to placeholder strings like ``CYCLONE`` / ``CUSTOMER
   SERVICE`` / ``RECEIVER`` which Edifabric accepts but which the
   operator rightly refused as not production-ready.
2. Re-parses via :func:`cyclone.parsers.parse_837.parse`.
3. Re-serializes via
   :func:`cyclone.parsers.serialize_837.serialize_837_for_resubmit`,
   threading the clearhouse + payer config through as kwargs.
4. Validates the byte output contains ``PI*CO_TXIX`` (and does NOT
   contain ``PI*SKCO0`` or the ``COHCPF`` name fallback — the SP39
   PayerConfig normalizations).
5. Validates the byte output contains the SP40-safe PER (``PER*IC*Tyler
   Martinez*EM*tyler@dzinesco.com``) and SBR (``SBR*P*18*******MC``).

**Filename format** — HCPF X12 File Naming Standards Quick Guide:
``tp{tpid}-{transaction_type}-yyyymmddhhmmssSSS-1of1.x12``
(example: ``tp11525703-837P-20260708132243505-1of1.x12``).

Built via :func:`cyclone.edi.filenames.build_outbound_filename` (the
single canonical HCPF outbound builder) so the 17-digit
millisecond-precision timestamp + "1of1" sequence + ".x12" extension
are guaranteed. Each file gets a unique millisecond slot by spreading
the batch 1ms apart over the run, so 363 files per regen land in
~363ms with millisecond-level uniqueness — well within the
single-second "1of1" window the spec allows.

The output is written to
``ingest/corrected-v2/v2-batch-<batch-id>-<N>-claims/``.

The original ``ingest/corrected/batch-*/`` tree is preserved
(postmortem anchor); the new tree is the production-ready set the
operator pushes to Gainwell.

Run::

    cd /home/tyler/dev/unbilled-july2026
    /home/tyler/dev/cyclone/backend/.venv/bin/python scripts/regen_corrected_files.py
"""
from __future__ import annotations

import sys
from pathlib import Path

# Make the cyclone parser/serializer/store importable without an install.
sys.path.insert(0, '/home/tyler/dev/cyclone/backend/src')

from cyclone import db as cycl_db  # noqa: E402
from cyclone.edi.filenames import build_outbound_filename  # noqa: E402
from cyclone.parsers.parse_837 import parse as parse_837_text  # noqa: E402
from cyclone.parsers.payer import PayerConfig  # noqa: E402
from cyclone.parsers.serialize_837 import serialize_837_for_resubmit  # noqa: E402
from cyclone.store import store as cycl_store  # noqa: E402

CYCLONE_ROOT = Path('/home/tyler/dev/cyclone')
INGEST_ROOT = CYCLONE_ROOT / 'ingest' / 'corrected'
OUT_ROOT = CYCLONE_ROOT / 'ingest' / 'corrected-v2'


def _batch_id_from_dirname(d: Path) -> str:
    """``batch-<id>-<N>-claims`` -> ``<id>`` (the first hyphen-delimited chunk)."""
    return d.name.split('-')[1]


def _interchange_index_for(global_counter: int) -> int:
    """The serialize helper needs a unique ISA13 per file; offset by
    1000 so the resulting ISA13s stay well above the seeded fixture
    range (0-999)."""
    return 1000 + global_counter


def _load_resubmit_kwargs() -> dict:
    """Build the ``serialize_837`` kwargs from the live clearhouse +
    CO_TXIX payer config. Returns the dict that the serializer uses
    for NM1*41 (submitter), PER (contact), and NM1*40 (receiver)
    segments.

    Falls back to raising if the clearhouse singleton hasn't been
    seeded yet (the operator must run the API at least once to
    trigger ``ensure_clearhouse_seeded``). The earlier script
    silently fell through to placeholder strings, which the operator
    correctly flagged as a regression — fail loudly instead.
    """
    # Bootstrap the DB schema + seed the clearhouse if missing.
    cycl_db.init_db()
    cycl_store.ensure_clearhouse_seeded()

    with cycl_db.SessionLocal()() as s:
        from cyclone.db import ClearhouseORM, PayerConfigORM  # noqa: F401
        ch = s.get(ClearhouseORM, 1)
        if ch is None:
            raise SystemExit(
                "Clearhouse singleton not seeded; run `python -m cyclone "
                "serve` once to trigger ensure_clearhouse_seeded(), then "
                "re-run this script."
            )
        # The 837P config for CO_TXIX carries submitter + receiver info.
        pco = s.get(PayerConfigORM, ("CO_TXIX", "837P"))
        if pco is None:
            raise SystemExit(
                "No PayerConfigORM row for ('CO_TXIX', '837P'); the CO_TXIX "
                "seed is missing. Check ensure_clearhouse_seeded()."
            )

    cfg = pco.config_json or {}

    # Map config_json -> serializer kwargs. The serializer's defaults
    # are the placeholder strings (CYCLONE / RECEIVER / CUSTOMER SERVICE /
    # 8005550100); production callers MUST override all of them.
    return {
        "sender_id": ch.tpid,                          # e.g. "11525703"
        "submitter_name": ch.submitter_name,           # e.g. "Dzinesco"
        "submitter_contact_name": ch.submitter_contact_name,    # "Tyler Martinez"
        "submitter_contact_email": ch.submitter_contact_email,  # "tyler@dzinesco.com"
        "receiver_name": cfg.get("receiver_name") or pco.receiver_name,
        "receiver_id": cfg.get("receiver_id") or pco.receiver_id,
        "claim_filing_indicator_code": cfg.get("sbr09_default") or "MC",
    }


def main() -> int:
    """Walk every ``batch-*/.x12`` under INGEST_ROOT, re-serialize
    through the fixed serializer with real submitter + receiver info,
    validate the output, and write to OUT_ROOT. Returns 0 on a clean
    run, 1 if any file failed or validation missed.

    Filenames follow the HCPF X12 File Naming Standards Quick Guide:
    ``tp{tpid}-837P-yyyymmddhhmmssSSS-1of1.x12``. The 17-digit
    millisecond-precision timestamp comes from
    :func:`build_outbound_filename` so the spec format is guaranteed
    (no ``-cyc-`` segment, no trailing ``-{counter}-`` suffix, no
    abbreviation). Each file's timestamp is offset by 1ms so all 363
    files land in distinct millisecond slots.
    """
    payer_cfg_co = PayerConfig.co_medicaid()
    resubmit_kwargs = _load_resubmit_kwargs()
    tpid = resubmit_kwargs["sender_id"]  # ISA06 — used in the filename
    assert tpid.isdigit(), f"TPID must be digits, got {tpid!r}"

    print("Resubmit kwargs (real submitter + receiver):")
    for k, v in resubmit_kwargs.items():
        print(f"  {k}={v!r}")
    print(f"Filename template: tp{tpid}-837P-yyyymmddhhmmssSSS-1of1.x12")
    print(f"  (built via cyclone.edi.filenames.build_outbound_filename)", flush=True)

    global_counter = 0
    ok = 0
    failed = 0
    failures: list[dict] = []

    batch_dirs = sorted(p for p in INGEST_ROOT.iterdir()
                        if p.is_dir() and p.name.startswith('batch-'))
    print(f'Found {len(batch_dirs)} batch dirs under {INGEST_ROOT}', flush=True)

    for batch_dir in batch_dirs:
        batch_id = _batch_id_from_dirname(batch_dir)
        files = sorted(batch_dir.glob('*.x12')) + sorted(batch_dir.glob('*.txt'))
        if not files:
            continue
        n = len(files)
        out_dir = OUT_ROOT / f'v2-batch-{batch_id}-{n}-claims'
        out_dir.mkdir(parents=True, exist_ok=True)
        print(f'  {batch_id[:12]}... ({n} files) -> {out_dir}', flush=True)

        for src in files:
            global_counter += 1
            # Each file's filename = real HCPF format with a unique
            # millisecond slot (1ms stride from now onward).
            fname = build_outbound_filename(
                tpid, "837P",
                now_mt=_now_mt_plus_ms(global_counter),
            )

            raw = src.read_text()
            try:
                parsed = parse_837_text(raw, payer_cfg_co)
            except Exception as exc:  # noqa: BLE001
                failed += 1
                failures.append({'file': src.name, 'reason': f'parse: {exc}'})
                continue
            if not parsed.claims:
                failed += 1
                failures.append({'file': src.name, 'reason': 'no claims parsed'})
                continue
            try:
                text = serialize_837_for_resubmit(
                    parsed.claims[0],
                    interchange_index=_interchange_index_for(global_counter),
                    **resubmit_kwargs,
                )
            except Exception as exc:  # noqa: BLE001
                failed += 1
                failures.append({'file': src.name, 'reason': f'serialize: {exc}'})
                continue
            content = text.encode()
            if b'PI*CO_TXIX' not in content:
                failed += 1
                failures.append({'file': src.name, 'reason': 'PI*CO_TXIX missing'})
                continue
            if b'PI*SKCO0' in content or b'*COHCPF****' in content:
                failed += 1
                failures.append({'file': src.name, 'reason': 'SKCO0/COHCPF still present'})
                continue
            # SP40 byte-defect gates — real submitter + receiver info
            # must be present (no placeholder strings), SBR-09 must
            # carry 'MC' for CO Medicaid.
            submitter_contact_email = (
                resubmit_kwargs["submitter_contact_email"] or ""
            ).encode()
            receiver_id = (resubmit_kwargs["receiver_id"] or "").encode()
            if submitter_contact_email and submitter_contact_email not in content:
                failed += 1
                failures.append({
                    'file': src.name,
                    'reason': f'submitter_contact_email '
                              f'{submitter_contact_email!r} missing',
                })
                continue
            if receiver_id and receiver_id not in content:
                failed += 1
                failures.append({
                    'file': src.name,
                    'reason': f'receiver_id {receiver_id!r} missing',
                })
                continue
            if b'SBR*P*18*******MC' not in content:
                failed += 1
                failures.append({'file': src.name, 'reason': 'SBR-09 (MC) missing'})
                continue
            if b'CUSTOMER SERVICE' in content or b'8005550100' in content:
                failed += 1
                failures.append({'file': src.name, 'reason': 'placeholder strings leaked'})
                continue
            # Final HCPF-spec filename validation guard (cheaper to
            # re-check the filename than to discover a malformed
            # filename on upload to Gainwell).
            from cyclone.edi.filenames import is_outbound_filename
            if not is_outbound_filename(fname):
                failed += 1
                failures.append({'file': src.name,
                                 'reason': f'filename {fname!r} does not match '
                                           f'HCPF OUTBOUND_RE (SPEC VIOLATION)'})
                continue
            (out_dir / fname).write_text(text)
            ok += 1

    print(f'\nDONE: ok={ok} failed={failed}', flush=True)
    if failures:
        print('First failures:')
        for f in failures[:5]:
            print(f'  {f}')
    return 0 if failed == 0 else 1


def _now_mt_plus_ms(ms_offset: int):
    """Return the current Mountain Time as a tz-aware datetime, offset
    by ``ms_offset`` milliseconds. Used to spread the regen run across
    distinct millisecond slots so each file's filename is unique
    within the spec's 17-digit yyyymmddhhmmssSSS window."""
    from datetime import datetime, timedelta
    from zoneinfo import ZoneInfo
    return datetime.now(ZoneInfo("America/Denver")) + timedelta(milliseconds=ms_offset)


if __name__ == '__main__':
    sys.exit(main())