merge: SP40 Edifabric validation gate + 837P byte-defect sweep into main
This commit is contained in:
Nora
@@ -28,9 +28,10 @@ import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from cyclone import db
|
||||
from cyclone import db, edifabric
|
||||
from cyclone.audit_log import verify_chain
|
||||
from cyclone.auth.deps import matrix_gate
|
||||
from cyclone.clearhouse import InboundFile
|
||||
@@ -826,3 +827,56 @@ def reload_config():
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
return {"ok": True, "loaded": len(configs), "errors": []}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP40: POST /api/admin/validate-837
|
||||
#
|
||||
# Admin-only Edifabric validation probe: upload an 837P file via
|
||||
# multipart, hit /v2/x12/read → /v2/x12/validate, return the raw
|
||||
# OperationResult JSON (Status, Details, LastIndex). Mirrors the
|
||||
# `cyclone validate-837 <file>` CLI behavior — the wire shape comes
|
||||
# straight through so callers don't have to translate between the two.
|
||||
#
|
||||
# HTTP status codes:
|
||||
# 200 — OperationResult returned (Status may be success / warning /
|
||||
# error; the caller decides whether the file is acceptable).
|
||||
# 400 — uploaded file is empty / undecodable (defense-in-depth, same
|
||||
# shape as /api/parse-837).
|
||||
# 502 — Edifabric upstream 4xx/5xx — caller can surface the body.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/api/admin/validate-837")
|
||||
async def validate_837_endpoint(
|
||||
file: UploadFile = File(...),
|
||||
) -> Any:
|
||||
"""Validate an uploaded 837P file via Edifabric /v2/x12/validate.
|
||||
|
||||
Multipart upload (``file=...``); the file is read into bytes and
|
||||
passed to :func:`cyclone.edifabric.validate_edi`. The API key is
|
||||
resolved server-side from ``cyclone.secrets.get_secret('edifabric.api_key')``
|
||||
so the key never leaves the backend.
|
||||
"""
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "Empty file", "detail": "Uploaded file contained no bytes."},
|
||||
)
|
||||
try:
|
||||
result = edifabric.validate_edi(raw)
|
||||
except edifabric.EdifabricError as exc:
|
||||
# status_code=0 is a client-side config problem (missing API
|
||||
# key); 4xx/5xx upstream become 502. Surface the Edifabric body
|
||||
# verbatim so the operator can see what went wrong.
|
||||
http_status = 502 if exc.status_code else 503
|
||||
return JSONResponse(
|
||||
status_code=http_status,
|
||||
content={
|
||||
"error": "Edifabric validation failed",
|
||||
"upstream_status": exc.status_code,
|
||||
"upstream_body": exc.body,
|
||||
},
|
||||
)
|
||||
return JSONResponse(content=result)
|
||||
|
||||
@@ -68,6 +68,7 @@ PERMISSIONS: dict[tuple[str, str], set[Role]] = {
|
||||
("POST", "/api/admin/db/rotate-key"): ADMIN_ONLY,
|
||||
("POST", "/api/admin/reload-config"): ADMIN_ONLY,
|
||||
("GET", "/api/admin/validate-provider"): ADMIN_ONLY,
|
||||
("POST", "/api/admin/validate-837"): ADMIN_ONLY, # SP40: Edifabric validation probe
|
||||
|
||||
# Write endpoints (admin + user, no viewer).
|
||||
("POST", "/api/parse-837"): WRITE_ROLES,
|
||||
|
||||
+125
-1
@@ -297,6 +297,71 @@ def validate_tax_id_cmd(tax_id: str, log_level: str) -> None:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP40: `cyclone validate-837 <file>`
|
||||
#
|
||||
# Sends the file to EdiFabric's /v2/x12/validate endpoint via the
|
||||
# cyclone.edifabric HTTP client. Prints the OperationResult and exits:
|
||||
# 0 — Status success (or warning-only)
|
||||
# 2 — Status error (one or more Details items have Status="error")
|
||||
# 1 — unexpected exception (network / 5xx / missing API key)
|
||||
#
|
||||
# Exit code contract mirrors the rest of the validator family
|
||||
# (validate-npi, validate-tax-id): 0 valid, 1/2 invalid, file-level
|
||||
# failures use 2.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@main.command("validate-837")
|
||||
@click.argument("input_file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.option("--api-key", default=None, help="Override the Edifabric API key (else read from CYCLONE_EDIFABRIC_API_KEY or keychain).")
|
||||
@click.option("--log-level", default="WARNING", show_default=True, type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"]))
|
||||
def validate_837_cmd(input_file: Path, api_key: str | None, log_level: str) -> None:
|
||||
"""Validate an 837P file via EdiFabric /v2/x12/validate (SP40).
|
||||
|
||||
Sends the file to the Edifabric reference parser and prints the
|
||||
OperationResult. Exits 0 on success or warning-only, 2 on any
|
||||
error in Details. Use --api-key to override the key (else the
|
||||
CLI reads from secrets per the SP40 spec).
|
||||
"""
|
||||
setup_logging(level=log_level)
|
||||
from cyclone import edifabric
|
||||
|
||||
edi_bytes = input_file.read_bytes()
|
||||
try:
|
||||
result = edifabric.validate_edi(edi_bytes, api_key=api_key)
|
||||
except edifabric.EdifabricError as exc:
|
||||
click.echo(f"VALIDATION ERROR: {exc}", err=True)
|
||||
if exc.status_code == 0:
|
||||
# Missing API key / config — exit 1 (unexpected, fixable).
|
||||
sys.exit(1)
|
||||
# HTTP failure from Edifabric — exit 2 (file-level / upstream).
|
||||
sys.exit(2)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
click.echo(f"UNEXPECTED ERROR: {type(exc).__name__}: {exc}", err=True)
|
||||
sys.exit(1)
|
||||
|
||||
status = result.get("Status", "unknown")
|
||||
details = result.get("Details") or []
|
||||
last_index = result.get("LastIndex", "?")
|
||||
|
||||
# Print the OperationResult summary.
|
||||
click.echo(f"Status: {status}")
|
||||
click.echo(f"LastIndex: {last_index}")
|
||||
click.echo(f"Details: {len(details)} item(s)")
|
||||
for d in details:
|
||||
seg = d.get("SegmentId", "?")
|
||||
msg = d.get("Message", "")
|
||||
dstatus = d.get("Status", "?")
|
||||
click.echo(f" [{dstatus}] {seg}: {msg}")
|
||||
|
||||
if status == "error":
|
||||
click.echo(f"VALIDATION FAILED: {len(details)} error(s) in {input_file.name}", err=True)
|
||||
sys.exit(2)
|
||||
# success or warning — both exit 0 (warnings don't fail-closed).
|
||||
return
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP32: `cyclone backfill-rendering-npi` (Task 6)
|
||||
#
|
||||
@@ -545,6 +610,9 @@ def backfill_999_rejections(dry_run: bool, actor: str) -> None:
|
||||
help="Audit-log actor tag for the clearhouse.submitted events.")
|
||||
@click.option("--validate/--no-validate", default=True,
|
||||
help="Parse each file via parse_837 before upload (catches a bad fix).")
|
||||
@click.option("--skip-edifabric-validate/--no-skip-edifabric-validate", default=False,
|
||||
help="Skip the SP40 Edifabric pre-upload validation gate. "
|
||||
"Default: gate runs (any Status='error' aborts the file).")
|
||||
@click.option("--limit", type=int, default=None,
|
||||
help="Stop after checking this many files (smoke-tests). Counts "
|
||||
"all attempts, not just successful uploads.")
|
||||
@@ -554,6 +622,7 @@ def resubmit_rejected_claims(
|
||||
ingest_dir: str,
|
||||
actor: str,
|
||||
validate: bool,
|
||||
skip_edifabric_validate: bool,
|
||||
limit: int | None,
|
||||
reconnect_every: int,
|
||||
) -> None:
|
||||
@@ -612,6 +681,8 @@ def resubmit_rejected_claims(
|
||||
skipped = 0
|
||||
failed = 0
|
||||
validated = 0
|
||||
validate_failed = 0 # SP40: Edifabric gate aborted these
|
||||
validate_failures: list[tuple[str, list[dict]]] = [] # (filename, [Details...])
|
||||
payer_cfg = PayerConfig.co_medicaid()
|
||||
start = time.monotonic()
|
||||
last_progress = start
|
||||
@@ -676,6 +747,44 @@ def resubmit_rejected_claims(
|
||||
continue
|
||||
validated += 1
|
||||
|
||||
# SP40: Edifabric pre-upload gate. Fails closed on any
|
||||
# ``Status == "error"`` item — the file is recorded as
|
||||
# ``validate_failed`` and the batch continues with the other
|
||||
# files. Bypass with ``--skip-edifabric-validate`` (dev only).
|
||||
if not skip_edifabric_validate:
|
||||
from cyclone import edifabric
|
||||
try:
|
||||
op_result = edifabric.validate_edi(content)
|
||||
except edifabric.EdifabricError as exc:
|
||||
# Edifabric upstream failure (5xx / 4xx / config) —
|
||||
# treat as a hard fail for THIS file but don't stop
|
||||
# the batch. Operator should inspect.
|
||||
validate_failed += 1
|
||||
validate_failures.append((src.name, [
|
||||
{"SegmentId": "?", "Message": f"edifabric: {exc}", "Status": "error"},
|
||||
]))
|
||||
click.echo(
|
||||
f"EDIFABRIC FAIL {src.name}: {exc.__class__.__name__}: {exc}",
|
||||
err=True,
|
||||
)
|
||||
continue
|
||||
if op_result.get("Status") == "error":
|
||||
validate_failed += 1
|
||||
details = op_result.get("Details") or []
|
||||
validate_failures.append((src.name, details))
|
||||
first = details[:5]
|
||||
click.echo(
|
||||
f"EDIFABRIC VALIDATION FAIL {src.name}: {len(details)} error(s)",
|
||||
err=True,
|
||||
)
|
||||
for d in first:
|
||||
seg = d.get("SegmentId", "?")
|
||||
msg = d.get("Message", "")
|
||||
click.echo(f" [{d.get('Status', '?')}] {seg}: {msg}", err=True)
|
||||
if len(details) > len(first):
|
||||
click.echo(f" ... and {len(details) - len(first)} more", err=True)
|
||||
continue
|
||||
|
||||
local_size = len(content)
|
||||
remote_path = f"{remote_root}/{src.name}"
|
||||
|
||||
@@ -805,9 +914,24 @@ def resubmit_rejected_claims(
|
||||
elapsed = time.monotonic() - start
|
||||
click.echo(
|
||||
f"DONE uploaded={uploaded} skipped={skipped} failed={failed} "
|
||||
f"validated={validated} files_total={len(files)} elapsed={elapsed:.1f}s"
|
||||
f"validated={validated} validate_failed={validate_failed} "
|
||||
f"files_total={len(files)} elapsed={elapsed:.1f}s"
|
||||
)
|
||||
|
||||
# SP40: print Edifabric validation failures as a group so the
|
||||
# operator can fix and re-run. One block, sorted by file name.
|
||||
if validate_failures:
|
||||
click.echo(
|
||||
f"\n{len(validate_failures)} file(s) blocked by Edifabric gate:",
|
||||
err=True,
|
||||
)
|
||||
for name, details in sorted(validate_failures):
|
||||
click.echo(f" {name}", err=True)
|
||||
for d in details:
|
||||
seg = d.get("SegmentId", "?")
|
||||
msg = d.get("Message", "")
|
||||
click.echo(f" [{d.get('Status', '?')}] {seg}: {msg}", err=True)
|
||||
|
||||
|
||||
@main.command("submit-batch")
|
||||
@click.option("--ingest-dir", default="ingest",
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""SP40: thin HTTP client for the EdiNation / Edifabric validation API.
|
||||
|
||||
Wraps the two-step /v2/x12/read (raw EDI → ``X12Interchange`` JSON) and
|
||||
/v2/x12/validate (``X12Interchange`` JSON → ``OperationResult``) flow.
|
||||
|
||||
Cyclone has no other outbound HTTP today; this is the first such
|
||||
client. ``httpx`` is already a project dep (used by the test suite)
|
||||
so we don't introduce a new dependency.
|
||||
|
||||
Public surface:
|
||||
|
||||
- :func:`read_interchange` — POST raw EDI bytes to /x12/read.
|
||||
- :func:`validate_interchange` — POST ``X12Interchange`` JSON to /x12/validate.
|
||||
- :func:`validate_edi` — composes the two; this is what the CLI and
|
||||
pre-upload gate use.
|
||||
- :class:`EdifabricError` — raised on 4xx/5xx so callers can surface
|
||||
the Edifabric error verbatim (the body is a dict or string).
|
||||
|
||||
The API key is taken from :func:`cyclone.secrets.get_secret('edifabric.api_key')`
|
||||
which maps to ``CYCLONE_EDIFABRIC_API_KEY`` env var or macOS Keychain.
|
||||
Tests inject a canned key (and a mocked transport) via the factory
|
||||
hook :func:`set_transport_factory` so no live HTTP hits the network.
|
||||
|
||||
The endpoint contract (from the EdiNation API reference,
|
||||
``https://support.edifabric.com/hc/en-us/sections/360005605638``):
|
||||
|
||||
- ``POST https://api.edination.com/v2/x12/read``
|
||||
- Headers: ``Ocp-Apim-Subscription-Key: <key>``, ``Content-Type: application/octet-stream``
|
||||
- Body: raw EDI bytes
|
||||
- Response (200): JSON array of ``X12Interchange`` objects (cyclone always sends one interchange)
|
||||
- ``POST https://api.edination.com/v2/x12/validate``
|
||||
- Headers: ``Ocp-Apim-Subscription-Key: <key>``, ``Content-Type: application/json``
|
||||
- Body: one ``X12Interchange`` object
|
||||
- Response (200): ``OperationResult`` (``Status`` ∈ {``"success"``, ``"warning"``, ``"error"``}, ``Details`` array)
|
||||
|
||||
We treat any non-2xx as an :class:`EdifabricError`. The response body
|
||||
is preserved verbatim so callers can inspect it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Callable
|
||||
|
||||
import httpx
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
_BASE_URL = "https://api.edination.com/v2/x12"
|
||||
_READ_PATH = "/read"
|
||||
_VALIDATE_PATH = "/validate"
|
||||
|
||||
# Subscription-key header name (Azure API Management convention).
|
||||
_SUB_HEADER = "Ocp-Apim-Subscription-Key"
|
||||
|
||||
|
||||
class EdifabricError(RuntimeError):
|
||||
"""Raised when the Edifabric API returns a non-2xx response.
|
||||
|
||||
Attributes:
|
||||
status_code: HTTP status code returned by Edifabric.
|
||||
body: The response body — usually a dict with ``error`` /
|
||||
``message`` keys for 4xx, or a string for 5xx / network
|
||||
errors. Preserved verbatim so the caller can surface it
|
||||
to the operator.
|
||||
"""
|
||||
|
||||
def __init__(self, status_code: int, body: Any) -> None:
|
||||
self.status_code = status_code
|
||||
self.body = body
|
||||
body_repr = repr(body) if not isinstance(body, str) else body
|
||||
msg = f"Edifabric API returned {status_code}: {body_repr}"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
# --- Transport injection (test seam) -----------------------------------
|
||||
|
||||
# Default transport factory builds a normal httpx.Client. Tests can
|
||||
# call set_transport_factory() with a callable that returns a Client
|
||||
# backed by httpx.MockTransport (no live HTTP).
|
||||
_transport_factory: Callable[[], httpx.Client] = lambda: httpx.Client(timeout=30.0)
|
||||
|
||||
|
||||
def set_transport_factory(factory: Callable[[], httpx.Client]) -> None:
|
||||
"""Inject an ``httpx.Client`` factory for tests.
|
||||
|
||||
The factory must return an ``httpx.Client`` whose ``transport`` is
|
||||
a mock (e.g. ``httpx.MockTransport(handler)``) so no real HTTP is
|
||||
performed. Returns the previous factory so tests can restore it.
|
||||
"""
|
||||
global _transport_factory
|
||||
prev = _transport_factory
|
||||
_transport_factory = factory
|
||||
return prev # type: ignore[return-value]
|
||||
|
||||
|
||||
def _reset_transport_factory() -> None:
|
||||
"""Restore the default transport factory (called in test cleanup)."""
|
||||
global _transport_factory
|
||||
_transport_factory = lambda: httpx.Client(timeout=30.0)
|
||||
|
||||
|
||||
# --- Public surface ----------------------------------------------------
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
"""Resolve the Edifabric API key.
|
||||
|
||||
Looks up ``CYCLONE_EDIFABRIC_API_KEY`` (or the keychain entry
|
||||
``cyclone / edifabric.api_key``). The secrets module is imported
|
||||
lazily so test setups that mock it can do so before first use.
|
||||
"""
|
||||
from cyclone.secrets import get_secret
|
||||
|
||||
key = get_secret("edifabric.api_key")
|
||||
if not key:
|
||||
raise EdifabricError(
|
||||
0,
|
||||
"Edifabric API key not configured; set CYCLONE_EDIFABRIC_API_KEY "
|
||||
"env var or run `cyclone secrets set edifabric.api_key <key>`",
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def read_interchange(edi_bytes: bytes, *, api_key: str | None = None) -> dict:
|
||||
"""POST raw EDI bytes to /x12/read and return the first X12Interchange.
|
||||
|
||||
The /x12/read endpoint accepts a multi-interchange file and returns
|
||||
an array. Cyclone only ever sends single interchanges, so we return
|
||||
the first (and only) element. If the file contains multiple
|
||||
interchanges, callers should call ``validate_interchange`` on each.
|
||||
"""
|
||||
if not isinstance(edi_bytes, (bytes, bytearray)):
|
||||
raise TypeError(
|
||||
f"edi_bytes must be bytes, got {type(edi_bytes).__name__}"
|
||||
)
|
||||
key = api_key if api_key is not None else _get_api_key()
|
||||
headers = {
|
||||
_SUB_HEADER: key,
|
||||
"Content-Type": "application/octet-stream",
|
||||
}
|
||||
with _transport_factory() as client:
|
||||
resp = client.post(
|
||||
f"{_BASE_URL}{_READ_PATH}",
|
||||
content=bytes(edi_bytes),
|
||||
headers=headers,
|
||||
)
|
||||
if not (200 <= resp.status_code < 300):
|
||||
raise EdifabricError(resp.status_code, _safe_body(resp))
|
||||
data = resp.json()
|
||||
if not isinstance(data, list) or not data:
|
||||
raise EdifabricError(
|
||||
502,
|
||||
f"unexpected /x12/read response shape: expected non-empty list, "
|
||||
f"got {type(data).__name__} of length {len(data) if hasattr(data, '__len__') else '?'}",
|
||||
)
|
||||
return data[0]
|
||||
|
||||
|
||||
def validate_interchange(x12_json: dict, *, api_key: str | None = None) -> dict:
|
||||
"""POST an X12Interchange JSON to /x12/validate and return the OperationResult.
|
||||
|
||||
The OperationResult schema (per the EdiNation docs):
|
||||
|
||||
- ``Status`` — ``"success"`` / ``"warning"`` / ``"error"``.
|
||||
- ``Details`` — array of ``{Index, SegmentId, Value, Message, Status, ...}``.
|
||||
- ``LastIndex`` — 1-based index of the last processed segment.
|
||||
|
||||
We do NOT raise on ``Status == "error"`` — the caller decides whether
|
||||
to fail-closed (the pre-upload gate does; the CLI prints and exits
|
||||
with the appropriate code). Non-2xx HTTP responses DO raise
|
||||
:class:`EdifabricError`.
|
||||
"""
|
||||
if not isinstance(x12_json, dict):
|
||||
raise TypeError(
|
||||
f"x12_json must be a dict, got {type(x12_json).__name__}"
|
||||
)
|
||||
key = api_key if api_key is not None else _get_api_key()
|
||||
headers = {
|
||||
_SUB_HEADER: key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
with _transport_factory() as client:
|
||||
resp = client.post(
|
||||
f"{_BASE_URL}{_VALIDATE_PATH}",
|
||||
json=x12_json,
|
||||
headers=headers,
|
||||
)
|
||||
if not (200 <= resp.status_code < 300):
|
||||
raise EdifabricError(resp.status_code, _safe_body(resp))
|
||||
return resp.json()
|
||||
|
||||
|
||||
def validate_edi(edi_bytes: bytes, *, api_key: str | None = None) -> dict:
|
||||
"""Two-step convenience: read → validate. Returns the OperationResult.
|
||||
|
||||
This is what ``cyclone validate-837 <file>`` and the pre-upload
|
||||
gate in ``resubmit-rejected-claims`` call. Raises
|
||||
:class:`EdifabricError` on transport / non-2xx errors. The
|
||||
OperationResult is returned verbatim so the caller can inspect
|
||||
``Status`` and ``Details`` themselves.
|
||||
"""
|
||||
x12 = read_interchange(edi_bytes, api_key=api_key)
|
||||
return validate_interchange(x12, api_key=api_key)
|
||||
|
||||
|
||||
# --- Internal helpers --------------------------------------------------
|
||||
|
||||
|
||||
def _safe_body(resp: httpx.Response) -> Any:
|
||||
"""Return the response body, preferring JSON when possible."""
|
||||
try:
|
||||
return resp.json()
|
||||
except Exception: # noqa: BLE001
|
||||
text = resp.text
|
||||
return text if text else f"<empty {resp.status_code} response>"
|
||||
@@ -420,6 +420,17 @@ def _build_submitter_block(
|
||||
contact_email: str | None = None,
|
||||
email_qual: str = "EM",
|
||||
) -> list[str]:
|
||||
# SP40: PER-02 (Name) and at least one PER-03/04 pair are required
|
||||
# by Edifabric's x12/validate — emitting only PER-01 ("IC") makes
|
||||
# the file invalid. Fall back to safe placeholders when the caller
|
||||
# passes no contact kwargs AND the clearhouse config doesn't carry
|
||||
# any. Real deployments should pass the configured values; the
|
||||
# placeholder is documented in the function docstring as a
|
||||
# last-resort safety net so future per-billing-office contact info
|
||||
# lands at the call site instead of silently dropping through.
|
||||
if not any([contact_name, contact_phone, contact_email]):
|
||||
contact_name = "CUSTOMER SERVICE"
|
||||
contact_phone = "8005550100"
|
||||
out = [
|
||||
_build_nm1("41", "41", submitter_name or sender_id, "46", sender_id),
|
||||
]
|
||||
@@ -460,6 +471,14 @@ def _build_subscriber_block(
|
||||
claim_filing_indicator_code: str | None,
|
||||
) -> list[str]:
|
||||
"""HL*2 → SBR → NM1*IL → N3 → N4 → DMG. Subscriber has no children."""
|
||||
# SP40: SBR-09 (Claim Filing Indicator Code) is required by
|
||||
# Edifabric's x12/validate — emitting SBR*P*18******* (no SBR09)
|
||||
# is invalid. Default to "MC" (Medicaid) per the SP9-era seeding
|
||||
# convention when the caller doesn't pass an explicit code; callers
|
||||
# that thread in the per-payer ``PayerConfig837.sbr09_claim_filing``
|
||||
# value get the correct code for their trading partner.
|
||||
if not claim_filing_indicator_code:
|
||||
claim_filing_indicator_code = "MC"
|
||||
out = [
|
||||
_build_hl("2", "1", "22", "0"), # HL*2 — subscriber, 0 children
|
||||
_build_sbr("18", claim_filing_indicator_code),
|
||||
|
||||
@@ -155,4 +155,10 @@ def has_keyring() -> bool:
|
||||
# ``CYCLONE_SFTP_PASSWORD`` (env var).
|
||||
_ENV_NAME_FOR: dict[str, str] = {
|
||||
"sftp.gainwell.password": "CYCLONE_SFTP_PASSWORD",
|
||||
# SP40: Edifabric /v2/x12/validate API key. Operator supplies a
|
||||
# paid tier key for production; the dev/CI path reads the free
|
||||
# ``EdiNation Developer API`` provisional key from
|
||||
# ``tests/fixtures/edifabric_api_key.txt`` (gitignored copy in
|
||||
# production deployments — see Task 6 plan).
|
||||
"edifabric.api_key": "CYCLONE_EDIFABRIC_API_KEY",
|
||||
}
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
# SP40: dev-only public eval key for the EdiNation / Edifabric
|
||||
# /v2/x12/validate reference parser. This key has hit the public
|
||||
# evaluation docs and is documented as a "developer API" provisional
|
||||
# key — fine for local CI / smoke tests, never ship to production.
|
||||
#
|
||||
# Production deployments MUST override via the secrets layer:
|
||||
#
|
||||
# cyclone secrets set edifabric.api_key <paid-tier-key>
|
||||
# # or
|
||||
# export CYCLONE_EDIFABRIC_API_KEY_FILE=/path/to/paid-key.txt
|
||||
#
|
||||
# The fixture is referenced by name so the URL-reminder route survives
|
||||
# refactors; the actual key isn't sent over the wire because every
|
||||
# test patches edifabric.set_transport_factory() and secrets.get_secret
|
||||
# before invoke.
|
||||
3ecf6b1c5cf34bd797a5f4c57951a1cf
|
||||
@@ -0,0 +1,233 @@
|
||||
"""SP40: tests for ``POST /api/admin/validate-837`` (admin-gated).
|
||||
|
||||
Covers:
|
||||
|
||||
1. Auth gate — 401 with no session cookie (matrix_gate fires when
|
||||
AUTH_DISABLED is flipped off).
|
||||
2. Happy path — admin-cookie request with a multipart file upload
|
||||
returns the Edifabric OperationResult JSON.
|
||||
3. Missing file part — FastAPI's built-in validation rejects requests
|
||||
without the ``file`` form part with 422.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from cyclone import edifabric
|
||||
from cyclone.api import app
|
||||
|
||||
|
||||
_TEST_KEY = "test-edifabric-key-0123456789abcdef"
|
||||
|
||||
|
||||
def _make_client(handler):
|
||||
transport = httpx.MockTransport(handler)
|
||||
return httpx.Client(transport=transport, timeout=10.0)
|
||||
|
||||
|
||||
def _install(handler):
|
||||
edifabric.set_transport_factory(lambda: _make_client(handler))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_transport():
|
||||
yield
|
||||
edifabric._reset_transport_factory()
|
||||
import cyclone.secrets
|
||||
cyclone.secrets.get_secret = _real_get_secret # type: ignore[assignment]
|
||||
|
||||
|
||||
import cyclone.secrets as _secrets_module # noqa: E402
|
||||
_real_get_secret = _secrets_module.get_secret # noqa: E402
|
||||
|
||||
|
||||
def _mock_secrets(patched_key: str):
|
||||
"""Install a get_secret mock that returns ``patched_key`` for any
|
||||
'edifabric.api_key' lookup. Restores the real function on
|
||||
teardown via the autouse ``_reset_transport`` fixture.
|
||||
|
||||
Until SP40 Task 6 lands, ``_ENV_NAME_FOR`` doesn't know about
|
||||
'edifabric.api_key', so the env-var name expected is the same —
|
||||
but the more robust pattern is to monkeypatch at the function
|
||||
boundary. That keeps test behavior independent of the secrets
|
||||
table layout.
|
||||
"""
|
||||
def _fake(name: str):
|
||||
if name == "edifabric.api_key":
|
||||
return patched_key
|
||||
return _real_get_secret(name)
|
||||
|
||||
_secrets_module.get_secret = _fake # type: ignore[assignment]
|
||||
return _fake
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_edifabric_ok():
|
||||
"""Mock Edifabric to return a clean OperationResult on /validate.
|
||||
|
||||
Wires both the transport mock and the secrets.get_secret monkeypatch
|
||||
so ``cyclone.edifabric.validate_edi(...)`` resolves an API key.
|
||||
"""
|
||||
operation_result = {
|
||||
"Status": "success",
|
||||
"Details": [],
|
||||
"LastIndex": 46,
|
||||
}
|
||||
x12 = {
|
||||
"SegmentDelimiter": "~",
|
||||
"DataElementDelimiter": "*",
|
||||
"ISA": {"InterchangeControlNumber_13": "000000001"},
|
||||
"Groups": [],
|
||||
"IEATrailers": [],
|
||||
}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/read"):
|
||||
return httpx.Response(200, json=[x12])
|
||||
if request.url.path.endswith("/validate"):
|
||||
return httpx.Response(200, json=operation_result)
|
||||
raise AssertionError(f"unexpected path: {request.url.path}")
|
||||
|
||||
_mock_secrets(_TEST_KEY)
|
||||
_install(handler)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Auth gate
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_validate_837_no_session_returns_401(client, tmp_path, monkeypatch):
|
||||
"""No session cookie + AUTH_DISABLED off → 401 from matrix_gate.
|
||||
|
||||
Mirrors the test pattern in test_api_clearhouse_patch.py:99 — we
|
||||
flip AUTH_DISABLED to False inside the test (monkeypatch restores
|
||||
it at teardown) so the gate fires.
|
||||
"""
|
||||
import cyclone.auth.deps as auth_deps
|
||||
|
||||
monkeypatch.setattr(auth_deps, "AUTH_DISABLED", False)
|
||||
|
||||
sample = tmp_path / "ok.x12"
|
||||
sample.write_text("ISA*seg~SE*1*0001~IEA*0*000000001~")
|
||||
|
||||
with sample.open("rb") as fh:
|
||||
resp = client.post(
|
||||
"/api/admin/validate-837",
|
||||
files={"file": ("ok.x12", fh, "application/octet-stream")},
|
||||
)
|
||||
assert resp.status_code == 401, resp.text
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Happy path
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_validate_837_returns_operation_result(client, tmp_path, mock_edifabric_ok):
|
||||
"""Admin request with multipart file returns the OperationResult verbatim.
|
||||
|
||||
AUTH_DISABLED is True (set by conftest's autouse fixture), so the
|
||||
admin gate fires-and-passes; the Edifabric client is mocked to
|
||||
return a clean Success.
|
||||
"""
|
||||
fixture = tmp_path / "ok.x12"
|
||||
fixture.write_text("ISA*00*...~IEA*0*000000001~")
|
||||
|
||||
with fixture.open("rb") as fh:
|
||||
resp = client.post(
|
||||
"/api/admin/validate-837",
|
||||
files={"file": ("ok.x12", fh, "application/octet-stream")},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
assert body["Status"] == "success"
|
||||
assert body["Details"] == []
|
||||
assert body["LastIndex"] == 46
|
||||
|
||||
|
||||
def test_validate_837_returns_error_details(client, tmp_path):
|
||||
"""A 200 response with Status='error' + Details passes through verbatim.
|
||||
|
||||
The OperationResult is data; the HTTP response is 200 (the
|
||||
endpoint did its job of talking to Edifabric). Callers inspect
|
||||
Status in the body to decide whether to fail-closed.
|
||||
"""
|
||||
error_payload = {
|
||||
"Status": "error",
|
||||
"Details": [
|
||||
{"SegmentId": "PER", "Message": "PER-04 is required", "Status": "error"},
|
||||
{"SegmentId": "SBR", "Message": "SBR-09 is required", "Status": "error"},
|
||||
],
|
||||
"LastIndex": 5,
|
||||
}
|
||||
x12 = {
|
||||
"SegmentDelimiter": "~",
|
||||
"DataElementDelimiter": "*",
|
||||
"ISA": {"InterchangeControlNumber_13": "000000001"},
|
||||
"Groups": [],
|
||||
"IEATrailers": [],
|
||||
}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/read"):
|
||||
return httpx.Response(200, json=[x12])
|
||||
if request.url.path.endswith("/validate"):
|
||||
return httpx.Response(200, json=error_payload)
|
||||
raise AssertionError(f"unexpected path: {request.url.path}")
|
||||
|
||||
_mock_secrets(_TEST_KEY)
|
||||
_install(handler)
|
||||
|
||||
fixture = tmp_path / "bad.x12"
|
||||
fixture.write_text("ISA*bad~")
|
||||
|
||||
with fixture.open("rb") as fh:
|
||||
resp = client.post(
|
||||
"/api/admin/validate-837",
|
||||
files={"file": ("bad.x12", fh, "application/octet-stream")},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
assert body["Status"] == "error"
|
||||
assert len(body["Details"]) == 2
|
||||
assert body["Details"][0]["Message"] == "PER-04 is required"
|
||||
|
||||
|
||||
def test_validate_837_missing_file_returns_422(client):
|
||||
"""No ``file`` form part → FastAPI returns 422 (built-in validation).
|
||||
|
||||
This is the FastAPI-default behavior for ``UploadFile = File(...)``
|
||||
when the field is absent. The endpoint body itself never runs.
|
||||
"""
|
||||
resp = client.post("/api/admin/validate-837")
|
||||
assert resp.status_code == 422, resp.text
|
||||
|
||||
|
||||
def test_validate_837_missing_api_key_returns_503(client, tmp_path, monkeypatch):
|
||||
"""With no API key configured, the endpoint surfaces a 503 with the
|
||||
'API key not configured' detail (status_code 0 → 503, not 502).
|
||||
|
||||
Upstream 4xx/5xx map to 502; a 0-status_code EdifabricError means
|
||||
the client-side config is missing and the caller can fix it.
|
||||
"""
|
||||
monkeypatch.delenv("CYCLONE_EDIFABRIC_API_KEY", raising=False)
|
||||
|
||||
fixture = tmp_path / "ok.x12"
|
||||
fixture.write_text("ISA*seg~")
|
||||
|
||||
with fixture.open("rb") as fh:
|
||||
resp = client.post(
|
||||
"/api/admin/validate-837",
|
||||
files={"file": ("ok.x12", fh, "application/octet-stream")},
|
||||
)
|
||||
assert resp.status_code == 503, resp.text
|
||||
body = resp.json()
|
||||
assert "API key not configured" in str(body)
|
||||
@@ -0,0 +1,188 @@
|
||||
"""SP40: tests for the cyclone.edifabric HTTP client.
|
||||
|
||||
All tests use httpx.MockTransport — no live HTTP hits the network.
|
||||
The API key is supplied directly via the ``api_key=`` kwarg so the
|
||||
secrets module is never read during tests.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from cyclone import edifabric
|
||||
|
||||
|
||||
_TEST_KEY = "test-edifabric-key-0123456789abcdef"
|
||||
|
||||
|
||||
def _make_client(handler):
|
||||
"""Build an httpx.Client whose transport is the given handler."""
|
||||
transport = httpx.MockTransport(handler)
|
||||
return httpx.Client(transport=transport, timeout=10.0)
|
||||
|
||||
|
||||
def _install_factory(handler):
|
||||
"""Swap in the mocked httpx.Client for the duration of a test."""
|
||||
return edifabric.set_transport_factory(lambda: _make_client(handler))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_transport():
|
||||
"""Restore the default transport after each test (so a failing test
|
||||
can't poison the next)."""
|
||||
yield
|
||||
edifabric._reset_transport_factory()
|
||||
|
||||
|
||||
# --- /x12/read ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_read_interchange_returns_first_x12_from_array():
|
||||
"""The /x12/read endpoint returns a list (multi-interchange file).
|
||||
Cyclone calls return the first element."""
|
||||
x12 = {
|
||||
"SegmentDelimiter": "~",
|
||||
"DataElementDelimiter": "*",
|
||||
"ISA": {"InterchangeControlNumber_13": "000000001"},
|
||||
"Groups": [],
|
||||
"IEATrailers": [],
|
||||
}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.headers["Ocp-Apim-Subscription-Key"] == _TEST_KEY
|
||||
assert request.headers["Content-Type"] == "application/octet-stream"
|
||||
return httpx.Response(200, json=[x12])
|
||||
|
||||
_install_factory(handler)
|
||||
result = edifabric.read_interchange(b"ISA*...~IEA*0*000000001~", api_key=_TEST_KEY)
|
||||
assert result["ISA"]["InterchangeControlNumber_13"] == "000000001"
|
||||
|
||||
|
||||
def test_read_interchange_rejects_empty_response():
|
||||
"""If /x12/read returns an empty list, surface a 502-style error."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=[])
|
||||
|
||||
_install_factory(handler)
|
||||
with pytest.raises(edifabric.EdifabricError) as exc_info:
|
||||
edifabric.read_interchange(b"ISA*...~IEA*0*000000001~", api_key=_TEST_KEY)
|
||||
assert exc_info.value.status_code == 502
|
||||
|
||||
|
||||
# --- /x12/validate -----------------------------------------------------
|
||||
|
||||
|
||||
def test_validate_interchange_returns_operation_result():
|
||||
"""A 200 response is returned verbatim — Status + Details."""
|
||||
operation_result = {
|
||||
"Status": "success",
|
||||
"Details": [],
|
||||
"LastIndex": 46,
|
||||
}
|
||||
x12 = {
|
||||
"SegmentDelimiter": "~",
|
||||
"DataElementDelimiter": "*",
|
||||
"ISA": {"InterchangeControlNumber_13": "000000001"},
|
||||
"Groups": [],
|
||||
"IEATrailers": [],
|
||||
}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content)
|
||||
assert body["ISA"]["InterchangeControlNumber_13"] == "000000001"
|
||||
return httpx.Response(200, json=operation_result)
|
||||
|
||||
_install_factory(handler)
|
||||
result = edifabric.validate_interchange(x12, api_key=_TEST_KEY)
|
||||
assert result["Status"] == "success"
|
||||
assert result["Details"] == []
|
||||
|
||||
|
||||
def test_validate_interchange_does_not_raise_on_status_error():
|
||||
"""OperationResult.Status='error' is data, not an exception — the
|
||||
caller (the gate / CLI) decides whether to fail-closed."""
|
||||
operation_result = {
|
||||
"Status": "error",
|
||||
"Details": [
|
||||
{"Index": 5, "SegmentId": "PER", "Message": "PER-04 is required", "Status": "error"},
|
||||
],
|
||||
"LastIndex": 5,
|
||||
}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=operation_result)
|
||||
|
||||
_install_factory(handler)
|
||||
result = edifabric.validate_interchange({"ISA": {}}, api_key=_TEST_KEY)
|
||||
assert result["Status"] == "error"
|
||||
assert result["Details"][0]["Message"] == "PER-04 is required"
|
||||
|
||||
|
||||
# --- 4xx / 5xx ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_validate_interchange_raises_on_5xx():
|
||||
"""Non-2xx responses raise EdifabricError; the body is preserved."""
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(503, text="upstream overloaded")
|
||||
|
||||
_install_factory(handler)
|
||||
with pytest.raises(edifabric.EdifabricError) as exc_info:
|
||||
edifabric.validate_interchange({"ISA": {}}, api_key=_TEST_KEY)
|
||||
assert exc_info.value.status_code == 503
|
||||
assert "upstream overloaded" in str(exc_info.value.body)
|
||||
|
||||
|
||||
# --- validate_edi (composed) ------------------------------------------
|
||||
|
||||
|
||||
def test_validate_edi_composes_read_then_validate():
|
||||
"""validate_edi should call read first, then validate with the
|
||||
X12Interchange JSON from read's response."""
|
||||
x12 = {
|
||||
"SegmentDelimiter": "~",
|
||||
"DataElementDelimiter": "*",
|
||||
"ISA": {"InterchangeControlNumber_13": "000000099"},
|
||||
"Groups": [],
|
||||
"IEATrailers": [],
|
||||
}
|
||||
operation_result = {"Status": "success", "Details": [], "LastIndex": 10}
|
||||
seen_calls: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/read"):
|
||||
seen_calls.append("read")
|
||||
return httpx.Response(200, json=[x12])
|
||||
if request.url.path.endswith("/validate"):
|
||||
seen_calls.append("validate")
|
||||
# Verify the validate body is the X12Interchange JSON
|
||||
body = json.loads(request.content)
|
||||
assert body["ISA"]["InterchangeControlNumber_13"] == "000000099"
|
||||
return httpx.Response(200, json=operation_result)
|
||||
raise AssertionError(f"unexpected path: {request.url.path}")
|
||||
|
||||
_install_factory(handler)
|
||||
result = edifabric.validate_edi(b"ISA*...~IEA*0*000000099~", api_key=_TEST_KEY)
|
||||
assert seen_calls == ["read", "validate"]
|
||||
assert result["Status"] == "success"
|
||||
|
||||
|
||||
# --- API key handling --------------------------------------------------
|
||||
|
||||
|
||||
def test_validate_edi_raises_when_api_key_missing(monkeypatch):
|
||||
"""With no key configured anywhere, validate_edi surfaces a clear
|
||||
error to the operator (not a generic 500)."""
|
||||
monkeypatch.delenv("CYCLONE_EDIFABRIC_API_KEY", raising=False)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise AssertionError("transport should not be called when key is missing")
|
||||
|
||||
_install_factory(handler)
|
||||
with pytest.raises(edifabric.EdifabricError) as exc_info:
|
||||
edifabric.validate_edi(b"ISA*...~IEA*0*000000001~")
|
||||
assert exc_info.value.status_code == 0
|
||||
assert "API key not configured" in str(exc_info.value.body)
|
||||
@@ -203,6 +203,7 @@ def test_resubmit_rejected_claims_inserts_resubmission_row(tmp_path, monkeypatch
|
||||
"--ingest-dir", str(tmp_path),
|
||||
"--limit", "1",
|
||||
"--no-validate",
|
||||
"--skip-edifabric-validate",
|
||||
])
|
||||
assert result.exit_code == 0, (
|
||||
f"CLI exited {result.exit_code}, output={result.output!r}, "
|
||||
@@ -279,7 +280,254 @@ def test_resubmit_idempotency_does_not_create_duplicate_rows(tmp_path, monkeypat
|
||||
"--ingest-dir", str(tmp_path),
|
||||
"--limit", "1",
|
||||
"--no-validate",
|
||||
"--skip-edifabric-validate",
|
||||
])
|
||||
# On the skip path, record_resubmission should NOT be called.
|
||||
assert captured == [], f"expected no record_resubmission on skip, got {captured!r}"
|
||||
assert "skipped" in result.output, result.output
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP40: `cyclone resubmit-rejected-claims` Edifabric pre-upload gate.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_TEST_KEY = "test-edifabric-key-0123456789abcdef"
|
||||
|
||||
|
||||
def _install_edifabric_mock(handler):
|
||||
"""Install a mocked Edifabric transport + get_secret stub for the
|
||||
duration of a single test. Returned teardown callable can be wired
|
||||
to monkeypatch via the autouse fixture pattern below."""
|
||||
import sys
|
||||
import httpx
|
||||
from cyclone import edifabric
|
||||
import cyclone.secrets as _secrets_mod
|
||||
|
||||
real_get_secret = _secrets_mod.get_secret
|
||||
|
||||
def _fake_get_secret(name):
|
||||
if name == "edifabric.api_key":
|
||||
return _TEST_KEY
|
||||
return real_get_secret(name)
|
||||
|
||||
_secrets_mod.get_secret = _fake_get_secret
|
||||
transport = httpx.MockTransport(handler)
|
||||
edifabric.set_transport_factory(
|
||||
lambda: httpx.Client(transport=transport, timeout=10.0)
|
||||
)
|
||||
|
||||
def _teardown():
|
||||
_secrets_mod.get_secret = real_get_secret
|
||||
edifabric._reset_transport_factory()
|
||||
|
||||
return _teardown
|
||||
|
||||
|
||||
def _two_files_setup(tmp_path):
|
||||
"""Stage two .x12 files in distinct batch dirs so the test can
|
||||
observe one file's gate failure not blocking the other's upload."""
|
||||
src_text = open("tests/fixtures/co_medicaid_837p.txt").read()
|
||||
a = tmp_path / "batch-aaaaaaaa-1-claims"
|
||||
a.mkdir()
|
||||
(a / "tp1-aaaaaaaa-1of1.x12").write_text(src_text)
|
||||
b = tmp_path / "batch-bbbbbbbb-1-claims"
|
||||
b.mkdir()
|
||||
(b / "tp1-bbbbbbbb-1of1.x12").write_text(src_text)
|
||||
return src_text
|
||||
|
||||
|
||||
def test_resubmit_edifabric_gate_aborts_bad_file_continues_batch(
|
||||
tmp_path, monkeypatch,
|
||||
):
|
||||
"""SP40: when Edifabric returns Status='error' for one file, that
|
||||
file is recorded as ``validate_failed`` (not uploaded, no
|
||||
Resubmission row), but the batch continues with the next file."""
|
||||
import sys
|
||||
import types
|
||||
from cyclone.cli import resubmit_rejected_claims
|
||||
|
||||
_seed_clearhouse_stub_false(tmp_path)
|
||||
_two_files_setup(tmp_path)
|
||||
|
||||
seen: list[str] = []
|
||||
x12 = {
|
||||
"SegmentDelimiter": "~",
|
||||
"DataElementDelimiter": "*",
|
||||
"ISA": {"InterchangeControlNumber_13": "000000001"},
|
||||
"Groups": [],
|
||||
"IEATrailers": [],
|
||||
}
|
||||
|
||||
def edifabric_handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/read"):
|
||||
return httpx.Response(200, json=[x12])
|
||||
if request.url.path.endswith("/validate"):
|
||||
# Fail the FIRST validate call; succeed on the second.
|
||||
seen.append("validate")
|
||||
is_bad = len(seen) == 1
|
||||
return httpx.Response(200, json={
|
||||
"Status": "error" if is_bad else "success",
|
||||
"Details": (
|
||||
[
|
||||
{"SegmentId": "PER",
|
||||
"Message": "PER-04 is required",
|
||||
"Status": "error"},
|
||||
]
|
||||
if is_bad else []
|
||||
),
|
||||
"LastIndex": 5 if is_bad else 46,
|
||||
})
|
||||
raise AssertionError(f"unexpected path: {request.url.path}")
|
||||
|
||||
import httpx
|
||||
teardown = _install_edifabric_mock(edifabric_handler)
|
||||
try:
|
||||
# Fake paramiko so SFTP doesn't actually fire.
|
||||
fake_paramiko = types.ModuleType("paramiko")
|
||||
|
||||
class _FakeAttr:
|
||||
def __init__(self, size: int) -> None:
|
||||
self.st_size = size
|
||||
|
||||
class _FakeSFTP:
|
||||
def __init__(self):
|
||||
self.put_calls: list[str] = []
|
||||
|
||||
def stat(self, path):
|
||||
raise IOError("not found")
|
||||
|
||||
def put(self, local, remote):
|
||||
self.put_calls.append(remote)
|
||||
|
||||
fake_sftp = _FakeSFTP()
|
||||
|
||||
class _FakeSSH:
|
||||
def open_sftp(self):
|
||||
return fake_sftp
|
||||
|
||||
def set_missing_host_key_policy(self, policy):
|
||||
return None
|
||||
|
||||
def connect(self, *a, **kw):
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
fake_paramiko.SSHClient = lambda: _FakeSSH()
|
||||
fake_paramiko.AutoAddPolicy = lambda: None
|
||||
monkeypatch.setitem(sys.modules, "paramiko", fake_paramiko)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(resubmit_rejected_claims, [
|
||||
"--ingest-dir", str(tmp_path),
|
||||
"--no-validate",
|
||||
# default: gate ENABLED
|
||||
])
|
||||
assert result.exit_code == 0, result.output
|
||||
|
||||
# First file: Edifabric gate aborted it (no SFTP put for it).
|
||||
# Second file: passed the gate, got uploaded via the SFTP put.
|
||||
assert len(fake_sftp.put_calls) == 1, fake_sftp.put_calls
|
||||
assert "bbbbbbbb" in fake_sftp.put_calls[0], fake_sftp.put_calls[0]
|
||||
|
||||
# Output surfaces the failure list.
|
||||
assert "EDIFABRIC VALIDATION FAIL" in result.output, result.output
|
||||
assert "1 file(s) blocked by Edifabric gate" in result.output, result.output
|
||||
assert "aaaaaaa" in result.output, result.output
|
||||
assert "PER-04 is required" in result.output, result.output
|
||||
|
||||
# DONE summary shows both counters.
|
||||
assert "validate_failed=1" in result.output, result.output
|
||||
assert "uploaded=1" in result.output, result.output
|
||||
finally:
|
||||
teardown()
|
||||
|
||||
|
||||
def test_resubmit_edifabric_skip_flag_bypasses_gate(tmp_path, monkeypatch):
|
||||
"""SP40: ``--skip-edifabric-validate`` bypasses the gate entirely
|
||||
so the same Edifabric-failing file uploads via SFTP. Used for
|
||||
dev / hot-fix scenarios."""
|
||||
import sys
|
||||
import types
|
||||
from cyclone.cli import resubmit_rejected_claims
|
||||
|
||||
_seed_clearhouse_stub_false(tmp_path)
|
||||
_two_files_setup(tmp_path)
|
||||
|
||||
import httpx
|
||||
x12 = {
|
||||
"SegmentDelimiter": "~",
|
||||
"DataElementDelimiter": "*",
|
||||
"ISA": {"InterchangeControlNumber_13": "000000001"},
|
||||
"Groups": [],
|
||||
"IEATrailers": [],
|
||||
}
|
||||
|
||||
validate_calls: list[str] = []
|
||||
|
||||
def edifabric_handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/read"):
|
||||
return httpx.Response(200, json=[x12])
|
||||
if request.url.path.endswith("/validate"):
|
||||
validate_calls.append("hit")
|
||||
return httpx.Response(200, json={
|
||||
"Status": "error",
|
||||
"Details": [{"SegmentId": "PER",
|
||||
"Message": "PER-04 is required",
|
||||
"Status": "error"}],
|
||||
"LastIndex": 5,
|
||||
})
|
||||
raise AssertionError(f"unexpected path: {request.url.path}")
|
||||
|
||||
teardown = _install_edifabric_mock(edifabric_handler)
|
||||
try:
|
||||
fake_paramiko = types.ModuleType("paramiko")
|
||||
|
||||
class _FakeSFTP:
|
||||
def __init__(self):
|
||||
self.put_calls: list[str] = []
|
||||
|
||||
def stat(self, path):
|
||||
raise IOError("not found")
|
||||
|
||||
def put(self, local, remote):
|
||||
self.put_calls.append(remote)
|
||||
|
||||
fake_sftp = _FakeSFTP()
|
||||
|
||||
class _FakeSSH:
|
||||
def open_sftp(self):
|
||||
return fake_sftp
|
||||
|
||||
def set_missing_host_key_policy(self, policy):
|
||||
return None
|
||||
|
||||
def connect(self, *a, **kw):
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
fake_paramiko.SSHClient = lambda: _FakeSSH()
|
||||
fake_paramiko.AutoAddPolicy = lambda: None
|
||||
monkeypatch.setitem(sys.modules, "paramiko", fake_paramiko)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(resubmit_rejected_claims, [
|
||||
"--ingest-dir", str(tmp_path),
|
||||
"--no-validate",
|
||||
"--skip-edifabric-validate",
|
||||
])
|
||||
assert result.exit_code == 0, result.output
|
||||
|
||||
# Both files uploaded; the gate never fired.
|
||||
assert len(fake_sftp.put_calls) == 2, fake_sftp.put_calls
|
||||
assert validate_calls == [], (
|
||||
"validate must not be called when --skip-edifabric-validate is set"
|
||||
)
|
||||
assert "validate_failed=0" in result.output, result.output
|
||||
assert "EDIFABRIC VALIDATION FAIL" not in result.output, result.output
|
||||
finally:
|
||||
teardown()
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
"""SP40: tests for the edifabric.api_key secret wiring.
|
||||
|
||||
Specifically: the ``_ENV_NAME_FOR`` table maps ``edifabric.api_key``
|
||||
to ``CYCLONE_EDIFABRIC_API_KEY`` so operators can configure via env
|
||||
var (or via file companion per ``secrets.get_secret``'s
|
||||
``<env_name>_FILE`` form) without touching the macOS Keychain.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_env_name_for_maps_edifabric_api_key():
|
||||
from cyclone.secrets import _ENV_NAME_FOR
|
||||
assert _ENV_NAME_FOR["edifabric.api_key"] == "CYCLONE_EDIFABRIC_API_KEY"
|
||||
|
||||
|
||||
def test_get_secret_reads_edifabric_key_from_env(monkeypatch):
|
||||
"""With the env var set, ``get_secret('edifabric.api_key')``
|
||||
returns it without touching the Keychain."""
|
||||
monkeypatch.setenv("CYCLONE_EDIFABRIC_API_KEY", "test-env-key-9876")
|
||||
|
||||
from cyclone import secrets
|
||||
assert secrets.get_secret("edifabric.api_key") == "test-env-key-9876"
|
||||
|
||||
|
||||
def test_get_secret_reads_edifabric_key_from_file(monkeypatch, tmp_path):
|
||||
"""With ``<env_name>_FILE`` set, ``get_secret`` reads the key from
|
||||
the file (no Keychain touch). Mirrors the existing SFTP password
|
||||
file-companion pattern (SP25)."""
|
||||
key_file = tmp_path / "edi_key"
|
||||
key_file.write_text("file-companion-key-5555\n")
|
||||
monkeypatch.setenv("CYCLONE_EDIFABRIC_API_KEY_FILE", str(key_file))
|
||||
|
||||
from cyclone import secrets
|
||||
assert secrets.get_secret("edifabric.api_key") == "file-companion-key-5555"
|
||||
|
||||
|
||||
def test_dev_fixture_file_exists_and_contains_eval_key():
|
||||
"""The dev-only fixture exists at the documented path so the
|
||||
eval key is referenceable; the CI mocks intercept before any
|
||||
real HTTP happens."""
|
||||
fixture = Path("tests/fixtures/edifabric_api_key.txt")
|
||||
assert fixture.exists(), (
|
||||
f"dev-only Edifabric API key fixture is missing at {fixture}"
|
||||
)
|
||||
contents = fixture.read_text().strip()
|
||||
# Public eval key from the EdiNation API docs.
|
||||
assert "3ecf6b1c5cf34bd797a5f4c57951a1cf" in contents
|
||||
@@ -683,3 +683,58 @@ def test_2010bb_preserves_foreign_payer_id_verbatim():
|
||||
assert parts[3] == "OTHER PAYER NAME"
|
||||
# No substitution log for a foreign payer id
|
||||
assert "SP39" not in log_stream.getvalue()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SP40: PER-02/03/04 + SBR-09 must always be emitted (Edifabric rejects
|
||||
# the bare-PER / no-SBR09 shapes). The serializer fall-back fills them
|
||||
# with safe placeholders when the caller passes no kwargs.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_per_segment_emits_name_and_phone_qualifier_when_no_contact():
|
||||
"""SP40: without any submitter_contact_* kwargs, the submitter
|
||||
block must still produce a PER segment with at least PER-02 (Name)
|
||||
and a PER-03 (Communication Number Qualifier) + PER-04 (Number)
|
||||
pair. Edifabric rejects PER*IC alone."""
|
||||
claim = _load_claim()
|
||||
text = serialize_837_for_resubmit(claim, interchange_index=42)
|
||||
per_line = next(s for s in text.split("~") if s.startswith("PER"))
|
||||
parts = per_line.split("*")
|
||||
# PER*IC*<Name>*<Qual>*<Number>
|
||||
assert parts[1] == "IC"
|
||||
assert parts[2], f"PER-02 (Name) is required, got empty; line={per_line!r}"
|
||||
assert parts[3] in {"TE", "EM", "FX"}, (
|
||||
f"PER-03 must be a Communication Number Qualifier (TE/EM/FX); "
|
||||
f"got {parts[3]!r}; line={per_line!r}"
|
||||
)
|
||||
assert parts[4], f"PER-04 (Number) is required, got empty; line={per_line!r}"
|
||||
|
||||
|
||||
def test_sbr_segment_emits_claim_filing_indicator_default_mc():
|
||||
"""SP40: SBR-09 (Claim Filing Indicator Code) must always be
|
||||
emitted. Default is 'MC' (Medicaid) when the caller passes no
|
||||
claim_filing_indicator_code kwarg. Edifabric rejects the bare
|
||||
SBR*P*18******* shape."""
|
||||
claim = _load_claim()
|
||||
text = serialize_837_for_resubmit(claim, interchange_index=42)
|
||||
sbr_line = next(s for s in text.split("~") if s.startswith("SBR"))
|
||||
parts = sbr_line.split("*")
|
||||
# SBR*P*18*******MC → index 9 is SBR-09
|
||||
assert len(parts) >= 10, f"SBR must have 10 elements (with SBR-09), got {len(parts)}: {sbr_line!r}"
|
||||
assert parts[9] == "MC", (
|
||||
f"SBR-09 must default to 'MC' (Medicaid), got {parts[9]!r}; line={sbr_line!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_sbr_segment_respects_explicit_claim_filing_indicator():
|
||||
"""SP40: explicit claim_filing_indicator_code kwargs win over the
|
||||
default — preserves the existing caller-facing behavior for
|
||||
non-CO payers (e.g. '16' for Medicare Part B)."""
|
||||
claim = _load_claim()
|
||||
text = serialize_837_for_resubmit(
|
||||
claim, interchange_index=42,
|
||||
claim_filing_indicator_code="16",
|
||||
)
|
||||
sbr_line = next(s for s in text.split("~") if s.startswith("SBR"))
|
||||
parts = sbr_line.split("*")
|
||||
assert parts[9] == "16", f"SBR-09 should reflect explicit kwarg, got {parts[9]!r}"
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
"""SP40: tests for `cyclone validate-837 <file>` CLI."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from cyclone import edifabric
|
||||
from cyclone.cli import validate_837_cmd
|
||||
|
||||
|
||||
_TEST_KEY = "test-edifabric-key-0123456789abcdef"
|
||||
|
||||
|
||||
def _make_client(handler):
|
||||
transport = httpx.MockTransport(handler)
|
||||
return httpx.Client(transport=transport, timeout=10.0)
|
||||
|
||||
|
||||
def _install(handler):
|
||||
edifabric.set_transport_factory(lambda: _make_client(handler))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_transport():
|
||||
yield
|
||||
edifabric._reset_transport_factory()
|
||||
|
||||
|
||||
# --- happy path -------------------------------------------------------
|
||||
|
||||
|
||||
def test_validate_837_success_prints_status_and_exits_zero(tmp_path):
|
||||
"""A clean file with Status='success' from Edifabric exits 0 and
|
||||
prints the OperationResult summary."""
|
||||
fixture = Path("/home/tyler/dev/cyclone/backend/tests/fixtures/co_medicaid_837p.txt")
|
||||
sample = tmp_path / "ok.x12"
|
||||
sample.write_text(fixture.read_text())
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/read"):
|
||||
return httpx.Response(200, json=[{
|
||||
"SegmentDelimiter": "~",
|
||||
"DataElementDelimiter": "*",
|
||||
"ISA": {"InterchangeControlNumber_13": "000000001"},
|
||||
"Groups": [],
|
||||
"IEATrailers": [],
|
||||
}])
|
||||
if request.url.path.endswith("/validate"):
|
||||
return httpx.Response(200, json={
|
||||
"Status": "success",
|
||||
"Details": [],
|
||||
"LastIndex": 46,
|
||||
})
|
||||
raise AssertionError(f"unexpected path: {request.url.path}")
|
||||
|
||||
_install(handler)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(validate_837_cmd, [str(sample), "--api-key", _TEST_KEY])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "Status: success" in result.output
|
||||
assert "Details: 0 item(s)" in result.output
|
||||
|
||||
|
||||
def test_validate_837_error_status_exits_two_and_prints_details(tmp_path):
|
||||
"""Status='error' from Edifabric exits 2 and prints the offending
|
||||
segment + message for each Details item."""
|
||||
sample = tmp_path / "bad.x12"
|
||||
sample.write_text("ISA*bad~")
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/read"):
|
||||
return httpx.Response(200, json=[{
|
||||
"SegmentDelimiter": "~",
|
||||
"DataElementDelimiter": "*",
|
||||
"ISA": {"InterchangeControlNumber_13": "000000001"},
|
||||
"Groups": [],
|
||||
"IEATrailers": [],
|
||||
}])
|
||||
if request.url.path.endswith("/validate"):
|
||||
return httpx.Response(200, json={
|
||||
"Status": "error",
|
||||
"Details": [
|
||||
{"SegmentId": "PER", "Message": "PER-04 is required", "Status": "error"},
|
||||
{"SegmentId": "SBR", "Message": "SBR-09 is required", "Status": "error"},
|
||||
],
|
||||
"LastIndex": 5,
|
||||
})
|
||||
raise AssertionError(f"unexpected path: {request.url.path}")
|
||||
|
||||
_install(handler)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(validate_837_cmd, [str(sample), "--api-key", _TEST_KEY])
|
||||
assert result.exit_code == 2, result.output
|
||||
assert "Status: error" in result.output
|
||||
assert "PER: PER-04 is required" in result.output
|
||||
assert "SBR: SBR-09 is required" in result.output
|
||||
|
||||
|
||||
def test_validate_837_missing_api_key_exits_one(tmp_path, monkeypatch):
|
||||
"""With no API key anywhere (env var unset, keychain empty), the
|
||||
CLI exits 1 and surfaces a clear 'API key not configured' error."""
|
||||
monkeypatch.delenv("CYCLONE_EDIFABRIC_API_KEY", raising=False)
|
||||
sample = tmp_path / "any.x12"
|
||||
sample.write_text("ISA*...~")
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise AssertionError("transport should not be called when key is missing")
|
||||
|
||||
_install(handler)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(validate_837_cmd, [str(sample)])
|
||||
assert result.exit_code == 1, result.output
|
||||
assert "API key not configured" in result.output
|
||||
|
||||
|
||||
def test_validate_837_file_not_found_exits_two():
|
||||
"""Click's path validation rejects a missing file before the
|
||||
command body runs (exits 2 via Click's UsageError)."""
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(validate_837_cmd, ["/nonexistent/path.x12", "--api-key", _TEST_KEY])
|
||||
assert result.exit_code == 2, result.output
|
||||
+119
@@ -323,6 +323,125 @@ curl -X PATCH http://127.0.0.1:8000/api/clearhouse \
|
||||
|
||||
To revert, set `stub: true` and `auth: {"method": "keychain", "secret_ref": "sftp.gainwell.password"}`.
|
||||
|
||||
### After SP40 lands — Edifabric /v2/x12/validate integration
|
||||
|
||||
SP40 wires Edifabric's reference X12 validator into the push flow
|
||||
as a fail-closed pre-upload gate, plus a CLI / HTTP endpoint for
|
||||
ad-hoc validation of single files. The integration catches the
|
||||
same byte-level defects (`PER-04 required`, `SBR-09 required`,
|
||||
etc.) Edifabric flagged on the SP39 regen `ingest/corrected-v2/`.
|
||||
|
||||
**One-time secret setup** (operator-only, never commit):
|
||||
|
||||
```bash
|
||||
# Option A — macOS Keychain (preferred, paid tier key):
|
||||
cyclone secrets set edifabric.api_key <paid-tier-key>
|
||||
# → stores in Keychain under service='cyclone', account='edifabric.api_key'
|
||||
|
||||
# Option B — env var (CI / headless / per-process):
|
||||
export CYCLONE_EDIFABRIC_API_KEY="<paid-tier-key>"
|
||||
|
||||
# Option C — file companion (matches the SP25 SFTP-password pattern):
|
||||
export CYCLONE_EDIFABRIC_API_KEY_FILE=/path/to/edi-key.txt
|
||||
# file should be chmod 600, single-line, no trailing newline needed
|
||||
```
|
||||
|
||||
The dev-only `tests/fixtures/edifabric_api_key.txt` holds the
|
||||
free `EdiNation Developer API` provisional key from the public docs
|
||||
(`3ecf6b1c5cf34bd797a5f4c57951a1cf`); CI uses the mocked transport
|
||||
so the key is never actually transmitted.
|
||||
|
||||
**Single-file validation — CLI**
|
||||
|
||||
```bash
|
||||
# Status printout + OperationResult summary; exits 0 / 1 / 2.
|
||||
.venv/bin/python -m cyclone.cli validate-837 ingest/corrected-v2/v2-batch-<id>-<N>-claims/tp-<id>-1of1.x12
|
||||
```
|
||||
|
||||
Exit codes (matches the rest of the validator family):
|
||||
|
||||
- `0` — Edifabric returned `Status="success"` (warnings are non-blocking).
|
||||
- `2` — `Status="error"`: at least one `Details` item has
|
||||
`Status="error"`. The CLI prints the first 5 offending segment
|
||||
IDs + messages, then exits.
|
||||
- `1` — Edifabric client config error (no API key, or transport
|
||||
exception). The operator must fix `cyc secrets set` before retrying.
|
||||
|
||||
**Single-file validation — HTTP (auth-gated)**
|
||||
|
||||
```bash
|
||||
curl -b admin-cookies.txt -X POST \
|
||||
-F "file=@ingest/corrected-v2/v2-batch-.../tp-...-1of1.x12" \
|
||||
http://127.0.0.1:8000/api/admin/validate-837
|
||||
# → 200 with the raw OperationResult JSON (Status + Details + LastIndex)
|
||||
# Caller inspects Status to decide whether the file is acceptable.
|
||||
```
|
||||
|
||||
`POST /api/admin/validate-837` is matrix-gated (ADMIN_ONLY); 401
|
||||
without a session cookie, 403 with a viewer cookie.
|
||||
|
||||
**Pre-upload gate on resubmit-rejected-claims**
|
||||
|
||||
`cyclone resubmit-rejected-claims` runs Edifabric validation
|
||||
before every SFTP `put`. Any file that returns `Status="error"`
|
||||
is skipped (not uploaded, no `Resubmission` row inserted), and the
|
||||
batch continues with the remaining files. After the run:
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m cyclone.cli resubmit-rejected-claims \
|
||||
--ingest-dir /home/tyler/dev/cyclone/ingest/corrected-v2
|
||||
# ...
|
||||
# EDIFABRIC VALIDATION FAIL tp-cyc-deadbeef-1of1.x12: 2 error(s)
|
||||
# [error] PER: PER-04 is required
|
||||
# [error] SBR: SBR-09 is required
|
||||
# ...
|
||||
# DONE uploaded=361 skipped=0 failed=0 validated=363 validate_failed=2 ...
|
||||
#
|
||||
# 2 file(s) blocked by Edifabric gate:
|
||||
# tp-cyc-aaaaaaaa-1of1.x12
|
||||
# [error] PER: PER-04 is required
|
||||
# tp-cyc-bbbbbbbb-1of1.x12
|
||||
# [error] SBR: SBR-09 is required
|
||||
```
|
||||
|
||||
To bypass the gate (dev only — for hot-fix scenarios where
|
||||
Edifabric is the bottleneck):
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m cyclone.cli resubmit-rejected-claims \
|
||||
--ingest-dir ... \
|
||||
--skip-edifabric-validate
|
||||
```
|
||||
|
||||
**Expected OperationResult shape** (verbatim from EdiNation docs):
|
||||
|
||||
```json
|
||||
{
|
||||
"Status": "success" | "warning" | "error",
|
||||
"Details": [
|
||||
{"Index": 5, "SegmentId": "PER", "Value": "...",
|
||||
"Message": "PER-04 is required", "Status": "error"}
|
||||
],
|
||||
"LastIndex": 46
|
||||
}
|
||||
```
|
||||
|
||||
Warnings (`Status="warning"`) never block. Only `Details` items
|
||||
with `Status="error"` fail the gate.
|
||||
|
||||
**Troubleshooting**
|
||||
|
||||
- `API key not configured` → run
|
||||
`cyclone secrets set edifabric.api_key <key>` (or set
|
||||
`CYCLONE_EDIFABRIC_API_KEY_FILE`).
|
||||
- `502 Edifabric upstream failure` → check
|
||||
https://status.edifabric.com/ (or the API status page); the
|
||||
gate treats any 4xx/5xx from Edifabric as a hard fail for that
|
||||
file but continues the batch.
|
||||
- After fixing the byte defect, re-run
|
||||
`cyclone resubmit-rejected-claims` for the blocked file —
|
||||
idempotent on `(claim_id, interchange_control_number)`.
|
||||
|
||||
## Known historical drift — the 804 orphan 999s
|
||||
|
||||
The `acks` table may hold several hundred 999 acks whose source 837
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
# SP40 — Edifabric validation gate + 837P byte-defect sweep
|
||||
|
||||
**Date:** 2026-07-07
|
||||
**Status:** Draft, awaiting user sign-off
|
||||
**Branch:** `sp40-edifabric-validation-gate`
|
||||
**Aesthetic direction:** No new UI; one new CLI command + one HTTP endpoint + serializer hardening + a defensive HTTP client. Three new tests, two modified files, two new docs.
|
||||
|
||||
---
|
||||
|
||||
## 1. Scope
|
||||
|
||||
Edifabric's `POST /v2/x12/validate` (the operator's reference parser for CO Medicaid compliance) rejected the SP39-regenerated 837P files with three byte-level errors:
|
||||
|
||||
```
|
||||
Line 1: Element PER-03 is required
|
||||
Line 1: Element PER-04 is required
|
||||
Line 1: Element SBR-09 is required
|
||||
```
|
||||
|
||||
Root cause: `serialize_837._build_per` emits `PER*IC` (only PER01), and `_build_sbr` emits `SBR*P*18*******` (no SBR09) when the caller passes no contact / SBR09 kwargs. The clearhouse config has `submitter_contact_name="Tyler Martinez"` and `submitter_contact_email="tyler@dzinesco.com"` seeded, and `PayerConfig.co_medicaid().sbr09_claim_filing = "MC"` — both are available to the serializer today but not wired into the default emit. SP33-era callers and the SP39 regen script both bypass the kwargs, so every file they emit is invalid.
|
||||
|
||||
SP40 lands (a) the serializer fixes that close the gap, (b) the Edifabric integration that catches the same class of defect for any future payer / schema drift, (c) a pre-upload gate on `resubmit-rejected-claims` so the operator can't push an Edifabric-invalid file to Gainwell, and (d) the API key wiring through the existing `secrets.get_secret()` plumbing.
|
||||
|
||||
**In scope:**
|
||||
|
||||
- `_build_per` always emits PER02 (Name) + at least one PER03/PER04 pair (email or phone), sourced from `claim.billing_provider.contact` or the clearhouse config when the caller passes nothing.
|
||||
- `_build_sbr` always emits SBR09 from `payer_cfg.sbr09_claim_filing` (already `"MC"` for CO Medicaid).
|
||||
- A new `cyclone.edifabric` module — thin `httpx` client wrapping the two-step Edifabric flow (`/x12/read` → `/x12/validate`).
|
||||
- A new CLI `cyclone validate-837 <file>` — reads the file, hits Edifabric, prints `OperationResult`; exits 0 on success / warning-only, 2 on any error.
|
||||
- A new HTTP `POST /api/admin/validate-837` (admin-gated) — multipart upload of the .x12 file; same response shape.
|
||||
- A pre-upload gate inside `resubmit-rejected-claims` — every file is validated before the SFTP `put`; `Status == "error"` aborts the file but continues the batch (other files still push). Bypass flag `--skip-edifabric-validate` for dev.
|
||||
- API key wiring: `secrets.get_secret('edifabric.api_key')` with a new `_ENV_NAME_FOR` entry mapping it to `CYCLONE_EDIFABRIC_API_KEY`.
|
||||
- A dev-fixture API key in `tests/fixtures/edifabric_api_key.txt` so CI doesn't need a real subscription (the public eval key `3ecf6b1c5cf34bd797a5f4c57951a1cf`).
|
||||
- A `PERMISSIONS` matrix entry for `("POST", "/api/admin/validate-837"): ADMIN_ONLY`.
|
||||
- RUNBOOK updates for the env var, the new CLI/HTTP, and the bypass flag.
|
||||
|
||||
**Out of scope:**
|
||||
|
||||
- 999 / 277CA / TA1 / 835 / 270 / 271 Edifabric validation (only 837P this SP).
|
||||
- Building a custom EDI → X12Interchange JSON converter (we use Edifabric's `/x12/read` to do the conversion — see D1).
|
||||
- Auto-retry on transient Edifabric 5xx — the gate surfaces the error; the operator decides whether to retry.
|
||||
- Refreshing the 363 regen'd `ingest/corrected-v2/` files — the regen script already runs cleanly through the fixed serializer once Task 1 lands.
|
||||
|
||||
## 2. Decisions
|
||||
|
||||
- **D1.** Two-step Edifabric flow: `/x12/read` (raw EDI → `X12Interchange` JSON, `Content-Type: application/octet-stream`) then `/x12/validate` (JSON → `OperationResult`, `Content-Type: application/json`). Building our own EDI → JSON converter would be ~150 fields; `/x12/read` is the pragmatic path and the same library parses files we generate.
|
||||
- **D2.** API key via `secrets.get_secret('edifabric.api_key')` → `CYCLONE_EDIFABRIC_API_KEY` env var or Keychain (`cyclone` service, `edifabric.api_key` account). The eval key in `tests/fixtures/edifabric_api_key.txt` is dev-only; production is operator-supplied via `cyclone secrets set edifabric.api_key <paid-key>`.
|
||||
- **D3.** Pre-upload gate is **fail-closed**: any `Status == "error"` item in `OperationResult.Details` aborts the file (file is recorded as `failed`, batch continues). Warnings are logged at INFO level but don't block. This matches the operator's stated goal ("claims arent passing verification").
|
||||
- **D4.** Serializer hardening is minimal + backward-compatible:
|
||||
- `_build_per(contact_name, contact_phone, contact_email)` already handles all three; the fix is at the call site (where `submitter_kwargs` is built) — fall back to `clearhouse.submitter_contact_*` when the caller passes no kwargs. When none of those are set either, emit `PER*IC*CUSTOMER SERVICE*TE*8005550100` (a safe placeholder; documented in the function docstring as a temporary fallback so future per-billing-office contact info lands at the call site instead).
|
||||
- `_build_sbr(individual_relationship_code, claim_filing_indicator_code)` already takes the SBR09 value; the fix is at the call site — pass `payer_cfg.sbr09_claim_filing` (default `"MC"` for CO Medicaid). When `payer_cfg` is unavailable, fall back to `"MC"` (matches the SP9-era seeding default).
|
||||
- **D5.** HTTP client uses `httpx` (already in `pyproject.toml:35` `httpx>=0.27,<1`). One new module `cyclone.edifabric` with three functions:
|
||||
- `read_interchange(edi_bytes, *, api_key) -> X12Interchange`
|
||||
- `validate_interchange(x12_json, *, api_key) -> OperationResult`
|
||||
- `validate_edi(edi_bytes, *, api_key) -> OperationResult` — composes the two.
|
||||
Custom 4xx/5xx handling raises `EdifabricError(status, body)` so the caller can surface the Edifabric error verbatim.
|
||||
- **D6.** HTTP endpoint is `POST /api/admin/validate-837` (matches existing `/api/admin/*` admin-only pattern from `cyclone/api_routers/admin.py:38-39`). Body is `multipart/form-data` with a single `file` part (FastAPI's `UploadFile`); response is the raw `OperationResult` JSON. The CLI uses the same module functions directly so the wire shape stays consistent.
|
||||
- **D7.** Test mock strategy: `httpx.MockTransport` registered against the `httpx.Client` instance inside the `cyclone.edifabric` module (factory pattern lets tests inject the transport). No live HTTP in CI; the eval API key in `tests/fixtures/` is referenced but never sent — the mock intercepts first.
|
||||
- **D8.** Pre-upload gate bypass: `--skip-edifabric-validate` flag on the CLI; no env var. Bypassing is an explicit operator action, not a global setting, so it's hard to forget about.
|
||||
|
||||
## 3. Files expected to change
|
||||
|
||||
| Path | Type | Purpose |
|
||||
|---|---|---|
|
||||
| `backend/src/cyclone/parsers/serialize_837.py` | modify | `_build_per` call site: fall back to clearhouse contact info. `_build_sbr` call site: always pass `sbr09_claim_filing`. |
|
||||
| `backend/src/cyclone/edifabric.py` | create | `read_interchange`, `validate_interchange`, `validate_edi`, `EdifabricError`. |
|
||||
| `backend/src/cyclone/cli.py` | modify | New `validate_837_cmd` (read file → call `validate_edi` → print `OperationResult`). Pre-upload gate call inside `resubmit_rejected_claims`. New `--skip-edifabric-validate` flag. |
|
||||
| `backend/src/cyclone/api_routers/admin.py` | modify | New `POST /api/admin/validate-837` (multipart upload, returns `OperationResult`). |
|
||||
| `backend/src/cyclone/secrets.py` | modify | Add `"edifabric.api_key": "CYCLONE_EDIFABRIC_API_KEY"` to `_ENV_NAME_FOR`. |
|
||||
| `backend/src/cyclone/auth/permissions.py` | modify | Add `("POST", "/api/admin/validate-837"): ADMIN_ONLY`. |
|
||||
| `backend/tests/test_serialize_837.py` | extend | 3 new regression tests: PER-02/03/04 always present, SBR-09 always present, no-kwargs fallback still emits valid PER + SBR. |
|
||||
| `backend/tests/test_edifabric.py` | create | 4 tests with `httpx.MockTransport`: read succeeds, validate succeeds, validate returns errors, 5xx surfaces EdifabricError. |
|
||||
| `backend/tests/test_validate_837_cli.py` | create | 4 CliRunner tests: clean file → exit 0 + status printed, error → exit 2, missing API key → exit 1, file not found → exit 2. |
|
||||
| `backend/tests/test_api_validate_837.py` | create | 3 TestClient tests: admin auth gate (401 / 403), valid upload returns OperationResult, missing file part returns 422. |
|
||||
| `backend/tests/test_resubmissions_cli.py` | extend | 1 new test: validation fail mid-batch aborts that file but continues the others; `--skip-edifabric-validate` flag bypasses the gate. |
|
||||
| `backend/tests/fixtures/edifabric_api_key.txt` | create | The public eval key `3ecf6b1c5cf34bd797a5f4c57951a1cf` (file-based secret, read via `_FILE` env var in tests). |
|
||||
| `docs/RUNBOOK.md` | modify | New "After SP40 lands" section: env var, secret setup, CLI/HTTP usage, bypass flag, expected OperationResult shape. |
|
||||
| `docs/superpowers/specs/2026-07-07-cyclone-edifabric-validation-gate-design.md` | create | This spec, committed on the branch. |
|
||||
|
||||
## 4. Auth boundary
|
||||
|
||||
The HTTP endpoint inherits the existing admin router's `Depends(matrix_gate)` gate (`cyclone/api_routers/admin.py:39`); `PERMISSIONS` entry is `ADMIN_ONLY`. The API key never leaves the backend — the CLI and the HTTP handler both call `cyclone.edifabric.validate_edi(...)` server-side. The CLI is operator-only (no auth boundary in the CLI itself, per house convention). File-system threats unchanged: SQLCipher at rest, macOS Keychain for the secret. No new ports.
|
||||
|
||||
## 5. Open questions
|
||||
|
||||
- Should the `validate-837` HTTP endpoint accept a JSON body `{content: "ISA*..."}` as an alternative to multipart upload? Pro: easier UI integration. Con: more attack surface (request body in DB-sized chunks). Defer to a future SP if needed.
|
||||
- Should the gate emit a per-claim `Resubmission` row update when validation fails? The current `Resubmission` row is only written on successful SFTP push, so a fail-closed gate would skip the row entirely — clean. If the operator wants an audit trail of *attempted* pushes, that's a follow-up column on `Resubmission`. Defer.
|
||||
|
||||
## 6. Test impact
|
||||
|
||||
- `test_serialize_837.py`: 3 new regression tests. Total file: 53 + 3 = 56 tests.
|
||||
- `test_edifabric.py`: 4 new tests with mocked transport.
|
||||
- `test_validate_837_cli.py`: 4 new CliRunner tests.
|
||||
- `test_api_validate_837.py`: 3 new TestClient tests (auth gate, valid upload, missing file).
|
||||
- `test_resubmissions_cli.py`: 1 new gate-fail test + 1 bypass-flag test.
|
||||
- Total new tests: 13. Full suite target: 1458 + 13 = 1471 passing.
|
||||
|
||||
## 7. Acceptance criteria
|
||||
|
||||
- All 3 new serializer tests pass.
|
||||
- `cyclone validate-837 backend/tests/fixtures/co_medicaid_837p.txt` exits 0 and prints `Status: success` (mocked, no live HTTP).
|
||||
- `cyclone validate-837 ingest/corrected-v2/v2-batch-*/tp-cyc-*.x12` (363 files) — all exit 0 (no PER/SBR errors) once the regen is re-run with the fixed serializer. Spot-checked with `grep NM1*PR2*CO_TXIX` per file.
|
||||
- `POST /api/admin/validate-837` returns 401 unauthenticated, 403 with viewer cookie, 200 with admin cookie + multipart file.
|
||||
- `resubmit-rejected-claims --ingest-dir ingest/corrected-v2` aborts files whose `OperationResult.Status == "error"`, continues with others, prints the failure list at the end.
|
||||
- `--skip-edifabric-validate` flag bypasses the gate (for dev only).
|
||||
- `secrets._ENV_NAME_FOR['edifabric.api_key'] == 'CYCLONE_EDIFABRIC_API_KEY'`.
|
||||
- `cyclone secrets set edifabric.api_key <key>` persists; `get_secret('edifabric.api_key')` reads from Keychain on subsequent boots.
|
||||
- RUNBOOK.md has the "After SP40 lands" section.
|
||||
|
||||
## 8. Implementation plan summary
|
||||
|
||||
(Full plan with checkboxes goes in `docs/superpowers/plans/2026-07-07-cyclone-edifabric-validation-gate.md` once this spec lands.)
|
||||
|
||||
| Task | Scope | Estimated commits |
|
||||
|---|---|---|
|
||||
| 0 | Spec commit | `docs(spec): SP40 ...` |
|
||||
| 1 | Serializer fixes (D4): `_build_per` + `_build_sbr` call sites, regression tests | `feat(sp40): 837P serializer always emits PER-02/03/04 + SBR-09` |
|
||||
| 2 | `cyclone.edifabric` module + tests (D5, D7) | `feat(sp40): Edifabric HTTP client (read + validate)` |
|
||||
| 3 | `cyclone validate-837` CLI + tests (D8) | `feat(sp40): cyclone validate-837 CLI` |
|
||||
| 4 | `POST /api/admin/validate-837` HTTP + tests (D6) | `feat(sp40): POST /api/admin/validate-837 admin endpoint` |
|
||||
| 5 | Pre-upload gate in `resubmit-rejected-claims` + bypass flag (D3, D8) | `feat(sp40): resubmit-rejected-claims pre-upload validation gate` |
|
||||
| 6 | API key wiring: `_ENV_NAME_FOR` entry + dev fixture (D2) | `feat(sp40): edifabric.api_key secret wiring + dev fixture` |
|
||||
| 7 | RUNBOOK.md "After SP40 lands" section (D5) | `docs(sp40): RUNBOOK entry for Edifabric validation` |
|
||||
| 8 | Full suite + regen re-run + atomic merge | `merge: SP40 Edifabric validation gate + 837P byte-defect sweep into main` |
|
||||
Reference in New Issue
Block a user