merge: SP25 + SP26 SFTP polling + secret-file lookup into Version-1.0.0
Bring the SP25 SFTP Polling Enablement (PATCH /api/clearhouse, scheduler hot-reload, env-var-first secret lookup) and SP26 SFTP Password File Companion (Docker secret _FILE tier) onto the v1.0.0 release branch so the production stack can actually flip the Gainwell MFT poll from stub → real. Conflict resolved in docker-compose.yml: kept both the SP26 CYCLONE_SFTP_PASSWORD_FILE env var and the new 'Always bind to 0.0.0.0' comment.
This commit is contained in:
tyler
@@ -2604,6 +2604,72 @@ def get_clearhouse():
|
||||
return json.loads(ch.model_dump_json())
|
||||
|
||||
|
||||
@app.patch("/api/clearhouse", dependencies=[Depends(matrix_gate)])
|
||||
async def patch_clearhouse(body: dict) -> Any:
|
||||
"""Replace the singleton clearhouse row (SP25).
|
||||
|
||||
The full ``Clearhouse`` model is required — we don't accept partial
|
||||
updates because the operator-facing use case is "I'm switching the
|
||||
loop to real MFT" or "I'm pointing at a different MFT server",
|
||||
not "I'm tweaking one field at a time." Validation errors are
|
||||
returned as 422 (Pydantic default).
|
||||
|
||||
After a successful write, the running scheduler is hot-reloaded
|
||||
via ``scheduler.reconfigure_scheduler()`` so the next tick uses
|
||||
the new SftpBlock without a process restart.
|
||||
"""
|
||||
from cyclone import scheduler as _scheduler_mod
|
||||
from cyclone.providers import Clearhouse as _Clearhouse, SftpBlock as _SftpBlock
|
||||
|
||||
# Strict-validate the sftp_block sub-dict FIRST. Pydantic v2's
|
||||
# default mode coerces strings to bools (e.g. ``"stub": "yes"``
|
||||
# silently becomes True), which would hide a real operator
|
||||
# mistake. The Clearhouse model itself stays in loose mode so
|
||||
# ISO-string ``updated_at`` (the JSON round-trip shape) keeps
|
||||
# parsing.
|
||||
raw_sb = body.get("sftp_block", {})
|
||||
try:
|
||||
_SftpBlock.model_validate(raw_sb, strict=True)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"invalid sftp_block: {exc}",
|
||||
) from exc
|
||||
|
||||
# Now validate the full body in loose mode.
|
||||
try:
|
||||
parsed = _Clearhouse.model_validate(body)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
# SP25: when sftp_block.stub=false, the block must carry an auth
|
||||
# account name and a non-empty host. The Pydantic model catches
|
||||
# some of these; this catches the "empty password_keychain_account"
|
||||
# case (which Pydantic allows because it's a free-form dict).
|
||||
sb = parsed.sftp_block
|
||||
if not sb.stub:
|
||||
if not sb.host:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail="sftp_block.host is required when stub=false",
|
||||
)
|
||||
auth = sb.auth or {}
|
||||
if not auth.get("password_keychain_account") and not auth.get("key_file"):
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=(
|
||||
"sftp_block.auth must contain either "
|
||||
"'password_keychain_account' or 'key_file' when stub=false"
|
||||
),
|
||||
)
|
||||
|
||||
updated = store.update_clearhouse(parsed)
|
||||
await _scheduler_mod.reconfigure_scheduler(
|
||||
updated.sftp_block,
|
||||
sftp_block_name=updated.name or "default",
|
||||
)
|
||||
return json.loads(updated.model_dump_json())
|
||||
|
||||
|
||||
@app.post("/api/clearhouse/submit", dependencies=[Depends(matrix_gate)])
|
||||
def submit_to_clearhouse(request: Request, body: dict):
|
||||
"""Submit a batch of claims to the clearhouse (SFTP). SP9: stub.
|
||||
|
||||
@@ -69,6 +69,11 @@ STATUS_SKIPPED = "skipped"
|
||||
STATUS_PENDING = "pending"
|
||||
|
||||
|
||||
# How long reconfigure_scheduler waits for the in-flight tick to
|
||||
# drain before cancelling the old task. Matches Scheduler.stop().
|
||||
_DRAIN_TIMEOUT_SECONDS = 30
|
||||
|
||||
|
||||
# File types we know how to route. The HCPF set is broader (270/271/
|
||||
# 276/277/278/820/834/ENCR) but Cyclone's parser only covers the
|
||||
# four below. Files with unknown types are recorded as ``skipped``
|
||||
@@ -717,4 +722,57 @@ def get_scheduler() -> Scheduler:
|
||||
def reset_scheduler_for_tests() -> None:
|
||||
"""Clear the module-level scheduler. Test-only."""
|
||||
global _scheduler
|
||||
_scheduler = None
|
||||
_scheduler = None
|
||||
|
||||
|
||||
async def reconfigure_scheduler(
|
||||
sftp_block: SftpBlock,
|
||||
*,
|
||||
poll_interval_seconds: int = 60,
|
||||
sftp_block_name: str = "default",
|
||||
) -> Scheduler:
|
||||
"""Replace the singleton scheduler; restart if it was running. SP25.
|
||||
|
||||
Called from ``PATCH /api/clearhouse`` so the next scheduler tick
|
||||
uses the new SftpBlock without a process restart.
|
||||
|
||||
Lifecycle:
|
||||
* If the previous scheduler is running, ``stop()`` it (waits
|
||||
up to ``_DRAIN_TIMEOUT_SECONDS`` for the current tick to
|
||||
finish — matches the existing ``Scheduler.stop()`` timeout).
|
||||
* Replace the module-level ``_scheduler`` singleton with a
|
||||
fresh ``Scheduler`` against the new block.
|
||||
* If the previous one was running, ``start()`` the new one.
|
||||
|
||||
Threading: same constraint as ``configure_scheduler`` — must be
|
||||
called from the FastAPI event loop, not a worker thread.
|
||||
"""
|
||||
global _scheduler
|
||||
was_running = False
|
||||
if _scheduler is not None:
|
||||
was_running = _scheduler.is_running()
|
||||
if was_running:
|
||||
# Drain the in-flight tick. Scheduler.stop() already
|
||||
# applies the _DRAIN_TIMEOUT_SECONDS timeout internally;
|
||||
# we just await it.
|
||||
await _scheduler.stop()
|
||||
poll = int(
|
||||
os.environ.get("CYCLONE_SCHEDULER_POLL_SECONDS", poll_interval_seconds),
|
||||
)
|
||||
_scheduler = Scheduler(
|
||||
sftp_block,
|
||||
poll_interval_seconds=poll,
|
||||
sftp_block_name=sftp_block_name,
|
||||
)
|
||||
if was_running:
|
||||
await _scheduler.start()
|
||||
log.info(
|
||||
"Scheduler reconfigured",
|
||||
extra={
|
||||
"was_running": was_running,
|
||||
"sftp_block": sftp_block_name,
|
||||
"host": sftp_block.host,
|
||||
"stub": sftp_block.stub,
|
||||
},
|
||||
)
|
||||
return _scheduler
|
||||
@@ -14,11 +14,36 @@ Setup (one-time, by the operator):
|
||||
|
||||
Verification:
|
||||
security find-generic-password -s cyclone -a sftp.gainwell.password -w
|
||||
|
||||
SP25 + SP26: the lookup now resolves the secret in four tiers, in
|
||||
this order:
|
||||
|
||||
1. ``<env_name>_FILE`` env var set — highest priority. Reads the
|
||||
file at that path, strips whitespace, returns the contents.
|
||||
This is the standard Docker-secrets pattern: mount a file at
|
||||
``/run/secrets/<name>`` and point the env var at it. An
|
||||
operator who explicitly sets ``_FILE`` is making a positive
|
||||
statement about where the secret lives, so a missing file
|
||||
surfaces as a ``RuntimeError`` rather than silently falling
|
||||
through. Empty string is treated as unset.
|
||||
2. Plain env var named exactly ``<env_name>`` — second priority.
|
||||
Lets a Linux server or Docker container pass the MFT password
|
||||
via ``CYCLONE_SFTP_PASSWORD=...`` without touching the Keychain
|
||||
or a file mount. Trailing/leading whitespace (including the
|
||||
``\\n`` that .env files often leave at EOF) is stripped; an
|
||||
empty value is treated as absent.
|
||||
3. macOS Keychain via ``keyring`` — third priority. Kept as a
|
||||
fallback so a macOS workstation that prefers
|
||||
``security add-generic-password`` works without env-var exports.
|
||||
4. ``None`` — caller decides what to do (``SftpClient._connect``
|
||||
raises a precise ``RuntimeError`` on real-mode auth).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -36,24 +61,68 @@ except ImportError:
|
||||
_HAS_KEYRING = False
|
||||
|
||||
|
||||
def _env_var_for(name: str) -> str:
|
||||
"""Resolve the operator-facing env-var name for a secret.
|
||||
|
||||
Returns the entry from ``_ENV_NAME_FOR`` if present, otherwise
|
||||
returns ``name`` verbatim.
|
||||
"""
|
||||
return _ENV_NAME_FOR.get(name, name)
|
||||
|
||||
|
||||
def get_secret(name: str) -> Optional[str]:
|
||||
"""Fetch a secret from macOS Keychain by name.
|
||||
"""Fetch a secret by name. Four-tier lookup: _FILE, env var, Keychain, None.
|
||||
|
||||
Args:
|
||||
name: The Keychain account (e.g. "sftp.gainwell.password").
|
||||
name: The secret name. Matches the Keychain account name
|
||||
(e.g. ``"sftp.gainwell.password"``). The operator-facing
|
||||
env-var form is mapped via the ``_ENV_NAME_FOR`` table at
|
||||
the bottom of this module.
|
||||
|
||||
Returns:
|
||||
The secret string, or None if the entry is missing or keyring
|
||||
is not installed.
|
||||
The secret string (stripped), or ``None`` if no tier produced
|
||||
a value.
|
||||
|
||||
Raises:
|
||||
RuntimeError: if ``<env_name>_FILE`` is set but the file at
|
||||
that path is missing or unreadable. The operator made a
|
||||
positive statement about where the secret lives; silent
|
||||
fall-through would mask a misconfiguration.
|
||||
"""
|
||||
if not _HAS_KEYRING:
|
||||
log.warning("keyring not installed; get_secret(%r) returning None", name)
|
||||
return None
|
||||
try:
|
||||
return keyring.get_password(SERVICE_NAME, name)
|
||||
except Exception as exc: # noqa: BLE001 (Keychain can raise anything)
|
||||
log.warning("Keychain get_secret(%r) failed: %s", name, exc)
|
||||
return None
|
||||
env_name = _env_var_for(name)
|
||||
file_env = env_name + "_FILE"
|
||||
file_path_raw = os.environ.get(file_env)
|
||||
if file_path_raw:
|
||||
# An empty string is treated as unset (matches the SP25 rule
|
||||
# for plain env vars).
|
||||
stripped_path = file_path_raw.strip()
|
||||
if stripped_path:
|
||||
try:
|
||||
value = Path(stripped_path).read_text().strip()
|
||||
except OSError as exc:
|
||||
raise RuntimeError(
|
||||
f"failed to read {file_env}={stripped_path}: {exc}"
|
||||
) from exc
|
||||
if value:
|
||||
log.debug("Secret %r resolved from file %r", name, stripped_path)
|
||||
return value
|
||||
|
||||
raw = os.environ.get(env_name)
|
||||
if raw is not None:
|
||||
stripped = raw.strip()
|
||||
if stripped:
|
||||
log.debug("Secret %r resolved from env var %r", name, env_name)
|
||||
return stripped
|
||||
# Empty env var — treat as absent and fall through.
|
||||
|
||||
if _HAS_KEYRING:
|
||||
try:
|
||||
value = keyring.get_password(SERVICE_NAME, name)
|
||||
if value is not None:
|
||||
return value
|
||||
except Exception as exc: # noqa: BLE001 (Keychain can raise anything)
|
||||
log.warning("Keychain get_secret(%r) failed: %s", name, exc)
|
||||
return None
|
||||
|
||||
|
||||
def set_secret(name: str, value: str) -> bool:
|
||||
@@ -77,3 +146,13 @@ def has_keyring() -> bool:
|
||||
"""True if the ``keyring`` library is importable (regardless of whether
|
||||
the Keychain entry actually exists)."""
|
||||
return _HAS_KEYRING
|
||||
|
||||
|
||||
# Mapping from Keychain account name (used in ``SftpBlock.auth``) to
|
||||
# the operator-facing env var. Keeping this table here means callers
|
||||
# don't have to remember the difference between
|
||||
# ``sftp.gainwell.password`` (Keychain account) and
|
||||
# ``CYCLONE_SFTP_PASSWORD`` (env var).
|
||||
_ENV_NAME_FOR: dict[str, str] = {
|
||||
"sftp.gainwell.password": "CYCLONE_SFTP_PASSWORD",
|
||||
}
|
||||
|
||||
@@ -2302,6 +2302,49 @@ class CycloneStore:
|
||||
"updated_at": row.updated_at,
|
||||
})
|
||||
|
||||
def update_clearhouse(self, block: Clearhouse) -> Clearhouse:
|
||||
"""Replace the singleton clearhouse row. SP25.
|
||||
|
||||
Used by ``PATCH /api/clearhouse`` to flip ``sftp_block.stub``
|
||||
and adjust host/port/paths without touching SQLite directly.
|
||||
Raises ``LookupError`` if the singleton row is missing — the
|
||||
caller is expected to run the lifespan seed first.
|
||||
"""
|
||||
from cyclone.db import ClearhouseORM
|
||||
with db.SessionLocal()() as s:
|
||||
row = s.get(ClearhouseORM, 1)
|
||||
if row is None:
|
||||
raise LookupError(
|
||||
"clearhouse singleton row missing; run the lifespan "
|
||||
"seed (ensure_clearhouse_seeded) before PATCH /api/clearhouse"
|
||||
)
|
||||
row.name = block.name
|
||||
row.tpid = block.tpid
|
||||
row.submitter_name = block.submitter_name
|
||||
row.submitter_id_qual = block.submitter_id_qual
|
||||
row.submitter_contact_name = block.submitter_contact_name
|
||||
row.submitter_contact_email = block.submitter_contact_email
|
||||
row.filename_block_json = json.loads(
|
||||
json.dumps(block.filename_block.model_dump())
|
||||
)
|
||||
row.sftp_block_json = json.loads(
|
||||
json.dumps(block.sftp_block.model_dump())
|
||||
)
|
||||
row.updated_at = datetime.now(timezone.utc).isoformat()
|
||||
s.commit()
|
||||
return Clearhouse.model_validate({
|
||||
"id": 1,
|
||||
"name": row.name,
|
||||
"tpid": row.tpid,
|
||||
"submitter_name": row.submitter_name,
|
||||
"submitter_id_qual": row.submitter_id_qual,
|
||||
"submitter_contact_name": row.submitter_contact_name,
|
||||
"submitter_contact_email": row.submitter_contact_email,
|
||||
"filename_block": dict(row.filename_block_json),
|
||||
"sftp_block": dict(row.sftp_block_json),
|
||||
"updated_at": row.updated_at,
|
||||
})
|
||||
|
||||
def ensure_clearhouse_seeded(self) -> None:
|
||||
"""Insert the default clearhouse singleton + 3 providers + CO_TXIX payer
|
||||
if they don't exist. Idempotent. Called from the API lifespan."""
|
||||
|
||||
Reference in New Issue
Block a user