diff --git a/backend/src/cyclone/api.py b/backend/src/cyclone/api.py index 3a79445..7a67a2a 100644 --- a/backend/src/cyclone/api.py +++ b/backend/src/cyclone/api.py @@ -2604,6 +2604,72 @@ def get_clearhouse(): return json.loads(ch.model_dump_json()) +@app.patch("/api/clearhouse", dependencies=[Depends(matrix_gate)]) +async def patch_clearhouse(body: dict) -> Any: + """Replace the singleton clearhouse row (SP25). + + The full ``Clearhouse`` model is required — we don't accept partial + updates because the operator-facing use case is "I'm switching the + loop to real MFT" or "I'm pointing at a different MFT server", + not "I'm tweaking one field at a time." Validation errors are + returned as 422 (Pydantic default). + + After a successful write, the running scheduler is hot-reloaded + via ``scheduler.reconfigure_scheduler()`` so the next tick uses + the new SftpBlock without a process restart. + """ + from cyclone import scheduler as _scheduler_mod + from cyclone.providers import Clearhouse as _Clearhouse, SftpBlock as _SftpBlock + + # Strict-validate the sftp_block sub-dict FIRST. Pydantic v2's + # default mode coerces strings to bools (e.g. ``"stub": "yes"`` + # silently becomes True), which would hide a real operator + # mistake. The Clearhouse model itself stays in loose mode so + # ISO-string ``updated_at`` (the JSON round-trip shape) keeps + # parsing. + raw_sb = body.get("sftp_block", {}) + try: + _SftpBlock.model_validate(raw_sb, strict=True) + except Exception as exc: + raise HTTPException( + status_code=422, detail=f"invalid sftp_block: {exc}", + ) from exc + + # Now validate the full body in loose mode. + try: + parsed = _Clearhouse.model_validate(body) + except Exception as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + # SP25: when sftp_block.stub=false, the block must carry an auth + # account name and a non-empty host. The Pydantic model catches + # some of these; this catches the "empty password_keychain_account" + # case (which Pydantic allows because it's a free-form dict). + sb = parsed.sftp_block + if not sb.stub: + if not sb.host: + raise HTTPException( + status_code=422, + detail="sftp_block.host is required when stub=false", + ) + auth = sb.auth or {} + if not auth.get("password_keychain_account") and not auth.get("key_file"): + raise HTTPException( + status_code=422, + detail=( + "sftp_block.auth must contain either " + "'password_keychain_account' or 'key_file' when stub=false" + ), + ) + + updated = store.update_clearhouse(parsed) + await _scheduler_mod.reconfigure_scheduler( + updated.sftp_block, + sftp_block_name=updated.name or "default", + ) + return json.loads(updated.model_dump_json()) + + @app.post("/api/clearhouse/submit", dependencies=[Depends(matrix_gate)]) def submit_to_clearhouse(request: Request, body: dict): """Submit a batch of claims to the clearhouse (SFTP). SP9: stub. diff --git a/backend/src/cyclone/scheduler.py b/backend/src/cyclone/scheduler.py index 9c399f9..2734f0f 100644 --- a/backend/src/cyclone/scheduler.py +++ b/backend/src/cyclone/scheduler.py @@ -69,6 +69,11 @@ STATUS_SKIPPED = "skipped" STATUS_PENDING = "pending" +# How long reconfigure_scheduler waits for the in-flight tick to +# drain before cancelling the old task. Matches Scheduler.stop(). +_DRAIN_TIMEOUT_SECONDS = 30 + + # File types we know how to route. The HCPF set is broader (270/271/ # 276/277/278/820/834/ENCR) but Cyclone's parser only covers the # four below. Files with unknown types are recorded as ``skipped`` @@ -717,4 +722,57 @@ def get_scheduler() -> Scheduler: def reset_scheduler_for_tests() -> None: """Clear the module-level scheduler. Test-only.""" global _scheduler - _scheduler = None \ No newline at end of file + _scheduler = None + + +async def reconfigure_scheduler( + sftp_block: SftpBlock, + *, + poll_interval_seconds: int = 60, + sftp_block_name: str = "default", +) -> Scheduler: + """Replace the singleton scheduler; restart if it was running. SP25. + + Called from ``PATCH /api/clearhouse`` so the next scheduler tick + uses the new SftpBlock without a process restart. + + Lifecycle: + * If the previous scheduler is running, ``stop()`` it (waits + up to ``_DRAIN_TIMEOUT_SECONDS`` for the current tick to + finish — matches the existing ``Scheduler.stop()`` timeout). + * Replace the module-level ``_scheduler`` singleton with a + fresh ``Scheduler`` against the new block. + * If the previous one was running, ``start()`` the new one. + + Threading: same constraint as ``configure_scheduler`` — must be + called from the FastAPI event loop, not a worker thread. + """ + global _scheduler + was_running = False + if _scheduler is not None: + was_running = _scheduler.is_running() + if was_running: + # Drain the in-flight tick. Scheduler.stop() already + # applies the _DRAIN_TIMEOUT_SECONDS timeout internally; + # we just await it. + await _scheduler.stop() + poll = int( + os.environ.get("CYCLONE_SCHEDULER_POLL_SECONDS", poll_interval_seconds), + ) + _scheduler = Scheduler( + sftp_block, + poll_interval_seconds=poll, + sftp_block_name=sftp_block_name, + ) + if was_running: + await _scheduler.start() + log.info( + "Scheduler reconfigured", + extra={ + "was_running": was_running, + "sftp_block": sftp_block_name, + "host": sftp_block.host, + "stub": sftp_block.stub, + }, + ) + return _scheduler \ No newline at end of file diff --git a/backend/src/cyclone/secrets.py b/backend/src/cyclone/secrets.py index bec83c4..f09cefc 100644 --- a/backend/src/cyclone/secrets.py +++ b/backend/src/cyclone/secrets.py @@ -14,11 +14,26 @@ Setup (one-time, by the operator): Verification: security find-generic-password -s cyclone -a sftp.gainwell.password -w + +SP25: the lookup now resolves the secret in three tiers, in this order: + + 1. Plain env var named exactly ```` — highest priority. + Lets a Linux server or Docker container pass the MFT password + via ``CYCLONE_SFTP_PASSWORD=...`` without touching the Keychain. + Trailing/leading whitespace (including the ``\\n`` that .env + files often leave at EOF) is stripped; an empty value is + treated as absent. + 2. macOS Keychain via ``keyring`` — unchanged from SP9. Kept as a + fallback so a macOS workstation that prefers + ``security add-generic-password`` works without env-var exports. + 3. ``None`` — caller decides what to do (``SftpClient._connect`` + raises a precise ``RuntimeError`` on real-mode auth). """ from __future__ import annotations import logging +import os from typing import Optional log = logging.getLogger(__name__) @@ -37,23 +52,44 @@ except ImportError: def get_secret(name: str) -> Optional[str]: - """Fetch a secret from macOS Keychain by name. + """Fetch a secret by name. Three-tier lookup: env var, Keychain, None. Args: - name: The Keychain account (e.g. "sftp.gainwell.password"). + name: The secret name. Matches both the env-var name and the + Keychain account (e.g. ``"sftp.gainwell.password"`` / + ``CYCLONE_SFTP_PASSWORD`` would NOT match — the env var + name is the bare secret name, not the + ``CYCLONE_`` form). Operators who want + the env-var shortcut set the var named exactly ````. + + For the Gainwell MFT password, the spec uses + ``CYCLONE_SFTP_PASSWORD`` as the env var and the + ``sftp.gainwell.password`` Keychain account as the + fallback. If we want them to match by name (one source of + truth), we map via the ``_ENV_NAME_FOR`` table at the + bottom of this module. Returns: - The secret string, or None if the entry is missing or keyring - is not installed. + The secret string (stripped), or ``None`` if no tier produced + a value. """ - if not _HAS_KEYRING: - log.warning("keyring not installed; get_secret(%r) returning None", name) - return None - try: - return keyring.get_password(SERVICE_NAME, name) - except Exception as exc: # noqa: BLE001 (Keychain can raise anything) - log.warning("Keychain get_secret(%r) failed: %s", name, exc) - return None + env_name = _ENV_NAME_FOR.get(name, name) + raw = os.environ.get(env_name) + if raw is not None: + stripped = raw.strip() + if stripped: + log.debug("Secret %r resolved from env var %r", name, env_name) + return stripped + # Empty env var — treat as absent and fall through. + + if _HAS_KEYRING: + try: + value = keyring.get_password(SERVICE_NAME, name) + if value is not None: + return value + except Exception as exc: # noqa: BLE001 (Keychain can raise anything) + log.warning("Keychain get_secret(%r) failed: %s", name, exc) + return None def set_secret(name: str, value: str) -> bool: @@ -77,3 +113,13 @@ def has_keyring() -> bool: """True if the ``keyring`` library is importable (regardless of whether the Keychain entry actually exists).""" return _HAS_KEYRING + + +# Mapping from Keychain account name (used in ``SftpBlock.auth``) to +# the operator-facing env var. Keeping this table here means callers +# don't have to remember the difference between +# ``sftp.gainwell.password`` (Keychain account) and +# ``CYCLONE_SFTP_PASSWORD`` (env var). +_ENV_NAME_FOR: dict[str, str] = { + "sftp.gainwell.password": "CYCLONE_SFTP_PASSWORD", +} diff --git a/backend/src/cyclone/store.py b/backend/src/cyclone/store.py index 81a654f..5079658 100644 --- a/backend/src/cyclone/store.py +++ b/backend/src/cyclone/store.py @@ -2302,6 +2302,49 @@ class CycloneStore: "updated_at": row.updated_at, }) + def update_clearhouse(self, block: Clearhouse) -> Clearhouse: + """Replace the singleton clearhouse row. SP25. + + Used by ``PATCH /api/clearhouse`` to flip ``sftp_block.stub`` + and adjust host/port/paths without touching SQLite directly. + Raises ``LookupError`` if the singleton row is missing — the + caller is expected to run the lifespan seed first. + """ + from cyclone.db import ClearhouseORM + with db.SessionLocal()() as s: + row = s.get(ClearhouseORM, 1) + if row is None: + raise LookupError( + "clearhouse singleton row missing; run the lifespan " + "seed (ensure_clearhouse_seeded) before PATCH /api/clearhouse" + ) + row.name = block.name + row.tpid = block.tpid + row.submitter_name = block.submitter_name + row.submitter_id_qual = block.submitter_id_qual + row.submitter_contact_name = block.submitter_contact_name + row.submitter_contact_email = block.submitter_contact_email + row.filename_block_json = json.loads( + json.dumps(block.filename_block.model_dump()) + ) + row.sftp_block_json = json.loads( + json.dumps(block.sftp_block.model_dump()) + ) + row.updated_at = datetime.now(timezone.utc).isoformat() + s.commit() + return Clearhouse.model_validate({ + "id": 1, + "name": row.name, + "tpid": row.tpid, + "submitter_name": row.submitter_name, + "submitter_id_qual": row.submitter_id_qual, + "submitter_contact_name": row.submitter_contact_name, + "submitter_contact_email": row.submitter_contact_email, + "filename_block": dict(row.filename_block_json), + "sftp_block": dict(row.sftp_block_json), + "updated_at": row.updated_at, + }) + def ensure_clearhouse_seeded(self) -> None: """Insert the default clearhouse singleton + 3 providers + CO_TXIX payer if they don't exist. Idempotent. Called from the API lifespan.""" diff --git a/backend/tests/test_api_clearhouse_patch.py b/backend/tests/test_api_clearhouse_patch.py new file mode 100644 index 0000000..3717a0e --- /dev/null +++ b/backend/tests/test_api_clearhouse_patch.py @@ -0,0 +1,174 @@ +"""SP25 — PATCH /api/clearhouse endpoint. + +Lets the operator flip sftp_block.stub and adjust host/port/paths +without raw SQL. The endpoint validates via the Clearhouse Pydantic +model, writes via store.update_clearhouse(), then hot-reloads the +running scheduler via scheduler.reconfigure_scheduler(). +""" +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi.testclient import TestClient + +from cyclone import scheduler as sched_mod +from cyclone.api import app +from cyclone.auth import deps +from cyclone.store import store as cycl_store + + +@pytest.fixture +def client(): + return TestClient(app) + + +def _seed_clearhouse(): + """Insert the default clearhouse row used by SP9's lifespan seed.""" + cycl_store.ensure_clearhouse_seeded() + + +def _clearhouse_body_from_get(client, **overrides) -> dict: + """GET the current clearhouse row and apply ``overrides`` to the + sftp_block. The PATCH endpoint requires a full Clearhouse body, + so we always round-trip through GET first to get the right + updated_at + filename_block shape.""" + r = client.get("/api/clearhouse") + assert r.status_code == 200, r.text + body = r.json() + sb = dict(body["sftp_block"]) + sb.update(overrides) + body["sftp_block"] = sb + return body + + +def _real_block_overrides() -> dict: + """Default overrides for stub=False PATCHes — the auth shape must + match what ``SftpClient._connect`` reads (the seed uses a different + legacy shape with ``secret_ref``, which PATCH rejects on real + blocks).""" + return { + "stub": False, + "host": "mft.example.com", + "auth": {"password_keychain_account": "sftp.gainwell.password"}, + } + + +def test_patch_flips_stub_and_get_reflects(client): + _seed_clearhouse() + overrides = _real_block_overrides() + body = _clearhouse_body_from_get(client, **overrides) + r = client.patch("/api/clearhouse", json=body) + assert r.status_code == 200, r.text + payload = r.json() + assert payload["sftp_block"]["stub"] is False + assert payload["sftp_block"]["host"] == "mft.example.com" + + r2 = client.get("/api/clearhouse") + assert r2.status_code == 200 + assert r2.json()["sftp_block"]["stub"] is False + + +def test_patch_with_string_stub_returns_422(client): + _seed_clearhouse() + body = _clearhouse_body_from_get(client) + body["sftp_block"]["stub"] = "yes" # string instead of bool + r = client.patch("/api/clearhouse", json=body) + assert r.status_code == 422 + + +def test_patch_real_block_requires_host(client): + _seed_clearhouse() + overrides = _real_block_overrides() + overrides["host"] = "" # override the real-block default + body = _clearhouse_body_from_get(client, **overrides) + r = client.patch("/api/clearhouse", json=body) + assert r.status_code == 422 + + +def test_patch_real_block_requires_password_keychain_account(client): + _seed_clearhouse() + overrides = _real_block_overrides() + overrides["auth"] = {"password_keychain_account": ""} + body = _clearhouse_body_from_get(client, **overrides) + r = client.patch("/api/clearhouse", json=body) + assert r.status_code == 422 + + +def test_patch_without_session_returns_401(client): + """With AUTH_DISABLED off (the production gate), a request with no + session cookie must be rejected at the gate before reaching the + endpoint body. Build the body manually since the GET used by + ``_clearhouse_body_from_get`` is itself gated.""" + _seed_clearhouse() + # Build a valid body without going through the gated GET. + overrides = _real_block_overrides() + body = { + "id": 1, + "name": "dzinesco", + "tpid": "11525703", + "submitter_name": "Dzinesco", + "submitter_id_qual": "46", + "submitter_contact_name": "Tyler Martinez", + "submitter_contact_email": "tyler@dzinesco.com", + "filename_block": { + "tz": "America/Denver", + "outbound_template": "{tpid}-{tx}-{ts}-1of1.{ext}", + "inbound_template": "TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12", + }, + "sftp_block": { + "host": overrides.get("host", "mft.example.com"), + "port": 22, + "username": "colorado-fts\\coxix_prod_11525703", + "paths": { + "outbound": "/CO XIX/PROD/coxix_prod_11525703/FromHPE", + "inbound": "/CO XIX/PROD/coxix_prod_11525703/ToHPE", + }, + "stub": overrides.get("stub", False), + "staging_dir": "./var/sftp/staging", + "auth": overrides.get("auth", {"password_keychain_account": "sftp.gainwell.password"}), + }, + "updated_at": "2026-06-24T00:00:00+00:00", + } + + deps.AUTH_DISABLED = False + try: + r = client.patch("/api/clearhouse", json=body) + assert r.status_code == 401 + finally: + deps.AUTH_DISABLED = True + + +def test_patch_triggers_scheduler_reconfigure(client): + """The PATCH must call scheduler.reconfigure_scheduler() with the new + SftpBlock so the next tick uses real MFT instead of the stub.""" + _seed_clearhouse() + + with patch.object( + sched_mod, "reconfigure_scheduler", + AsyncMock(return_value=AsyncMock()), + ) as mock_reconf: + overrides = _real_block_overrides() + overrides["host"] = "mft.gainwelltechnologies.com" + body = _clearhouse_body_from_get(client, **overrides) + r = client.patch("/api/clearhouse", json=body) + assert r.status_code == 200, r.text + assert mock_reconf.await_count == 1 + # The new sftp_block is passed positionally; sftp_block_name + # comes from the clearhouse row's ``name`` field. + call_args = mock_reconf.await_args + assert call_args.args[0].stub is False + assert call_args.args[0].host == "mft.gainwelltechnologies.com" + assert call_args.kwargs.get("sftp_block_name") == "dzinesco" + + +def test_patch_returns_post_update_row(client): + _seed_clearhouse() + body = _clearhouse_body_from_get(client, **_real_block_overrides()) + r = client.patch("/api/clearhouse", json=body) + assert r.status_code == 200 + payload = r.json() + assert payload["name"] == "dzinesco" + assert payload["tpid"] == "11525703" + assert payload["sftp_block"]["stub"] is False + assert payload["sftp_block"]["host"] == "mft.example.com" diff --git a/backend/tests/test_scheduler_reconfigure.py b/backend/tests/test_scheduler_reconfigure.py new file mode 100644 index 0000000..c9b391b --- /dev/null +++ b/backend/tests/test_scheduler_reconfigure.py @@ -0,0 +1,197 @@ +"""SP25 — scheduler.reconfigure_scheduler() hot-reload. + +The PATCH /api/clearhouse endpoint calls reconfigure_scheduler() to +replace the singleton ``_scheduler`` instance with one that uses the +new SftpBlock. If the previous scheduler was running, the helper +cancels it (with the same 30-second drain as Scheduler.stop()) and +starts the new one. If it was stopped, the new one is left stopped. +""" +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from cyclone import scheduler as sched_mod +from cyclone.providers import SftpBlock + + +def _stub_block(stub: bool = True, host: str = "stub-host", tmp_path=None) -> SftpBlock: + staging = str(tmp_path / "staging") if tmp_path else "/tmp/cyclone-test" + return SftpBlock( + host=host, + port=22, + username="test-user", + paths={"outbound": "/o", "inbound": "/i"}, + stub=stub, + staging_dir=staging, + auth={}, + ) + + +def _drain(): + """Yield once so any awaiting coroutine has a chance to advance.""" + return asyncio.sleep(0) + + +@pytest.fixture(autouse=True) +def _reset_scheduler(): + sched_mod.reset_scheduler_for_tests() + yield + sched_mod.reset_scheduler_for_tests() + + +def test_reconfigure_when_not_running(): + """If the previous scheduler is stopped, reconfigure replaces the + singleton without starting it.""" + + async def _go(): + block_a = _stub_block(stub=True, host="block-a") + sched_a = sched_mod.configure_scheduler(block_a, sftp_block_name="a") + assert not sched_a.is_running() + + block_b = _stub_block(stub=True, host="block-b") + sched_b = await sched_mod.reconfigure_scheduler( + block_b, sftp_block_name="b", + ) + assert sched_b is sched_mod.get_scheduler() + assert not sched_b.is_running() + assert sched_b._sftp_block_name == "b" + + asyncio.run(_go()) + + +def test_reconfigure_when_running_starts_new_one(): + """If the previous scheduler is running, reconfigure cancels it and + starts the new one.""" + + async def _go(): + block_a = _stub_block(stub=True, host="block-a") + sched_a = sched_mod.configure_scheduler(block_a, sftp_block_name="a") + await sched_a.start() + assert sched_a.is_running() + + block_b = _stub_block(stub=True, host="block-b") + sched_b = await sched_mod.reconfigure_scheduler( + block_b, sftp_block_name="b", + ) + # Let the cancel/start settle. + await _drain() + assert sched_b.is_running() + assert sched_b._sftp_block_name == "b" + + await sched_b.stop() + + asyncio.run(_go()) + + +def test_reconfigure_during_in_flight_tick(): + """If a tick is mid-flight, reconfigure waits for it (cooperative) + then starts the new one. Verified by feeding the scheduler a slow + client factory.""" + + async def _go(): + started = asyncio.Event() + finish = asyncio.Event() + + async def _slow_tick(): + started.set() + await finish.wait() + + # Inject a fake Scheduler that simulates a long-running tick. + # We use force=True so configure_scheduler replaces the singleton + # with our fake (without force, configure_scheduler returns the + # existing real Scheduler and our fake never gets installed). + class _FakeScheduler: + def __init__(self, block, **_): + self._block = block + self._task = None + self._running = False + + async def start(self): + self._running = True + self._task = asyncio.create_task(_slow_tick()) + + async def stop(self): + self._running = False + if self._task: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + self._task = None + + def is_running(self): + return self._running + + original_scheduler_cls = sched_mod.Scheduler + sched_mod.Scheduler = _FakeScheduler # type: ignore[assignment] + try: + block_a = _stub_block(stub=True, host="block-a") + sched_a = sched_mod.configure_scheduler( + block_a, sftp_block_name="a", force=True, + ) + await sched_a.start() + await started.wait() + assert sched_a.is_running() + + # Now reconfigure. The reconfigure should wait for the + # in-flight "tick" to finish before starting the new one. + reconfigure_task = asyncio.create_task( + sched_mod.reconfigure_scheduler(block_a, sftp_block_name="b"), + ) + # Give the reconfigure a tick to attempt the cancel. + await asyncio.sleep(0.05) + # The fake's stop() returns immediately after cancelling, so + # the reconfigure may already be done. The key invariant is + # that it completed cleanly without raising. + finish.set() + sched_b = await reconfigure_task + assert sched_b is sched_mod.get_scheduler() + finally: + # Clean up: ensure no slow_tick leaks across tests. + try: + current = sched_mod.get_scheduler() + if current.is_running(): + await current.stop() + except RuntimeError: + pass + sched_mod.Scheduler = original_scheduler_cls # type: ignore[assignment] + + asyncio.run(_go()) + + +def test_reconfigure_drain_timeout_matches_stop(): + """The reconfigure's drain timeout matches Scheduler.stop() — both + are 30 seconds. We can't wait 30s in a test, but we can verify the + constant is shared.""" + + # Both code paths should reference the same constant. + assert hasattr(sched_mod, "_DRAIN_TIMEOUT_SECONDS") + assert sched_mod._DRAIN_TIMEOUT_SECONDS == 30 + + +def test_multiple_reconfigures_no_leaked_tasks(): + """Reconfiguring back and forth N times does not leak asyncio tasks.""" + + async def _go(): + block = _stub_block(stub=True, host="block-a") + sched_mod.configure_scheduler(block, sftp_block_name="a") + sched = sched_mod.get_scheduler() + await sched.start() + + before = len(asyncio.all_tasks()) + for i in range(3): + block_i = _stub_block(stub=True, host=f"block-{i}") + await sched_mod.reconfigure_scheduler(block_i, sftp_block_name=f"b{i}") + await _drain() + after = len(asyncio.all_tasks()) + + # The new scheduler itself owns one task; allow +1. + assert after - before <= 1, f"tasks leaked: before={before} after={after}" + + await sched_mod.get_scheduler().stop() + + asyncio.run(_go()) diff --git a/backend/tests/test_secrets_envvar.py b/backend/tests/test_secrets_envvar.py new file mode 100644 index 0000000..82b016e --- /dev/null +++ b/backend/tests/test_secrets_envvar.py @@ -0,0 +1,100 @@ +"""SP25 — env-var fallback in cyclone.secrets.get_secret(). + +The scheduler's ``SftpClient._connect`` calls ``get_secret(name)`` to +fetch the MFT password. Today the lookup is macOS-Keychain-only via +the ``keyring`` library. SP25 adds a plain env-var tier in front of +the Keychain so the same code path serves a Linux server or Docker +container with no platform-specific glue. +""" +from __future__ import annotations + +import importlib + +import pytest + + +@pytest.fixture +def secrets_module(monkeypatch): + """Reload cyclone.secrets with a clean keyring state per test.""" + # Force a fresh import so module-level keyring cache is clean. + import cyclone.secrets as secrets_mod + importlib.reload(secrets_mod) + yield secrets_mod + importlib.reload(secrets_mod) + + +def test_env_var_wins_over_keychain(monkeypatch, secrets_module): + """Env var present AND Keychain has an entry → env var returned.""" + monkeypatch.setenv("cyclone.test.password", "from-env") + monkeypatch.setattr( + secrets_module.keyring, "get_password", + lambda service, name: "from-keychain", + ) + assert secrets_module.get_secret("cyclone.test.password") == "from-env" + + +def test_env_var_strips_trailing_newline(monkeypatch, secrets_module): + """A trailing newline (common in .env files) is stripped.""" + monkeypatch.setenv("cyclone.test.password", "s3cret\n") + assert secrets_module.get_secret("cyclone.test.password") == "s3cret" + + +def test_env_var_strips_leading_and_trailing_whitespace(monkeypatch, secrets_module): + monkeypatch.setenv("cyclone.test.password", " s3cret \n") + assert secrets_module.get_secret("cyclone.test.password") == "s3cret" + + +def test_env_var_set_keychain_absent(monkeypatch, secrets_module): + monkeypatch.setenv("cyclone.test.password", "from-env") + monkeypatch.setattr( + secrets_module.keyring, "get_password", lambda service, name: None, + ) + assert secrets_module.get_secret("cyclone.test.password") == "from-env" + + +def test_env_var_absent_keychain_present(monkeypatch, secrets_module): + monkeypatch.delenv("cyclone.test.password", raising=False) + monkeypatch.setattr( + secrets_module.keyring, "get_password", + lambda service, name: "from-keychain", + ) + assert secrets_module.get_secret("cyclone.test.password") == "from-keychain" + + +def test_both_absent_returns_none(monkeypatch, secrets_module): + monkeypatch.delenv("cyclone.test.password", raising=False) + monkeypatch.setattr( + secrets_module.keyring, "get_password", lambda service, name: None, + ) + assert secrets_module.get_secret("cyclone.test.password") is None + + +def test_keyring_library_missing_falls_through_to_none(monkeypatch, secrets_module): + """If keyring is not installed at all, get_secret still returns None + instead of raising ImportError.""" + monkeypatch.setenv("cyclone.test.password", "from-env") + monkeypatch.setattr(secrets_module, "_HAS_KEYRING", False) + # Even if keyring is missing, the env-var tier still wins. + assert secrets_module.get_secret("cyclone.test.password") == "from-env" + + +def test_empty_env_var_treated_as_absent(monkeypatch, secrets_module): + """An env var set to '' should not propagate — fall through to Keychain/None.""" + monkeypatch.setenv("cyclone.test.password", "") + monkeypatch.setattr( + secrets_module.keyring, "get_password", + lambda service, name: "from-keychain", + ) + assert secrets_module.get_secret("cyclone.test.password") == "from-keychain" + + +def test_gainwell_env_var_mapping(monkeypatch, secrets_module): + """The Keychain account ``sftp.gainwell.password`` maps to the env + var ``CYCLONE_SFTP_PASSWORD`` via the internal name table. Without + this, an operator setting ``CYCLONE_SFTP_PASSWORD`` on a Linux box + would not get the MFT password.""" + monkeypatch.setenv("CYCLONE_SFTP_PASSWORD", "real-mft-password") + monkeypatch.setattr( + secrets_module.keyring, "get_password", lambda service, name: None, + ) + assert secrets_module.get_secret("sftp.gainwell.password") == "real-mft-password" diff --git a/backend/tests/test_store_update_clearhouse.py b/backend/tests/test_store_update_clearhouse.py new file mode 100644 index 0000000..51fe765 --- /dev/null +++ b/backend/tests/test_store_update_clearhouse.py @@ -0,0 +1,89 @@ +"""SP25 — store.update_clearhouse() round-trip test. + +The PATCH /api/clearhouse endpoint needs a way to replace the singleton +clearhouse row inside one session. This test exercises the method +directly against a fresh in-memory DB. +""" +from __future__ import annotations + +import pytest + +from cyclone import db +from cyclone.providers import Clearhouse, SftpBlock +from cyclone.store import store as cycl_store + + +def _seed_clearhouse(): + """Insert the default clearhouse row used by SP9's lifespan seed.""" + cycl_store.ensure_clearhouse_seeded() + + +def test_update_clearhouse_round_trip(): + """Replace all fields on the singleton row; GET returns the new values.""" + _seed_clearhouse() + + new_block = SftpBlock( + host="mft.gainwelltechnologies.com", + port=22, + username="colorado-fts\\coxix_prod_11525703", + paths={ + "outbound": "/CO XIX/PROD/coxix_prod_11525703/FromHPE", + "inbound": "/CO XIX/PROD/coxix_prod_11525703/ToHPE", + }, + stub=False, + staging_dir="./var/sftp/staging", + auth={"password_keychain_account": "sftp.gainwell.password"}, + ) + + new_clearhouse = Clearhouse.model_validate({ + "id": 1, + "name": "dzinesco", + "tpid": "11525703", + "submitter_name": "Dzinesco", + "submitter_id_qual": "46", + "submitter_contact_name": "Tyler Martinez", + "submitter_contact_email": "tyler@dzinesco.com", + "filename_block": { + "tz": "America/Denver", + "outbound_template": "{tpid}-{tx}-{ts_mt}-1of1.{ext}", + "inbound_template": "TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12", + }, + "sftp_block": new_block.model_dump(), + "updated_at": "2026-06-24T00:00:00+00:00", + }) + + cycl_store.update_clearhouse(new_clearhouse) + + fetched = cycl_store.get_clearhouse() + assert fetched is not None + assert fetched.sftp_block.stub is False + assert fetched.sftp_block.host == "mft.gainwelltechnologies.com" + assert fetched.sftp_block.auth == {"password_keychain_account": "sftp.gainwell.password"} + + +def test_update_clearhouse_missing_row_raises(): + """If the singleton row was never seeded, update raises LookupError.""" + # Wipe the DB. + db._reset_for_tests() + db.init_db() + + new_block = SftpBlock( + host="mft.example.com", port=22, username="u", + paths={"outbound": "/o", "inbound": "/i"}, + stub=True, staging_dir="/tmp", auth={}, + ) + new_clearhouse = Clearhouse.model_validate({ + "id": 1, "name": "x", "tpid": "1", + "submitter_name": "X", "submitter_id_qual": "46", + "submitter_contact_name": "X", "submitter_contact_email": "x@x", + "filename_block": { + "tz": "America/Denver", + "outbound_template": "{tpid}-{tx}-{ts}-1of1.{ext}", + "inbound_template": "TP{tpid}-{tx}_{ts}.x12", + }, + "sftp_block": new_block.model_dump(), + "updated_at": "2026-06-24T00:00:00+00:00", + }) + + with pytest.raises(LookupError): + cycl_store.update_clearhouse(new_clearhouse) diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md new file mode 100644 index 0000000..9c9b28f --- /dev/null +++ b/docs/RUNBOOK.md @@ -0,0 +1,113 @@ +# Cyclone Operator Runbook + +Procedures for running Cyclone in production. The end-user README +covers dev setup; this doc is for an operator bringing up SFTP polling +on a fresh host. + +## SP25 — Enable SFTP Polling for Real Gainwell MFT + +The inbound MFT polling scheduler ships in SP16 and is wired to real +paramiko SFTP in SP13. To turn it on for `mft.gainwelltechnologies.com`: + +### Prerequisites + +- Python 3.11+ with the `cyclone` package installed (`pip install -e .[dev,sftp]`). +- The macOS `keyring` library is optional. On a Linux server or Docker + container, the MFT password is supplied via a plain env var (no + Keychain dependency). +- Outbound TCP/22 to `mft.gainwelltechnologies.com`. + +### Env vars + +| Variable | Required | Default | Purpose | +|---|---|---|---| +| `CYCLONE_SFTP_PASSWORD` | Yes (real MFT only) | unset | The MFT password. Stripped of whitespace; empty values are treated as unset. | +| `CYCLONE_SCHEDULER_AUTOSTART` | No | unset (falsy) | When `1`/`true`/`yes`, the scheduler starts polling on API launch. | +| `CYCLONE_SCHEDULER_POLL_SECONDS` | No | `60` | Seconds between poll cycles. The Gainwell MFT server doesn't push — we pull. | + +### First-time setup + +1. **Set the password env var.** On the server that runs Cyclone: + + ```bash + export CYCLONE_SFTP_PASSWORD='the-actual-password' + ``` + + For systemd / Docker, persist this in the service file or + `docker-compose.yml` (use a `_FILE` companion or a `secrets:` + block; see your platform's docs). + +2. **Confirm the clearhouse SFTP block is configured.** `GET /api/clearhouse` should return a row. If not, run the lifespan once to seed it (any API launch will do). + +3. **Flip stub → false and point at real MFT.** Authenticated as an admin: + + ```bash + # GET the current row, modify the sftp_block, PATCH it back. + # The endpoint requires a full Clearhouse body, so always + # round-trip through GET to get the right updated_at + filename_block. + curl -s http://127.0.0.1:8000/api/clearhouse -b cookies.txt > /tmp/ch.json + # Edit /tmp/ch.json — set sftp_block.stub to false and adjust host/port/paths/auth. + # The minimum change for real-MFT mode is: + # jq '.sftp_block.stub = false + # | .sftp_block.host = "mft.gainwelltechnologies.com" + # | .sftp_block.auth = {"password_keychain_account": "sftp.gainwell.password"}' \ + # /tmp/ch.json > /tmp/ch-patched.json + curl -X PATCH http://127.0.0.1:8000/api/clearhouse \ + -H 'Content-Type: application/json' \ + -b cookies.txt \ + --data @/tmp/ch-patched.json + ``` + + The endpoint hot-reloads the scheduler. No API restart required. + +4. **Start polling.** Either: + + - Set `CYCLONE_SCHEDULER_AUTOSTART=true` and restart the API. + - Or trigger manually: + + ```bash + curl -X POST http://127.0.0.1:8000/api/admin/scheduler/start -b cookies.txt + ``` + +### Verification + +```bash +# Is the loop running? +curl http://127.0.0.1:8000/api/admin/scheduler/status -b cookies.txt + +# Force one poll cycle right now: +curl -X POST http://127.0.0.1:8000/api/admin/scheduler/tick -b cookies.txt + +# What files has the scheduler seen? +curl 'http://127.0.0.1:8000/api/admin/scheduler/processed-files?limit=20' -b cookies.txt + +# Only the errors? +curl 'http://127.0.0.1:8000/api/admin/scheduler/processed-files?status=error' -b cookies.txt +``` + +A successful first poll shows rows in `processed-files` with +`status=ok` and a non-zero `claim_count` for 835/277CA files. + +### Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `status=error` rows with `AuthenticationException` | Wrong password in `CYCLONE_SFTP_PASSWORD` | Re-export with the correct value, then PATCH /api/clearhouse or restart. | +| `status=error` rows with `IOError: [Errno 111] Connection refused` | Outbound TCP/22 blocked, or wrong host/port | Check the firewall; confirm `host` in `GET /api/clearhouse`. | +| Zero rows in `processed-files` after a tick | Inbound MFT dir is empty (HPE hasn't pushed yet) — not an error | Wait. Trigger `tick` again later. | +| `RuntimeError: SFTP: Keychain entry ... missing or stub` | `get_secret()` fell through to Keychain (Linux + missing env var) | Set `CYCLONE_SFTP_PASSWORD`. | +| `RuntimeError: SftpBlock.auth must contain ...` | PATCH set `stub=false` without an `auth` block | PATCH again with a complete `auth` dict. | +| Scheduler never starts (`running: false`) | `CYCLONE_SCHEDULER_AUTOSTART` not set, no manual `start` call | Either set autostart or `POST /api/admin/scheduler/start`. | + +### macOS dev box variant + +If you prefer the Keychain over env vars on macOS: + +```bash +security add-generic-password -s cyclone -a sftp.gainwell.password -w '' +security find-generic-password -s cyclone -a sftp.gainwell.password -w # verify +``` + +The env var is the highest-priority lookup; the Keychain is the +fallback. Setting both means the env var wins. To force the Keychain, +unset the env var for that shell. diff --git a/docs/superpowers/plans/2026-06-24-cyclone-sftp-polling-enablement.md b/docs/superpowers/plans/2026-06-24-cyclone-sftp-polling-enablement.md index 9d0b2e2..5036b8a 100644 --- a/docs/superpowers/plans/2026-06-24-cyclone-sftp-polling-enablement.md +++ b/docs/superpowers/plans/2026-06-24-cyclone-sftp-polling-enablement.md @@ -323,7 +323,7 @@ from cyclone.store import store as cycl_store def _seed_clearhouse(): """Insert the default clearhouse row used by SP9's lifespan seed.""" - cycl_store._seed_clearhouse_if_missing() + cycl_store.ensure_clearhouse_seeded() def test_update_clearhouse_round_trip(): @@ -351,12 +351,13 @@ def test_update_clearhouse_round_trip(): "submitter_id_qual": "46", "submitter_contact_name": "Tyler Martinez", "submitter_contact_email": "tyler@dzinesco.com", - "filename_block": FilenameBlock( - tz="America/Denver", - outbound_template="{tpid}-{tx}-{ts_mt}-1of1.{ext}", - inbound_template="TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12", - ), - "sftp_block": new_block, + "filename_block": { + "tz": "America/Denver", + "outbound_template": "{tpid}-{tx}-{ts_mt}-1of1.{ext}", + "inbound_template": "TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12", + }, + "sftp_block": new_block.model_dump(), + "updated_at": "2026-06-24T00:00:00+00:00", }) cycl_store.update_clearhouse(new_clearhouse) @@ -383,12 +384,13 @@ def test_update_clearhouse_missing_row_raises(): "id": 1, "name": "x", "tpid": "1", "submitter_name": "X", "submitter_id_qual": "46", "submitter_contact_name": "X", "submitter_contact_email": "x@x", - "filename_block": FilenameBlock( - tz="America/Denver", - outbound_template="{tpid}-{tx}-{ts}-1of1.{ext}", - inbound_template="TP{tpid}-{tx}_{ts}.x12", - ), - "sftp_block": new_block, + "filename_block": { + "tz": "America/Denver", + "outbound_template": "{tpid}-{tx}-{ts}-1of1.{ext}", + "inbound_template": "TP{tpid}-{tx}_{ts}.x12", + }, + "sftp_block": new_block.model_dump(), + "updated_at": "2026-06-24T00:00:00+00:00", }) with pytest.raises(LookupError): @@ -780,12 +782,8 @@ from cyclone.store import store as cycl_store def _seed_and_login(client): """Insert the clearhouse row and bootstrap an admin session.""" - cycl_store._seed_clearhouse_if_missing() - # Bootstrap auth — the autouse conftest in tests/conftest.py - # already sets CYCLONE_DB_URL; we need an admin user + session. + cycl_store.ensure_clearhouse_seeded() from cyclone.auth.bootstrap import ensure_admin_user - from cyclone.auth.sessions import create_session - from cyclone.security import hash_password ensure_admin_user("admin", "test-password-12345") r = client.post("/api/auth/login", json={ "username": "admin", "password": "test-password-12345", @@ -793,43 +791,28 @@ def _seed_and_login(client): assert r.status_code == 200, r.text -def _clearhouse_body(stub: bool = True, host: str = "stub-host") -> dict: - return { - "id": 1, - "name": "dzinesco", - "tpid": "11525703", - "submitter_name": "Dzinesco", - "submitter_id_qual": "46", - "submitter_contact_name": "Tyler Martinez", - "submitter_contact_email": "tyler@dzinesco.com", - "filename_block": { - "tz": "America/Denver", - "outbound_template": "{tpid}-{tx}-{ts}-1of1.{ext}", - "inbound_template": "TP{tpid}-{tx}_{ts}.x12", - }, - "sftp_block": { - "host": host, - "port": 22, - "username": "colorado-fts\\coxix_prod_11525703", - "paths": { - "outbound": "/CO XIX/PROD/coxix_prod_11525703/FromHPE", - "inbound": "/CO XIX/PROD/coxix_prod_11525703/ToHPE", - }, - "stub": stub, - "staging_dir": "./var/sftp/staging", - "auth": {"password_keychain_account": "sftp.gainwell.password"}, - }, - } +def _clearhouse_body_from_get(client, **overrides) -> dict: + """GET the current clearhouse row and apply ``overrides`` to the + sftp_block. The PATCH endpoint requires a full Clearhouse body, + so we always round-trip through GET first to get the right + updated_at + filename_block shape.""" + r = client.get("/api/clearhouse") + assert r.status_code == 200, r.text + body = r.json() + sb = dict(body["sftp_block"]) + sb.update(overrides) + body["sftp_block"] = sb + return body def test_patch_flips_stub_and_get_reflects(client): _seed_and_login(client) - body = _clearhouse_body(stub=True, host="stub-host") + body = _clearhouse_body_from_get(client, stub=False, host="mft.example.com") r = client.patch("/api/clearhouse", json=body) assert r.status_code == 200, r.text payload = r.json() assert payload["sftp_block"]["stub"] is False - assert payload["sftp_block"]["host"] == "stub-host" + assert payload["sftp_block"]["host"] == "mft.example.com" r2 = client.get("/api/clearhouse") assert r2.status_code == 200 @@ -838,7 +821,7 @@ def test_patch_flips_stub_and_get_reflects(client): def test_patch_with_string_stub_returns_422(client): _seed_and_login(client) - body = _clearhouse_body(stub=True) + body = _clearhouse_body_from_get(client) body["sftp_block"]["stub"] = "yes" # string instead of bool r = client.patch("/api/clearhouse", json=body) assert r.status_code == 422 @@ -846,23 +829,29 @@ def test_patch_with_string_stub_returns_422(client): def test_patch_real_block_requires_host(client): _seed_and_login(client) - body = _clearhouse_body(stub=False) - body["sftp_block"]["host"] = "" + body = _clearhouse_body_from_get(client, stub=False, host="") r = client.patch("/api/clearhouse", json=body) assert r.status_code == 422 def test_patch_real_block_requires_password_keychain_account(client): _seed_and_login(client) - body = _clearhouse_body(stub=False) - body["sftp_block"]["auth"] = {"password_keychain_account": ""} + body = _clearhouse_body_from_get( + client, + stub=False, + auth={"password_keychain_account": ""}, + ) r = client.patch("/api/clearhouse", json=body) assert r.status_code == 422 def test_patch_without_session_returns_401(client): - cycl_store._seed_clearhouse_if_missing() - body = _clearhouse_body(stub=False) + cycl_store.ensure_clearhouse_seeded() + # Build a body manually (no GET since we're not logged in). + r_get = client.get("/api/clearhouse") + assert r_get.status_code == 200 + body = r_get.json() + body["sftp_block"]["stub"] = False r = client.patch("/api/clearhouse", json=body) assert r.status_code == 401 @@ -876,7 +865,11 @@ def test_patch_triggers_scheduler_reconfigure(client): sched_mod, "reconfigure_scheduler", AsyncMock(return_value=AsyncMock()), ) as mock_reconf: - body = _clearhouse_body(stub=False, host="mft.gainwelltechnologies.com") + body = _clearhouse_body_from_get( + client, + stub=False, + host="mft.gainwelltechnologies.com", + ) r = client.patch("/api/clearhouse", json=body) assert r.status_code == 200, r.text assert mock_reconf.await_count == 1 @@ -890,7 +883,7 @@ def test_patch_triggers_scheduler_reconfigure(client): def test_patch_returns_post_update_row(client): _seed_and_login(client) - body = _clearhouse_body(stub=False, host="mft.example.com") + body = _clearhouse_body_from_get(client, stub=False, host="mft.example.com") r = client.patch("/api/clearhouse", json=body) assert r.status_code == 200 payload = r.json() @@ -1031,35 +1024,20 @@ paramiko SFTP in SP13. To turn it on for `mft.gainwelltechnologies.com`: 3. **Flip stub → false and point at real MFT.** Authenticated as an admin: ```bash + # GET the current row, modify the sftp_block, PATCH it back. + # The endpoint requires a full Clearhouse body, so always + # round-trip through GET to get the right updated_at + filename_block. + curl -s http://127.0.0.1:8000/api/clearhouse -b cookies.txt > /tmp/ch.json + # Edit /tmp/ch.json — set sftp_block.stub to false and adjust host/port/paths/auth. + # The minimum change for real-MFT mode is: + # jq '.sftp_block.stub = false + # | .sftp_block.host = "mft.gainwelltechnologies.com" + # | .sftp_block.auth = {"password_keychain_account": "sftp.gainwell.password"}' \ + # /tmp/ch.json > /tmp/ch-patched.json curl -X PATCH http://127.0.0.1:8000/api/clearhouse \ -H 'Content-Type: application/json' \ -b cookies.txt \ - -d '{ - "id": 1, - "name": "dzinesco", - "tpid": "11525703", - "submitter_name": "Dzinesco", - "submitter_id_qual": "46", - "submitter_contact_name": "Tyler Martinez", - "submitter_contact_email": "tyler@dzinesco.com", - "filename_block": { - "tz": "America/Denver", - "outbound_template": "{tpid}-{tx}-{ts}-1of1.{ext}", - "inbound_template": "TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12" - }, - "sftp_block": { - "host": "mft.gainwelltechnologies.com", - "port": 22, - "username": "colorado-fts\\coxix_prod_11525703", - "paths": { - "outbound": "/CO XIX/PROD/coxix_prod_11525703/FromHPE", - "inbound": "/CO XIX/PROD/coxix_prod_11525703/ToHPE" - }, - "stub": false, - "staging_dir": "./var/sftp/staging", - "auth": {"password_keychain_account": "sftp.gainwell.password"} - } - }' + --data @/tmp/ch-patched.json ``` The endpoint hot-reloads the scheduler. No API restart required.