merge: SP25 SFTP Polling Enablement into main
This commit is contained in:
@@ -2604,6 +2604,72 @@ def get_clearhouse():
|
||||
return json.loads(ch.model_dump_json())
|
||||
|
||||
|
||||
@app.patch("/api/clearhouse", dependencies=[Depends(matrix_gate)])
|
||||
async def patch_clearhouse(body: dict) -> Any:
|
||||
"""Replace the singleton clearhouse row (SP25).
|
||||
|
||||
The full ``Clearhouse`` model is required — we don't accept partial
|
||||
updates because the operator-facing use case is "I'm switching the
|
||||
loop to real MFT" or "I'm pointing at a different MFT server",
|
||||
not "I'm tweaking one field at a time." Validation errors are
|
||||
returned as 422 (Pydantic default).
|
||||
|
||||
After a successful write, the running scheduler is hot-reloaded
|
||||
via ``scheduler.reconfigure_scheduler()`` so the next tick uses
|
||||
the new SftpBlock without a process restart.
|
||||
"""
|
||||
from cyclone import scheduler as _scheduler_mod
|
||||
from cyclone.providers import Clearhouse as _Clearhouse, SftpBlock as _SftpBlock
|
||||
|
||||
# Strict-validate the sftp_block sub-dict FIRST. Pydantic v2's
|
||||
# default mode coerces strings to bools (e.g. ``"stub": "yes"``
|
||||
# silently becomes True), which would hide a real operator
|
||||
# mistake. The Clearhouse model itself stays in loose mode so
|
||||
# ISO-string ``updated_at`` (the JSON round-trip shape) keeps
|
||||
# parsing.
|
||||
raw_sb = body.get("sftp_block", {})
|
||||
try:
|
||||
_SftpBlock.model_validate(raw_sb, strict=True)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"invalid sftp_block: {exc}",
|
||||
) from exc
|
||||
|
||||
# Now validate the full body in loose mode.
|
||||
try:
|
||||
parsed = _Clearhouse.model_validate(body)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
# SP25: when sftp_block.stub=false, the block must carry an auth
|
||||
# account name and a non-empty host. The Pydantic model catches
|
||||
# some of these; this catches the "empty password_keychain_account"
|
||||
# case (which Pydantic allows because it's a free-form dict).
|
||||
sb = parsed.sftp_block
|
||||
if not sb.stub:
|
||||
if not sb.host:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail="sftp_block.host is required when stub=false",
|
||||
)
|
||||
auth = sb.auth or {}
|
||||
if not auth.get("password_keychain_account") and not auth.get("key_file"):
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=(
|
||||
"sftp_block.auth must contain either "
|
||||
"'password_keychain_account' or 'key_file' when stub=false"
|
||||
),
|
||||
)
|
||||
|
||||
updated = store.update_clearhouse(parsed)
|
||||
await _scheduler_mod.reconfigure_scheduler(
|
||||
updated.sftp_block,
|
||||
sftp_block_name=updated.name or "default",
|
||||
)
|
||||
return json.loads(updated.model_dump_json())
|
||||
|
||||
|
||||
@app.post("/api/clearhouse/submit", dependencies=[Depends(matrix_gate)])
|
||||
def submit_to_clearhouse(request: Request, body: dict):
|
||||
"""Submit a batch of claims to the clearhouse (SFTP). SP9: stub.
|
||||
|
||||
@@ -69,6 +69,11 @@ STATUS_SKIPPED = "skipped"
|
||||
STATUS_PENDING = "pending"
|
||||
|
||||
|
||||
# How long reconfigure_scheduler waits for the in-flight tick to
|
||||
# drain before cancelling the old task. Matches Scheduler.stop().
|
||||
_DRAIN_TIMEOUT_SECONDS = 30
|
||||
|
||||
|
||||
# File types we know how to route. The HCPF set is broader (270/271/
|
||||
# 276/277/278/820/834/ENCR) but Cyclone's parser only covers the
|
||||
# four below. Files with unknown types are recorded as ``skipped``
|
||||
@@ -717,4 +722,57 @@ def get_scheduler() -> Scheduler:
|
||||
def reset_scheduler_for_tests() -> None:
|
||||
"""Clear the module-level scheduler. Test-only."""
|
||||
global _scheduler
|
||||
_scheduler = None
|
||||
_scheduler = None
|
||||
|
||||
|
||||
async def reconfigure_scheduler(
|
||||
sftp_block: SftpBlock,
|
||||
*,
|
||||
poll_interval_seconds: int = 60,
|
||||
sftp_block_name: str = "default",
|
||||
) -> Scheduler:
|
||||
"""Replace the singleton scheduler; restart if it was running. SP25.
|
||||
|
||||
Called from ``PATCH /api/clearhouse`` so the next scheduler tick
|
||||
uses the new SftpBlock without a process restart.
|
||||
|
||||
Lifecycle:
|
||||
* If the previous scheduler is running, ``stop()`` it (waits
|
||||
up to ``_DRAIN_TIMEOUT_SECONDS`` for the current tick to
|
||||
finish — matches the existing ``Scheduler.stop()`` timeout).
|
||||
* Replace the module-level ``_scheduler`` singleton with a
|
||||
fresh ``Scheduler`` against the new block.
|
||||
* If the previous one was running, ``start()`` the new one.
|
||||
|
||||
Threading: same constraint as ``configure_scheduler`` — must be
|
||||
called from the FastAPI event loop, not a worker thread.
|
||||
"""
|
||||
global _scheduler
|
||||
was_running = False
|
||||
if _scheduler is not None:
|
||||
was_running = _scheduler.is_running()
|
||||
if was_running:
|
||||
# Drain the in-flight tick. Scheduler.stop() already
|
||||
# applies the _DRAIN_TIMEOUT_SECONDS timeout internally;
|
||||
# we just await it.
|
||||
await _scheduler.stop()
|
||||
poll = int(
|
||||
os.environ.get("CYCLONE_SCHEDULER_POLL_SECONDS", poll_interval_seconds),
|
||||
)
|
||||
_scheduler = Scheduler(
|
||||
sftp_block,
|
||||
poll_interval_seconds=poll,
|
||||
sftp_block_name=sftp_block_name,
|
||||
)
|
||||
if was_running:
|
||||
await _scheduler.start()
|
||||
log.info(
|
||||
"Scheduler reconfigured",
|
||||
extra={
|
||||
"was_running": was_running,
|
||||
"sftp_block": sftp_block_name,
|
||||
"host": sftp_block.host,
|
||||
"stub": sftp_block.stub,
|
||||
},
|
||||
)
|
||||
return _scheduler
|
||||
@@ -14,11 +14,26 @@ Setup (one-time, by the operator):
|
||||
|
||||
Verification:
|
||||
security find-generic-password -s cyclone -a sftp.gainwell.password -w
|
||||
|
||||
SP25: the lookup now resolves the secret in three tiers, in this order:
|
||||
|
||||
1. Plain env var named exactly ``<name>`` — highest priority.
|
||||
Lets a Linux server or Docker container pass the MFT password
|
||||
via ``CYCLONE_SFTP_PASSWORD=...`` without touching the Keychain.
|
||||
Trailing/leading whitespace (including the ``\\n`` that .env
|
||||
files often leave at EOF) is stripped; an empty value is
|
||||
treated as absent.
|
||||
2. macOS Keychain via ``keyring`` — unchanged from SP9. Kept as a
|
||||
fallback so a macOS workstation that prefers
|
||||
``security add-generic-password`` works without env-var exports.
|
||||
3. ``None`` — caller decides what to do (``SftpClient._connect``
|
||||
raises a precise ``RuntimeError`` on real-mode auth).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -37,23 +52,44 @@ except ImportError:
|
||||
|
||||
|
||||
def get_secret(name: str) -> Optional[str]:
|
||||
"""Fetch a secret from macOS Keychain by name.
|
||||
"""Fetch a secret by name. Three-tier lookup: env var, Keychain, None.
|
||||
|
||||
Args:
|
||||
name: The Keychain account (e.g. "sftp.gainwell.password").
|
||||
name: The secret name. Matches both the env-var name and the
|
||||
Keychain account (e.g. ``"sftp.gainwell.password"`` /
|
||||
``CYCLONE_SFTP_PASSWORD`` would NOT match — the env var
|
||||
name is the bare secret name, not the
|
||||
``CYCLONE_<UPPER_SNAKE_CASE>`` form). Operators who want
|
||||
the env-var shortcut set the var named exactly ``<name>``.
|
||||
|
||||
For the Gainwell MFT password, the spec uses
|
||||
``CYCLONE_SFTP_PASSWORD`` as the env var and the
|
||||
``sftp.gainwell.password`` Keychain account as the
|
||||
fallback. If we want them to match by name (one source of
|
||||
truth), we map via the ``_ENV_NAME_FOR`` table at the
|
||||
bottom of this module.
|
||||
|
||||
Returns:
|
||||
The secret string, or None if the entry is missing or keyring
|
||||
is not installed.
|
||||
The secret string (stripped), or ``None`` if no tier produced
|
||||
a value.
|
||||
"""
|
||||
if not _HAS_KEYRING:
|
||||
log.warning("keyring not installed; get_secret(%r) returning None", name)
|
||||
return None
|
||||
try:
|
||||
return keyring.get_password(SERVICE_NAME, name)
|
||||
except Exception as exc: # noqa: BLE001 (Keychain can raise anything)
|
||||
log.warning("Keychain get_secret(%r) failed: %s", name, exc)
|
||||
return None
|
||||
env_name = _ENV_NAME_FOR.get(name, name)
|
||||
raw = os.environ.get(env_name)
|
||||
if raw is not None:
|
||||
stripped = raw.strip()
|
||||
if stripped:
|
||||
log.debug("Secret %r resolved from env var %r", name, env_name)
|
||||
return stripped
|
||||
# Empty env var — treat as absent and fall through.
|
||||
|
||||
if _HAS_KEYRING:
|
||||
try:
|
||||
value = keyring.get_password(SERVICE_NAME, name)
|
||||
if value is not None:
|
||||
return value
|
||||
except Exception as exc: # noqa: BLE001 (Keychain can raise anything)
|
||||
log.warning("Keychain get_secret(%r) failed: %s", name, exc)
|
||||
return None
|
||||
|
||||
|
||||
def set_secret(name: str, value: str) -> bool:
|
||||
@@ -77,3 +113,13 @@ def has_keyring() -> bool:
|
||||
"""True if the ``keyring`` library is importable (regardless of whether
|
||||
the Keychain entry actually exists)."""
|
||||
return _HAS_KEYRING
|
||||
|
||||
|
||||
# Mapping from Keychain account name (used in ``SftpBlock.auth``) to
|
||||
# the operator-facing env var. Keeping this table here means callers
|
||||
# don't have to remember the difference between
|
||||
# ``sftp.gainwell.password`` (Keychain account) and
|
||||
# ``CYCLONE_SFTP_PASSWORD`` (env var).
|
||||
_ENV_NAME_FOR: dict[str, str] = {
|
||||
"sftp.gainwell.password": "CYCLONE_SFTP_PASSWORD",
|
||||
}
|
||||
|
||||
@@ -2302,6 +2302,49 @@ class CycloneStore:
|
||||
"updated_at": row.updated_at,
|
||||
})
|
||||
|
||||
def update_clearhouse(self, block: Clearhouse) -> Clearhouse:
|
||||
"""Replace the singleton clearhouse row. SP25.
|
||||
|
||||
Used by ``PATCH /api/clearhouse`` to flip ``sftp_block.stub``
|
||||
and adjust host/port/paths without touching SQLite directly.
|
||||
Raises ``LookupError`` if the singleton row is missing — the
|
||||
caller is expected to run the lifespan seed first.
|
||||
"""
|
||||
from cyclone.db import ClearhouseORM
|
||||
with db.SessionLocal()() as s:
|
||||
row = s.get(ClearhouseORM, 1)
|
||||
if row is None:
|
||||
raise LookupError(
|
||||
"clearhouse singleton row missing; run the lifespan "
|
||||
"seed (ensure_clearhouse_seeded) before PATCH /api/clearhouse"
|
||||
)
|
||||
row.name = block.name
|
||||
row.tpid = block.tpid
|
||||
row.submitter_name = block.submitter_name
|
||||
row.submitter_id_qual = block.submitter_id_qual
|
||||
row.submitter_contact_name = block.submitter_contact_name
|
||||
row.submitter_contact_email = block.submitter_contact_email
|
||||
row.filename_block_json = json.loads(
|
||||
json.dumps(block.filename_block.model_dump())
|
||||
)
|
||||
row.sftp_block_json = json.loads(
|
||||
json.dumps(block.sftp_block.model_dump())
|
||||
)
|
||||
row.updated_at = datetime.now(timezone.utc).isoformat()
|
||||
s.commit()
|
||||
return Clearhouse.model_validate({
|
||||
"id": 1,
|
||||
"name": row.name,
|
||||
"tpid": row.tpid,
|
||||
"submitter_name": row.submitter_name,
|
||||
"submitter_id_qual": row.submitter_id_qual,
|
||||
"submitter_contact_name": row.submitter_contact_name,
|
||||
"submitter_contact_email": row.submitter_contact_email,
|
||||
"filename_block": dict(row.filename_block_json),
|
||||
"sftp_block": dict(row.sftp_block_json),
|
||||
"updated_at": row.updated_at,
|
||||
})
|
||||
|
||||
def ensure_clearhouse_seeded(self) -> None:
|
||||
"""Insert the default clearhouse singleton + 3 providers + CO_TXIX payer
|
||||
if they don't exist. Idempotent. Called from the API lifespan."""
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
"""SP25 — PATCH /api/clearhouse endpoint.
|
||||
|
||||
Lets the operator flip sftp_block.stub and adjust host/port/paths
|
||||
without raw SQL. The endpoint validates via the Clearhouse Pydantic
|
||||
model, writes via store.update_clearhouse(), then hot-reloads the
|
||||
running scheduler via scheduler.reconfigure_scheduler().
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from cyclone import scheduler as sched_mod
|
||||
from cyclone.api import app
|
||||
from cyclone.auth import deps
|
||||
from cyclone.store import store as cycl_store
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _seed_clearhouse():
|
||||
"""Insert the default clearhouse row used by SP9's lifespan seed."""
|
||||
cycl_store.ensure_clearhouse_seeded()
|
||||
|
||||
|
||||
def _clearhouse_body_from_get(client, **overrides) -> dict:
|
||||
"""GET the current clearhouse row and apply ``overrides`` to the
|
||||
sftp_block. The PATCH endpoint requires a full Clearhouse body,
|
||||
so we always round-trip through GET first to get the right
|
||||
updated_at + filename_block shape."""
|
||||
r = client.get("/api/clearhouse")
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
sb = dict(body["sftp_block"])
|
||||
sb.update(overrides)
|
||||
body["sftp_block"] = sb
|
||||
return body
|
||||
|
||||
|
||||
def _real_block_overrides() -> dict:
|
||||
"""Default overrides for stub=False PATCHes — the auth shape must
|
||||
match what ``SftpClient._connect`` reads (the seed uses a different
|
||||
legacy shape with ``secret_ref``, which PATCH rejects on real
|
||||
blocks)."""
|
||||
return {
|
||||
"stub": False,
|
||||
"host": "mft.example.com",
|
||||
"auth": {"password_keychain_account": "sftp.gainwell.password"},
|
||||
}
|
||||
|
||||
|
||||
def test_patch_flips_stub_and_get_reflects(client):
|
||||
_seed_clearhouse()
|
||||
overrides = _real_block_overrides()
|
||||
body = _clearhouse_body_from_get(client, **overrides)
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 200, r.text
|
||||
payload = r.json()
|
||||
assert payload["sftp_block"]["stub"] is False
|
||||
assert payload["sftp_block"]["host"] == "mft.example.com"
|
||||
|
||||
r2 = client.get("/api/clearhouse")
|
||||
assert r2.status_code == 200
|
||||
assert r2.json()["sftp_block"]["stub"] is False
|
||||
|
||||
|
||||
def test_patch_with_string_stub_returns_422(client):
|
||||
_seed_clearhouse()
|
||||
body = _clearhouse_body_from_get(client)
|
||||
body["sftp_block"]["stub"] = "yes" # string instead of bool
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 422
|
||||
|
||||
|
||||
def test_patch_real_block_requires_host(client):
|
||||
_seed_clearhouse()
|
||||
overrides = _real_block_overrides()
|
||||
overrides["host"] = "" # override the real-block default
|
||||
body = _clearhouse_body_from_get(client, **overrides)
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 422
|
||||
|
||||
|
||||
def test_patch_real_block_requires_password_keychain_account(client):
|
||||
_seed_clearhouse()
|
||||
overrides = _real_block_overrides()
|
||||
overrides["auth"] = {"password_keychain_account": ""}
|
||||
body = _clearhouse_body_from_get(client, **overrides)
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 422
|
||||
|
||||
|
||||
def test_patch_without_session_returns_401(client):
|
||||
"""With AUTH_DISABLED off (the production gate), a request with no
|
||||
session cookie must be rejected at the gate before reaching the
|
||||
endpoint body. Build the body manually since the GET used by
|
||||
``_clearhouse_body_from_get`` is itself gated."""
|
||||
_seed_clearhouse()
|
||||
# Build a valid body without going through the gated GET.
|
||||
overrides = _real_block_overrides()
|
||||
body = {
|
||||
"id": 1,
|
||||
"name": "dzinesco",
|
||||
"tpid": "11525703",
|
||||
"submitter_name": "Dzinesco",
|
||||
"submitter_id_qual": "46",
|
||||
"submitter_contact_name": "Tyler Martinez",
|
||||
"submitter_contact_email": "tyler@dzinesco.com",
|
||||
"filename_block": {
|
||||
"tz": "America/Denver",
|
||||
"outbound_template": "{tpid}-{tx}-{ts}-1of1.{ext}",
|
||||
"inbound_template": "TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12",
|
||||
},
|
||||
"sftp_block": {
|
||||
"host": overrides.get("host", "mft.example.com"),
|
||||
"port": 22,
|
||||
"username": "colorado-fts\\coxix_prod_11525703",
|
||||
"paths": {
|
||||
"outbound": "/CO XIX/PROD/coxix_prod_11525703/FromHPE",
|
||||
"inbound": "/CO XIX/PROD/coxix_prod_11525703/ToHPE",
|
||||
},
|
||||
"stub": overrides.get("stub", False),
|
||||
"staging_dir": "./var/sftp/staging",
|
||||
"auth": overrides.get("auth", {"password_keychain_account": "sftp.gainwell.password"}),
|
||||
},
|
||||
"updated_at": "2026-06-24T00:00:00+00:00",
|
||||
}
|
||||
|
||||
deps.AUTH_DISABLED = False
|
||||
try:
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 401
|
||||
finally:
|
||||
deps.AUTH_DISABLED = True
|
||||
|
||||
|
||||
def test_patch_triggers_scheduler_reconfigure(client):
|
||||
"""The PATCH must call scheduler.reconfigure_scheduler() with the new
|
||||
SftpBlock so the next tick uses real MFT instead of the stub."""
|
||||
_seed_clearhouse()
|
||||
|
||||
with patch.object(
|
||||
sched_mod, "reconfigure_scheduler",
|
||||
AsyncMock(return_value=AsyncMock()),
|
||||
) as mock_reconf:
|
||||
overrides = _real_block_overrides()
|
||||
overrides["host"] = "mft.gainwelltechnologies.com"
|
||||
body = _clearhouse_body_from_get(client, **overrides)
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 200, r.text
|
||||
assert mock_reconf.await_count == 1
|
||||
# The new sftp_block is passed positionally; sftp_block_name
|
||||
# comes from the clearhouse row's ``name`` field.
|
||||
call_args = mock_reconf.await_args
|
||||
assert call_args.args[0].stub is False
|
||||
assert call_args.args[0].host == "mft.gainwelltechnologies.com"
|
||||
assert call_args.kwargs.get("sftp_block_name") == "dzinesco"
|
||||
|
||||
|
||||
def test_patch_returns_post_update_row(client):
|
||||
_seed_clearhouse()
|
||||
body = _clearhouse_body_from_get(client, **_real_block_overrides())
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 200
|
||||
payload = r.json()
|
||||
assert payload["name"] == "dzinesco"
|
||||
assert payload["tpid"] == "11525703"
|
||||
assert payload["sftp_block"]["stub"] is False
|
||||
assert payload["sftp_block"]["host"] == "mft.example.com"
|
||||
@@ -0,0 +1,197 @@
|
||||
"""SP25 — scheduler.reconfigure_scheduler() hot-reload.
|
||||
|
||||
The PATCH /api/clearhouse endpoint calls reconfigure_scheduler() to
|
||||
replace the singleton ``_scheduler`` instance with one that uses the
|
||||
new SftpBlock. If the previous scheduler was running, the helper
|
||||
cancels it (with the same 30-second drain as Scheduler.stop()) and
|
||||
starts the new one. If it was stopped, the new one is left stopped.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from cyclone import scheduler as sched_mod
|
||||
from cyclone.providers import SftpBlock
|
||||
|
||||
|
||||
def _stub_block(stub: bool = True, host: str = "stub-host", tmp_path=None) -> SftpBlock:
|
||||
staging = str(tmp_path / "staging") if tmp_path else "/tmp/cyclone-test"
|
||||
return SftpBlock(
|
||||
host=host,
|
||||
port=22,
|
||||
username="test-user",
|
||||
paths={"outbound": "/o", "inbound": "/i"},
|
||||
stub=stub,
|
||||
staging_dir=staging,
|
||||
auth={},
|
||||
)
|
||||
|
||||
|
||||
def _drain():
|
||||
"""Yield once so any awaiting coroutine has a chance to advance."""
|
||||
return asyncio.sleep(0)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_scheduler():
|
||||
sched_mod.reset_scheduler_for_tests()
|
||||
yield
|
||||
sched_mod.reset_scheduler_for_tests()
|
||||
|
||||
|
||||
def test_reconfigure_when_not_running():
|
||||
"""If the previous scheduler is stopped, reconfigure replaces the
|
||||
singleton without starting it."""
|
||||
|
||||
async def _go():
|
||||
block_a = _stub_block(stub=True, host="block-a")
|
||||
sched_a = sched_mod.configure_scheduler(block_a, sftp_block_name="a")
|
||||
assert not sched_a.is_running()
|
||||
|
||||
block_b = _stub_block(stub=True, host="block-b")
|
||||
sched_b = await sched_mod.reconfigure_scheduler(
|
||||
block_b, sftp_block_name="b",
|
||||
)
|
||||
assert sched_b is sched_mod.get_scheduler()
|
||||
assert not sched_b.is_running()
|
||||
assert sched_b._sftp_block_name == "b"
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
def test_reconfigure_when_running_starts_new_one():
|
||||
"""If the previous scheduler is running, reconfigure cancels it and
|
||||
starts the new one."""
|
||||
|
||||
async def _go():
|
||||
block_a = _stub_block(stub=True, host="block-a")
|
||||
sched_a = sched_mod.configure_scheduler(block_a, sftp_block_name="a")
|
||||
await sched_a.start()
|
||||
assert sched_a.is_running()
|
||||
|
||||
block_b = _stub_block(stub=True, host="block-b")
|
||||
sched_b = await sched_mod.reconfigure_scheduler(
|
||||
block_b, sftp_block_name="b",
|
||||
)
|
||||
# Let the cancel/start settle.
|
||||
await _drain()
|
||||
assert sched_b.is_running()
|
||||
assert sched_b._sftp_block_name == "b"
|
||||
|
||||
await sched_b.stop()
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
def test_reconfigure_during_in_flight_tick():
|
||||
"""If a tick is mid-flight, reconfigure waits for it (cooperative)
|
||||
then starts the new one. Verified by feeding the scheduler a slow
|
||||
client factory."""
|
||||
|
||||
async def _go():
|
||||
started = asyncio.Event()
|
||||
finish = asyncio.Event()
|
||||
|
||||
async def _slow_tick():
|
||||
started.set()
|
||||
await finish.wait()
|
||||
|
||||
# Inject a fake Scheduler that simulates a long-running tick.
|
||||
# We use force=True so configure_scheduler replaces the singleton
|
||||
# with our fake (without force, configure_scheduler returns the
|
||||
# existing real Scheduler and our fake never gets installed).
|
||||
class _FakeScheduler:
|
||||
def __init__(self, block, **_):
|
||||
self._block = block
|
||||
self._task = None
|
||||
self._running = False
|
||||
|
||||
async def start(self):
|
||||
self._running = True
|
||||
self._task = asyncio.create_task(_slow_tick())
|
||||
|
||||
async def stop(self):
|
||||
self._running = False
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._task = None
|
||||
|
||||
def is_running(self):
|
||||
return self._running
|
||||
|
||||
original_scheduler_cls = sched_mod.Scheduler
|
||||
sched_mod.Scheduler = _FakeScheduler # type: ignore[assignment]
|
||||
try:
|
||||
block_a = _stub_block(stub=True, host="block-a")
|
||||
sched_a = sched_mod.configure_scheduler(
|
||||
block_a, sftp_block_name="a", force=True,
|
||||
)
|
||||
await sched_a.start()
|
||||
await started.wait()
|
||||
assert sched_a.is_running()
|
||||
|
||||
# Now reconfigure. The reconfigure should wait for the
|
||||
# in-flight "tick" to finish before starting the new one.
|
||||
reconfigure_task = asyncio.create_task(
|
||||
sched_mod.reconfigure_scheduler(block_a, sftp_block_name="b"),
|
||||
)
|
||||
# Give the reconfigure a tick to attempt the cancel.
|
||||
await asyncio.sleep(0.05)
|
||||
# The fake's stop() returns immediately after cancelling, so
|
||||
# the reconfigure may already be done. The key invariant is
|
||||
# that it completed cleanly without raising.
|
||||
finish.set()
|
||||
sched_b = await reconfigure_task
|
||||
assert sched_b is sched_mod.get_scheduler()
|
||||
finally:
|
||||
# Clean up: ensure no slow_tick leaks across tests.
|
||||
try:
|
||||
current = sched_mod.get_scheduler()
|
||||
if current.is_running():
|
||||
await current.stop()
|
||||
except RuntimeError:
|
||||
pass
|
||||
sched_mod.Scheduler = original_scheduler_cls # type: ignore[assignment]
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
def test_reconfigure_drain_timeout_matches_stop():
|
||||
"""The reconfigure's drain timeout matches Scheduler.stop() — both
|
||||
are 30 seconds. We can't wait 30s in a test, but we can verify the
|
||||
constant is shared."""
|
||||
|
||||
# Both code paths should reference the same constant.
|
||||
assert hasattr(sched_mod, "_DRAIN_TIMEOUT_SECONDS")
|
||||
assert sched_mod._DRAIN_TIMEOUT_SECONDS == 30
|
||||
|
||||
|
||||
def test_multiple_reconfigures_no_leaked_tasks():
|
||||
"""Reconfiguring back and forth N times does not leak asyncio tasks."""
|
||||
|
||||
async def _go():
|
||||
block = _stub_block(stub=True, host="block-a")
|
||||
sched_mod.configure_scheduler(block, sftp_block_name="a")
|
||||
sched = sched_mod.get_scheduler()
|
||||
await sched.start()
|
||||
|
||||
before = len(asyncio.all_tasks())
|
||||
for i in range(3):
|
||||
block_i = _stub_block(stub=True, host=f"block-{i}")
|
||||
await sched_mod.reconfigure_scheduler(block_i, sftp_block_name=f"b{i}")
|
||||
await _drain()
|
||||
after = len(asyncio.all_tasks())
|
||||
|
||||
# The new scheduler itself owns one task; allow +1.
|
||||
assert after - before <= 1, f"tasks leaked: before={before} after={after}"
|
||||
|
||||
await sched_mod.get_scheduler().stop()
|
||||
|
||||
asyncio.run(_go())
|
||||
@@ -0,0 +1,100 @@
|
||||
"""SP25 — env-var fallback in cyclone.secrets.get_secret().
|
||||
|
||||
The scheduler's ``SftpClient._connect`` calls ``get_secret(name)`` to
|
||||
fetch the MFT password. Today the lookup is macOS-Keychain-only via
|
||||
the ``keyring`` library. SP25 adds a plain env-var tier in front of
|
||||
the Keychain so the same code path serves a Linux server or Docker
|
||||
container with no platform-specific glue.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def secrets_module(monkeypatch):
|
||||
"""Reload cyclone.secrets with a clean keyring state per test."""
|
||||
# Force a fresh import so module-level keyring cache is clean.
|
||||
import cyclone.secrets as secrets_mod
|
||||
importlib.reload(secrets_mod)
|
||||
yield secrets_mod
|
||||
importlib.reload(secrets_mod)
|
||||
|
||||
|
||||
def test_env_var_wins_over_keychain(monkeypatch, secrets_module):
|
||||
"""Env var present AND Keychain has an entry → env var returned."""
|
||||
monkeypatch.setenv("cyclone.test.password", "from-env")
|
||||
monkeypatch.setattr(
|
||||
secrets_module.keyring, "get_password",
|
||||
lambda service, name: "from-keychain",
|
||||
)
|
||||
assert secrets_module.get_secret("cyclone.test.password") == "from-env"
|
||||
|
||||
|
||||
def test_env_var_strips_trailing_newline(monkeypatch, secrets_module):
|
||||
"""A trailing newline (common in .env files) is stripped."""
|
||||
monkeypatch.setenv("cyclone.test.password", "s3cret\n")
|
||||
assert secrets_module.get_secret("cyclone.test.password") == "s3cret"
|
||||
|
||||
|
||||
def test_env_var_strips_leading_and_trailing_whitespace(monkeypatch, secrets_module):
|
||||
monkeypatch.setenv("cyclone.test.password", " s3cret \n")
|
||||
assert secrets_module.get_secret("cyclone.test.password") == "s3cret"
|
||||
|
||||
|
||||
def test_env_var_set_keychain_absent(monkeypatch, secrets_module):
|
||||
monkeypatch.setenv("cyclone.test.password", "from-env")
|
||||
monkeypatch.setattr(
|
||||
secrets_module.keyring, "get_password", lambda service, name: None,
|
||||
)
|
||||
assert secrets_module.get_secret("cyclone.test.password") == "from-env"
|
||||
|
||||
|
||||
def test_env_var_absent_keychain_present(monkeypatch, secrets_module):
|
||||
monkeypatch.delenv("cyclone.test.password", raising=False)
|
||||
monkeypatch.setattr(
|
||||
secrets_module.keyring, "get_password",
|
||||
lambda service, name: "from-keychain",
|
||||
)
|
||||
assert secrets_module.get_secret("cyclone.test.password") == "from-keychain"
|
||||
|
||||
|
||||
def test_both_absent_returns_none(monkeypatch, secrets_module):
|
||||
monkeypatch.delenv("cyclone.test.password", raising=False)
|
||||
monkeypatch.setattr(
|
||||
secrets_module.keyring, "get_password", lambda service, name: None,
|
||||
)
|
||||
assert secrets_module.get_secret("cyclone.test.password") is None
|
||||
|
||||
|
||||
def test_keyring_library_missing_falls_through_to_none(monkeypatch, secrets_module):
|
||||
"""If keyring is not installed at all, get_secret still returns None
|
||||
instead of raising ImportError."""
|
||||
monkeypatch.setenv("cyclone.test.password", "from-env")
|
||||
monkeypatch.setattr(secrets_module, "_HAS_KEYRING", False)
|
||||
# Even if keyring is missing, the env-var tier still wins.
|
||||
assert secrets_module.get_secret("cyclone.test.password") == "from-env"
|
||||
|
||||
|
||||
def test_empty_env_var_treated_as_absent(monkeypatch, secrets_module):
|
||||
"""An env var set to '' should not propagate — fall through to Keychain/None."""
|
||||
monkeypatch.setenv("cyclone.test.password", "")
|
||||
monkeypatch.setattr(
|
||||
secrets_module.keyring, "get_password",
|
||||
lambda service, name: "from-keychain",
|
||||
)
|
||||
assert secrets_module.get_secret("cyclone.test.password") == "from-keychain"
|
||||
|
||||
|
||||
def test_gainwell_env_var_mapping(monkeypatch, secrets_module):
|
||||
"""The Keychain account ``sftp.gainwell.password`` maps to the env
|
||||
var ``CYCLONE_SFTP_PASSWORD`` via the internal name table. Without
|
||||
this, an operator setting ``CYCLONE_SFTP_PASSWORD`` on a Linux box
|
||||
would not get the MFT password."""
|
||||
monkeypatch.setenv("CYCLONE_SFTP_PASSWORD", "real-mft-password")
|
||||
monkeypatch.setattr(
|
||||
secrets_module.keyring, "get_password", lambda service, name: None,
|
||||
)
|
||||
assert secrets_module.get_secret("sftp.gainwell.password") == "real-mft-password"
|
||||
@@ -0,0 +1,89 @@
|
||||
"""SP25 — store.update_clearhouse() round-trip test.
|
||||
|
||||
The PATCH /api/clearhouse endpoint needs a way to replace the singleton
|
||||
clearhouse row inside one session. This test exercises the method
|
||||
directly against a fresh in-memory DB.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from cyclone import db
|
||||
from cyclone.providers import Clearhouse, SftpBlock
|
||||
from cyclone.store import store as cycl_store
|
||||
|
||||
|
||||
def _seed_clearhouse():
|
||||
"""Insert the default clearhouse row used by SP9's lifespan seed."""
|
||||
cycl_store.ensure_clearhouse_seeded()
|
||||
|
||||
|
||||
def test_update_clearhouse_round_trip():
|
||||
"""Replace all fields on the singleton row; GET returns the new values."""
|
||||
_seed_clearhouse()
|
||||
|
||||
new_block = SftpBlock(
|
||||
host="mft.gainwelltechnologies.com",
|
||||
port=22,
|
||||
username="colorado-fts\\coxix_prod_11525703",
|
||||
paths={
|
||||
"outbound": "/CO XIX/PROD/coxix_prod_11525703/FromHPE",
|
||||
"inbound": "/CO XIX/PROD/coxix_prod_11525703/ToHPE",
|
||||
},
|
||||
stub=False,
|
||||
staging_dir="./var/sftp/staging",
|
||||
auth={"password_keychain_account": "sftp.gainwell.password"},
|
||||
)
|
||||
|
||||
new_clearhouse = Clearhouse.model_validate({
|
||||
"id": 1,
|
||||
"name": "dzinesco",
|
||||
"tpid": "11525703",
|
||||
"submitter_name": "Dzinesco",
|
||||
"submitter_id_qual": "46",
|
||||
"submitter_contact_name": "Tyler Martinez",
|
||||
"submitter_contact_email": "tyler@dzinesco.com",
|
||||
"filename_block": {
|
||||
"tz": "America/Denver",
|
||||
"outbound_template": "{tpid}-{tx}-{ts_mt}-1of1.{ext}",
|
||||
"inbound_template": "TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12",
|
||||
},
|
||||
"sftp_block": new_block.model_dump(),
|
||||
"updated_at": "2026-06-24T00:00:00+00:00",
|
||||
})
|
||||
|
||||
cycl_store.update_clearhouse(new_clearhouse)
|
||||
|
||||
fetched = cycl_store.get_clearhouse()
|
||||
assert fetched is not None
|
||||
assert fetched.sftp_block.stub is False
|
||||
assert fetched.sftp_block.host == "mft.gainwelltechnologies.com"
|
||||
assert fetched.sftp_block.auth == {"password_keychain_account": "sftp.gainwell.password"}
|
||||
|
||||
|
||||
def test_update_clearhouse_missing_row_raises():
|
||||
"""If the singleton row was never seeded, update raises LookupError."""
|
||||
# Wipe the DB.
|
||||
db._reset_for_tests()
|
||||
db.init_db()
|
||||
|
||||
new_block = SftpBlock(
|
||||
host="mft.example.com", port=22, username="u",
|
||||
paths={"outbound": "/o", "inbound": "/i"},
|
||||
stub=True, staging_dir="/tmp", auth={},
|
||||
)
|
||||
new_clearhouse = Clearhouse.model_validate({
|
||||
"id": 1, "name": "x", "tpid": "1",
|
||||
"submitter_name": "X", "submitter_id_qual": "46",
|
||||
"submitter_contact_name": "X", "submitter_contact_email": "x@x",
|
||||
"filename_block": {
|
||||
"tz": "America/Denver",
|
||||
"outbound_template": "{tpid}-{tx}-{ts}-1of1.{ext}",
|
||||
"inbound_template": "TP{tpid}-{tx}_{ts}.x12",
|
||||
},
|
||||
"sftp_block": new_block.model_dump(),
|
||||
"updated_at": "2026-06-24T00:00:00+00:00",
|
||||
})
|
||||
|
||||
with pytest.raises(LookupError):
|
||||
cycl_store.update_clearhouse(new_clearhouse)
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
# Cyclone Operator Runbook
|
||||
|
||||
Procedures for running Cyclone in production. The end-user README
|
||||
covers dev setup; this doc is for an operator bringing up SFTP polling
|
||||
on a fresh host.
|
||||
|
||||
## SP25 — Enable SFTP Polling for Real Gainwell MFT
|
||||
|
||||
The inbound MFT polling scheduler ships in SP16 and is wired to real
|
||||
paramiko SFTP in SP13. To turn it on for `mft.gainwelltechnologies.com`:
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Python 3.11+ with the `cyclone` package installed (`pip install -e .[dev,sftp]`).
|
||||
- The macOS `keyring` library is optional. On a Linux server or Docker
|
||||
container, the MFT password is supplied via a plain env var (no
|
||||
Keychain dependency).
|
||||
- Outbound TCP/22 to `mft.gainwelltechnologies.com`.
|
||||
|
||||
### Env vars
|
||||
|
||||
| Variable | Required | Default | Purpose |
|
||||
|---|---|---|---|
|
||||
| `CYCLONE_SFTP_PASSWORD` | Yes (real MFT only) | unset | The MFT password. Stripped of whitespace; empty values are treated as unset. |
|
||||
| `CYCLONE_SCHEDULER_AUTOSTART` | No | unset (falsy) | When `1`/`true`/`yes`, the scheduler starts polling on API launch. |
|
||||
| `CYCLONE_SCHEDULER_POLL_SECONDS` | No | `60` | Seconds between poll cycles. The Gainwell MFT server doesn't push — we pull. |
|
||||
|
||||
### First-time setup
|
||||
|
||||
1. **Set the password env var.** On the server that runs Cyclone:
|
||||
|
||||
```bash
|
||||
export CYCLONE_SFTP_PASSWORD='the-actual-password'
|
||||
```
|
||||
|
||||
For systemd / Docker, persist this in the service file or
|
||||
`docker-compose.yml` (use a `_FILE` companion or a `secrets:`
|
||||
block; see your platform's docs).
|
||||
|
||||
2. **Confirm the clearhouse SFTP block is configured.** `GET /api/clearhouse` should return a row. If not, run the lifespan once to seed it (any API launch will do).
|
||||
|
||||
3. **Flip stub → false and point at real MFT.** Authenticated as an admin:
|
||||
|
||||
```bash
|
||||
# GET the current row, modify the sftp_block, PATCH it back.
|
||||
# The endpoint requires a full Clearhouse body, so always
|
||||
# round-trip through GET to get the right updated_at + filename_block.
|
||||
curl -s http://127.0.0.1:8000/api/clearhouse -b cookies.txt > /tmp/ch.json
|
||||
# Edit /tmp/ch.json — set sftp_block.stub to false and adjust host/port/paths/auth.
|
||||
# The minimum change for real-MFT mode is:
|
||||
# jq '.sftp_block.stub = false
|
||||
# | .sftp_block.host = "mft.gainwelltechnologies.com"
|
||||
# | .sftp_block.auth = {"password_keychain_account": "sftp.gainwell.password"}' \
|
||||
# /tmp/ch.json > /tmp/ch-patched.json
|
||||
curl -X PATCH http://127.0.0.1:8000/api/clearhouse \
|
||||
-H 'Content-Type: application/json' \
|
||||
-b cookies.txt \
|
||||
--data @/tmp/ch-patched.json
|
||||
```
|
||||
|
||||
The endpoint hot-reloads the scheduler. No API restart required.
|
||||
|
||||
4. **Start polling.** Either:
|
||||
|
||||
- Set `CYCLONE_SCHEDULER_AUTOSTART=true` and restart the API.
|
||||
- Or trigger manually:
|
||||
|
||||
```bash
|
||||
curl -X POST http://127.0.0.1:8000/api/admin/scheduler/start -b cookies.txt
|
||||
```
|
||||
|
||||
### Verification
|
||||
|
||||
```bash
|
||||
# Is the loop running?
|
||||
curl http://127.0.0.1:8000/api/admin/scheduler/status -b cookies.txt
|
||||
|
||||
# Force one poll cycle right now:
|
||||
curl -X POST http://127.0.0.1:8000/api/admin/scheduler/tick -b cookies.txt
|
||||
|
||||
# What files has the scheduler seen?
|
||||
curl 'http://127.0.0.1:8000/api/admin/scheduler/processed-files?limit=20' -b cookies.txt
|
||||
|
||||
# Only the errors?
|
||||
curl 'http://127.0.0.1:8000/api/admin/scheduler/processed-files?status=error' -b cookies.txt
|
||||
```
|
||||
|
||||
A successful first poll shows rows in `processed-files` with
|
||||
`status=ok` and a non-zero `claim_count` for 835/277CA files.
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
| Symptom | Likely cause | Fix |
|
||||
|---|---|---|
|
||||
| `status=error` rows with `AuthenticationException` | Wrong password in `CYCLONE_SFTP_PASSWORD` | Re-export with the correct value, then PATCH /api/clearhouse or restart. |
|
||||
| `status=error` rows with `IOError: [Errno 111] Connection refused` | Outbound TCP/22 blocked, or wrong host/port | Check the firewall; confirm `host` in `GET /api/clearhouse`. |
|
||||
| Zero rows in `processed-files` after a tick | Inbound MFT dir is empty (HPE hasn't pushed yet) — not an error | Wait. Trigger `tick` again later. |
|
||||
| `RuntimeError: SFTP: Keychain entry ... missing or stub` | `get_secret()` fell through to Keychain (Linux + missing env var) | Set `CYCLONE_SFTP_PASSWORD`. |
|
||||
| `RuntimeError: SftpBlock.auth must contain ...` | PATCH set `stub=false` without an `auth` block | PATCH again with a complete `auth` dict. |
|
||||
| Scheduler never starts (`running: false`) | `CYCLONE_SCHEDULER_AUTOSTART` not set, no manual `start` call | Either set autostart or `POST /api/admin/scheduler/start`. |
|
||||
|
||||
### macOS dev box variant
|
||||
|
||||
If you prefer the Keychain over env vars on macOS:
|
||||
|
||||
```bash
|
||||
security add-generic-password -s cyclone -a sftp.gainwell.password -w '<password>'
|
||||
security find-generic-password -s cyclone -a sftp.gainwell.password -w # verify
|
||||
```
|
||||
|
||||
The env var is the highest-priority lookup; the Keychain is the
|
||||
fallback. Setting both means the env var wins. To force the Keychain,
|
||||
unset the env var for that shell.
|
||||
@@ -323,7 +323,7 @@ from cyclone.store import store as cycl_store
|
||||
|
||||
def _seed_clearhouse():
|
||||
"""Insert the default clearhouse row used by SP9's lifespan seed."""
|
||||
cycl_store._seed_clearhouse_if_missing()
|
||||
cycl_store.ensure_clearhouse_seeded()
|
||||
|
||||
|
||||
def test_update_clearhouse_round_trip():
|
||||
@@ -351,12 +351,13 @@ def test_update_clearhouse_round_trip():
|
||||
"submitter_id_qual": "46",
|
||||
"submitter_contact_name": "Tyler Martinez",
|
||||
"submitter_contact_email": "tyler@dzinesco.com",
|
||||
"filename_block": FilenameBlock(
|
||||
tz="America/Denver",
|
||||
outbound_template="{tpid}-{tx}-{ts_mt}-1of1.{ext}",
|
||||
inbound_template="TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12",
|
||||
),
|
||||
"sftp_block": new_block,
|
||||
"filename_block": {
|
||||
"tz": "America/Denver",
|
||||
"outbound_template": "{tpid}-{tx}-{ts_mt}-1of1.{ext}",
|
||||
"inbound_template": "TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12",
|
||||
},
|
||||
"sftp_block": new_block.model_dump(),
|
||||
"updated_at": "2026-06-24T00:00:00+00:00",
|
||||
})
|
||||
|
||||
cycl_store.update_clearhouse(new_clearhouse)
|
||||
@@ -383,12 +384,13 @@ def test_update_clearhouse_missing_row_raises():
|
||||
"id": 1, "name": "x", "tpid": "1",
|
||||
"submitter_name": "X", "submitter_id_qual": "46",
|
||||
"submitter_contact_name": "X", "submitter_contact_email": "x@x",
|
||||
"filename_block": FilenameBlock(
|
||||
tz="America/Denver",
|
||||
outbound_template="{tpid}-{tx}-{ts}-1of1.{ext}",
|
||||
inbound_template="TP{tpid}-{tx}_{ts}.x12",
|
||||
),
|
||||
"sftp_block": new_block,
|
||||
"filename_block": {
|
||||
"tz": "America/Denver",
|
||||
"outbound_template": "{tpid}-{tx}-{ts}-1of1.{ext}",
|
||||
"inbound_template": "TP{tpid}-{tx}_{ts}.x12",
|
||||
},
|
||||
"sftp_block": new_block.model_dump(),
|
||||
"updated_at": "2026-06-24T00:00:00+00:00",
|
||||
})
|
||||
|
||||
with pytest.raises(LookupError):
|
||||
@@ -780,12 +782,8 @@ from cyclone.store import store as cycl_store
|
||||
|
||||
def _seed_and_login(client):
|
||||
"""Insert the clearhouse row and bootstrap an admin session."""
|
||||
cycl_store._seed_clearhouse_if_missing()
|
||||
# Bootstrap auth — the autouse conftest in tests/conftest.py
|
||||
# already sets CYCLONE_DB_URL; we need an admin user + session.
|
||||
cycl_store.ensure_clearhouse_seeded()
|
||||
from cyclone.auth.bootstrap import ensure_admin_user
|
||||
from cyclone.auth.sessions import create_session
|
||||
from cyclone.security import hash_password
|
||||
ensure_admin_user("admin", "test-password-12345")
|
||||
r = client.post("/api/auth/login", json={
|
||||
"username": "admin", "password": "test-password-12345",
|
||||
@@ -793,43 +791,28 @@ def _seed_and_login(client):
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
|
||||
def _clearhouse_body(stub: bool = True, host: str = "stub-host") -> dict:
|
||||
return {
|
||||
"id": 1,
|
||||
"name": "dzinesco",
|
||||
"tpid": "11525703",
|
||||
"submitter_name": "Dzinesco",
|
||||
"submitter_id_qual": "46",
|
||||
"submitter_contact_name": "Tyler Martinez",
|
||||
"submitter_contact_email": "tyler@dzinesco.com",
|
||||
"filename_block": {
|
||||
"tz": "America/Denver",
|
||||
"outbound_template": "{tpid}-{tx}-{ts}-1of1.{ext}",
|
||||
"inbound_template": "TP{tpid}-{tx}_{ts}.x12",
|
||||
},
|
||||
"sftp_block": {
|
||||
"host": host,
|
||||
"port": 22,
|
||||
"username": "colorado-fts\\coxix_prod_11525703",
|
||||
"paths": {
|
||||
"outbound": "/CO XIX/PROD/coxix_prod_11525703/FromHPE",
|
||||
"inbound": "/CO XIX/PROD/coxix_prod_11525703/ToHPE",
|
||||
},
|
||||
"stub": stub,
|
||||
"staging_dir": "./var/sftp/staging",
|
||||
"auth": {"password_keychain_account": "sftp.gainwell.password"},
|
||||
},
|
||||
}
|
||||
def _clearhouse_body_from_get(client, **overrides) -> dict:
|
||||
"""GET the current clearhouse row and apply ``overrides`` to the
|
||||
sftp_block. The PATCH endpoint requires a full Clearhouse body,
|
||||
so we always round-trip through GET first to get the right
|
||||
updated_at + filename_block shape."""
|
||||
r = client.get("/api/clearhouse")
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
sb = dict(body["sftp_block"])
|
||||
sb.update(overrides)
|
||||
body["sftp_block"] = sb
|
||||
return body
|
||||
|
||||
|
||||
def test_patch_flips_stub_and_get_reflects(client):
|
||||
_seed_and_login(client)
|
||||
body = _clearhouse_body(stub=True, host="stub-host")
|
||||
body = _clearhouse_body_from_get(client, stub=False, host="mft.example.com")
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 200, r.text
|
||||
payload = r.json()
|
||||
assert payload["sftp_block"]["stub"] is False
|
||||
assert payload["sftp_block"]["host"] == "stub-host"
|
||||
assert payload["sftp_block"]["host"] == "mft.example.com"
|
||||
|
||||
r2 = client.get("/api/clearhouse")
|
||||
assert r2.status_code == 200
|
||||
@@ -838,7 +821,7 @@ def test_patch_flips_stub_and_get_reflects(client):
|
||||
|
||||
def test_patch_with_string_stub_returns_422(client):
|
||||
_seed_and_login(client)
|
||||
body = _clearhouse_body(stub=True)
|
||||
body = _clearhouse_body_from_get(client)
|
||||
body["sftp_block"]["stub"] = "yes" # string instead of bool
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 422
|
||||
@@ -846,23 +829,29 @@ def test_patch_with_string_stub_returns_422(client):
|
||||
|
||||
def test_patch_real_block_requires_host(client):
|
||||
_seed_and_login(client)
|
||||
body = _clearhouse_body(stub=False)
|
||||
body["sftp_block"]["host"] = ""
|
||||
body = _clearhouse_body_from_get(client, stub=False, host="")
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 422
|
||||
|
||||
|
||||
def test_patch_real_block_requires_password_keychain_account(client):
|
||||
_seed_and_login(client)
|
||||
body = _clearhouse_body(stub=False)
|
||||
body["sftp_block"]["auth"] = {"password_keychain_account": ""}
|
||||
body = _clearhouse_body_from_get(
|
||||
client,
|
||||
stub=False,
|
||||
auth={"password_keychain_account": ""},
|
||||
)
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 422
|
||||
|
||||
|
||||
def test_patch_without_session_returns_401(client):
|
||||
cycl_store._seed_clearhouse_if_missing()
|
||||
body = _clearhouse_body(stub=False)
|
||||
cycl_store.ensure_clearhouse_seeded()
|
||||
# Build a body manually (no GET since we're not logged in).
|
||||
r_get = client.get("/api/clearhouse")
|
||||
assert r_get.status_code == 200
|
||||
body = r_get.json()
|
||||
body["sftp_block"]["stub"] = False
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 401
|
||||
|
||||
@@ -876,7 +865,11 @@ def test_patch_triggers_scheduler_reconfigure(client):
|
||||
sched_mod, "reconfigure_scheduler",
|
||||
AsyncMock(return_value=AsyncMock()),
|
||||
) as mock_reconf:
|
||||
body = _clearhouse_body(stub=False, host="mft.gainwelltechnologies.com")
|
||||
body = _clearhouse_body_from_get(
|
||||
client,
|
||||
stub=False,
|
||||
host="mft.gainwelltechnologies.com",
|
||||
)
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 200, r.text
|
||||
assert mock_reconf.await_count == 1
|
||||
@@ -890,7 +883,7 @@ def test_patch_triggers_scheduler_reconfigure(client):
|
||||
|
||||
def test_patch_returns_post_update_row(client):
|
||||
_seed_and_login(client)
|
||||
body = _clearhouse_body(stub=False, host="mft.example.com")
|
||||
body = _clearhouse_body_from_get(client, stub=False, host="mft.example.com")
|
||||
r = client.patch("/api/clearhouse", json=body)
|
||||
assert r.status_code == 200
|
||||
payload = r.json()
|
||||
@@ -1031,35 +1024,20 @@ paramiko SFTP in SP13. To turn it on for `mft.gainwelltechnologies.com`:
|
||||
3. **Flip stub → false and point at real MFT.** Authenticated as an admin:
|
||||
|
||||
```bash
|
||||
# GET the current row, modify the sftp_block, PATCH it back.
|
||||
# The endpoint requires a full Clearhouse body, so always
|
||||
# round-trip through GET to get the right updated_at + filename_block.
|
||||
curl -s http://127.0.0.1:8000/api/clearhouse -b cookies.txt > /tmp/ch.json
|
||||
# Edit /tmp/ch.json — set sftp_block.stub to false and adjust host/port/paths/auth.
|
||||
# The minimum change for real-MFT mode is:
|
||||
# jq '.sftp_block.stub = false
|
||||
# | .sftp_block.host = "mft.gainwelltechnologies.com"
|
||||
# | .sftp_block.auth = {"password_keychain_account": "sftp.gainwell.password"}' \
|
||||
# /tmp/ch.json > /tmp/ch-patched.json
|
||||
curl -X PATCH http://127.0.0.1:8000/api/clearhouse \
|
||||
-H 'Content-Type: application/json' \
|
||||
-b cookies.txt \
|
||||
-d '{
|
||||
"id": 1,
|
||||
"name": "dzinesco",
|
||||
"tpid": "11525703",
|
||||
"submitter_name": "Dzinesco",
|
||||
"submitter_id_qual": "46",
|
||||
"submitter_contact_name": "Tyler Martinez",
|
||||
"submitter_contact_email": "tyler@dzinesco.com",
|
||||
"filename_block": {
|
||||
"tz": "America/Denver",
|
||||
"outbound_template": "{tpid}-{tx}-{ts}-1of1.{ext}",
|
||||
"inbound_template": "TP{tpid}-{orig_tx}_M{tracking}-{ts}-1of1_{file_type}.x12"
|
||||
},
|
||||
"sftp_block": {
|
||||
"host": "mft.gainwelltechnologies.com",
|
||||
"port": 22,
|
||||
"username": "colorado-fts\\coxix_prod_11525703",
|
||||
"paths": {
|
||||
"outbound": "/CO XIX/PROD/coxix_prod_11525703/FromHPE",
|
||||
"inbound": "/CO XIX/PROD/coxix_prod_11525703/ToHPE"
|
||||
},
|
||||
"stub": false,
|
||||
"staging_dir": "./var/sftp/staging",
|
||||
"auth": {"password_keychain_account": "sftp.gainwell.password"}
|
||||
}
|
||||
}'
|
||||
--data @/tmp/ch-patched.json
|
||||
```
|
||||
|
||||
The endpoint hot-reloads the scheduler. No API restart required.
|
||||
|
||||
Reference in New Issue
Block a user