feat(sp16): live MFT polling scheduler

Adds an asyncio-based background scheduler that polls the Gainwell
MFT inbound path, downloads new files, and routes them through the
appropriate parser (999 / 835 / 277CA / TA1). Idempotent (re-ticks
and restarts skip already-processed files via the new
processed_inbound_files table). Crash-safe (per-file try/except so
one bad file doesn't stop the loop).

Lifespan auto-configures from the seeded dzinesco clearhouse's SFTP
block; auto-start is opt-in via CYCLONE_SCHEDULER_AUTOSTART.

Five admin endpoints added:
  GET  /api/admin/scheduler/status
  POST /api/admin/scheduler/start
  POST /api/admin/scheduler/stop
  POST /api/admin/scheduler/tick
  GET  /api/admin/scheduler/processed-files?status=&limit=

20 new tests (15 unit + 5 API).
This commit is contained in:
Tyler
2026-06-21 09:20:58 -06:00
parent 1267a341e3
commit 40f184c858
10 changed files with 1424 additions and 11 deletions
+152
View File
@@ -0,0 +1,152 @@
"""SP16 — Admin scheduler API endpoint tests.
The endpoints under /api/admin/scheduler/* are thin wrappers around
:class:`cyclone.scheduler.Scheduler`. These tests exercise them via
the FastAPI TestClient to confirm wiring (auth-free admin endpoints
work, response shapes match, idempotency holds).
"""
from __future__ import annotations
import asyncio
import json
from pathlib import Path
import pytest
@pytest.fixture
def _stub_scheduler_env(tmp_path, monkeypatch):
"""Set up: a stub-mode SFTP block, scheduler configured.
Yields (staging_dir, scheduler_singleton). We deliberately do
NOT enable SQLCipher encryption in this fixture — the scheduler
doesn't care about encryption, and patching ``db_crypto.get_secret``
here would cause the lifespan handler to rebuild the engine with
SQLCipher on a plain-SQLite test file (which raises "file is not
a database"). The encryption-at-rest tests live in
``test_db_crypto.py``.
"""
from cyclone import db
from cyclone import scheduler as sched_mod
from cyclone.providers import SftpBlock
monkeypatch.setenv("CYCLONE_DB_URL", f"sqlite:///{tmp_path}/test.db")
db._reset_for_tests()
staging = tmp_path / "staging"
inbound = staging / "ToHPE"
inbound.mkdir(parents=True)
sftp_block = SftpBlock(
host="mft.example.com",
port=22,
username="test",
paths={"outbound": "/FromHPE", "inbound": "/ToHPE"},
stub=True,
staging_dir=str(staging),
poll_seconds=60,
auth={"method": "keychain", "secret_ref": "test.password"},
)
sched_mod.reset_scheduler_for_tests()
sched = sched_mod.configure_scheduler(sftp_block, sftp_block_name="t")
yield staging, sched
sched_mod.reset_scheduler_for_tests()
db._reset_for_tests()
def _drop_file(staging: Path, name: str, body: bytes) -> Path:
p = staging / "ToHPE" / name
p.write_bytes(body)
return p
def test_scheduler_status_starts_not_running(_stub_scheduler_env):
from fastapi.testclient import TestClient
from cyclone.api import app
_, sched = _stub_scheduler_env
with TestClient(app) as client:
r = client.get("/api/admin/scheduler/status")
assert r.status_code == 200
body = r.json()
assert body["running"] is False
assert body["poll_interval_seconds"] == 60
assert body["sftp_block_name"] == "t"
def test_scheduler_start_then_status_then_stop(_stub_scheduler_env):
from fastapi.testclient import TestClient
from cyclone.api import app
with TestClient(app) as client:
r1 = client.post("/api/admin/scheduler/start")
assert r1.status_code == 200
assert r1.json()["status"]["running"] is True
r2 = client.get("/api/admin/scheduler/status")
assert r2.json()["running"] is True
r3 = client.post("/api/admin/scheduler/stop")
assert r3.status_code == 200
assert r3.json()["status"]["running"] is False
def test_scheduler_tick_processes_one_file(_stub_scheduler_env):
from fastapi.testclient import TestClient
from cyclone.api import app
staging, _ = _stub_scheduler_env
_drop_file(
staging,
"TP11525703-837P_M019048402-20260618130000000-1of1_TA1.x12",
(Path(__file__).parent / "fixtures" / "minimal_ta1.txt").read_bytes(),
)
with TestClient(app) as client:
r = client.post("/api/admin/scheduler/tick")
assert r.status_code == 200
body = r.json()
assert body["ok"] is True
assert body["tick"]["files_seen"] == 1
assert body["tick"]["files_processed"] == 1
def test_scheduler_processed_files_lists_history(_stub_scheduler_env):
from fastapi.testclient import TestClient
from cyclone.api import app
staging, _ = _stub_scheduler_env
_drop_file(
staging,
"TP11525703-837P_M019048402-20260618130000000-1of1_TA1.x12",
(Path(__file__).parent / "fixtures" / "minimal_ta1.txt").read_bytes(),
)
with TestClient(app) as client:
client.post("/api/admin/scheduler/tick")
r = client.get("/api/admin/scheduler/processed-files")
assert r.status_code == 200
body = r.json()
assert body["count"] == 1
f = body["files"][0]
assert f["status"] == "ok"
assert f["parser_used"] == "parse_ta1"
assert "TP11525703" in f["name"]
def test_scheduler_processed_files_filters_by_status(_stub_scheduler_env):
from fastapi.testclient import TestClient
from cyclone.api import app
staging, _ = _stub_scheduler_env
# Drop a file with a type Cyclone doesn't parse — gets recorded as
# "skipped".
_drop_file(
staging,
"TP11525703-837P_M019048402-20260618130000000-1of1_270.x12",
b"some bytes",
)
with TestClient(app) as client:
client.post("/api/admin/scheduler/tick")
r_all = client.get("/api/admin/scheduler/processed-files")
r_skipped = client.get(
"/api/admin/scheduler/processed-files?status=skipped",
)
r_ok = client.get(
"/api/admin/scheduler/processed-files?status=ok",
)
assert r_all.json()["count"] == 1
assert r_skipped.json()["count"] == 1
assert r_ok.json()["count"] == 0