feat(sp27): surface consecutive_failures + last_error in Scheduler.status()
This commit is contained in:
Nora
@@ -0,0 +1,229 @@
|
||||
"""SP27 Task 9: scheduler status surfaces SFTP failures.
|
||||
|
||||
The 06/25 silent hang exposed a gap: ``Scheduler.status()`` had no
|
||||
signal that polling had stalled. The operator's UI couldn't tell
|
||||
"all quiet on the MFT front" from "we've been unable to reach the
|
||||
MFT server for 3 hours". This pins the fix — the status dict
|
||||
carries ``consecutive_failures``, ``last_error``, ``last_error_at``,
|
||||
and ``last_sftp_attempt_at`` so the operator pill can flip to
|
||||
destructive after 3 (or however many) consecutive failures.
|
||||
|
||||
Discriminator under test: ``TickResult.sftp_failed`` is the flag
|
||||
``tick()`` keys off — NOT ``TickResult.errors``. Per-file processing
|
||||
errors append to ``errors`` but must NOT bump ``consecutive_failures``.
|
||||
The test suite pins both halves of the discriminator (SFTP failure
|
||||
bumps; per-file failure doesn't).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from cyclone.clearhouse import InboundFile
|
||||
from cyclone.providers import SftpBlock
|
||||
from cyclone.scheduler import Scheduler
|
||||
|
||||
|
||||
def _block() -> SftpBlock:
|
||||
return SftpBlock(
|
||||
host="mft.example.com",
|
||||
port=22,
|
||||
username="user",
|
||||
auth={"password_keychain_account": "x"},
|
||||
paths={"inbound": "/inbound", "outbound": "/outbound"},
|
||||
stub=True,
|
||||
)
|
||||
|
||||
|
||||
# ---- pre-state: a fresh scheduler has no errors -------------------------
|
||||
|
||||
|
||||
def test_status_starts_with_zero_failures_and_no_error():
|
||||
"""A fresh scheduler must report no errors — pins that the new
|
||||
fields default cleanly and don't trip on first read."""
|
||||
sched = Scheduler(_block())
|
||||
status = sched.status()
|
||||
assert status.consecutive_failures == 0
|
||||
assert status.last_error is None
|
||||
assert status.last_error_at is None
|
||||
assert status.last_sftp_attempt_at is None
|
||||
|
||||
|
||||
# ---- a failed SFTP call bumps consecutive_failures -----------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_records_last_error_after_failure():
|
||||
"""A failed tick bumps consecutive_failures and records the error.
|
||||
|
||||
Simulates a broken SFTP client. Pins the post-Task-8 behavior
|
||||
where ``_tick_impl`` catches the error and ``tick()`` records
|
||||
it on the scheduler's status.
|
||||
"""
|
||||
class BrokenClient:
|
||||
def list_inbound(self):
|
||||
raise RuntimeError("simulated outage")
|
||||
|
||||
# SP27 Task 8: scheduler calls async_list_inbound() (the
|
||||
# wait_for-wrapped variant). The broken stub raises before
|
||||
# the wait_for wrapper runs — that's a real SFTP failure
|
||||
# either way.
|
||||
async def async_list_inbound(self):
|
||||
raise RuntimeError("simulated outage")
|
||||
|
||||
sched = Scheduler(_block(), sftp_client_factory=lambda b: BrokenClient())
|
||||
|
||||
before = datetime.now(timezone.utc)
|
||||
await sched.tick()
|
||||
after = datetime.now(timezone.utc)
|
||||
|
||||
status = sched.status()
|
||||
assert status.consecutive_failures == 1
|
||||
assert status.last_error is not None
|
||||
assert "simulated outage" in status.last_error
|
||||
# last_error_at is set to roughly the time of the failed tick
|
||||
assert status.last_error_at is not None
|
||||
assert before <= status.last_error_at <= after
|
||||
# last_sftp_attempt_at is also bumped — separate from last_error_at
|
||||
# when the failure was an SFTP-side issue (not a parse error).
|
||||
assert status.last_sftp_attempt_at is not None
|
||||
assert before <= status.last_sftp_attempt_at <= after
|
||||
|
||||
|
||||
# ---- a successful tick clears consecutive_failures -----------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_tick_clears_consecutive_failures():
|
||||
"""A tick that succeeds (or skips cleanly) resets the counter.
|
||||
|
||||
Without this, a transient blip would lock the destructive pill
|
||||
on forever. Pins the recovery path.
|
||||
"""
|
||||
class BrokenClient:
|
||||
def list_inbound(self):
|
||||
raise RuntimeError("transient")
|
||||
|
||||
async def async_list_inbound(self):
|
||||
raise RuntimeError("transient")
|
||||
|
||||
class HealthyClient:
|
||||
def list_inbound(self):
|
||||
return []
|
||||
|
||||
async def async_list_inbound(self):
|
||||
return []
|
||||
|
||||
# First: two failing ticks to bump the counter
|
||||
sched = Scheduler(_block(), sftp_client_factory=lambda b: BrokenClient())
|
||||
await sched.tick()
|
||||
await sched.tick()
|
||||
assert sched.status().consecutive_failures == 2
|
||||
|
||||
# Then: switch to a healthy client and tick again
|
||||
sched._sftp_client_factory = lambda b: HealthyClient()
|
||||
await sched.tick()
|
||||
status = sched.status()
|
||||
assert status.consecutive_failures == 0
|
||||
# last_error is preserved (audit trail) but last_error_at is from
|
||||
# the last failure, not the success.
|
||||
assert status.last_error is not None # the historical error stays
|
||||
assert "transient" in status.last_error
|
||||
|
||||
|
||||
# ---- last_sftp_attempt_at moves on every attempt, not just failures ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_last_sftp_attempt_at_advances_on_every_tick():
|
||||
"""Even a successful tick should bump last_sftp_attempt_at.
|
||||
|
||||
Without this, the operator can't tell when the scheduler last
|
||||
*tried* to reach the MFT — only when it last failed.
|
||||
"""
|
||||
sched = Scheduler(_block()) # default stub returns []
|
||||
before = datetime.now(timezone.utc)
|
||||
await sched.tick()
|
||||
after = datetime.now(timezone.utc)
|
||||
first = sched.status().last_sftp_attempt_at
|
||||
assert first is not None
|
||||
assert before <= first <= after
|
||||
|
||||
# Wait a moment, tick again — last_sftp_attempt_at should move forward.
|
||||
await asyncio.sleep(0.01)
|
||||
await sched.tick()
|
||||
second = sched.status().last_sftp_attempt_at
|
||||
assert second is not None
|
||||
assert second > first
|
||||
|
||||
|
||||
# ---- as_dict surfaces the new fields for the API -----------------------
|
||||
|
||||
|
||||
def test_status_as_dict_includes_new_fields():
|
||||
"""The API endpoint (/api/health or /api/admin/scheduler) reads
|
||||
status.as_dict() — the new fields must be present and serialized."""
|
||||
sched = Scheduler(_block())
|
||||
d = sched.status().as_dict()
|
||||
assert "consecutive_failures" in d
|
||||
assert d["consecutive_failures"] == 0
|
||||
assert "last_error" in d
|
||||
assert d["last_error"] is None
|
||||
assert "last_error_at" in d
|
||||
assert d["last_error_at"] is None
|
||||
assert "last_sftp_attempt_at" in d
|
||||
assert d["last_sftp_attempt_at"] is None
|
||||
|
||||
|
||||
# ---- per-file processing errors must NOT bump consecutive_failures ------
|
||||
#
|
||||
# This pins the SP27 Task 9 discriminator (``TickResult.sftp_failed``
|
||||
# vs. ``TickResult.errors``). Without this test, a future refactor
|
||||
# that broadens ``result.errors`` semantics — or rewires `tick()` to
|
||||
# key off it again — would silently re-break the discrimination.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_per_file_error_does_not_bump_consecutive_failures(monkeypatch):
|
||||
"""Per-file processing errors must NOT flip the destructive pill.
|
||||
|
||||
The listing step succeeds but ``_handle_one`` simulates the
|
||||
real handler's error path: it appends to ``result.errors``
|
||||
(the way ``_handle_one`` does on parser exceptions) but does
|
||||
NOT touch ``result.sftp_failed``. The pill must stay clean.
|
||||
"""
|
||||
|
||||
class FilesClient:
|
||||
async def async_list_inbound(self):
|
||||
return [
|
||||
InboundFile(
|
||||
name="tp123-999_MTRACKING-20260525001606060-1of1.x12",
|
||||
size=100,
|
||||
modified_at=datetime.now(timezone.utc),
|
||||
# ``local_path`` is None because fake_handle_one
|
||||
# never reaches the download step.
|
||||
local_path=None,
|
||||
)
|
||||
]
|
||||
|
||||
sched = Scheduler(_block(), sftp_client_factory=lambda b: FilesClient())
|
||||
|
||||
async def fake_handle_one(f, result):
|
||||
# Mirrors the real ``_handle_one`` parser-failure path:
|
||||
# appends to result.errors, does NOT set sftp_failed.
|
||||
result.errors.append(f"{f.name}: ValueError: garbage")
|
||||
|
||||
monkeypatch.setattr(sched, "_handle_one", fake_handle_one)
|
||||
|
||||
await sched.tick()
|
||||
status = sched.status()
|
||||
|
||||
# Counter stays clean — per-file error is not an SFTP outage.
|
||||
assert status.consecutive_failures == 0
|
||||
# ``last_error`` is reserved for SFTP-side failures.
|
||||
assert status.last_error is None
|
||||
# But ``last_sftp_attempt_at`` moves forward regardless — the
|
||||
# scheduler did try, it just failed to ingest the file.
|
||||
assert status.last_sftp_attempt_at is not None
|
||||
Reference in New Issue
Block a user