feat(sp27): surface consecutive_failures + last_error in Scheduler.status()

backend/src/cyclone/scheduler.py

@@ -95,6 +95,12 @@ class TickResult:
     files_skipped: int = 0
     files_errored: int = 0
     errors: list[str] = field(default_factory=list)
+    # SP27 Task 9: ``sftp_failed`` is the discriminator that separates
+    # SFTP-side failures (the listing step) from per-file processing
+    # errors. ``tick()`` keys off this flag — NOT ``result.errors`` —
+    # so a single malformed 999 cannot flip the operator's destructive
+    # pill. Set in ``_tick_impl``'s listing-error branches only.
+    sftp_failed: bool = False
 
     def as_dict(self) -> dict[str, Any]:
         return {
@@ -107,12 +113,22 @@ class TickResult:
             "files_skipped": self.files_skipped,
             "files_errored": self.files_errored,
             "errors": list(self.errors),
+            "sftp_failed": self.sftp_failed,
         }
 
 
 @dataclass
 class SchedulerStatus:
-    """Snapshot of the scheduler's runtime state."""
+    """Snapshot of the scheduler's runtime state.
+
+    SP27 Task 9 added the four SFTP-error fields
+    (``consecutive_failures``, ``last_error``, ``last_error_at``,
+    ``last_sftp_attempt_at``) so the operator's UI can tell "all
+    quiet on the MFT front" from "we've been unable to reach the
+    MFT server for 3 hours". The previous 06/25 incident went
+    unnoticed because the status dict had no signal for a hung
+    SFTP poll.
+    """
 
     running: bool
     poll_interval_seconds: int
@@ -123,6 +139,13 @@ class SchedulerStatus:
     total_skipped: int
     total_errored: int
     last_tick: Optional[TickResult] = None
+    # SP27 Task 9: SFTP-error surfacing. The UI flips to a destructive
+    # pill when ``consecutive_failures >= 3`` (``CYCLONE_SCHEDULER_FAIL_THRESHOLD``
+    # in the operator pill config; not enforced server-side).
+    consecutive_failures: int = 0
+    last_error: Optional[str] = None
+    last_error_at: Optional[datetime] = None
+    last_sftp_attempt_at: Optional[datetime] = None
 
     def as_dict(self) -> dict[str, Any]:
         return {
@@ -137,6 +160,15 @@ class SchedulerStatus:
             "total_skipped": self.total_skipped,
             "total_errored": self.total_errored,
             "last_tick": self.last_tick.as_dict() if self.last_tick else None,
+            "consecutive_failures": self.consecutive_failures,
+            "last_error": self.last_error,
+            "last_error_at": (
+                self.last_error_at.isoformat() if self.last_error_at else None
+            ),
+            "last_sftp_attempt_at": (
+                self.last_sftp_attempt_at.isoformat()
+                if self.last_sftp_attempt_at else None
+            ),
         }
 
 
@@ -231,6 +263,40 @@ class Scheduler:
         # ticks stack up; the next tick fires only after the previous
         # one finishes).
         self._tick_in_progress = False
+        # SP27 Task 9: SFTP-error surfacing. ``_consecutive_failures``
+        # counts back-to-back tick failures (cleared on the next
+        # successful tick). The other three fields are the most
+        # recent failure for the operator's pill — preserved across
+        # successes so the audit trail doesn't disappear the moment
+        # the MFT server recovers.
+        self._consecutive_failures = 0
+        self._last_error: Optional[str] = None
+        self._last_error_at: Optional[datetime] = None
+        self._last_sftp_attempt_at: Optional[datetime] = None
+
+    def _record_sftp_outcome(
+        self, *, success: bool, error: Optional[str] = None,
+    ) -> None:
+        """Update the SFTP-error state after a tick.
+
+        ``success=True`` resets the consecutive-failure counter; the
+        last-error fields are preserved as an audit trail (the
+        operator's UI can still surface "last failure: 2 hours ago"
+        after a recovery).
+
+        ``success=False`` bumps the counter and records the error
+        message + timestamp. ``last_sftp_attempt_at`` is bumped in
+        both cases so the operator can tell when the scheduler last
+        *tried* to reach the MFT (not just when it last failed).
+        """
+        now = datetime.now(timezone.utc)
+        self._last_sftp_attempt_at = now
+        if success:
+            self._consecutive_failures = 0
+            return
+        self._consecutive_failures += 1
+        self._last_error = error
+        self._last_error_at = now
 
     # ---- Public API -------------------------------------------------------
 
@@ -278,6 +344,10 @@ class Scheduler:
             total_skipped=self._total_skipped,
             total_errored=self._total_errored,
             last_tick=self._last_tick,
+            consecutive_failures=self._consecutive_failures,
+            last_error=self._last_error,
+            last_error_at=self._last_error_at,
+            last_sftp_attempt_at=self._last_sftp_attempt_at,
         )
 
     def is_running(self) -> bool:
@@ -291,6 +361,13 @@ class Scheduler:
         SFTP server from a stampede when the operator hits
         ``/api/admin/scheduler/tick`` while a scheduled tick is
         already running.
+
+        SP27 Task 9: the tick outcome is also recorded on the
+        scheduler's SFTP-error state via
+        :meth:`_record_sftp_outcome`. The discriminator is
+        ``TickResult.sftp_failed`` (set by ``_tick_impl``'s
+        listing-error branches only), not ``TickResult.errors``
+        which is also populated by per-file processing errors.
         """
         while self._tick_in_progress:
             await asyncio.sleep(0.05)
@@ -303,6 +380,20 @@ class Scheduler:
             self._total_processed += result.files_processed
             self._total_skipped += result.files_skipped
             self._total_errored += result.files_errored
+            # SFTP-error surfacing: discriminator is ``result.sftp_failed``,
+            # NOT ``result.errors``. ``sftp_failed`` is set by
+            # ``_tick_impl``'s listing-error branches (timeout, connect
+            # refused). Per-file processing errors append to
+            # ``result.errors`` but do NOT set ``sftp_failed`` — a
+            # single malformed 999 in a 5-file batch must not flip
+            # the operator's pill to destructive.
+            if result.sftp_failed:
+                self._record_sftp_outcome(
+                    success=False,
+                    error="; ".join(result.errors),
+                )
+            else:
+                self._record_sftp_outcome(success=True)
             return result
         finally:
             self._tick_in_progress = False
@@ -323,6 +414,11 @@ class Scheduler:
         Honors ``_stop_event`` between files. Concurrent calls are
         coalesced the same way :meth:`tick` does, so a slow handler
         can't be stampeded by a parallel admin invocation.
+
+        SP27 Task 9: deliberately does NOT update the SFTP-error
+        state (no listing, no SFTP). The consecutive-failure counter
+        is the signal for scheduled SFTP polling; operator-initiated
+        batches shouldn't be able to flip it.
         """
         while self._tick_in_progress:
             await asyncio.sleep(0.05)
@@ -386,11 +482,13 @@ class Scheduler:
         except asyncio.TimeoutError:
             log.exception("SFTP list_inbound timed out")
             result.errors.append("list_inbound: timeout")
+            result.sftp_failed = True
             result.finished_at = datetime.now(timezone.utc)
             return result
         except Exception as exc:  # noqa: BLE001
             log.exception("SFTP list_inbound failed")
             result.errors.append(f"list_inbound: {exc}")
+            result.sftp_failed = True
             result.finished_at = datetime.now(timezone.utc)
             return result