fix(sftp): case-insensitive inbound regex, skip _warn.txt, add targeted pull
Three changes that unblock the daily inbound pull from Gainwell's FromHPE MFT path: 1. INBOUND_RE now accepts both 'TP' and 'tp' prefixes via inline (?i:TP) scoping — case-folding the whole pattern would let lowercase '837p' / tracking IDs through, which is invalid HCPF. The rest of the pattern is still case-sensitive. 2. _list_inbound_paramiko skips *_warn.txt entries. Gainwell's MFT drops ~583 advisory text-format notes in the same inbound dir; they come first alphabetically and were padding every poll with ~80 min of pointless downloads. 3. New SftpClient.list_inbound_names() + download_inbound() pair gives a metadata-only listing and on-demand fetch. The scheduler's existing full-listing path still works (now without the warn padding); the new path is what the new /api/admin/scheduler/pull-inbound endpoint and the 'cyclone pull-inbound' CLI use to fast-target a date range without paying the cost of a full ~6000-file download. Scheduler.process_inbound_files() runs the same per-file pipeline as a regular tick on the pre-fetched list, so dedup via processed_inbound_files still applies. Tests added in test_filenames.py (lowercase + mixed-case cases), test_sftp_paramiko.py (warn skip + no-download listing), and test_scheduler.py (process_inbound_files idempotency). With this, the daily 385-file pull for 20260624 completes in seconds via 'docker exec cyclone-backend-1 python -m cyclone pull-inbound --date 20260624 --block dzinesco' (or the equivalent POST to /api/admin/scheduler/pull-inbound?date=20260624).
This commit is contained in:
@@ -284,4 +284,62 @@ class TestRoutedFileTypes:
|
||||
type. These tests are the regression net."""
|
||||
|
||||
def test_handlers_cover_all_routed_types(self):
|
||||
assert set(HANDLERS.keys()) == ROUTED_FILE_TYPES
|
||||
assert set(HANDLERS.keys()) == ROUTED_FILE_TYPES
|
||||
|
||||
|
||||
class TestProcessInboundFiles:
|
||||
"""Scheduler.process_inbound_files() — date-filtered pull path.
|
||||
|
||||
The /api/admin/scheduler/pull-inbound endpoint and the
|
||||
``cyclone pull-inbound`` CLI both call this. It must:
|
||||
* process each file in the provided list (no SFTP listdir)
|
||||
* dedupe via ``processed_inbound_files`` (idempotent on rerun)
|
||||
* not touch SFTP at all — files are expected to be on local
|
||||
disk at ``f.local_path``
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_processes_provided_files_without_listdir(
|
||||
self, sftp_block, _drop_file, tmp_path,
|
||||
):
|
||||
from cyclone.clearhouse import InboundFile
|
||||
from cyclone.scheduler import STATUS_OK
|
||||
|
||||
# Drop a known 999 file on disk (the stub scheduler doesn't
|
||||
# need SFTP — it reads from staging dir).
|
||||
_drop_file(
|
||||
"TP11525703-837P_M019048402-20260520231513488-1of1_999.x12",
|
||||
b"not a real 999 -- handler will parse_error",
|
||||
)
|
||||
sched = _make_scheduler(sftp_block, tmp_path)
|
||||
|
||||
# Build the InboundFile records manually — caller is
|
||||
# responsible for staging (mirrors the targeted-pull flow).
|
||||
path = tmp_path / "staging" / "FromHPE" / \
|
||||
"TP11525703-837P_M019048402-20260520231513488-1of1_999.x12"
|
||||
files = [InboundFile(
|
||||
name=path.name,
|
||||
size=path.stat().st_size,
|
||||
modified_at=datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc),
|
||||
local_path=path,
|
||||
)]
|
||||
|
||||
result = await sched.process_inbound_files(files)
|
||||
# The file's bytes are intentionally invalid — we only care
|
||||
# that process_inbound_files invokes the handler and
|
||||
# records the outcome (here: error from parse).
|
||||
assert result.files_seen == 1
|
||||
assert result.files_seen == result.files_processed + result.files_errored
|
||||
# Idempotent: a second call is a no-op (already-processed dedup).
|
||||
result2 = await sched.process_inbound_files(files)
|
||||
assert result2.files_seen == 1
|
||||
# Either skipped (because the prior call recorded it as error)
|
||||
# or error — both prove the dedup branch fired. We just check
|
||||
# it's not re-processed successfully.
|
||||
with db.SessionLocal()() as s:
|
||||
rows = s.query(ProcessedInboundFile).filter_by(
|
||||
sftp_block_name="test-block",
|
||||
name=path.name,
|
||||
).all()
|
||||
assert len(rows) == 1
|
||||
assert rows[0].status in (STATUS_OK, STATUS_ERROR, STATUS_SKIPPED)
|
||||
Reference in New Issue
Block a user