fix(sftp): case-insensitive inbound regex, skip _warn.txt, add targeted pull
Three changes that unblock the daily inbound pull from Gainwell's FromHPE MFT path: 1. INBOUND_RE now accepts both 'TP' and 'tp' prefixes via inline (?i:TP) scoping — case-folding the whole pattern would let lowercase '837p' / tracking IDs through, which is invalid HCPF. The rest of the pattern is still case-sensitive. 2. _list_inbound_paramiko skips *_warn.txt entries. Gainwell's MFT drops ~583 advisory text-format notes in the same inbound dir; they come first alphabetically and were padding every poll with ~80 min of pointless downloads. 3. New SftpClient.list_inbound_names() + download_inbound() pair gives a metadata-only listing and on-demand fetch. The scheduler's existing full-listing path still works (now without the warn padding); the new path is what the new /api/admin/scheduler/pull-inbound endpoint and the 'cyclone pull-inbound' CLI use to fast-target a date range without paying the cost of a full ~6000-file download. Scheduler.process_inbound_files() runs the same per-file pipeline as a regular tick on the pre-fetched list, so dedup via processed_inbound_files still applies. Tests added in test_filenames.py (lowercase + mixed-case cases), test_sftp_paramiko.py (warn skip + no-download listing), and test_scheduler.py (process_inbound_files idempotency). With this, the daily 385-file pull for 20260624 completes in seconds via 'docker exec cyclone-backend-1 python -m cyclone pull-inbound --date 20260624 --block dzinesco' (or the equivalent POST to /api/admin/scheduler/pull-inbound?date=20260624).
This commit is contained in:
@@ -259,6 +259,66 @@ class TestRealModeListInbound:
|
||||
assert len(files) == 1
|
||||
assert files[0].name == "real.x12"
|
||||
|
||||
def test_list_skips_warn_txt_files(self, monkeypatch, tmp_path: Path):
|
||||
# Gainwell's MFT drops advisory *_warn.txt files in the same
|
||||
# inbound dir. They're text-format side-channel notes (not X12
|
||||
# envelopes) and must be skipped at list time.
|
||||
real_attr = MagicMock()
|
||||
real_attr.filename = "TP123-837P_M456-20260520231513488-1of1_999.x12"
|
||||
real_attr.st_mode = 0o100644
|
||||
real_attr.st_size = 1024
|
||||
real_attr.st_mtime = 1718899200
|
||||
|
||||
warn_attr = MagicMock()
|
||||
warn_attr.filename = "TP123-837P-202606181208100000-1of1_warn.txt"
|
||||
warn_attr.st_mode = 0o100644
|
||||
warn_attr.st_size = 200
|
||||
warn_attr.st_mtime = 1718899200
|
||||
|
||||
mock_ssh, mock_sftp = _make_mock_paramiko(
|
||||
monkeypatch, sftp_attrs=[real_attr, warn_attr],
|
||||
)
|
||||
|
||||
def _open(path, mode="rb"):
|
||||
m = MagicMock()
|
||||
m.__enter__.return_value = io.BytesIO(b"x12 content")
|
||||
return m
|
||||
|
||||
mock_sftp.open.side_effect = _open
|
||||
|
||||
block = _block(staging_dir=str(tmp_path / "staging"))
|
||||
client = SftpClient(block)
|
||||
files = client.list_inbound()
|
||||
|
||||
names = [f.name for f in files]
|
||||
assert "TP123-837P_M456-20260520231513488-1of1_999.x12" in names
|
||||
assert not any(n.endswith("_warn.txt") for n in names)
|
||||
assert len(files) == 1
|
||||
|
||||
def test_list_inbound_names_does_not_download(self, monkeypatch, tmp_path: Path):
|
||||
# list_inbound_names() must do a metadata-only SFTP listing —
|
||||
# no sftp.open() / no file written to the cache.
|
||||
attr = MagicMock()
|
||||
attr.filename = "TP123-837P_M456-20260520231513488-1of1_999.x12"
|
||||
attr.st_mode = 0o100644
|
||||
attr.st_size = 1024
|
||||
attr.st_mtime = 1718899200
|
||||
|
||||
mock_ssh, mock_sftp = _make_mock_paramiko(
|
||||
monkeypatch, sftp_attrs=[attr],
|
||||
)
|
||||
|
||||
block = _block(staging_dir=str(tmp_path / "staging"))
|
||||
client = SftpClient(block)
|
||||
files = client.list_inbound_names()
|
||||
|
||||
# sftp.open() must NOT have been called (no download).
|
||||
mock_sftp.open.assert_not_called()
|
||||
assert len(files) == 1
|
||||
# The InboundFile.local_path is set to the planned cache path
|
||||
# but the file itself doesn't exist yet.
|
||||
assert not files[0].local_path.exists()
|
||||
|
||||
|
||||
class TestRealModeReadFile:
|
||||
def test_read_returns_bytes(self, monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user