fix(sftp): case-insensitive inbound regex, skip _warn.txt, add targeted pull
Three changes that unblock the daily inbound pull from Gainwell's FromHPE MFT path: 1. INBOUND_RE now accepts both 'TP' and 'tp' prefixes via inline (?i:TP) scoping — case-folding the whole pattern would let lowercase '837p' / tracking IDs through, which is invalid HCPF. The rest of the pattern is still case-sensitive. 2. _list_inbound_paramiko skips *_warn.txt entries. Gainwell's MFT drops ~583 advisory text-format notes in the same inbound dir; they come first alphabetically and were padding every poll with ~80 min of pointless downloads. 3. New SftpClient.list_inbound_names() + download_inbound() pair gives a metadata-only listing and on-demand fetch. The scheduler's existing full-listing path still works (now without the warn padding); the new path is what the new /api/admin/scheduler/pull-inbound endpoint and the 'cyclone pull-inbound' CLI use to fast-target a date range without paying the cost of a full ~6000-file download. Scheduler.process_inbound_files() runs the same per-file pipeline as a regular tick on the pre-fetched list, so dedup via processed_inbound_files still applies. Tests added in test_filenames.py (lowercase + mixed-case cases), test_sftp_paramiko.py (warn skip + no-download listing), and test_scheduler.py (process_inbound_files idempotency). With this, the daily 385-file pull for 20260624 completes in seconds via 'docker exec cyclone-backend-1 python -m cyclone pull-inbound --date 20260624 --block dzinesco' (or the equivalent POST to /api/admin/scheduler/pull-inbound?date=20260624).
This commit is contained in:
@@ -107,6 +107,48 @@ def test_parse_inbound_277():
|
||||
assert parsed.file_type == "277"
|
||||
|
||||
|
||||
def test_parse_inbound_lowercase_tp_prefix_999():
|
||||
# Gainwell's production filer uses lowercase `tp` for inbound 999/TA1.
|
||||
# The inbound regex must accept both casings on the TP prefix.
|
||||
name = "tp11525703-837P_M019048402-20260520231513488-1of1_999.x12"
|
||||
parsed = parse_inbound_filename(name)
|
||||
assert parsed.tpid == "11525703"
|
||||
assert parsed.orig_tx == "837P"
|
||||
assert parsed.tracking == "M019048402"
|
||||
assert parsed.file_type == "999"
|
||||
assert parsed.ext == "x12"
|
||||
|
||||
|
||||
def test_parse_inbound_lowercase_tp_prefix_ta1():
|
||||
name = "tp11525703-837P_M019044969-20260520180505477-1of1_TA1.x12"
|
||||
parsed = parse_inbound_filename(name)
|
||||
assert parsed.file_type == "TA1"
|
||||
|
||||
|
||||
def test_is_inbound_filename_accepts_both_cases():
|
||||
# is_inbound_filename() is the fast path used by the scheduler to
|
||||
# filter the listing. It must accept both casings.
|
||||
upper = "TP11525703-837P_M019048402-20260520231513488-1of1_999.x12"
|
||||
lower = "tp11525703-837P_M019048402-20260520231513488-1of1_999.x12"
|
||||
assert is_inbound_filename(upper)
|
||||
assert is_inbound_filename(lower)
|
||||
|
||||
|
||||
def test_parse_inbound_rejects_mixed_case_tracking():
|
||||
# Tracking value must stay uppercase alnum; the case-insensitive
|
||||
# flag is intentionally scoped to the TP prefix by the file_type
|
||||
# and timestamp constraints, so a mixed-case tracking should still
|
||||
# be rejected (it'd be invalid HCPF).
|
||||
# We exercise the obvious "totally lowercase" rejection to confirm
|
||||
# the rest of the pattern is still strict.
|
||||
with pytest.raises(ValueError, match="Not a valid HCPF inbound"):
|
||||
# Lowercase orig_tx; the orig_tx class is [A-Z0-9]+ so it
|
||||
# must be uppercase.
|
||||
parse_inbound_filename(
|
||||
"tp11525703-837p_M019048402-20260520231513488-1of1_999.x12"
|
||||
)
|
||||
|
||||
|
||||
def test_parse_inbound_rejects_missing_tp_prefix():
|
||||
with pytest.raises(ValueError, match="Not a valid HCPF inbound"):
|
||||
parse_inbound_filename("11525703-837P_M019048402-20260520231513488-1of1_999.x12")
|
||||
@@ -157,8 +199,10 @@ def test_is_outbound_filename():
|
||||
|
||||
def test_is_inbound_filename():
|
||||
assert is_inbound_filename("TP11525703-837P_M019048402-20260520231513488-1of1_999.x12")
|
||||
# Lowercase tp prefix is the outbound shape, not inbound
|
||||
assert not is_inbound_filename("tp11525703-837P_M019048402-20260520231513488-1of1_999.x12")
|
||||
# Lowercase tp prefix is now accepted too — Gainwell's filer has
|
||||
# used both casings on inbound 999/TA1 files.
|
||||
assert is_inbound_filename("tp11525703-837P_M019048402-20260520231513488-1of1_999.x12")
|
||||
# Outbound shape is still rejected (no tracking/ts/file_type).
|
||||
assert not is_inbound_filename("11525703-837P-20260620132243505-1of1.x12")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user