feat(backend): X12 segment tokenizer with ISA delimiter detection

backend/tests/test_segments.py

@@ -0,0 +1,69 @@
+import pytest
+from cyclone.parsers.exceptions import CycloneParseError
+from cyclone.parsers.segments import Delimiters, tokenize
+
+ISA_PREFIX = "ISA*00*          *00*          *ZZ*11525703       *ZZ*COMEDASSISTPROG*260611*0814*"
+
+
+def test_tokenize_parses_isa_delimiters():
+    text = ISA_PREFIX + "*^00501*991102977*1*P*:~"
+    segments = tokenize(text)
+    assert segments[0][0] == "ISA"
+    assert len(segments) == 1
+
+
+def test_tokenize_splits_segments():
+    text = ISA_PREFIX + "*^00501*991102977*1*P*:~ST*837*0001~SE*2*0001~"
+    segments = tokenize(text)
+    assert [s[0] for s in segments] == ["ISA", "ST", "SE"]
+
+
+def test_tokenize_splits_elements():
+    text = ISA_PREFIX + "*^00501*991102977*1*P*:~NM1*85*2*Acme*****XX*1234567890~"
+    segments = tokenize(text)
+    nm1 = [s for s in segments if s[0] == "NM1"][0]
+    assert nm1[1] == "85"
+    assert nm1[8] == "XX"
+    assert nm1[9] == "1234567890"
+
+
+def test_tokenize_preserves_empty_elements():
+    """Empty elements (e.g. 'NM1*IL*1*Doe*John****MI*X') must stay as empty strings."""
+    text = ISA_PREFIX + "*^00501*991102977*1*P*:~NM1*IL*1*Doe*John****MI*X~"
+    segments = tokenize(text)
+    nm1 = [s for s in segments if s[0] == "NM1"][0]
+    assert nm1[5] == ""  # position 5 is the 4th consecutive empty after suffix
+    assert nm1[6] == ""
+
+
+def test_tokenize_handles_component_separator():
+    """SV101 can be 'HC:99213' which is component-split at ':'."""
+    text = ISA_PREFIX + "*^00501*991102977*1*P*:~SV1*HC:99213*100*UN*1***1~"
+    segments = tokenize(text)
+    sv1 = [s for s in segments if s[0] == "SV1"][0]
+    # SV101 is element index 1; components 1, 2 are 'HC', '99213'
+    assert sv1[1] == "HC:99213"
+
+
+def test_tokenize_strips_trailing_crlf():
+    text = ISA_PREFIX + "*^00501*991102977*1*P*:~\r\n"
+    segments = tokenize(text)
+    assert [s[0] for s in segments] == ["ISA"]
+
+
+def test_tokenize_raises_on_missing_isa():
+    with pytest.raises(CycloneParseError, match="ISA segment not found"):
+        tokenize("hello world")
+
+
+def test_tokenize_raises_on_short_isa():
+    with pytest.raises(CycloneParseError, match="ISA segment too short"):
+        tokenize("ISA*00*00~")
+
+
+def test_delimiters_dataclass():
+    d = Delimiters(element="*", repetition="^", component=":", segment="~")
+    assert d.element == "*"
+    assert d.repetition == "^"
+    assert d.component == ":"
+    assert d.segment == "~"