feat(backend): X12 segment tokenizer with ISA delimiter detection

backend/src/cyclone/parsers/segments.py

@@ -0,0 +1,80 @@
+"""X12 segment tokenizer.
+
+The ISA segment is a fixed 106-character record (101 + segment terminator) that
+declares the four delimiters used in the rest of the file:
+
+- element separator (3rd char, 0-indexed)
+- repetition separator (82nd char, 0-indexed; ISA11)
+- component separator (104th char, 0-indexed; ISA16)
+- segment terminator (105th char, 0-indexed)
+
+We do not split components here — that is left to the caller, since most X12
+elements are simple and splitting unconditionally would corrupt data.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from cyclone.parsers.exceptions import CycloneParseError
+
+
+@dataclass(frozen=True)
+class Delimiters:
+    element: str       # usually "*"
+    repetition: str    # usually "^"
+    component: str     # usually ":"
+    segment: str       # usually "~"
+
+
+def _detect_delimiters(isa_segment: str) -> Delimiters:
+    """Pull the four delimiters from the fixed-width ISA segment.
+
+    The ISA segment must be exactly 106 characters (including the terminator).
+    """
+    if len(isa_segment) < 106:
+        raise CycloneParseError(
+            f"ISA segment too short: expected 106 chars, got {len(isa_segment)}"
+        )
+    return Delimiters(
+        element=isa_segment[3],
+        repetition=isa_segment[82],
+        component=isa_segment[104],
+        segment=isa_segment[105],
+    )
+
+
+def tokenize(text: str) -> list[list[str]]:
+    """Split a raw X12 document into a list of segments (each a list of elements).
+
+    The first segment must be an ISA. Raises :class:`CycloneParseError` on
+    missing or malformed envelopes.
+    """
+    # Normalize line endings; the segment terminator handles real segmentation.
+    text = text.replace("\r\n", "").replace("\n", "").replace("\r", "")
+
+    # Find the first ISA by scanning for the literal "ISA" + element separator.
+    if not text.startswith("ISA") or len(text) < 4:
+        raise CycloneParseError("ISA segment not found at start of file")
+
+    # The first 106 chars are the ISA segment.
+    isa_segment = text[:106]
+    try:
+        delimiters = _detect_delimiters(isa_segment)
+    except IndexError as exc:  # pragma: no cover - guarded by length check
+        raise CycloneParseError(f"Could not parse ISA delimiters: {exc}") from exc
+
+    # Slice off the ISA, then split the rest on the segment terminator.
+    body = text[106:]
+    raw_segments = body.split(delimiters.segment)
+
+    # Prepend the ISA so callers always see it as the first segment.
+    isa_elements = isa_segment.split(delimiters.element)
+    out: list[list[str]] = [isa_elements]
+
+    for raw in raw_segments:
+        if not raw:
+            continue
+        out.append(raw.split(delimiters.element))
+
+    return out