test: isolate pipeline-stage branches only hit via the full corpus

Several branches were only reached through tests/v2/test_cases.py's
full-pipeline corpus, so a regression scoped to one stage wouldn't be
pinpointed by that stage's own test file: assign's SUFFIX_COMMA
structure, segment's cosmetic-vs-structural empty-bucket handling
(single trailing comma vs. a leading comma), group's
extra_suffix_delimiters entry-splitting in a FAMILY_COMMA tail
segment, and tokenize's interleaving of two simultaneously active
extracted regions (review finding on #288).

Author: derek73

tests/v2/pipeline/test_assign.py

@@ -101,6 +101,19 @@ def test_suffix_comma_and_extra_segments() -> None:
     assert _by_role(out, Role.SUFFIX) == "Jr."
 
 
+def test_suffix_comma_structure() -> None:
+    # Structure.SUFFIX_COMMA (segment.py): >1 word before the first
+    # comma AND every post-first segment is entirely lenient-suffix.
+    # ('Smith, John, Jr.' above only has ONE word before its first
+    # comma, so it is FAMILY_COMMA with a trailing suffix segment, not
+    # this branch.) Previously this assign() branch (tail=1) was only
+    # reached via the full-corpus test, not its own isolated case.
+    out = _assigned("John Smith, PhD")
+    assert _by_role(out, Role.GIVEN) == "John"
+    assert _by_role(out, Role.FAMILY) == "Smith"
+    assert _by_role(out, Role.SUFFIX) == "PhD"
+
+
 def test_trailing_suffix_run_no_comma() -> None:
     out = _assigned("John Jack Kennedy PhD MD")
     assert _by_role(out, Role.MIDDLE) == "Jack"

tests/v2/pipeline/test_group.py

@@ -22,8 +22,10 @@
 # real vocabulary overlap; the subset rule only constrains particles.
 
 
-def _grouped(text: str) -> ParseState:
-    state = ParseState(original=text, lexicon=_LEX, policy=Policy())
+def _grouped(text: str, policy: Policy | None = None,
+             lexicon: Lexicon | None = None) -> ParseState:
+    state = ParseState(original=text, lexicon=lexicon or _LEX,
+                       policy=policy or Policy())
     return group(classify(segment(tokenize(extract_delimited(state)))))
 
 
@@ -128,6 +130,30 @@ def test_initials_do_not_count_as_rootnames_for_conjunction_carveout() -> None:
     assert _piece_texts(out) == [["J.", "Ruiz", "y", "Gomez"]]
 
 
+def test_extra_suffix_delimiter_splits_tail_entries() -> None:
+    # v1 expand_suffix_delimiter parity (#191): a configured delimiter
+    # is transparent in a FAMILY_COMMA tail segment -- it separates
+    # suffix ENTRIES (each rendered independently) and is itself
+    # dropped, rather than becoming a suffix token or fusing the
+    # entries on either side into one "joined" run.
+    lex = Lexicon(suffix_acronyms=frozenset({"phd"}),
+                  suffix_words=frozenset({"jr", "v", "md"}))
+    out = _grouped("Smith, John, V MD / PhD",
+                   Policy(extra_suffix_delimiters=frozenset({"/"})),
+                   lexicon=lex)
+    # one tail SEGMENT, split into three suffix pieces: "/" is dropped
+    # rather than surviving as its own piece
+    assert _piece_texts(out) == [["Smith"], ["John"], ["V", "MD", "PhD"]]
+    slash_idx = next(i for i, t in enumerate(out.tokens) if t.text == "/")
+    assert slash_idx in out.dropped
+    # "MD" continues the "V MD" entry (joined); "PhD" starts a fresh
+    # entry across the dropped delimiter, so it does NOT get "joined"
+    md_tok = next(t for t in out.tokens if t.text == "MD")
+    phd_tok = next(t for t in out.tokens if t.text == "PhD")
+    assert "joined" in md_tok.tags
+    assert "joined" not in phd_tok.tags
+
+
 def test_suffix_comma_name_segment_gets_no_additional_count() -> None:
     # v1 parity: additional_parts_count applies to FAMILY_COMMA parts only;
     # ', PhD' must not tip the single-letter-conjunction carve-out

tests/v2/pipeline/test_segment.py

@@ -78,6 +78,25 @@ def test_comma_only_input_is_no_comma_structure() -> None:
     assert out.segments == ()
 
 
+def test_single_trailing_comma_is_cosmetic_and_dropped() -> None:
+    # v1 parity: exactly ONE trailing comma is cosmetic and stripped
+    # (segment.py's "pop the trailing empty bucket" step), so
+    # 'Smith, John,' segments identically to 'Smith, John' -- unlike a
+    # genuinely empty INTERIOR bucket ('Doe,, Jr.'), which stays
+    # structural and keeps its position.
+    out = _segmented("Smith, John,")
+    assert out.structure is Structure.FAMILY_COMMA
+    assert [_texts(out, s) for s in out.segments] == [["Smith"], ["John"]]
+
+
+def test_leading_comma_yields_empty_first_segment() -> None:
+    # A leading comma produces a non-trailing empty bucket, which is
+    # structural (not cosmetic) and keeps its position as segment 0.
+    out = _segmented(",John Smith")
+    assert out.structure is Structure.FAMILY_COMMA
+    assert [_texts(out, s) for s in out.segments] == [[], ["John", "Smith"]]
+
+
 def test_strict_comma_suffixes_veto_lenient_only_members() -> None:
     # lenient_comma_suffixes=False: the post-comma test drops back to
     # the strict predicate, so initial-shaped suffix words no longer

tests/v2/pipeline/test_tokenize.py

@@ -52,6 +52,25 @@ def test_extracted_regions_are_skipped_and_tokenized_with_role() -> None:
     assert starts == sorted(starts)
 
 
+def test_two_simultaneous_extracted_regions_interleave_correctly() -> None:
+    # tokenize's masked/extracted loops each run over ALL their regions
+    # before the final span-sort; every other test here exercises at
+    # most one masked region at a time. With two active extraction
+    # types (nickname parens AND a configured maiden bracket) on the
+    # same string, main-stream and both extracted regions must still
+    # interleave into the correct overall span order.
+    pol = Policy(maiden_delimiters=frozenset({("[", "]")}))
+    out = _tokenized("John (Jack) [Doe] Smith", pol)
+    assert [(t.text, t.role) for t in out.tokens] == [
+        ("John", None),
+        ("Jack", Role.NICKNAME),
+        ("Doe", Role.MAIDEN),
+        ("Smith", None),
+    ]
+    starts = [t.span.start for t in out.tokens]
+    assert starts == sorted(starts)
+
+
 def test_comma_inside_extracted_region_is_not_an_offset() -> None:
     out = _tokenized('John "Jack, Jr" Kim')
     assert out.comma_offsets == ()