Stripped invisibles are transparent to the interpunct flank guard

The B7 flank guard read text[i-1]/text[i+1] raw, so a policy-stripped
invisible adjacent to the dot defeated it: parse("威廉‏·莎士比亚")
left the literal dot in the family field, lost the order suppression,
and the polluted token defeated the script license too. That input is
the real-world case strip_bidi exists for -- RTL documents quoting
Chinese transcriptions get U+200E/200F inserted at script transitions,
in text visually identical to the clean form.

The guard now walks each flank past characters the strip policy
removes (bidi when strip_bidi, emoji when strip_emoji) before
classifying: a character that vanishes from the token stream cannot
occupy a flank position either. Whitespace, commas, and the name-dot
separators remain guard-defeating -- a dot beside a space is not
between characters -- and the region-local bounds are unchanged, so
the pinned edge behavior stays.

Also pins the guard's edge coverage the surviving mutants demanded:
the trailing-dot upper bound (whose loss is an IndexError, not a
wrong answer), the minimal one-char flanks, the one-sided flank
(the AND, and that no spurious offset records), and hangul flanks
(마틴·루터·킹 -- Korean writes interpunct transcriptions too).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: derek73

nameparser/_pipeline/_tokenize.py

@@ -65,6 +65,29 @@
 _classified_char = _script_matcher(*_SCRIPT_RANGES, whole=True)
 
 
+def _is_emoji(ch: str) -> bool:
+    cp = ord(ch)
+    return any(lo <= cp <= hi for lo, hi in _EMOJI_RANGES)
+
+
+def _flank(text: str, i: int, step: int, limit: int,
+           state: ParseState) -> str | None:
+    """The nearest flank character the strip policy would keep, or
+    None at the region bound. Stripped invisibles (bidi, emoji) are
+    TRANSPARENT here: they vanish from the token stream, so they must
+    not occupy a flank position either -- an RTL document quoting a
+    transcription puts U+200F beside the dot in visually identical
+    text. Whitespace and the other separators stay guard-defeating:
+    a dot beside a space is not between characters."""
+    while i != limit:
+        ch = text[i]
+        if not (state.policy.strip_bidi and _BIDI.match(ch)) \
+                and not (state.policy.strip_emoji and _is_emoji(ch)):
+            return ch
+        i += step
+    return None
+
+
 def _ignorable(ch: str, state: ParseState) -> bool:
     if ch.isspace():
         return True
@@ -90,8 +113,7 @@ def _ignorable(ch: str, state: ParseState) -> bool:
     if state.policy.strip_bidi and _BIDI.match(ch):
         return True
     if state.policy.strip_emoji:
-        cp = ord(ch)
-        return any(lo <= cp <= hi for lo, hi in _EMOJI_RANGES)
+        return _is_emoji(ch)
     return False
 
 
@@ -105,17 +127,23 @@ def _tokenize_region(state: ParseState, start: int, end: int,
         ch = text[i]
         is_separator = ch in COMMA_CHARS or _ignorable(ch, state)
         if not is_separator and ch == _INTERPUNCT:
-            # flanks are region-local: a B7 at a region edge has a
-            # masked or absent neighbor and stays token text.
-            # Region-local on purpose and defensively: under the
-            # default delimiters a masked span's edge character is
-            # never classified, so the two bounds are indistinguishable
-            # -- but a custom delimiter ending in a classified
-            # character would otherwise let a B7 split and record
-            # across a mask seam.
-            is_separator = (start < i < end - 1
-                            and _classified_char(text[i - 1])
-                            and _classified_char(text[i + 1]))
+            # flanks are region-local (the _flank walks stop at the
+            # region bounds): a B7 at a region edge has a masked or
+            # absent neighbor and stays token text. Region-local on
+            # purpose and defensively: under the default delimiters a
+            # masked span's edge character is never classified, so
+            # the two bounds are indistinguishable -- but a custom
+            # delimiter ending in a classified character would
+            # otherwise let a B7 split and record across a mask seam.
+            # The flank scan reads raw text like segmentation matching
+            # does (#272's stance): NFD hangul degrades to no-split,
+            # never wrong-split -- classification NFC-normalizes but
+            # the guard does not.
+            left = _flank(text, i - 1, -1, start - 1, state)
+            right = _flank(text, i + 1, +1, end, state)
+            is_separator = (left is not None and right is not None
+                            and _classified_char(left)
+                            and _classified_char(right))
         if is_separator:
             if tok_start is not None:
                 tokens.append(WorkToken(text[tok_start:i],

tests/v2/pipeline/test_tokenize.py

@@ -139,6 +139,58 @@ def test_interpunct_at_edges_stays() -> None:
     assert [t.text for t in state.tokens] == ["·威廉"]
 
 
+def test_stripped_bidi_mark_is_transparent_to_the_flank_guard() -> None:
+    # an RTL document quoting a transcription: U+200F beside the dot,
+    # visually identical to the clean form -- the stripped mark must
+    # not occupy a flank position
+    state = _tokenized("威廉‏·莎士比亚")
+    assert [t.text for t in state.tokens] == ["威廉", "莎士比亚"]
+    assert state.interpunct_offsets != ()
+
+
+def test_stripped_emoji_is_transparent_to_the_flank_guard() -> None:
+    state = _tokenized("威廉\U0001f600·莎士比亚")
+    assert [t.text for t in state.tokens] == ["威廉", "莎士比亚"]
+
+
+def test_unstripped_bidi_mark_defeats_the_guard() -> None:
+    # the contrast: with strip_bidi off the mark is real token text,
+    # an unclassified flank -- the dot stays interior
+    state = _tokenized("威廉‏·莎士比亚",
+                       policy=Policy(strip_bidi=False))
+    assert state.interpunct_offsets == ()
+
+
+def test_interpunct_trailing_edge_stays() -> None:
+    # the upper bound: a trailing dot has no right flank -- and
+    # dropping the bound is an IndexError, not a wrong answer
+    state = _tokenized("威廉·")
+    assert [t.text for t in state.tokens] == ["威廉·"]
+
+
+def test_interpunct_minimal_flanks_split() -> None:
+    state = _tokenized("威·廉")
+    assert [t.text for t in state.tokens] == ["威", "廉"]
+    assert state.interpunct_offsets == (1,)
+
+
+def test_interpunct_one_sided_flank_stays() -> None:
+    # the AND in the guard: one classified neighbor is not enough,
+    # and no offset records (a spurious record would fake the
+    # transcription marker downstream)
+    state = _tokenized("Anna·威")
+    assert [t.text for t in state.tokens] == ["Anna·威"]
+    assert state.interpunct_offsets == ()
+
+
+def test_hangul_flanks_split_and_record() -> None:
+    # Korean writes transcribed foreign names with the interpunct too
+    # (마틴·루터·킹) -- hangul flanks are classified flanks
+    state = _tokenized("김·민준")
+    assert [t.text for t in state.tokens] == ["김", "민준"]
+    assert state.interpunct_offsets == (1,)
+
+
 def test_interpunct_in_delimited_regions_splits_but_does_not_record() -> None:
     # a B7 inside a nickname must not mark the OUTER name as a
     # transcription -- same flag that keeps nickname commas out of