Point UNBALANCED_DELIMITER at the token it landed in

Last gap from the emitter audit. The kind carried no tokens, so the
ambiguity was locatable only by parsing an offset back out of its
detail string -- which is prose, not an API. The stray character does
survive into a token ('"Nick', 'Smith)'); nothing was pointing at it.

extract_delimited runs before tokens exist, so PendingAmbiguity gains
an `origin` character offset that tokenize resolves to the containing
token's index once the stream is built. Stages after tokenize set
indices directly and leave origin None.

Every emitted kind now carries its tokens:

    'Jon "Nick Smith'         unbalanced-delimiter  ['"Nick']
    'John Smith)'             unbalanced-delimiter  ['Smith)']
    'Van Johnson'             particle-or-given     ['Van']
    'John Smith MA'           suffix-or-family      ['MA']
    'JEFFREY (JD) BRICKEN'    suffix-or-nickname    ['JD']
    'Smith, John, Extra, Jr.' comma-structure       ['Extra']

The docstring's "May carry no tokens" is narrowed to what remains true:
a character inside a masked region belongs to no token.

test_stage_field_ownership caught tokenize writing a field it did not
own before -- fixed, and classify's entry corrected in the same pass,
since its SUFFIX_OR_NICKNAME emitter had the same omission and only
passed because no case-table row triggers it yet.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: derek73

nameparser/_pipeline/_extract.py

@@ -104,6 +104,7 @@ def _unmatched(open_: str, offset: int) -> tuple[int, PendingAmbiguity]:
     return (offset, PendingAmbiguity(
         AmbiguityKind.UNBALANCED_DELIMITER,
         f"unmatched {open_!r} at offset {offset}; treated as literal text",
+        origin=offset,
     ))
 
 
@@ -216,7 +217,7 @@ def extract_delimited(state: ParseState) -> ParseState:
             ambiguities.append(PendingAmbiguity(
                 AmbiguityKind.UNBALANCED_DELIMITER,
                 f"unmatched {close!r} at offset {j}; treated as "
-                f"literal text"))
+                f"literal text", origin=j))
     return dataclasses.replace(
         state, extracted=tuple(extracted), masked=tuple(masked),
         ambiguities=state.ambiguities + tuple(ambiguities))

nameparser/_pipeline/_state.py

@@ -47,11 +47,18 @@ class Structure(Enum):
 @dataclass(frozen=True, slots=True)
 class PendingAmbiguity:
     """An ambiguity recorded mid-pipeline by token INDEX; assemble
-    materializes real Ambiguity objects over the final tokens."""
+    materializes real Ambiguity objects over the final tokens.
+
+    ``origin`` is for the one stage that runs BEFORE tokens exist:
+    extract_delimited knows only a character offset, so it records that
+    and tokenize resolves it to the containing token's index. Stages
+    after tokenize set ``indices`` directly and leave ``origin`` None.
+    """
 
     kind: AmbiguityKind
     detail: str
     indices: tuple[int, ...] = ()
+    origin: int | None = None
 
 
 @dataclass(frozen=True, slots=True)

nameparser/_pipeline/_tokenize.py

@@ -90,5 +90,19 @@ def tokenize(state: ParseState) -> ParseState:
         _tokenize_region(state, inner.start, inner.end, role, False,
                          tokens, commas)
     tokens.sort(key=lambda t: t.span)
+    # extract_delimited runs before tokens exist, so its ambiguities
+    # carry a character offset instead of an index. Resolve them now
+    # that the stray character has landed in a token ('"Nick', 'Smith)')
+    # -- without this the ambiguity is locatable only by parsing the
+    # offset back out of its detail string. An offset inside a masked
+    # region belongs to no token; those keep an empty tuple, which the
+    # kind's contract already allows.
+    ambiguities = tuple(
+        a if a.origin is None else dataclasses.replace(
+            a, indices=tuple(
+                i for i, t in enumerate(tokens)
+                if t.span.start <= a.origin < t.span.end))
+        for a in state.ambiguities)
     return dataclasses.replace(state, tokens=tuple(tokens),
-                               comma_offsets=tuple(sorted(commas)))
+                               comma_offsets=tuple(sorted(commas)),
+                               ambiguities=ambiguities)

nameparser/_types.py

@@ -252,8 +252,10 @@ class AmbiguityKind(StrEnum):
     #: particle in other names.
     PARTICLE_OR_GIVEN = "particle-or-given"
     #: A nickname/maiden delimiter opened without closing (or closed
-    #: without opening); the text was kept as literal name content.
-    #: May carry no tokens.
+    #: without opening); the text was kept as literal name content, so
+    #: the tokens are the one the stray character ended up inside. Rare
+    #: exception: a character that lands in no token at all (inside a
+    #: masked region) leaves the tuple empty.
     UNBALANCED_DELIMITER = "unbalanced-delimiter"
     #: More comma-separated segments than any recognized name shape;
     #: the parse is best-effort over the extra segments.

tests/v2/pipeline/test_extract.py

@@ -239,3 +239,20 @@ def test_unmatched_open_is_not_double_reported_as_a_close() -> None:
     # still produce exactly one ambiguity
     assert len(parse('John " Smith').ambiguities) == 1
     assert len(parse('Jon "Nick Smith').ambiguities) == 1
+
+
+@pytest.mark.parametrize(("text", "expected"), [
+    ('Jon "Nick Smith', '"Nick'),
+    ("John (Jack Smith", "(Jack"),
+    ("John Smith)", "Smith)"),
+    ("John Jack) Smith", "Jack)"),
+])
+def test_unbalanced_delimiter_points_at_its_token(
+    text: str, expected: str
+) -> None:
+    # the kind was documented as possibly carrying no tokens, but the
+    # stray character does survive into one -- without it the ambiguity
+    # is locatable only by parsing an offset back out of the detail
+    # string, which is not an API
+    (amb,) = parse(text).ambiguities
+    assert [t.text for t in amb.tokens] == [expected]

tests/v2/pipeline/test_state.py

@@ -46,9 +46,15 @@ def test_stage_field_ownership() -> None:
 
     ownership = {
         "extract_delimited": {"extracted", "masked", "ambiguities"},
-        "tokenize": {"tokens", "comma_offsets"},
+        # tokenize also rewrites ambiguities: extract_delimited runs
+        # before tokens exist, so its UNBALANCED_DELIMITER entries carry
+        # a character offset that tokenize resolves to a token index
+        "tokenize": {"tokens", "comma_offsets", "ambiguities"},
         "segment": {"segments", "structure", "ambiguities"},
-        "classify": {"tokens"},
+        # classify also emits SUFFIX_OR_NICKNAME: the delimiter escape
+        # that decides it lives in extract_delimited, which has no token
+        # index to point at, so the report is raised here instead
+        "classify": {"tokens", "ambiguities"},
         "group": {"tokens", "pieces", "piece_tags", "dropped"},
         "assign": {"tokens", "ambiguities"},
         "post_rules": {"tokens"},