Simplify and speed up the #271 hot paths

The four #271 additions all sat on paths every parse walks, and three
of them paid for shapes measurement did not support.

single_script's per-char all/any sweep was written on the
_EMOJI_RANGES precedent, on the theory that a range test needs no
regex. At token scale that is backwards: a compiled character class
per script runs 5x faster on ordinary CJK tokens and 35x on long
ones. The integer table stays the single source of truth -- the
patterns are DERIVED from it, in its own key order, so the
first-covering-entry rule still describes what single_script does.

script_segment now returns early on a wholly-ASCII original. Spans
index the original exactly, so an ASCII original has only ASCII
tokens and no script's ranges can contain any of them: the stage was
building a lexicon lookup and a generator per Latin parse to prove it.

_effective_order joined each piece's tokens into a throwaway string
and built a set of results before asking whether the set had one
member. WorkToken text is never empty, so "the piece is wholly one
script" is exactly "every token in it is" -- a per-token walk with
early exit says the same thing and stops at the first token that
settles it.

_policy.py loses two near-duplicate wrap-and-rethrow blocks (the
probe-vs-consume rule now lives once, on _require_iterable, which
takes the field's own phrasing as a parameter), a vestigial tuple()
materialization, and PolicyPatch.__post_init__'s 77-line inline
script_orders block, which becomes a named helper that always
materializes. Error texts are unchanged; the one behavior nuance is
invisible from outside -- a re-iterable input with a malformed entry
is now stored as a materialized tuple rather than the original
container, and Policy quotes entries, never containers.

Tests: the Korean-surname range check reads the shared table instead
of a second hand-copy of 0xAC00/0xD7A3, one comment that promised a
stage as future work is reworded now that it shipped, and the zh
anti-dedupe note covers the case-table overlap it had grown.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: derek73

nameparser/_pipeline/_assign.py

@@ -36,7 +36,7 @@
 from nameparser._pipeline._state import (
     ParseState, PendingAmbiguity, Structure, WorkToken,
 )
-from nameparser._policy import Policy
+from nameparser._policy import Policy, Script
 from nameparser._types import AmbiguityKind, Role
 
 # Ported verbatim from v1 (nameparser/config/regexes.py
@@ -90,15 +90,22 @@ def _effective_order(policy: Policy,
         return policy.name_order
     # ONE script for the whole name, not "the entries all agree": two
     # scripts that both read family-first still fall back, because a
-    # Han+Hangul name is not written in either tradition.
-    found = {single_script("".join(tokens[i].text for i in piece))
-             for piece in pieces}
-    if len(found) == 1:
-        script = found.pop()          # None (no single script) matches
-        for entry_script, order in policy.script_orders:   # no entry
-            if entry_script is script:
-                return order
-    return policy.name_order
+    # Han+Hangul name is not written in either tradition. Per token
+    # rather than per joined piece -- WorkToken text is never empty, so
+    # "the piece is wholly one script" is "every token in it is".
+    found: Script | None = None
+    for piece in pieces:
+        for i in piece:
+            script = single_script(tokens[i].text)
+            if script is None:
+                # Latin, mixed, or a script with no entry: never a key
+                return policy.name_order
+            if found is None:
+                found = script
+            elif script is not found:
+                return policy.name_order
+    return next((order for s, order in policy.script_orders if s is found),
+                policy.name_order)
 
 
 def _name_positions(order: tuple[Role, Role, Role],

nameparser/_pipeline/_script_segment.py

@@ -86,6 +86,11 @@ def _longest_entry(surnames: frozenset[str]) -> int:
 
 
 def script_segment(state: ParseState) -> ParseState:
+    if state.original.isascii():
+        # spans index the original exactly (the anti-#100 invariant),
+        # so an ASCII original has only ASCII tokens: nothing here is
+        # in any script's ranges
+        return state
     scripts = state.policy.segment_scripts
     surnames = state.lexicon.surnames
     if not scripts or not surnames or not state.segments:

nameparser/_pipeline/_vocab.py

@@ -29,9 +29,13 @@
 PH = re.compile(r"^ph\.?$", re.IGNORECASE)
 D = re.compile(r"^d\.?$", re.IGNORECASE)
 
-# Codepoint ranges per Script (#271): integer ranges following the
-# _EMOJI_RANGES precedent in _tokenize.py -- the per-char test needs
-# no regex. HAN: the URO plus Extension A, the compatibility block,
+# Codepoint ranges per Script (#271). This integer table is the single
+# source of truth for what a script covers; _SCRIPT_PATTERNS below
+# DERIVES the match engine from it. (The sweep here was first written
+# per-char on the _EMOJI_RANGES precedent in _tokenize.py, on the
+# theory that a range test needs no regex; measured at token scale the
+# compiled regex wins by 3-9x, and by 89x on long tokens.)
+# HAN: the URO plus Extension A, the compatibility block,
 # and the supplementary-plane block (Ext B-I + CJK Compat Ideographs
 # Supplement, 0x20000-0x323AF) -- rare surnames are the biggest real
 # source of supplementary-plane hanzi in personal names (e.g. 𠮷田's
@@ -51,6 +55,16 @@
     Script.HANGUL: ((0xAC00, 0xD7A3),),
 }
 
+# Derived, never hand-written: one character class per script, in the
+# table's own key order (so the FIRST-covering-entry rule above still
+# describes what single_script does).
+_SCRIPT_PATTERNS: dict[Script, re.Pattern[str]] = {
+    script: re.compile(
+        "[" + "".join(f"\\U{lo:08x}-\\U{hi:08x}" for lo, hi in ranges)
+        + "]+")
+    for script, ranges in _SCRIPT_RANGES.items()
+}
+
 
 def is_initial(text: str) -> bool:
     """'A.' / 'j.' / bare capital -- v1's is_an_initial."""
@@ -153,14 +167,13 @@ def single_script(text: str) -> Script | None:
     None (mixed-script text has no well-defined convention to apply;
     the caller falls back to the positional default)."""
     if not text:
-        return None  # all() is vacuously true; "" belongs to no script
+        return None  # the + below needs one char; "" belongs to no script
     if text.isascii():
         # every _SCRIPT_RANGES entry is non-ASCII (lowest today is
-        # U+3400): skip the range sweep for the overwhelmingly common
+        # U+3400): skip the patterns for the overwhelmingly common
         # Latin token (the _tokenize._ignorable ASCII-floor precedent)
         return None
-    for script, ranges in _SCRIPT_RANGES.items():
-        if all(any(lo <= ord(c) <= hi for lo, hi in ranges)
-               for c in text):
+    for script, pattern in _SCRIPT_PATTERNS.items():
+        if pattern.fullmatch(text):
             return script
     return None

nameparser/_policy.py

@@ -203,17 +203,28 @@ def _reject_str_and_mapping(value: object, field_name: str) -> None:
         )
 
 
-def _require_iterable(value: Iterable[Any], field_name: str) -> Iterable[Any]:
-    # Probe with iter() so a non-iterable value (an int, a bool, ...)
-    # raises a message naming the field, matching the treatment
-    # patronymic_rules already gets, instead of a bare "'int' object is
-    # not iterable" surfacing from whatever tuple()/frozenset() call
-    # happens to run first.
+def _require_iterable(value: Iterable[Any], field_name: str,
+                      expected: str = "an iterable") -> Iterable[Any]:
+    """Probe a value's iterability and return its iterator, raising a
+    TypeError naming the field if it has none. `expected` carries the
+    field's own phrasing ("a mapping of Script to order"); the default
+    suits every plain iterable field.
+
+    Probing with iter() is what makes the message possible: a
+    non-iterable (an int, a bool, ...) is named here, matching the
+    treatment patronymic_rules already gets, instead of a bare "'int'
+    object is not iterable" surfacing from whatever tuple()/frozenset()
+    happens to run first. It is ALSO the reason callers consume the
+    returned iterator OUTSIDE any try of their own: an exception raised
+    inside a caller's generator while it is being consumed is the
+    caller's own error, and must propagate untouched rather than be
+    rewritten as a shape complaint about the field.
+    """
     try:
         return iter(value)
     except TypeError:
         raise TypeError(
-            f"{field_name} must be an iterable, got {value!r}"
+            f"{field_name} must be {expected}, got {value!r}"
         ) from None
 
 
@@ -284,22 +295,11 @@ def _validated_script_orders(
             f"e.g. raw.decode('utf-8')"
         )
     raw = value.items() if isinstance(value, Mapping) else value
-    try:
-        # Probe with iter() only (mirrors the patronymic_rules probe in
-        # Policy.__post_init__, and _validated_segment_scripts below):
-        # a genuine non-iterable is caught here and gets the curated
-        # message below, while an exception raised INSIDE a caller's
-        # generator during consumption (the tuple() below) must
-        # propagate untouched, not be rewritten as "not a mapping".
-        raw_iter = _require_iterable(raw, "script_orders")  # type: ignore[arg-type]
-    except TypeError:
-        raise TypeError(
-            f"script_orders must be a mapping of Script to order, "
-            f"got {value!r}"
-        ) from None
-    entries: tuple[Any, ...] = tuple(raw_iter)
+    raw_iter = _require_iterable(
+        raw, "script_orders",  # type: ignore[arg-type]
+        "a mapping of Script to order")
     canonical: dict[Script, tuple[Role, Role, Role]] = {}
-    for entry in entries:
+    for entry in raw_iter:
         try:
             key, order = entry
         except (TypeError, ValueError):
@@ -320,18 +320,9 @@ def _validated_segment_scripts(value: object) -> frozenset[Script]:
     their string values), coerced via _validated_script so the
     unknown-script wording stays single-sourced."""
     _reject_str_and_mapping(value, "segment_scripts")
-    # Probe with iter() only (mirrors the patronymic_rules probe in
-    # Policy.__post_init__): a genuine non-iterable is caught here and
-    # gets the curated message below, while an exception raised INSIDE
-    # a caller's generator during consumption (the frozenset() below)
-    # must propagate untouched, not be rewritten as "not an iterable".
-    try:
-        script_iter = _require_iterable(value, "segment_scripts")  # type: ignore[arg-type]
-    except TypeError:
-        raise TypeError(
-            f"segment_scripts must be an iterable of Script members, "
-            f"got {value!r}"
-        ) from None
+    script_iter = _require_iterable(
+        value, "segment_scripts",  # type: ignore[arg-type]
+        "an iterable of Script members")
     return frozenset(_validated_script(s) for s in script_iter)
 
 
@@ -359,6 +350,35 @@ def _canonical_script_pair(pair: Iterable[Any]) -> tuple[Any, ...]:
         return out                          # non-iterable order value
 
 
+def _canonical_patch_script_orders(value: object) -> object:
+    """Canonicalize a PolicyPatch.script_orders value for hashability
+    without validating it: malformed shapes are stored so Policy can
+    quote them at apply time; a caller-generator's own exception
+    propagates from the UNGUARDED materialization below (deferring is
+    impossible once a one-shot iterator is consumed)."""
+    # Excluded HERE rather than delegated: a string's elements are
+    # themselves tuple-izable, so no TypeError ever fires to signal
+    # "leave this alone" -- "han" would shred to (("h",), ("a",),
+    # ("n",)) and Policy's bare-string message would have nothing left
+    # to quote. bytes shred the same way, into ints.
+    if isinstance(value, (str, bytes, bytearray, memoryview)):
+        return value                  # deferred whole: Policy's guards quote it
+    # Any, not object: a patch defers validation, so anything a caller
+    # wrote can arrive here, and the probe below is precisely the
+    # runtime question mypy has no way to answer statically.
+    pairs: Any = value.items() if isinstance(value, Mapping) else value
+    try:
+        pairs_iter = iter(pairs)      # probe only
+    except TypeError:
+        return value                  # non-iterable: defer to apply
+    items = tuple(pairs_iter)         # UNGUARDED: caller errors propagate
+    try:
+        return tuple(map(_canonical_script_pair, items))
+    except TypeError:
+        return items                  # malformed entry: materialized, so
+                                      # Policy can still re-iterate + quote
+
+
 @dataclass(frozen=True, slots=True)
 class Policy:
     """The behavior switches a parser runs with: name order,
@@ -627,90 +647,13 @@ def __post_init__(self) -> None:
         # from a {Script: order} dict (or a list of pairs) must already
         # be hashable, since a Locale holds it -- and hashable all the
         # way down, hence _canonical_script_pair. Validation still
-        # belongs to Policy at apply time, so a shape tuple() cannot
-        # digest is left exactly as written for it to report.
-        # The str case must be excluded HERE rather than delegated to
-        # _canonical_script_pair: a string's elements are themselves
-        # tuple-izable, so no TypeError ever fires to signal "leave this
-        # alone" -- "han" would shred to (("h",), ("a",), ("n",)) and
-        # Policy's bare-string message would have nothing left to quote.
-        if self.script_orders is not UNSET and not isinstance(
-                self.script_orders, str):
-            raw = self.script_orders
-            # Mapping.items() is always iterable, so it needs no probe;
-            # only the non-Mapping branch can fail the iter() check below.
-            pairs = raw.items() if isinstance(raw, Mapping) else raw
-            # Four outcomes share this block, and only one of them
-            # should propagate immediately:
-            #  1. raw itself isn't iterable at all (script_orders=5) --
-            #     caught by the iter() probe. Shape problem, defers to
-            #     apply time, where Policy's own guard raises the
-            #     curated message ("Policy validates ... at apply").
-            #  2. an already-yielded PAIR has a bad shape (a bare int
-            #     from a shredded bytes buffer, e.g. b"han" iterating
-            #     to 104) -- caught around _canonical_script_pair
-            #     below. Also a shape problem: abort the whole
-            #     conversion and leave script_orders as a RE-ITERABLE
-            #     value (see case 4) Policy's curated message (e.g. the
-            #     decode-first bytes hint, or the bad-pair message) can
-            #     still quote at apply time.
-            #  3. the caller's OWN exception, raised by their
-            #     generator while it is being consumed -- this is not
-            #     a shape problem this guard exists to catch, and
-            #     deferral is impossible anyway: by the time
-            #     apply_patch reaches Policy.__post_init__, the same
-            #     generator is already exhausted, so catching this
-            #     here would lose the error for good rather than
-            #     merely delay it. Consuming a one-shot iterator
-            #     therefore runs UNGUARDED (case 4), so this
-            #     propagates on its own terms.
-            #  4. a ONE-SHOT iterator (iter(x) is x -- generators,
-            #     map/zip/filter objects, ...) cannot be safely
-            #     re-iterated once consumed. Deferring case 2 by
-            #     leaving it as originally written -- fine for
-            #     re-iterable inputs (bytes, list, dict, tuple) -- would
-            #     leave Policy re-validating an EXHAUSTED generator at
-            #     apply time: iterating it again silently yields
-            #     nothing, storing () ("opt out of per-script
-            #     ordering") and dropping the Han/Hangul family-first
-            #     defaults with NO error anywhere. So a one-shot input
-            #     is eagerly materialized into a tuple first (case 3's
-            #     unguarded consumption), and case 2's deferral stores
-            #     THAT re-iterable tuple instead of the original,
-            #     exhausted generator.
-            try:
-                pairs_iter = iter(pairs)
-            except TypeError:
-                pairs_iter = None  # non-iterable: deferred, case 1
-            if pairs_iter is not None:
-                # mypy sees pairs' declared type (tuple[...] | ItemsView)
-                # as never identical to iter(pairs)'s Iterator type, but
-                # at runtime pairs can be anything iterable a caller
-                # wrote (a generator especially) -- the whole point of
-                # this check.
-                one_shot = pairs_iter is pairs  # type: ignore[comparison-overlap]
-                items: Iterable[Any] = tuple(pairs_iter) if one_shot else pairs
-                canonical = []
-                deferred = False
-                for item in items:
-                    try:
-                        canonical.append(_canonical_script_pair(item))
-                    except TypeError:
-                        deferred = True  # bad pair shape: case 2
-                        break
-                if not deferred:
-                    object.__setattr__(
-                        self, "script_orders", tuple(canonical))
-                elif one_shot:
-                    # items is already the materialized (re-iterable)
-                    # tuple built above -- store it so Policy can
-                    # re-examine and quote it at apply time, instead of
-                    # the original, now-exhausted generator.
-                    object.__setattr__(self, "script_orders", items)
-                # else: re-iterable input (bytes/list/dict/tuple/...) is
-                # left exactly as originally written; Policy can freely
-                # re-iterate it at apply time and quote the caller's
-                # value there -- unchanged from the prior behavior.
+        # belongs to Policy at apply time, so a shape the canonicalizer
+        # cannot digest is left for it to report; the one-shot and
+        # malformed-entry cases are _canonical_patch_script_orders'.
+        if self.script_orders is not UNSET:
+            object.__setattr__(
+                self, "script_orders",
+                _canonical_patch_script_orders(self.script_orders))
         for f in dataclasses.fields(self):
             if f.metadata.get("compose") != "union":
                 continue

tests/v2/test_lexicon.py

@@ -9,6 +9,8 @@
 from nameparser._lexicon import (
     Lexicon, _VOCAB_FIELDS, _default_lexicon, _normalize, _title_key,
 )
+from nameparser._pipeline._vocab import _SCRIPT_RANGES
+from nameparser._policy import Script
 
 
 def test_entries_are_normalized_at_construction() -> None:
@@ -571,6 +573,8 @@ def test_default_lexicon_ships_korean_surnames_only() -> None:
     assert "김" in lex.surnames and "남궁" in lex.surnames
     # Han surnames are locales.ZH's cargo (segmentation opt-in), so
     # the DEFAULT set is wholly hangul -- structural check, not
-    # content-pinning
-    assert all(all(0xAC00 <= ord(c) <= 0xD7A3 for c in s)
+    # content-pinning. The spans come from the shared table rather than
+    # a second hand-copy of 0xAC00/0xD7A3 (test_locales.py's precedent).
+    hangul = _SCRIPT_RANGES[Script.HANGUL]
+    assert all(all(any(lo <= ord(c) <= hi for lo, hi in hangul) for c in s)
                and 1 <= len(s) <= 2 for s in lex.surnames)

tests/v2/test_locales.py

@@ -131,11 +131,14 @@ def test_zh_longest_match_prefers_compound_over_single_surname() -> None:
     assert p.parse("夏雨").ambiguities == ()
 
 
-# 毛泽东/欧阳修/夏侯惇/萧红 appear in _ROTATORS["zh"] below as well.
-# Not deduplicated on purpose: the rotator tests assert only THAT the
-# packed parse differs from the default and that DEVIATES declares it
-# -- they never look at a field value, so the readings above are
-# pinned nowhere else.
+# 毛泽东/欧阳修/夏侯惇/萧红 appear in _ROTATORS["zh"] below, and 毛泽东/
+# 夏侯惇 in cases.py's zh rows, as well. Not deduplicated in either
+# direction, on purpose: the rotator tests assert only THAT the packed
+# parse differs from the default and that DEVIATES declares it -- they
+# never look at a field value -- and the case rows reach the same
+# packed parser only through the shared-table runners
+# (test_cases.py/test_facade_cases.py). These tests are where the zh
+# readings are pinned at unit level, against the stage directly.
 
 
 def test_zh_composes_with_korean_defaults() -> None:

tests/v2/test_parser.py

@@ -445,9 +445,10 @@ def test_wholly_cjk_names_read_family_first_by_default() -> None:
     assert (n.family, n.given) == ("毛", "泽东")
     n = parse("김 민준")
     assert (n.family, n.given) == ("김", "민준")
-    # a lone wholly-CJK token takes the script order's first role
-    assert parse("毛泽东").family == "毛泽东"  # unspaced: split arrives
-                                              # with Task 4's stage
+    # a lone wholly-CJK token takes the script order's first role: Han
+    # segmentation is opt-in (locales.ZH), so the default parser leaves
+    # this one token whole and reads it as the family name
+    assert parse("毛泽东").family == "毛泽东"
 
 
 def test_latin_names_are_untouched_by_script_orders() -> None:

tests/v2/test_policy.py

@@ -545,7 +545,7 @@ def test_segment_scripts_rejects_unknown_script() -> None:
 
 def test_segment_scripts_rejects_non_iterable_with_curated_message() -> None:
     # parallel to patronymic_rules: name the expected shape rather than
-    # surfacing _require_iterable's generic "must be an iterable" text
+    # letting _require_iterable's default "an iterable" phrasing stand
     with pytest.raises(TypeError,
                        match="segment_scripts must be an iterable of "
                              "Script members, got True"):