Simplify: hoist the JA union, merge factory closures, trim duplicated prose

* Hoist _JA_SCRIPTS into _policy.py beside _SCRIPT_RANGES: the union
  was defined twice (_pipeline/_vocab.py frozenset, locales/ja.py
  tuple) with nothing pinning their membership equality -- the same
  drift class this branch already deleted at span level. One tuple
  now serves the kana license, the ja pack's DEVIATES, and the
  adapter's repertoire guard; the katakana rationale moves onto the
  ja predicates it explains, and _vocab's naming-defense comment
  (guarding a constraint that never existed) goes away.
* Merge _script_matcher's twin closures: one compiled [cls]+ pattern
  with the bound method chosen by `whole` -- search over [cls]+ is
  exactly contains-any, so the second pattern and closure bought
  nothing.
* Reorder ja adapter guards: the O(1) length check now leads the
  chain, so routine one-char tokens from spaced input ("林 太郎") no
  longer pay the regex fullmatch and the 〆 scan (measured 4x) before
  being declined. All guards return the same None, so behavior is
  unchanged.
* Trim four prose duplications: the 〆 paragraph's copy of the 々
  closer in _policy's table comment (plus an orphan line wrap), the
  triple-stated ordering point on _SCRIPT_MATCHERS, locales.rst's
  restatement of the marker-classification contract the factory
  docstring already carries, and the twice-narrated namedivider
  mis-cut measurement in test_locales.py (the authoritative account
  stays at ja.py's decline comment).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: derek73

docs/locales.rst

@@ -283,11 +283,8 @@ by ``tests/v2/test_locales.py``:
    not — when in doubt, ``DEVIATES`` should say yes. A pack whose
    scope is a *script* builds the predicate with ``_script_matcher``
    from ``nameparser/_policy.py``, the way ``zh`` and ``ja`` do —
-   never by compiling its own character ranges: the table in
-   ``_policy.py`` is the single source for script codepoint spans, and
-   the contract test reads any module-level ``re.Pattern`` in a pack
-   as a marker regex needing branch coverage (a pack that imports
-   ``re`` without exposing one fails outright).
+   never by compiling its own character ranges: the factory's
+   docstring explains how the pack-contract test enforces this.
 #. Add a rotator list to ``tests/v2/test_locales.py``. Every pack needs
    one, but what it has to contain follows from how the pack declares
    its scope. A pack declaring by *marker regex* (``ru``, ``tr_az``)

nameparser/_pipeline/_vocab.py

@@ -13,7 +13,8 @@
 from collections.abc import Callable, Iterable
 
 from nameparser._lexicon import Lexicon, _normalize
-from nameparser._policy import Script, _SCRIPT_RANGES, _script_matcher
+from nameparser._policy import (Script, _JA_SCRIPTS, _SCRIPT_RANGES,
+                                _script_matcher)
 
 # Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus
 # its empty-string alternative -- WorkToken text is never empty. Kept in
@@ -39,26 +40,15 @@
 # compiled regex wins by 3-9x, and by roughly 70x on long tokens.)
 # Derived, never hand-written -- even the class construction goes
 # through the shared factory: one wholly-of predicate per script, in
-# the table's own key order (a dict comprehension over _SCRIPT_RANGES
-# preserves it, so single_script's FIRST-covering-entry rule still
-# describes what it does; the disjointness requirement that makes
-# that well-defined is stated with the table in _policy).
+# the table's key order -- the FIRST-covering-entry rule _classify
+# documents.
 _SCRIPT_MATCHERS: dict[Script, Callable[[str], bool]] = {
     script: _script_matcher(script, whole=True)
     for script in _SCRIPT_RANGES
 }
 
-#: The Japanese repertoire: the union effective_script's kana license
-#: quantifies over -- HAN, HIRAGANA, KATAKANA (HANGUL simply omitted).
-#: A frozenset, not the tuple this started as: resolve_script_set
-#: below is the "later task that needs membership" the tuple's
-#: original comment anticipated. Membership doesn't care about order,
-#: and the matcher built below doesn't either (a regex character
-#: class matches the same set regardless of the order its ranges are
-#: written in).
-_JA_SCRIPTS = frozenset({Script.HAN, Script.HIRAGANA, Script.KATAKANA})
-# Not _wholly_japanese: that name belongs to the ja pack's own
-# predicate over the same union (nameparser/locales/ja.py).
+# The whole-token matcher over _policy's _JA_SCRIPTS union, backing
+# effective_script's kana license.
 _wholly_ja = _script_matcher(*_JA_SCRIPTS, whole=True)
 
 
@@ -261,6 +251,6 @@ def resolve_script_set(scripts: Iterable[Script]) -> Script | None:
     found = frozenset(scripts)
     if len(found) <= 1:
         return next(iter(found), None)
-    if found <= _JA_SCRIPTS:
+    if found.issubset(_JA_SCRIPTS):
         return Script.HIRAGANA
     return None

nameparser/_policy.py

@@ -67,8 +67,8 @@ class Script(StrEnum):
 # below (the table lives here rather than in the pipeline because
 # packs must not import the pipeline).
 # HAN: the ideographic iteration mark U+3005 and the shime mark
-# U+3006, the URO plus Extension
-# A, the compatibility block, and the supplementary-plane block
+# U+3006, the URO plus Extension A, the compatibility block, and the
+# supplementary-plane block
 # (Ext B-I + CJK Compat Ideographs Supplement, 0x20000-0x323AF) --
 # rare surnames are the biggest real source of supplementary-plane
 # hanzi in personal names (e.g. 𠮷田's 𠮷, U+20BB7), so leaving them
@@ -92,8 +92,6 @@ class Script(StrEnum):
 # Shimeki, 〆谷 Shimetani, 〆野) -- its other uses (the envelope
 # closing mark, 〆切) never reach a name parser -- and it appears in
 # no other script's names.
-# Leaving it out made 〆木 a mixed-script token: the name reversed and
-# never gated into segmentation.
 # HANGUL: precomposed syllables only -- modern Korean
 # text never writes names as bare jamo.
 # HIRAGANA/KATAKANA (#272): the two kana blocks, each in full. There
@@ -138,6 +136,12 @@ class Script(StrEnum):
     Script.KATAKANA: ((0x30A0, 0x30FF),),
 }
 
+#: The Japanese repertoire: the three scripts Japanese names draw on.
+#: The kana license (_pipeline/_vocab.py's effective_script), the ja
+#: pack's DEVIATES, and the segmenter adapter's repertoire guard all
+#: quantify over this one union (HANGUL simply omitted).
+_JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA)
+
 
 def _script_matcher(*scripts: Script,
                     whole: bool = False) -> Callable[[str], bool]:
@@ -167,17 +171,14 @@ def _script_matcher(*scripts: Script,
     cls = "".join(f"\\U{lo:08x}-\\U{hi:08x}"
                   for script in scripts
                   for lo, hi in _SCRIPT_RANGES[script])
-    if whole:
-        pattern = re.compile(f"[{cls}]+")
-
-        def wholly(text: str) -> bool:
-            return pattern.fullmatch(text) is not None
-        return wholly
-    single = re.compile(f"[{cls}]")
-
-    def contains(text: str) -> bool:
-        return single.search(text) is not None
-    return contains
+    # one pattern serves both modes: fullmatch of [cls]+ is wholly-of,
+    # and search over [cls]+ is exactly contains-any
+    pattern = re.compile(f"[{cls}]+")
+    match = pattern.fullmatch if whole else pattern.search
+
+    def matcher(text: str) -> bool:
+        return match(text) is not None
+    return matcher
 
 
 # Order-spec constants (#270). Each reads as its contents because roles

nameparser/locales/ja.py

@@ -59,7 +59,8 @@
 
 from nameparser._lexicon import Lexicon
 from nameparser._locale import Locale
-from nameparser._policy import PolicyPatch, Script, _script_matcher
+from nameparser._policy import (PolicyPatch, Script, _JA_SCRIPTS,
+                                _script_matcher)
 from nameparser._types import Segmentation
 
 if TYPE_CHECKING:
@@ -72,21 +73,18 @@
         segment_scripts=frozenset({Script.HAN, Script.HIRAGANA})),
 )
 
-#: The scripts DEVIATES and the adapter quantify over. KATAKANA is
-#: here even though the pack never ACTIVATES it: a katakana-bearing
-#: token can still be a kana-licensed composite the pack acts on
-#: (山田エミ) -- DEVIATES must declare it, and the adapter must not
-#: decline it. A pure-katakana token never reaches THE ADAPTER:
-#: KATAKANA is in no activation set, so the stage's own gate stops it
-#: before the segmenter is consulted. DEVIATES still declares one,
-#: and must: it is a predicate over any name at all, called before
-#: any gate runs, and over-declaring is its safe direction by design.
-_JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA)
-
-# Both compiled by _policy's factory from the shared codepoint table,
-# so the pack carries no spans of its own to drift out of sync: the
-# contains-any form backs DEVIATES, the whole-token form is the
-# adapter's repertoire guard.
+# Both compiled by _policy's factory from the shared codepoint table
+# and its _JA_SCRIPTS union, so the pack carries no spans of its own
+# to drift out of sync: the contains-any form backs DEVIATES, the
+# whole-token form is the adapter's repertoire guard. Both match
+# KATAKANA even though the pack never ACTIVATES it: a katakana-bearing
+# token can still be a kana-licensed composite the pack acts on
+# (山田エミ) -- DEVIATES must declare it, and the adapter must not
+# decline it. A pure-katakana token never reaches THE ADAPTER:
+# KATAKANA is in no activation set, so the stage's own gate stops it
+# before the segmenter is consulted. DEVIATES still declares one,
+# and must: it is a predicate over any name at all, called before
+# any gate runs, and over-declaring is its safe direction by design.
 _has_japanese = _script_matcher(*_JA_SCRIPTS)
 _wholly_japanese = _script_matcher(*_JA_SCRIPTS, whole=True)
 
@@ -160,6 +158,12 @@ def ja_segmenter(*, gbdt: bool = False) -> Segmenter:
                else namedivider.BasicNameDivider())
 
     def segment(text: str) -> Segmentation | None:
+        # namedivider RAISES below two characters ("Name length needs
+        # at least 2 chars"), and a segmenter's exceptions propagate by
+        # contract, so a one-character token -- routine in spaced input
+        # like "林 太郎" -- must be declined here, not passed on.
+        if len(text) < 2:
+            return None
         # segment_scripts UNIONS under pack composition, so this can
         # receive tokens of any other activated script (hangul, under
         # the default policy). Decline anything outside the Japanese
@@ -181,12 +185,6 @@ def segment(text: str) -> Segmentation | None:
         # the safe direction and mid-token 〆 is vanishingly rare.
         if "〆" in text:
             return None
-        # namedivider RAISES below two characters ("Name length needs
-        # at least 2 chars"), and a segmenter's exceptions propagate by
-        # contract, so a one-character token -- routine in spaced input
-        # like "林 太郎" -- must be declined here, not passed on.
-        if len(text) < 2:
-            return None
         result = divider.divide_name(text)
         if result.family + result.given != text:
             return None          # defensive: never trust reconstruction

tests/v2/test_locales.py

@@ -276,12 +276,9 @@ def test_ja_adapter_guard_stack_against_a_stub(
 def test_ja_adapter_declines_shime_tokens(
         monkeypatch: pytest.MonkeyPatch) -> None:
     # Since U+3006 classifies as Han, 〆木太郎 is wholly Japanese and
-    # reaches the adapter -- but namedivider 0.4.x has no knowledge of
-    # the shime mark: measured, its rule path cuts every attested 〆
-    # surname at offset 1 (〆木太郎 -> 〆 | 木太郎, score 1.0,
-    # algorithm='rule'), a wrong family arriving with NO SEGMENTATION
-    # report because 1.0 clears the floor. The adapter declines these
-    # tokens before the divider is ever consulted.
+    # reaches the adapter, but the divider is never consulted for 〆
+    # tokens -- see the decline comment in ja.py for the measured
+    # mis-cut it prevents.
     calls: list[str] = []
 
     def divide(text: str) -> _FakeDivided:
@@ -491,10 +488,9 @@ def test_ja_divides_an_astral_kanji_name() -> None:
 
 @_needs_ja
 def test_namedivider_still_miscuts_shime_names() -> None:
-    # The canary behind the adapter's 〆 decline: namedivider 0.4.x's
-    # rule path cuts every attested 〆 surname at offset 1 with
-    # confidence 1.0. When this fails, namedivider learned the mark --
-    # revisit the decline in ja.py rather than deleting this test.
+    # The canary behind the adapter's 〆 decline: when this fails,
+    # namedivider learned the mark -- revisit the decline in ja.py
+    # rather than deleting this test.
     import namedivider
     result = namedivider.BasicNameDivider().divide_name("〆木太郎")
     assert (result.family, result.given) == ("〆", "木太郎")