From 29d2e4e860704e4c99cca9c0b8d424e397454cb9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 14:31:26 -0700 Subject: [PATCH 1/5] Actually exercise the surname split under drawn lexicons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manufactured 'surname + 민준' token was in the pool but rarely in the name: one entry among ~30, against a name of 1-8 pieces. Measured, the lexicon and policy agree on ~14 draws in 250 but the token reached the name only 0-4 times -- and every time it did, the split fired. The shortfall was sampling, not the stage, so insert one at a drawn position when the draw admits it. Twelve randomized runs of 250 now give 0-19 fires (median 8) where they gave 0-2. Also corrects this comment's own claim. It called the surname half 'structurally inert, not luck' on the strength of a single derandomize=True run reporting zero -- but that run is one sample and repeating it cannot disagree with itself. The structural part is real and narrower than stated: a non-hangul surname makes a mixed-script token that can never be a site. The rest was luck. --- tests/v2/test_properties.py | 77 +++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 5c85caa..cb94d8e 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -19,6 +19,7 @@ from nameparser._lexicon import _VOCAB_FIELDS from nameparser._pipeline import run from nameparser._pipeline._state import ParseState +from nameparser._pipeline._vocab import effective_script from nameparser._types import AmbiguityKind, Role from .conftest import differential_corpus @@ -190,7 +191,9 @@ def test_particle_fork_is_never_double_reported(text: str) -> None: # The CJK tail (#271) is what lets a drawn `surnames` set activate # script_segment at all: hangul is the script segmented by default, so # "김"/"남궁" are what make the stage fire (see _names_using, which -# supplies the unspaced token to fire it ON), and the Han rows ride +# supplies the unspaced token to fire it ON -- and, since the drawn +# policy picks its own segment_scripts, only when that policy +# activates hangul too), and the Han rows ride # along for script_orders -- Han segmentation is opt-in via # locales.ZH, which these policies do not draw. _VOCAB = st.sampled_from([ @@ -301,13 +304,18 @@ def _policies(draw: st.DrawFn) -> Policy: @st.composite -def _names_using(draw: st.DrawFn, lexicon: Lexicon) -> str: +def _names_using(draw: st.DrawFn, lexicon: Lexicon, + policy: Policy) -> str: """Build the input out of the lexicon's OWN words. Fuzzing configuration while feeding unrelated text tests almost nothing: a randomly generated string essentially never contains a randomly generated vocabulary entry, so every configured set would sit unused and the parse would take the same path every time. + + Takes the POLICY as well as the lexicon because one of the shapes + below is only reachable when the two agree -- see the segmentation + note. """ vocab = sorted({w for name in _SET_FIELDS for w in getattr(lexicon, name)}) @@ -327,22 +335,40 @@ def _names_using(draw: st.DrawFn, lexicon: Lexicon) -> str: # the stage bails on a wholly-ASCII original, so the non-Latin stem # is what admits '민준jr'. Both fires observed under drawn lexicons # are of exactly that shape. - # Neither derivation guarantees a fire on any given run: the piece - # is one of ~30 in the pool, so it competes with the bare vocabulary - # words, and a bare drawn surname standing earlier in the name takes - # the surname site before the unspaced token can. Instrument before - # concluding either half is exercised -- the counts above are what - # that costs to find out, and the two halves are NOT in the same - # state. Measured over this test's 250 examples: the peel fires - # twice, the surname half ZERO times -- currently inert. Structural, - # not luck: `w + "민준"` on a non-hangul surname is a MIXED-script - # token, whose effective_script is None, so it can never be an - # activated surname site at all; only a drawn HANGUL surname makes - # one, and across the run exactly one such token was sampled - # ('남궁민준'), into an example whose drawn policy had HANGUL out of - # segment_scripts. Deriving the token was still the right move -- - # it took the half off structurally-unreachable -- but a fix worth - # having would derive it in the script the policy activated. + # Being in the POOL is not the same as being in the NAME, and for + # the surname half that gap was the whole story. Measured with the + # token merely offered: over 250 examples the lexicon and policy + # agree often enough (13-19 runs of 250) but the token is drawn into + # the name only 0-4 times, because it is one entry among ~30 and a + # name takes 1-8 pieces. Every time it IS drawn the split fires -- + # the conversion from sampled to fired is 100% -- so the shortfall + # was sampling, never the stage. Hence the forced insertion below + # rather than a wider pool. + # An earlier version of this comment called the surname half + # "structurally inert, not luck" on the strength of a single + # derandomize=True run reporting zero. That run is one sample, and + # repeating it cannot disagree with itself; randomized runs give + # 0, 1, 1, 2, 2, 4. The structural part is real but narrower than + # claimed: `w + "민준"` on a NON-hangul surname is a mixed-script + # token whose effective_script is None, so those candidates can + # never be a site whatever the policy says. Only a drawn hangul + # surname makes a usable one, which is what `activatable` selects. + # With the insertion the surname half measures 0, 2, 5, 6, 6, 7, 9, + # 9, 10, 12, 15, 19 over twelve randomized runs of 250 -- roughly + # eight times what it was, and NOT a guarantee. It cannot be one: + # the insertion only fires when the drawn lexicon carries a hangul + # surname AND the drawn policy activates hangul, which is about 14 + # draws in 250, and a run that draws few can still reach zero. What + # changed is that alignment now converts to a fire every time + # instead of one time in five. + # The peel still rides on sampling alone (0-5 per run): it is + # saturated by case rows and stage tests, so a second forced piece + # was not worth the distribution shift. The mechanism generalizes + # if that ever changes. + # Re-measure rather than trusting these numbers: derandomize=True + # is set on the test, so repeating THAT run returns the same count + # forever and cannot disagree with itself. Drive these strategies + # under derandomize=False to see the spread. # sorted for the same reason `vocab` above is: frozenset iteration # order is not stable across runs, and an unsorted pool shifts # every index sampled_from draws -- which would defeat @@ -353,7 +379,18 @@ def _names_using(draw: st.DrawFn, lexicon: Lexicon) -> str: # pool is never empty even for an empty lexicon pieces = st.sampled_from( vocab + unspaced + glued + ["John", "Smith", "Q.", ",", "(", "'"]) - return " ".join(draw(st.lists(pieces, min_size=1, max_size=8))) + drawn = draw(st.lists(pieces, min_size=1, max_size=8)) + # The one shape the pool cannot deliver on its own. Inserted at a + # drawn position rather than the front: the stage takes the first + # ACTIVATED-script token, not the first token, so a leading Latin + # word must not hide it -- and both placements are worth covering. + activatable = sorted( + w + "민준" for w in lexicon.surnames + if effective_script(w + "민준") in policy.segment_scripts) + if activatable: + drawn.insert(draw(st.integers(0, len(drawn))), + draw(st.sampled_from(activatable))) + return " ".join(drawn) @given(_lexicons(), _policies(), st.data()) @@ -363,7 +400,7 @@ def test_any_valid_config_still_parses_totally( # Building the parser is part of the contract: a Lexicon and Policy # that each constructed must also combine. parser = Parser(lexicon=lexicon, policy=policy) - text = data.draw(_names_using(lexicon)) + text = data.draw(_names_using(lexicon, policy)) parsed = parser.parse(text) # must not raise, ever # the anti-#100 invariant, under configuration rather than under # the default vocabulary: spans index the original exactly From 2af134019a53fcb16d01f47e4b4e0529560bff29 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 23:34:27 -0700 Subject: [PATCH 2/5] Correct the Han claim the vocabulary note makes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The note said Han segmentation is "opt-in via locales.ZH, which these policies do not draw", implying the Han entries are inert here. True of locale PACKS, and false of what `_policies` actually generates: it draws `segment_scripts` freely, and HAN lands in 37.7% of drawn policies (measured over 20000 draws). An activated Han token is not scenery either -- standing earlier in the name it takes the surname site and declines, so "欧阳 김민준" does not split where "김민준 欧阳" does. Pre-existing, and two lines from where this branch has been editing. Co-Authored-By: Claude Opus 5 --- tests/v2/test_properties.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index cb94d8e..ab6e443 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -193,9 +193,13 @@ def test_particle_fork_is_never_double_reported(text: str) -> None: # "김"/"남궁" are what make the stage fire (see _names_using, which # supplies the unspaced token to fire it ON -- and, since the drawn # policy picks its own segment_scripts, only when that policy -# activates hangul too), and the Han rows ride -# along for script_orders -- Han segmentation is opt-in via -# locales.ZH, which these policies do not draw. +# activates hangul too). The Han rows ride along for script_orders, +# and are not inert here either -- `_policies` draws segment_scripts +# freely, so HAN is activated in 37.7% of drawn policies (measured +# over 20000 draws), and an activated Han token STANDING EARLIER takes +# the surname site: "欧阳 김민준" does not split, where "김민준 欧阳" +# does. What these policies never draw is a locale PACK, which is the +# only way shipped configuration turns Han segmentation on. _VOCAB = st.sampled_from([ "van", "de", "la", "bin", "abdul", "abu", "dr", "sir", "prof", "md", "jr", "iii", "esq", "ma", "do", "and", "y", "née", "geb", From d834f014f05fc32a1d393b8fb54760fb7b53b4b1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 23:34:57 -0700 Subject: [PATCH 3/5] Put the segmentation fork under a drawn lexicon at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit script_segment's multi-match fork -- two vocabulary entries match the same token, longest-first chooses, and the stage reports the reading it passed over -- was fuzzed by nothing. The cause is structural rather than statistical: the pool's only hangul entries were 김 and 남궁, and no entry was a proper PREFIX of another, so `matches` could never hold more than one length whatever the draw did. 남 is a shipped Korean surname and a prefix of 남궁, so a lexicon drawn with both makes 남궁민준 match twice and sends the parse down the ambiguity-emission-plus-index-remap path. Instrumented rather than assumed, because reachable is not the same as reached: both entries have to land in the same drawn `surnames` (0.8% of draws), so the fork fires about once in 900 examples -- 42 over 36000 measured -- and the committed derandomize=True seed does not reach it. A randomized run is what sees it. Said in the note so the next reader does not conclude from one green run that the entry is doing nothing. Co-Authored-By: Claude Opus 5 --- tests/v2/test_properties.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index ab6e443..8e49d19 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -190,10 +190,18 @@ def test_particle_fork_is_never_double_reported(text: str) -> None: # would be a dead entry that trips Lexicon's multi-word warning. # The CJK tail (#271) is what lets a drawn `surnames` set activate # script_segment at all: hangul is the script segmented by default, so -# "김"/"남궁" are what make the stage fire (see _names_using, which +# "김"/"남궁"/"남" are what make the stage fire (see _names_using, which # supplies the unspaced token to fire it ON -- and, since the drawn # policy picks its own segment_scripts, only when that policy -# activates hangul too). The Han rows ride along for script_orders, +# activates hangul too). "남" is there for the stage's multi-match +# FORK, which nothing else in this pool can reach: it is a proper +# PREFIX of "남궁", so a lexicon drawn with both makes "남궁민준" match +# twice, and longest-first then has to choose and report the reading +# it passed over. Reachable is all it is -- both entries have to land +# in the same drawn `surnames`, 0.8% of draws, so the fork fires +# roughly once in 900 examples (42 over 36000 measured) and the +# committed 250-example seed does not reach it at all; a randomized +# run is what sees it. The Han rows ride along for script_orders, # and are not inert here either -- `_policies` draws segment_scripts # freely, so HAN is activated in 37.7% of drawn policies (measured # over 20000 draws), and an activated Han token STANDING EARLIER takes @@ -204,7 +212,7 @@ def test_particle_fork_is_never_double_reported(text: str) -> None: "van", "de", "la", "bin", "abdul", "abu", "dr", "sir", "prof", "md", "jr", "iii", "esq", "ma", "do", "and", "y", "née", "geb", "a", "b", "ph.d", "عبد", "фон", "μεγα", - "김", "남궁", "毛", "欧阳", + "김", "남궁", "남", "毛", "欧阳", ]) # given_name_titles is the one field matched as a space-joined run, so From 007c6b66cf07eb852aa4688fddbd8bd48063753b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 23:35:22 -0700 Subject: [PATCH 4/5] Build the name list instead of mutating Hypothesis's `drawn.insert(...)` writes into the list `st.lists` handed back. Safe today -- that strategy builds a fresh list per draw -- but the hazard class is the kind a fuzzer should not carry: a strategy that ever handed back a shared or memoized list would turn this into a bug in the test harness, reported as a bug in the code under test. The rebuild costs nothing and draws in the same order, so the committed seed still counts 23 surname splits and one peel. Also names the shape honestly in the comment above it: the pool CAN produce that token, it just loses the draw. "cannot deliver on its own" contradicted the paragraph four lines up. Co-Authored-By: Claude Opus 5 --- tests/v2/test_properties.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 8e49d19..00d2e9d 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -392,16 +392,22 @@ def _names_using(draw: st.DrawFn, lexicon: Lexicon, pieces = st.sampled_from( vocab + unspaced + glued + ["John", "Smith", "Q.", ",", "(", "'"]) drawn = draw(st.lists(pieces, min_size=1, max_size=8)) - # The one shape the pool cannot deliver on its own. Inserted at a - # drawn position rather than the front: the stage takes the first - # ACTIVATED-script token, not the first token, so a leading Latin - # word must not hide it -- and both placements are worth covering. + # The one shape the pool does not deliver RELIABLY -- it is in + # there, it just loses the draw. Inserted at a drawn position + # rather than the front: the stage takes the first ACTIVATED-script + # token, not the first token, so a leading Latin word cannot hide + # it -- and both placements are worth covering. activatable = sorted( w + "민준" for w in lexicon.surnames if effective_script(w + "민준") in policy.segment_scripts) if activatable: - drawn.insert(draw(st.integers(0, len(drawn))), - draw(st.sampled_from(activatable))) + # rebuilt rather than list.insert()d: st.lists hands back a + # fresh list today, so mutating it is safe today, and a + # strategy that ever memoized one would make this a bug in the + # fuzzer rather than in the code under test + at = draw(st.integers(0, len(drawn))) + token = draw(st.sampled_from(activatable)) + drawn = [*drawn[:at], token, *drawn[at:]] return " ".join(drawn) From 8fd2eb33b09e1784dfe158f27f0acc69fb498bd7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 23:35:58 -0700 Subject: [PATCH 5/5] Say what the forced insertion is worth, in numbers that were measured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third pass at this comment, and the first two both overstated the same half from samples too small to carry the claim. What it said and what measurement says: * "the conversion from sampled to fired is 100%" -- it is about four in five (350 of 450 aligned examples over 24 randomized runs of 250). The conclusion it supported was right and is kept; the number is gone, replaced by the decline paths, which are all the stage deciding: a FAMILY_COMMA opts the stage out, or an earlier script-written token takes the surname site -- a bare drawn surname ('김 김민준'), a hangul post-nominal, or a Han token under a policy that activated HAN. That last group is what the rewrite deleted and this restores; a Latin word never takes the site (zero occurrences in 1082 aligned examples), so the note that only Latin blocks it named the one blocker that does not bite. * "13-19 [per] 250" alignment, "0-4 times" in the name, "one entry among ~30", "0-5 per run" for the peel -- measured 5.5% of draws over 20000, a couple per 250, a pool of median size 23, and 0-7 peels per randomized run. Ranges taken from a handful of runs are dropped or widened. * "roughly eight times what it was" -- follows from no measurement I can reproduce; dropped for the fire counts themselves. Leads instead with the one figure that cannot rot: instrument _split under the committed derandomize=True seed and origin/master counts zero surname splits against two peels, this tree 23 against one. One command, same answer every time. Adds what the change does NOT buy, which is the honest headline: this layer asserts totality and span-exactness only, so every behavioral mutant in the stage passes here before and after and is killed by tests/v2/pipeline/test_script_segment.py. The one defect class it can catch is span arithmetic in _split, and the peel already reached that path -- writing `base + end` as `base + end + 1`, origin/master finds it in 8 runs of 10 at a median 1561 shrink calls and 14.6 seconds, this tree in 10 of 10 at 346 calls and 3.6 seconds. And two costs that went unmentioned: the insertion drops the stage's i-is-None early return under a live configuration from 1.3% of examples to 0.1% (it survives where a drawn quote pair carries the token into a nickname), and an insertion at the end can suppress a peel, since the peel site is the last non-post-nominal token. Correction to this branch's first commit message, which is pushed and not being amended: it gave the pre-change baseline as "0-2" where the comment said 0, 1, 1, 2, 2, 4. Neither is the range -- 36 randomized runs of 250 give 0-9. Co-Authored-By: Claude Opus 5 --- tests/v2/test_properties.py | 115 ++++++++++++++++++++++++------------ 1 file changed, 78 insertions(+), 37 deletions(-) diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 00d2e9d..91cbfaa 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -201,8 +201,8 @@ def test_particle_fork_is_never_double_reported(text: str) -> None: # in the same drawn `surnames`, 0.8% of draws, so the fork fires # roughly once in 900 examples (42 over 36000 measured) and the # committed 250-example seed does not reach it at all; a randomized -# run is what sees it. The Han rows ride along for script_orders, -# and are not inert here either -- `_policies` draws segment_scripts +# run is what sees it. The Han rows ride along for script_orders, and +# are not inert here either -- `_policies` draws segment_scripts # freely, so HAN is activated in 37.7% of drawn policies (measured # over 20000 draws), and an activated Han token STANDING EARLIER takes # the surname site: "欧阳 김민준" does not split, where "김민준 欧阳" @@ -345,42 +345,83 @@ def _names_using(draw: st.DrawFn, lexicon: Lexicon, # mixed-script token the surname half correctly declines, and for # the peel it is the one shape that reaches an ASCII tail at all -- # the stage bails on a wholly-ASCII original, so the non-Latin stem - # is what admits '민준jr'. Both fires observed under drawn lexicons - # are of exactly that shape. + # is what admits '민준jr'. Every peel fire observed under a drawn + # lexicon is of exactly that shape -- the committed run's is + # '민준de'. + # What the forced insertion below is worth, in the one number that + # cannot rot: instrument _split under this test's committed + # derandomize=True seed -- peel and surname split are told apart by + # its tail_tag argument -- and origin/master's version of this + # strategy counts ZERO surname splits against two peels, while this + # one counts 23 against one. That figure is reproducible by anyone + # in one command and moves only when these strategies do. # Being in the POOL is not the same as being in the NAME, and for - # the surname half that gap was the whole story. Measured with the - # token merely offered: over 250 examples the lexicon and policy - # agree often enough (13-19 runs of 250) but the token is drawn into - # the name only 0-4 times, because it is one entry among ~30 and a - # name takes 1-8 pieces. Every time it IS drawn the split fires -- - # the conversion from sampled to fired is 100% -- so the shortfall - # was sampling, never the stage. Hence the forced insertion below - # rather than a wider pool. - # An earlier version of this comment called the surname half - # "structurally inert, not luck" on the strength of a single - # derandomize=True run reporting zero. That run is one sample, and - # repeating it cannot disagree with itself; randomized runs give - # 0, 1, 1, 2, 2, 4. The structural part is real but narrower than - # claimed: `w + "민준"` on a NON-hangul surname is a mixed-script - # token whose effective_script is None, so those candidates can - # never be a site whatever the policy says. Only a drawn hangul - # surname makes a usable one, which is what `activatable` selects. - # With the insertion the surname half measures 0, 2, 5, 6, 6, 7, 9, - # 9, 10, 12, 15, 19 over twelve randomized runs of 250 -- roughly - # eight times what it was, and NOT a guarantee. It cannot be one: - # the insertion only fires when the drawn lexicon carries a hangul - # surname AND the drawn policy activates hangul, which is about 14 - # draws in 250, and a run that draws few can still reach zero. What - # changed is that alignment now converts to a fire every time - # instead of one time in five. - # The peel still rides on sampling alone (0-5 per run): it is - # saturated by case rows and stage tests, so a second forced piece - # was not worth the distribution shift. The mechanism generalizes - # if that ever changes. - # Re-measure rather than trusting these numbers: derandomize=True - # is set on the test, so repeating THAT run returns the same count - # forever and cannot disagree with itself. Drive these strategies - # under derandomize=False to see the spread. + # the surname half that gap was the whole story. Two things must + # coincide before the shape is legal at all -- a HANGUL surname + # drawn AND a policy that activates hangul, together 5.5% of draws + # over 20000 -- and the token then has to win a place in a 1-8 + # piece name against a pool of median size 23. Merely offered, it + # reached the name a couple of times per 250. The shortfall was + # sampling, never the stage. + # What that buys is COVERAGE, not detection, and the two are worth + # separating because the first is the easier to oversell. This + # layer asserts totality and span-exactness only, so every + # BEHAVIORAL mutant in the stage -- the site's last-token scan, + # shortest-first, the whole-token guard, the activation gate, the + # post-nominal decline, the prefix cap, the segment remap -- passes + # here with the insertion and without it, and is killed by + # tests/v2/pipeline/test_script_segment.py instead. The one defect + # class this layer CAN catch is span arithmetic inside _split, and + # the peel already reached that path. Writing `base + end` as + # `base + end + 1` and giving each tree ten randomized runs of 250: + # origin/master finds it in 8 runs of 10, at a median 1561 shrink + # calls and 14.6 seconds; this tree finds it in 10 of 10, at 346 + # calls and 3.6 seconds, and shrinks to '김민준 김' where master + # shrinks to the peel's '민준van'. st.integers shrinks the + # insertion index toward 0, which walks the token into the position + # likeliest to fire instead of away from it. + # Alignment is still not a guarantee: with the token forced in the + # split fires in about four aligned examples in five (350 of 450, + # over 24 randomized runs of 250). Every decline is the stage + # deciding, not waste. A FAMILY_COMMA opts the stage out whole -- a + # ',' piece is in the pool. Otherwise an EARLIER script-written + # token takes the surname site: a bare drawn surname ('김 김민준' + # leaves 김민준 unsplit -- the whole-token guard), a drawn hangul + # post-nominal (the leading post-nominal decline), or a Han token + # under a policy that activated HAN. A LATIN word never takes it -- + # 'John 김민준' still splits, zero occurrences in 1082 aligned + # examples -- which is why the insertion index is drawn rather than + # pinned to 0. + # Two earlier versions of this comment overstated this half from + # small samples -- one calling it "structurally inert, not luck" on + # a single derandomize=True run reporting zero, one putting the + # conversion above at 100%. So: a derandomize=True run is one + # sample and cannot disagree with itself, and every randomized + # figure quoted here was taken under derandomize=False instead, + # which is how to re-measure them. The structural claim is real but + # narrower than it was made: `w + "민준"` on a NON-hangul surname is + # a mixed-script token whose effective_script is None, so those + # candidates can never be a site whatever the policy says. Only a + # drawn hangul surname makes a usable one, which is what + # `activatable` selects. + # Two shapes the insertion costs, both small, neither zero. It is + # unconditional once lexicon and policy align, so the stage's + # i-is-None early return under a LIVE configuration fell from 1.3% + # of examples to 0.1%: it survives only where a drawn quote pair + # carries the token off into a nickname, leaving segments[0] with + # nothing in an activated script ("prof ' Smith 남민준 '"). And the + # inserted token can SUPPRESS a peel, the peel site being the last + # non-post-nominal token and this token being no post-nominal: + # '김민준씨 김민준' leaves 씨 glued where '김민준 김민준씨' peels + # it. + # The peel gets no forced piece of its own -- it is saturated by + # case rows and stage tests, so a second one was not worth the + # distribution shift -- and its count here is a lottery either way + # (0-7 per randomized run of 250). On net the insertion nudges it + # UP, by a route worth knowing: a hangul token makes the original + # non-ASCII, which lifts the stage's ASCII bail off Latin tails + # that are otherwise unreachable -- under honorific_tails={'a'}, + # 'John la' does not peel and '김민준 la' does. # sorted for the same reason `vocab` above is: frozenset iteration # order is not stable across runs, and an unsorted pool shifts # every index sampled_from draws -- which would defeat