Merge pull request #304 from derek73/v2.1/cjk-corpus

Differential harness: CJK corpus generated from the case table

Author: derek73

tests/v2/test_regex_sync.py

@@ -12,9 +12,12 @@
 Layering is the usual reason for a copy but not the only one, so this
 module's scope is the PROMISE rather than that one pair of packages:
 the comma-set pin below reads _pipeline._state instead of config, and
-the last test reaches outside the package altogether, to a TOML file
-that could not import a Python constant if it wanted to.
+the last three tests reach outside the package altogether -- two to a
+TOML file that could not import a Python constant if it wanted to,
+one to a generated corpus whose generator can, and must stay run.
 """
+import importlib.util
+import json
 import re
 import tomllib
 from pathlib import Path
@@ -232,3 +235,64 @@ def test_differential_cjk_rule_matches_the_script_ranges() -> None:
     assert declared == expected, (
         f"{toml_path.name}'s CJK name_regex declares {sorted(declared)}; "
         f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}")
+
+
+def test_differential_compound_rule_matches_the_script_ranges() -> None:
+    """The fix(cjk-delimited-nickname) rule (#295) carries the SECOND
+    hand copy of the script spans in the toml: its require-a-classified-
+    codepoint lookahead exists so the delimiter set alone cannot claim a
+    Latin name's first/last regression ('John 「Jack」 Kennedy' -- the
+    corner brackets sit outside every classified span). Pin that copy to
+    the table exactly as the canonical CJK rule's is, and pin the
+    delimiter set itself, so widening either (to ASCII quotes, say) is
+    an explicit decision here rather than a silent absorption change.
+
+    Selection note for future rule authors: the canonical-CJK-rule pin
+    above selects by the literal '#271'/'#272' substrings and asserts
+    uniqueness -- this rule's slug avoids them on purpose, and any new
+    rule's must too.
+    """
+    toml_path = (Path(__file__).parents[2] / "tools" / "differential"
+                 / "expected_changes.toml")
+    rules = tomllib.loads(toml_path.read_text())["change"]
+    matched = [r for r in rules if "cjk-delimited-nickname" in r["issue"]]
+    assert len(matched) == 1
+    regex = matched[0]["name_regex"]
+    assert "[「」『』・･]" in regex, (
+        "the compound rule's delimiter set changed; decide deliberately")
+    declared = {
+        (int(lo, 16), int(hi, 16))
+        for lo, hi in re.findall(r"\\u([0-9A-Fa-f]{4})-\\u([0-9A-Fa-f]{4})",
+                                 regex)}
+    expected = {span
+                for spans in _policy._SCRIPT_RANGES.values()
+                for span in spans
+                if span[1] <= 0xFFFF} | {(0xFF65, 0xFF65)}
+    assert declared == expected, (
+        f"compound rule's codepoint lookahead declares {sorted(declared)}; "
+        f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}")
+
+
+def test_cjk_corpus_matches_the_case_table() -> None:
+    """corpus_cjk.jsonl is GENERATED, not curated (#295): every
+    distinct case-table text bearing a codepoint the script table
+    classifies, sorted -- see build_cjk_corpus.py for why the other
+    two corpora cannot carry these names. The checked-in file must
+    equal what the generator would write, so a CJK case row added
+    without regenerating fails HERE instead of silently narrowing
+    the differential gate back toward the blind spot #295 closed.
+    Same promise as the toml pin above, aimed at a generated artifact
+    instead of a hand copy.
+    """
+    tools = Path(__file__).parents[2] / "tools" / "differential"
+    spec = importlib.util.spec_from_file_location(
+        "build_cjk_corpus", tools / "build_cjk_corpus.py")
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    checked_in = [json.loads(line) for line in
+                  (tools / "corpus_cjk.jsonl")
+                  .read_text(encoding="utf-8").splitlines()]
+    assert checked_in == module.selected_names(), (
+        "corpus_cjk.jsonl is stale: regenerate with "
+        "`uv run python tools/differential/build_cjk_corpus.py`")

tools/differential/README.md

@@ -38,7 +38,7 @@ needs widening. The run must exit 0 before a 2.0 release; the classified
 summary it prints is the source for the "Behavior Changes" section of
 `docs/release_log.rst`.
 
-## The two corpora
+## The three corpora
 
 `compare.py` reads **every** `corpus*.jsonl` beside it by default
 (deduped), because a corpus you have to ask for by name is a corpus
@@ -48,11 +48,18 @@ that stops being run. Pass `--corpus PATH` (repeatable) to narrow it.
 |---|---|---|
 | `corpus.jsonl` | v1's own test suite at a pinned ref | anything 2.0 added — v1's authors had no reason to test a typographic nickname delimiter or a Cyrillic title |
 | `corpus_issues.jsonl` | name-like strings harvested from the GitHub issue tracker | anything nobody ever reported |
+| `corpus_cjk.jsonl` | the CJK-bearing rows of `tests/v2/cases.py`, via `build_cjk_corpus.py` (#295) | anything the case table itself missed — it re-witnesses reviewed expectations at the 1.4 boundary rather than discovering new shapes |
 
 They are deliberately separate rather than merged: `corpus.jsonl` is
 reproducible forever from an immutable git ref, while the issue
-tracker is mutable, so regenerating the second one is an explicit,
-reviewable act that can only add names.
+tracker is mutable, so regenerating `corpus_issues.jsonl` is an
+explicit, reviewable act that can only add names — and the CJK corpus
+exists because BOTH of those are structurally blind to unspaced CJK (v1's
+banks never tested it; `build_issues_corpus.py` requires an internal
+space, which unspaced names never have). It regenerates from the case
+table, and `tests/v2/test_regex_sync.py` pins the checked-in file
+against the generator's selection, so a CJK case row added without
+regenerating fails the suite instead of silently narrowing this gate.
 
 The issue corpus earned its place on the first run — 166 of its 198
 names were not in `corpus.jsonl`, and it immediately surfaced five

tools/differential/build_cjk_corpus.py

@@ -0,0 +1,62 @@
+"""Regenerate corpus_cjk.jsonl from the CJK rows of the case table.
+
+The third corpus, with the third provenance (#295): corpus.jsonl
+regenerates from v1's test banks at an immutable ref and
+corpus_issues.jsonl harvests the issue tracker, but neither can carry
+an unspaced CJK name -- v1's authors had no reason to test one, and
+build_issues_corpus.py requires an internal space, which is exactly
+the shape #271 classifies. This file derives from the reviewed case
+table instead: every distinct `text` in tests/v2/cases.py containing
+a character the script table classifies, regardless of the row's
+policy/locale context -- the corpus carries name STRINGS and the
+differential run parses them with the default facade, so a name from
+a zh-scoped row is simply one more CJK name to diff.
+
+Selection is by the shipped table (nameparser._policy._SCRIPT_RANGES,
+through the same _script_matcher the parser uses), so a script added
+to the table widens the harvest on the next regeneration, and a case
+row added for future CJK work (#298's 间隔号 forms, say) enters the
+corpus by being written down in the table everyone already reviews.
+
+Regenerate after editing CJK case rows:
+
+    uv run python tools/differential/build_cjk_corpus.py
+
+tests/v2/test_regex_sync.py pins the checked-in file against this
+module's selection, so a stale corpus fails the suite rather than
+silently narrowing the differential gate.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[1]
+sys.path.insert(0, str(ROOT))
+
+from nameparser._policy import _SCRIPT_RANGES, _script_matcher  # noqa: E402
+from tests.v2.cases import CASES  # noqa: E402
+
+OUT = HERE / "corpus_cjk.jsonl"
+
+_has_cjk = _script_matcher(*_SCRIPT_RANGES)
+
+
+def selected_names() -> list[str]:
+    """Every distinct case-table text bearing a classified codepoint,
+    sorted for a deterministic file."""
+    return sorted({case.text for case in CASES if _has_cjk(case.text)})
+
+
+def main() -> None:
+    names = selected_names()
+    with OUT.open("w", encoding="utf-8") as fh:
+        for name in names:
+            fh.write(json.dumps(name, ensure_ascii=False) + "\n")
+    print(f"wrote {len(names)} names to {OUT.name}")
+
+
+if __name__ == "__main__":
+    main()

tools/differential/corpus_cjk.jsonl

@@ -0,0 +1,31 @@
+"Dr 김민준, Jr."
+"John 王"
+"〆木 ひろ"
+"〆木 太郎"
+"〆木太郎"
+"みなみ"
+"マイケル"
+"マイケル ジャクソン"
+"マイケル・ジャクソン"
+"佐々木 太郎"
+"司马相如"
+"夏侯惇"
+"山田 エミ"
+"山田 太郎 (マイケル・ジャクソン)"
+"山田「タロ」太郎"
+"张伟"
+"毛 泽东"
+"毛 김"
+"毛泽东"
+"田中『ハナ』花子"
+"諸葛亮"
+"阿明"
+"高橋 みなみ"
+"高橋みなみ"
+"高橋・一郎"
+"高橋一郎"
+"김 민준"
+"김민준"
+"남궁"
+"남궁민수"
+"남궁민수, 지훈"

tools/differential/expected_changes.toml

@@ -44,15 +44,14 @@ issue = "fix(#271/#272) native-script CJK: family-first order, hangul segmentati
 # parity today. The dot is the whole mechanism; the dot is the whole
 # span.
 #
-# Expected to match nothing against the current corpora, which contain
-# no CJK at all -- build_issues_corpus.py requires an internal space,
-# and unspaced names are the shape this classifies. That blind spot is
-# tracked as #272's and #271's shared gap in issue #295, which also
-# records the provenance decision a fix needs; until it is closed the
-# behavioral guarantee lives in the fix(#271)/fix(#272) rows of
-# tests/v2/cases.py and the fix(shime-mark) rows, whose 〆 span rides
-# in on the same HAN entry, and this rule is kept ready for the moment
-# a CJK name lands in a corpus.
+# corpus_cjk.jsonl (#295) exists so this rule fires in every real
+# run: build_cjk_corpus.py harvests every CJK-bearing text in
+# tests/v2/cases.py -- the fix(#271)/fix(#272)/fix(shime-mark) rows
+# AND the parity rows, whose value here is pinning NON-diffs (マイケル
+# must not change against 1.4) -- closing the blind spot the other
+# two corpora have by construction: v1's banks had no reason to test
+# CJK, and build_issues_corpus.py requires an internal space, which
+# unspaced names never have.
 name_regex = "[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65]"
 fields = ["first", "middle", "last"]
 
@@ -123,6 +122,36 @@ issue = "feat(#273) typographic nickname delimiters recognized by default"
 name_regex = "[“”„«»「」『』（）]"
 fields = ["middle", "nickname"]
 
+[[change]]
+issue = "fix(cjk-delimited-nickname) delimiter recognition compounds with the CJK order flip"
+# '山田「タロ」太郎', '山田 太郎 (マイケル・ジャクソン)': one name, two
+# intended changes at once -- the corner-bracket/nakaguro handling
+# (2.0's typographic delimiters; 2.1's dot separator rendering the
+# nickname join with a space) changes `nickname`, while the same
+# name's wholly-CJK remainder takes the 2.1 family-first flip in
+# first/last. Neither single-change rule may claim the union (the
+# delimiter rule's fields exclude first/last, the CJK rule's exclude
+# nickname -- each on purpose, so a lone regression in the other's
+# fields stays loud). Both lookaheads are required: the delimiters
+# alone would match 'John 「Jack」 Kennedy' -- the brackets sit in CJK
+# Symbols and Punctuation, OUTSIDE every classified span, so a Latin
+# name can carry them -- and would then absorb a bare first/last
+# regression on it. Requiring a classified codepoint too confines the
+# rule to names the order flip can actually reach. The second
+# lookahead's class is the same hand copy of _SCRIPT_RANGES the rule
+# above carries, pinned by the same sync test. One absorption is
+# inherent and accepted: a nickname-ONLY diff on a nakaguro name
+# ('マイケル・ジャクソン') classifies here, because subset-matching
+# cannot REQUIRE a field to have changed. Positional note: this rule
+# overlaps the CJK rule above and the delimiter rule below; within
+# the name_regex tier file order decides which label a diff reports
+# under, so it sits after the tighter single-change rules on purpose.
+# The issue slug deliberately avoids the literal #271/#272 strings --
+# test_regex_sync's differential pin selects the canonical CJK rule
+# by those substrings and asserts it is unique.
+name_regex = "(?s)(?=.*[「」『』・･])(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])"
+fields = ["first", "last", "nickname"]
+
 [[change]]
 issue = "feat(#269) non-Latin titles/conjunctions recognized"
 # 'г-н Иван Петров' (Cyrillic title), 'Хосе И Мария Сантос' (Cyrillic