|
| 1 | +"""Harvest name-like strings from this project's GitHub issue tracker. |
| 2 | +
|
| 3 | +Companion to build_corpus.py, and deliberately a SEPARATE corpus with a |
| 4 | +separate builder. The two have different provenance and different |
| 5 | +blind spots: |
| 6 | +
|
| 7 | +- corpus.jsonl comes from v1's own test suite at a pinned git ref. It |
| 8 | + is reproducible forever, and it is structurally blind to anything |
| 9 | + 2.0 added -- v1's authors had no reason to write a test for a |
| 10 | + typographic nickname delimiter or a Cyrillic title. |
| 11 | +- corpus_issues.jsonl comes from what USERS REPORTED, which is the |
| 12 | + adversarial half: names that broke the parser in the field, in the |
| 13 | + reporter's own words. On its first run it surfaced five intended- |
| 14 | + but-unclassified 2.0 behaviors and one shape (a LEADING "Ph. D.") |
| 15 | + that no test had considered. |
| 16 | +
|
| 17 | +Reproducibility differs, and that is why these are not merged. A git |
| 18 | +ref is immutable; an issue tracker is not. Re-running this later can |
| 19 | +only ADD names as new issues arrive, so the checked-in file is the |
| 20 | +snapshot under test and regeneration is an explicit, reviewable act. |
| 21 | +
|
| 22 | +Over-collection is fine, same as build_corpus.py: the comparator just |
| 23 | +parses more names. Junk like 'Bridge (1.4)' costs one parse and |
| 24 | +produces no diff. |
| 25 | +
|
| 26 | +Regenerate with (requires `gh` authenticated to the repo): |
| 27 | + uv run python tools/differential/build_issues_corpus.py \\ |
| 28 | + > tools/differential/corpus_issues.jsonl |
| 29 | +""" |
| 30 | +import json |
| 31 | +import re |
| 32 | +import subprocess |
| 33 | +import sys |
| 34 | + |
| 35 | +# Either an explicit HumanName("...") call -- the reporter showing the |
| 36 | +# failing input -- or a quoted, capitalized phrase, which is how names |
| 37 | +# appear in prose ("parsing 'Hans "Hansi" Müller' gives ..."). |
| 38 | +_CANDIDATE = re.compile( |
| 39 | + r"""HumanName\(\s*["']([^"']{3,60})["']""" |
| 40 | + r"""|["']([A-Z][^"'\n]{4,60})["']""" |
| 41 | +) |
| 42 | +# Structural characters that mean we caught code or markup, not a name. |
| 43 | +_NOT_A_NAME = set('{}<>=/\\|') |
| 44 | + |
| 45 | + |
| 46 | +def _harvest(text: str) -> set[str]: |
| 47 | + found = set() |
| 48 | + for match in _CANDIDATE.finditer(text): |
| 49 | + value = (match.group(1) or match.group(2) or "").strip() |
| 50 | + # Require an internal space: a single word is a surname at best |
| 51 | + # and carries no structure to disagree about. |
| 52 | + if value and " " in value and not (_NOT_A_NAME & set(value)): |
| 53 | + found.add(value) |
| 54 | + return found |
| 55 | + |
| 56 | + |
| 57 | +def main() -> None: |
| 58 | + raw = subprocess.run( |
| 59 | + ["gh", "issue", "list", "--state", "all", "--limit", "1000", |
| 60 | + "--json", "number,title,body"], |
| 61 | + capture_output=True, text=True, check=True).stdout |
| 62 | + issues = json.loads(raw) |
| 63 | + names: set[str] = set() |
| 64 | + for issue in issues: |
| 65 | + names |= _harvest( |
| 66 | + f"{issue.get('title') or ''}\n{issue.get('body') or ''}") |
| 67 | + for name in sorted(names): |
| 68 | + print(json.dumps(name, ensure_ascii=False)) |
| 69 | + print(f"{len(names)} names from {len(issues)} issues", file=sys.stderr) |
| 70 | + |
| 71 | + |
| 72 | +if __name__ == "__main__": |
| 73 | + main() |
0 commit comments