Skip to content

Commit 8838474

Browse files
derek73claude
andcommitted
Give every config entry point the decode hint, not three of five
Reviewing the commit that added the hint found it had done to itself exactly what it was fixing: bytes reached Lexicon's vocabulary fields and Policy's collection fields with a decode hint, while capitalization_exceptions and name_order still reported the cryptic byte value ("name_order elements must be Role members, got 103" -- 103 is 'g'). Guarding one member of a family and leaving its siblings is the most repeated defect on this branch; that is now three rounds. Also widened to bytearray and memoryview. bytes was the only binary type checked, so memoryview(b'ab') still surfaced "got 97". One shared _reject_buffer() in _lexicon, called from both entry points, and the same check on all three Policy/PolicyPatch sites. The tests are parametrized over every entry point x every buffer type rather than one example each -- a per-example test is what let the gap through. The perf-shape coverage table said "at n=200"; the test runs at 800. Re-measured: the columns are identical at both, so the note now cites the base it is actually describing. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 53a68ae commit 8838474

5 files changed

Lines changed: 59 additions & 31 deletions

File tree

nameparser/_lexicon.py

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,21 @@ def _title_key(words: Iterable[str]) -> str:
9797
return " ".join(filter(None, (_normalize(w) for w in words)))
9898

9999

100+
def _reject_buffer(value: object, label: str, plural: str) -> None:
101+
"""Binary sequences iterate to INTS, so every downstream entry check
102+
reports a byte value -- "must be strings, got 100" for b'dr', where
103+
100 is 'd'. That names neither the cause nor the fix. v1 shipped a
104+
decode hint for this (#238), and parse() and the facade both carry
105+
one, so no config entry point should be the odd one out.
106+
"""
107+
if isinstance(value, (bytes, bytearray, memoryview)):
108+
raise TypeError(
109+
f"{label} must be an iterable of {plural}, not "
110+
f"{type(value).__name__} -- decode first, e.g. "
111+
f"raw.decode('utf-8')"
112+
)
113+
114+
100115
def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]:
101116
# Reject a bare str before iterating: iterating "dr" would silently
102117
# yield the single characters {'d', 'r'} -- the set(str) footgun on
@@ -106,16 +121,7 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]:
106121
f"Lexicon.{field_name} must be an iterable of strings, "
107122
f"not a bare string"
108123
)
109-
# bytes iterate to INTS, so without this the entry check below
110-
# reports "entries must be strings, got 100" -- the byte value of
111-
# 'd', naming neither the cause nor the fix. v1 shipped a decode
112-
# hint for exactly this (#238); parse() and the facade carry one,
113-
# so the config surface should not be the odd one out.
114-
if isinstance(entries, (bytes, bytearray)):
115-
raise TypeError(
116-
f"Lexicon.{field_name} must be an iterable of strings, not "
117-
f"bytes -- decode first, e.g. raw.decode('utf-8')"
118-
)
124+
_reject_buffer(entries, f"Lexicon.{field_name}", "strings")
119125
# A Mapping would silently contribute only its keys; a dict here
120126
# almost always means the caller confused this field with
121127
# capitalization_exceptions.
@@ -170,6 +176,7 @@ def _normpairs(
170176
"capitalization_exceptions must be a mapping or an "
171177
"iterable of (key, value) pairs, not a bare string"
172178
)
179+
_reject_buffer(raw, "capitalization_exceptions", "(key, value) pairs")
173180
pairs = raw.items() if isinstance(raw, Mapping) else raw
174181
try:
175182
pairs = iter(pairs)

nameparser/_policy.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,12 @@ def _reject_bare_string_order(value: object) -> None:
9898
f"name_order must be an iterable of three Roles, not a "
9999
f"mapping: {value!r}"
100100
)
101+
if isinstance(value, (bytes, bytearray, memoryview)):
102+
raise TypeError(
103+
f"name_order must be an iterable of three Roles, not "
104+
f"{type(value).__name__} -- decode first, e.g. "
105+
f"raw.decode('utf-8')"
106+
)
101107

102108

103109
def _reject_str_and_mapping(value: object, field_name: str) -> None:
@@ -118,10 +124,11 @@ def _reject_str_and_mapping(value: object, field_name: str) -> None:
118124
# bytes iterate to ints, so the entry check would report a byte
119125
# value and name neither the cause nor the fix; same decode hint
120126
# Lexicon and parse() give.
121-
if isinstance(value, (bytes, bytearray)):
127+
if isinstance(value, (bytes, bytearray, memoryview)):
122128
raise TypeError(
123-
f"{field_name} must be an iterable of strings, not bytes -- "
124-
f"decode first, e.g. raw.decode('utf-8')"
129+
f"{field_name} must be an iterable of strings, not "
130+
f"{type(value).__name__} -- decode first, e.g. "
131+
f"raw.decode('utf-8')"
125132
)
126133
if isinstance(value, Mapping):
127134
# Name the way out, not just the harm. "contributes only its

tests/v2/test_benchmark.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,17 +45,17 @@ def test_facade_thousand_names_under_a_second() -> None:
4545
# loop. Cheap per unit, so a superlinear stage shows up as growth rather
4646
# than as a slow parse.
4747
#
48-
# What each DIMENSION is covered by, at n=200 (measure before pruning a
49-
# shape that looks redundant -- several differ only in a column you
50-
# cannot see from the string):
48+
# What each DIMENSION is covered by, measured at _BASE. Several shapes
49+
# differ only in a column you cannot see from the string, so measure
50+
# before pruning one that looks redundant:
5151
#
5252
# dimension covered by
5353
# token count all nine
5454
# piece count unmatched_*, plain_tokens, commas, titles
5555
# SEGMENT count commas ONLY -- deleting it leaves every
5656
# segment-keyed regression unguarded
57-
# intra-piece accumulation particles (199-token piece),
58-
# conjunctions (200) -- the merge() quadratic
57+
# intra-piece accumulation particles (one 799-token piece),
58+
# conjunctions (800) -- the merge() quadratic
5959
# masked-span count delimiter_pairs, quote_pairs (0 pieces:
6060
# everything is consumed as a delimited run)
6161
_SHAPES = {

tests/v2/test_lexicon.py

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -444,12 +444,22 @@ def test_remove_error_offers_the_removal_remedy_too() -> None:
444444
lex.remove(particles={"van"})
445445

446446

447-
@pytest.mark.parametrize("value", [b"dr", bytearray(b"dr")])
448-
def test_vocab_field_rejects_bytes_with_a_decode_hint(value: object) -> None:
449-
# bytes iterate to INTS, so the generic entry check reported
450-
# "entries must be strings, got 100" -- the byte value of 'd',
451-
# naming neither the cause nor the fix. v1 shipped a decode hint
452-
# for exactly this (#238); parse() and the facade already carry
453-
# one, so the config surface should not be the odd one out.
447+
@pytest.mark.parametrize("value", [b"dr", bytearray(b"dr"), memoryview(b"dr")])
448+
@pytest.mark.parametrize("build", [
449+
lambda v: Lexicon(titles=v),
450+
lambda v: Lexicon.empty().add(titles=v),
451+
lambda v: Lexicon(capitalization_exceptions=v),
452+
])
453+
def test_every_lexicon_entry_point_rejects_a_buffer_with_a_decode_hint(
454+
build: Callable[[object], object], value: object) -> None:
455+
# Binary sequences iterate to INTS, so the generic entry check
456+
# reported "entries must be strings, got 100" -- the byte value of
457+
# 'd', naming neither the cause nor the fix. v1 shipped a decode
458+
# hint for exactly this (#238).
459+
#
460+
# Every entry point, because the first version of this fix covered
461+
# the vocabulary fields and left capitalization_exceptions behind:
462+
# guarding one member of a family and not its siblings is the
463+
# single most repeated defect on this branch.
454464
with pytest.raises(TypeError, match="decode"):
455-
Lexicon(titles=value) # type: ignore[arg-type]
465+
build(value)

tests/v2/test_policy.py

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -396,9 +396,13 @@ def test_a_pair_mapping_is_accepted_through_items(cls: type) -> None:
396396

397397

398398
@pytest.mark.parametrize("cls", [Policy, PolicyPatch])
399-
@pytest.mark.parametrize("value", [b" - ", bytearray(b" - ")])
400-
def test_collection_fields_reject_bytes_with_a_decode_hint(
401-
cls: type, value: object) -> None:
402-
# Same cryptic int message as Lexicon had: bytes iterate to ints.
399+
@pytest.mark.parametrize("value", [b" - ", bytearray(b" - "), memoryview(b" - ")])
400+
@pytest.mark.parametrize(
401+
"field", ["extra_suffix_delimiters", "nickname_delimiters",
402+
"maiden_delimiters", "patronymic_rules", "name_order"])
403+
def test_every_field_rejects_a_buffer_with_a_decode_hint(
404+
cls: type, field: str, value: object) -> None:
405+
# Same cryptic int message Lexicon had, and name_order was the one
406+
# field the first pass missed.
403407
with pytest.raises(TypeError, match="decode"):
404-
cls(extra_suffix_delimiters=value)
408+
cls(**{field: value})

0 commit comments

Comments
 (0)