Skip to content

Commit 697dbe3

Browse files
derek73claude
andcommitted
Eliminate the py/overly-large-range false positive at the source
The lgtm[...] suppression comment is not honored by the repo's CodeQL setup -- moving it only changed alert fingerprints and resurfaced the alerts. Root fix instead: py/overly-large-range fires on LITERAL astral character-class ranges (it decomposes them into overlapping surrogate pairs), so neither file has one anymore. _tokenize's per-char test becomes plain integer codepoint ranges (no regex at all, cheaper); regexes.py keeps its public compiled re_emoji surface but builds the class from the same codepoint pairs -- verified byte-identical to the previous pattern. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 6ee4681 commit 697dbe3

2 files changed

Lines changed: 21 additions & 14 deletions

File tree

nameparser/_pipeline/_tokenize.py

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,15 @@
2626
)
2727
from nameparser._types import Role, Span
2828

29-
# Ported verbatim from v1 (nameparser/config/regexes.py, "emoji" and
30-
# "bidi") -- layering forbids importing the config package here, so the
31-
# patterns are duplicated by design with this provenance note. When
32-
# editing, keep both copies in sync.
33-
_EMOJI = re.compile('['
34-
'\U0001F300-\U0001F64F' # lgtm[py/overly-large-range]
35-
'\U0001F680-\U0001F6FF'
36-
'\u2600-\u26FF\u2700-\u27BF]+')
29+
# Ported from v1 (nameparser/config/regexes.py, "emoji" and "bidi") --
30+
# layering forbids importing the config package here, so the tables are
31+
# duplicated by design with this provenance note. When editing, keep
32+
# both copies in sync (regexes.py builds its public re_emoji from the
33+
# SAME codepoint pairs). Integer ranges, not a regex character class:
34+
# the per-char test needs no regex, and CodeQL's py/overly-large-range
35+
# false-positives on literal astral ranges (surrogate decomposition).
36+
_EMOJI_RANGES = ((0x1F300, 0x1F64F), (0x1F680, 0x1F6FF),
37+
(0x2600, 0x26FF), (0x2700, 0x27BF))
3738
_BIDI = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+')
3839

3940

@@ -46,7 +47,10 @@ def _ignorable(ch: str, state: ParseState) -> bool:
4647
return False
4748
if state.policy.strip_bidi and _BIDI.match(ch):
4849
return True
49-
return bool(state.policy.strip_emoji and _EMOJI.match(ch))
50+
if state.policy.strip_emoji:
51+
cp = ord(ch)
52+
return any(lo <= cp <= hi for lo, hi in _EMOJI_RANGES)
53+
return False
5054

5155

5256
def _tokenize_region(state: ParseState, start: int, end: int,

nameparser/config/regexes.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
import re
22

3-
# emoji regex from https://stackoverflow.com/questions/26568722/remove-unicode-emoji-using-re-in-python
4-
re_emoji = re.compile('['
5-
'\U0001F300-\U0001F64F' # lgtm[py/overly-large-range]
6-
'\U0001F680-\U0001F6FF'
7-
'\u2600-\u26FF\u2700-\u27BF]+')
3+
# emoji ranges from https://stackoverflow.com/questions/26568722/remove-unicode-emoji-using-re-in-python
4+
# Built from codepoint pairs rather than a literal character class:
5+
# the compiled pattern is identical, but CodeQL's py/overly-large-range
6+
# false-positives on literal astral ranges (surrogate decomposition).
7+
_EMOJI_RANGES = ((0x1F300, 0x1F64F), (0x1F680, 0x1F6FF),
8+
(0x2600, 0x26FF), (0x2700, 0x27BF))
9+
re_emoji = re.compile(
10+
'[' + ''.join(f'{chr(lo)}-{chr(hi)}' for lo, hi in _EMOJI_RANGES) + ']+')
811

912
# Invisible bidirectional formatting characters: ALM, LRM, RLM, the
1013
# embedding/override marks (LRE/RLE/PDF/LRO/RLO) and the isolates

0 commit comments

Comments
 (0)