Merge pull request #276 from apoorva-01/fix/issue-266-bidi-control-chars

Strip invisible bidi control characters during preprocessing

Author: derek73

docs/customize.rst

@@ -501,6 +501,24 @@ You can turn this off by setting the ``emoji`` regex to ``False``.
     >>> str(hn)
     'Sam 😊 Smith'
 
+Don't Remove Bidi Control Characters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, invisible bidirectional control characters (the left-to-right and
+right-to-left marks and friends, common in copy-pasted right-to-left names) are
+removed from the input string before the name is parsed. You can turn this off
+by setting the ``bidi`` regex to ``False``.
+
+.. doctest::
+
+    >>> from nameparser import HumanName
+    >>> from nameparser.config import Constants
+    >>> constants = Constants()
+    >>> constants.regexes.bidi = False
+    >>> hn = HumanName("\u200fJohn\u200f Smith", constants=constants)
+    >>> hn.first == "\u200fJohn\u200f"
+    True
+
 Config Changes May Need Parse Refresh
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/release_log.rst

@@ -1,5 +1,9 @@
 Release Log
 ===========
+* 1.3.1 - Unreleased
+
+    - Fix invisible Unicode bidirectional control characters (LRM/RLM/ALM, the embedding/override marks, and the isolates U+2066–U+2069) surviving parsing and sticking to ``first``/``last``/etc., so a copy-pasted right-to-left name silently failed equality and dedup. They are now stripped in preprocessing like emoji; disable via ``CONSTANTS.regexes.bidi = False`` (closes #266)
+
 * 1.3.0 - July 5, 2026
 
     **Breaking Changes & Deprecations**

nameparser/config/regexes.py

@@ -6,6 +6,12 @@
     '\U0001F680-\U0001F6FF'
     '\u2600-\u26FF\u2700-\u27BF]+')
 
+# Invisible bidirectional formatting characters: ALM, LRM, RLM, the
+# embedding/override marks (LRE/RLE/PDF/LRO/RLO) and the isolates
+# (LRI/RLI/FSI/PDI). Copy-pasted RTL names often carry these; they render
+# as nothing but stick to name parts and break equality/lookups.
+re_bidi = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+')
+
 EMPTY_REGEX = re.compile('')
 
 REGEXES = {
@@ -20,6 +26,7 @@
     "no_vowels": re.compile(r'^[^aeyiuo]+$', re.I),
     "period_not_at_end": re.compile(r'.*\..+$', re.I),
     "emoji": re_emoji,
+    "bidi": re_bidi,
     "phd": re.compile(r'\s(ph\.?\s+d\.?)', re.I),
     "space_before_comma": re.compile(r'\s+,'),
     "east_slavic_patronymic": re.compile(

nameparser/parser.py

@@ -928,9 +928,11 @@ def pre_process(self) -> None:
         This method happens at the beginning of the :py:func:`parse_full_name`
         before any other processing of the string aside from unicode
         normalization, so it's a good place to do any custom handling in a
-        subclass. Runs :py:func:`parse_nicknames` and :py:func:`squash_emoji`.
+        subclass. Runs :py:func:`squash_bidi`, :py:func:`parse_nicknames` and
+        :py:func:`squash_emoji`.
 
         """
+        self.squash_bidi()
         self.fix_phd()
         self.parse_nicknames()
         self.squash_emoji()
@@ -1133,6 +1135,16 @@ def squash_emoji(self) -> None:
         if re_emoji and re_emoji.search(self._full_name):
             self._full_name = re_emoji.sub('', self._full_name)
 
+    def squash_bidi(self) -> None:
+        """
+        Remove invisible bidirectional control characters from the input
+        string. They carry no name content but stick to the parts they
+        surround, so parsed attributes stop comparing equal to the clean name.
+        """
+        re_bidi = self.C.regexes.bidi
+        if re_bidi and re_bidi.search(self._full_name):
+            self._full_name = re_bidi.sub('', self._full_name)
+
     def handle_firstnames(self) -> None:
         """
         If there are only two parts and one is a title, assume it's a last name

tests/test_output_format.py

@@ -136,3 +136,27 @@ def test_keep_emojis(self) -> None:
         self.m(hn.last, "Smith😊", hn)
         self.assertEqual(str(hn), "∫≜⩕ Smith😊")
         # test cleanup
+
+    def test_remove_bidi_control_chars(self) -> None:
+        # LRM/RLM and friends ride along with copy-pasted names and stick to
+        # the parts they surround. Covers every character in the bidi set.
+        for mark in ("\u200e", "\u200f", "\u061c", "\u202a", "\u202b",
+                     "\u202c", "\u202d", "\u202e", "\u2066", "\u2067",
+                     "\u2068", "\u2069"):
+            hn = HumanName(mark + "John" + mark + " Smith")
+            self.m(hn.first, "John", hn)
+            self.m(hn.last, "Smith", hn)
+
+    def test_bidi_stripped_name_compares_equal(self) -> None:
+        # The reported symptom: an invisible RLM around an RTL name makes the
+        # parsed part fail equality against the clean string (issue #266).
+        hn = HumanName("\u200fمحمد بن سلمان\u200f")
+        self.assertEqual(hn.first, "محمد")
+
+    def test_keep_bidi_control_chars(self) -> None:
+        from nameparser.config import Constants
+        constants = Constants()
+        constants.regexes.bidi = False  # type: ignore[assignment]
+        hn = HumanName("\u200fJohn\u200f Smith", constants)
+        self.m(hn.first, "\u200fJohn\u200f", hn)
+        self.m(hn.last, "Smith", hn)