internationalize word boundary checks

.coveragerc

@@ -1,2 +1,4 @@
 [run]
-omit = test/*
+omit = 
+    test/*
+    setup.py

README.rst

@@ -67,6 +67,24 @@ Case Sensitive example
     >>> keywords_found
     >>> # ['Bay Area']
 
+Span of keywords extracted
+    >>> from flashtext import KeywordProcessor
+    >>> keyword_processor = KeywordProcessor()
+    >>> keyword_processor.add_keyword('Big Apple', 'New York')
+    >>> keyword_processor.add_keyword('Bay Area')
+    >>> keywords_found = keyword_processor.extract_keywords('I love big Apple and Bay Area.', span_info=True)
+    >>> keywords_found
+    >>> # [('New York', 7, 16), ('Bay Area', 21, 29)]
+
+Get Extra information with keywords extracted
+    >>> from flashtext import KeywordProcessor
+    >>> kp = KeywordProcessor()
+    >>> kp.add_keyword('Taj Mahal', ('Monument', 'Taj Mahal'))
+    >>> kp.add_keyword('Delhi', ('Location', 'Delhi'))
+    >>> kp.extract_keywords('Taj Mahal is in Delhi.')
+    >>> # [('Monument', 'Taj Mahal'), ('Location', 'Delhi')]
+    >>> # NOTE: replace_keywords feature won't work with this.
+
 No clean name for Keywords
     >>> from flashtext import KeywordProcessor
     >>> keyword_processor = KeywordProcessor()
@@ -134,9 +152,9 @@ Get all keywords in dictionary
     >>> from flashtext import KeywordProcessor
     >>> keyword_processor = KeywordProcessor()
     >>> keyword_processor.add_keyword('j2ee', 'Java')
-    >>> keyword_processor.add_keyword('onGoing', 'rendom')
+    >>> keyword_processor.add_keyword('colour', 'color')
     >>> keyword_processor.get_all_keywords()
-    >>> # output: {'j2ee': 'Java', 'ongoing': 'rendom'}
+    >>> # output: {'colour': 'color', 'j2ee': 'Java'}
 
 For detecting Word Boundary currently any character other than this `\\w` `[A-Za-z0-9_]` is considered a word boundary.
 
@@ -199,11 +217,27 @@ The idea for this library came from the following `StackOverflow question
 <https://stackoverflow.com/questions/44178449/regex-replace-is-taking-time-for-millions-of-documents-how-to-make-it-faster>`_.
 
 
-References
+Citation
 ----------
 
 The original paper published on `FlashText algorithm <https://arxiv.org/abs/1711.00046>`_.
 
+::
+
+    @ARTICLE{2017arXiv171100046S,
+       author = {{Singh}, V.},
+        title = "{Replace or Retrieve Keywords In Documents at Scale}",
+      journal = {ArXiv e-prints},
+    archivePrefix = "arXiv",
+       eprint = {1711.00046},
+     primaryClass = "cs.DS",
+     keywords = {Computer Science - Data Structures and Algorithms},
+         year = 2017,
+        month = oct,
+       adsurl = {http://adsabs.harvard.edu/abs/2017arXiv171100046S},
+      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+    }
+
 The article published on `Medium freeCodeCamp <https://medium.freecodecamp.org/regex-was-taking-5-days-flashtext-does-it-in-15-minutes-55f04411025f>`_.
 
 

docs/index.rst

@@ -64,6 +64,24 @@ Case Sensitive example
     >>> keywords_found
     >>> # ['Bay Area']
 
+Span of keywords extracted
+    >>> from flashtext import KeywordProcessor
+    >>> keyword_processor = KeywordProcessor()
+    >>> keyword_processor.add_keyword('Big Apple', 'New York')
+    >>> keyword_processor.add_keyword('Bay Area')
+    >>> keywords_found = keyword_processor.extract_keywords('I love big Apple and Bay Area.', span_info=True)
+    >>> keywords_found
+    >>> # [('New York', 7, 16), ('Bay Area', 21, 29)]
+
+Get Extra information with keywords extracted
+    >>> from flashtext import KeywordProcessor
+    >>> kp = KeywordProcessor()
+    >>> kp.add_keyword('Taj Mahal', ('Monument', 'Taj Mahal'))
+    >>> kp.add_keyword('Delhi', ('Location', 'Delhi'))
+    >>> kp.extract_keywords('Taj Mahal is in Delhi.')
+    >>> # [('Monument', 'Taj Mahal'), ('Location', 'Delhi')]
+    >>> # NOTE: replace_keywords feature won't work with this.
+
 No clean name for Keywords
     >>> from flashtext import KeywordProcessor
     >>> keyword_processor = KeywordProcessor()
@@ -131,9 +149,9 @@ Get all keywords in dictionary
     >>> from flashtext import KeywordProcessor
     >>> keyword_processor = KeywordProcessor()
     >>> keyword_processor.add_keyword('j2ee', 'Java')
-    >>> keyword_processor.add_keyword('onGoing', 'rendom')
+    >>> keyword_processor.add_keyword('colour', 'color')
     >>> keyword_processor.get_all_keywords()
-    >>> # output: {'j2ee': 'Java', 'ongoing': 'rendom'}
+    >>> # output: {'colour': 'color', 'j2ee': 'Java'}
 
 For detecting Word Boundary currently any character other than this `\\w` `[A-Za-z0-9_]` is considered a word boundary.
 
@@ -207,11 +225,27 @@ The idea for this library came from the following `StackOverflow question
 <https://stackoverflow.com/questions/44178449/regex-replace-is-taking-time-for-millions-of-documents-how-to-make-it-faster>`_.
 
 
-References
+Citation
 ----------
 
 The original paper published on `FlashText algorithm <https://arxiv.org/abs/1711.00046>`_.
 
+::
+
+    @ARTICLE{2017arXiv171100046S,
+       author = {{Singh}, V.},
+        title = "{Replace or Retrieve Keywords In Documents at Scale}",
+      journal = {ArXiv e-prints},
+    archivePrefix = "arXiv",
+       eprint = {1711.00046},
+     primaryClass = "cs.DS",
+     keywords = {Computer Science - Data Structures and Algorithms},
+         year = 2017,
+        month = oct,
+       adsurl = {http://adsabs.harvard.edu/abs/2017arXiv171100046S},
+      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+    }
+
 The article published on `Medium freeCodeCamp <https://medium.freecodecamp.org/regex-was-taking-5-days-flashtext-does-it-in-15-minutes-55f04411025f>`_.
 
 

flashtext/keyword.py

@@ -1,5 +1,7 @@
 import os
 import string
+import io
+import re
 
 
 class KeywordProcessor(object):
@@ -8,8 +10,6 @@ class KeywordProcessor(object):
     Attributes:
         _keyword (str): Used as key to store keywords in trie dictionary.
             Defaults to '_keyword_'
-        non_word_boundaries (set(str)): Characters that will determine if the word is continuing.
-            Defaults to set([A-Za-z0-9_])
         keyword_trie_dict (dict): Trie dict built character by character, that is used for lookup
             Defaults to empty dictionary
         case_sensitive (boolean): if the search algorithm should be case sensitive or not.
@@ -34,20 +34,16 @@ class KeywordProcessor(object):
         * Idea came from this `Stack Overflow Question <https://stackoverflow.com/questions/44178449/regex-replace-is-taking-time-for-millions-of-documents-how-to-make-it-faster>`_.
     """
 
+    NON_WORD_CHAR_REGEX = re.compile(r'\W', re.UNICODE)
+
+
     def __init__(self, case_sensitive=False):
         """
         Args:
             case_sensitive (boolean): Keyword search should be case sensitive set or not.
                 Defaults to False
         """
         self._keyword = '_keyword_'
-        self._white_space_chars = set(['.', '\t', '\n', '\a', ' ', ','])
-        try:
-            # python 2.x
-            self.non_word_boundaries = set(string.digits + string.letters + '_')
-        except AttributeError:
-            # python 3.x
-            self.non_word_boundaries = set(string.digits + string.ascii_letters + '_')
         self.keyword_trie_dict = dict()
         self.case_sensitive = case_sensitive
         self._terms_in_trie = 0
@@ -173,8 +169,12 @@ def __delitem__(self, keyword):
                 if letter in current_dict:
                     character_trie_list.append((letter, current_dict))
                     current_dict = current_dict[letter]
-            # remove the charactes from trie dict if there are no other keywords with them
-            if self._keyword in current_dict:
+                else:
+                    # if character is not found, break out of the loop
+                    current_dict = None
+                    break
+            # remove the characters from trie dict if there are no other keywords with them
+            if current_dict and self._keyword in current_dict:
                 # we found a complete match for input keyword.
                 character_trie_list.append((self._keyword, current_dict))
                 character_trie_list.reverse()
@@ -197,26 +197,6 @@ def __iter__(self):
         """
         raise NotImplementedError("Please use get_all_keywords() instead")
 
-    def set_non_word_boundaries(self, non_word_boundaries):
-        """set of characters that will be considered as part of word.
-
-        Args:
-            non_word_boundaries (set(str)):
-                Set of characters that will be considered as part of word.
-
-        """
-        self.non_word_boundaries = non_word_boundaries
-
-    def add_non_word_boundary(self, character):
-        """add a character that will be considered as part of word.
-
-        Args:
-            character (char):
-                Character that will be considered as part of word.
-
-        """
-        self.non_word_boundaries.add(character)
-
     def add_keyword(self, keyword, clean_name=None):
         """To add one or more keywords to the dictionary
         pass the keyword and the clean name it maps to.
@@ -283,11 +263,12 @@ def get_keyword(self, word):
         """
         return self.__getitem__(word)
 
-    def add_keyword_from_file(self, keyword_file):
+    def add_keyword_from_file(self, keyword_file, encoding="utf-8"):
         """To add keywords from a file
 
         Args:
             keyword_file : path to keywords file
+            encoding : specify the encoding of the file
 
         Examples:
             keywords file format can be like:
@@ -311,7 +292,7 @@ def add_keyword_from_file(self, keyword_file):
         """
         if not os.path.isfile(keyword_file):
             raise IOError("Invalid file path {}".format(keyword_file))
-        with open(keyword_file)as f:
+        with io.open(keyword_file, encoding=encoding) as f:
             for line in f:
                 if '=>' in line:
                     keyword, clean_name = line.split('=>')
@@ -381,7 +362,7 @@ def add_keywords_from_list(self, keyword_list):
 
         """
         if not isinstance(keyword_list, list):
-                raise AttributeError("keyword_list should be a list")
+            raise AttributeError("keyword_list should be a list")
 
         for keyword in keyword_list:
             self.add_keyword(keyword)
@@ -410,7 +391,7 @@ def get_all_keywords(self, term_so_far='', current_dict=None):
 
         Args:
             term_so_far : string
-                term built so far by adding all previous charactes
+                term built so far by adding all previous characters
             current_dict : dict
                 current recursive position in dictionary
 
@@ -441,7 +422,7 @@ def get_all_keywords(self, term_so_far='', current_dict=None):
                     terms_present[key] = sub_values[key]
         return terms_present
 
-    def extract_keywords(self, sentence):
+    def extract_keywords(self, sentence, span_info=False):
         """Searches in the string for all keywords present in corpus.
         Keywords present are added to a list `keywords_extracted` and returned.
 
@@ -468,13 +449,15 @@ def extract_keywords(self, sentence):
         if not self.case_sensitive:
             sentence = sentence.lower()
         current_dict = self.keyword_trie_dict
+        sequence_start_pos = 0
         sequence_end_pos = 0
+        reset_current_dict = False
         idx = 0
         sentence_len = len(sentence)
         while idx < sentence_len:
             char = sentence[idx]
             # when we reach a character that might denote word end
-            if char not in self.non_word_boundaries:
+            if KeywordProcessor.NON_WORD_CHAR_REGEX.match(char):
 
                 # if end is present in current_dict
                 if self._keyword in current_dict or char in current_dict:
@@ -494,7 +477,7 @@ def extract_keywords(self, sentence):
                         idy = idx + 1
                         while idy < sentence_len:
                             inner_char = sentence[idy]
-                            if inner_char not in self.non_word_boundaries and self._keyword in current_dict_continued:
+                            if KeywordProcessor.NON_WORD_CHAR_REGEX.match(inner_char) and self._keyword in current_dict_continued:
                                 # update longest sequence found
                                 longest_sequence_found = current_dict_continued[self._keyword]
                                 sequence_end_pos = idy
@@ -515,32 +498,39 @@ def extract_keywords(self, sentence):
                             idx = sequence_end_pos
                     current_dict = self.keyword_trie_dict
                     if longest_sequence_found:
-                        keywords_extracted.append(longest_sequence_found)
-
+                        keywords_extracted.append((longest_sequence_found, sequence_start_pos, idx))
+                    reset_current_dict = True
                 else:
                     # we reset current_dict
                     current_dict = self.keyword_trie_dict
+                    reset_current_dict = True
             elif char in current_dict:
                 # we can continue from this char
                 current_dict = current_dict[char]
             else:
                 # we reset current_dict
                 current_dict = self.keyword_trie_dict
+                reset_current_dict = True
                 # skip to end of word
                 idy = idx + 1
                 while idy < sentence_len:
                     char = sentence[idy]
-                    if char not in self.non_word_boundaries:
+                    if KeywordProcessor.NON_WORD_CHAR_REGEX.match(char):
                         break
                     idy += 1
                 idx = idy
             # if we are end of sentence and have a sequence discovered
             if idx + 1 >= sentence_len:
                 if self._keyword in current_dict:
                     sequence_found = current_dict[self._keyword]
-                    keywords_extracted.append(sequence_found)
+                    keywords_extracted.append((sequence_found, sequence_start_pos, sentence_len))
             idx += 1
-        return keywords_extracted
+            if reset_current_dict:
+                reset_current_dict = False
+                sequence_start_pos = idx
+        if span_info:
+            return keywords_extracted
+        return [value[0] for value in keywords_extracted]
 
     def replace_keywords(self, sentence):
         """Searches in the string for all keywords present in corpus.
@@ -579,7 +569,7 @@ def replace_keywords(self, sentence):
             char = sentence[idx]
             current_word += orig_sentence[idx]
             # when we reach whitespace
-            if char not in self.non_word_boundaries:
+            if KeywordProcessor.NON_WORD_CHAR_REGEX.match(char):
                 current_white_space = char
                 # if end is present in current_dict
                 if self._keyword in current_dict or char in current_dict:
@@ -600,7 +590,7 @@ def replace_keywords(self, sentence):
                         while idy < sentence_len:
                             inner_char = sentence[idy]
                             current_word_continued += orig_sentence[idy]
-                            if inner_char not in self.non_word_boundaries and self._keyword in current_dict_continued:
+                            if KeywordProcessor.NON_WORD_CHAR_REGEX.match(inner_char) and self._keyword in current_dict_continued:
                                 # update longest sequence found
                                 current_white_space = inner_char
                                 longest_sequence_found = current_dict_continued[self._keyword]
@@ -648,7 +638,7 @@ def replace_keywords(self, sentence):
                 while idy < sentence_len:
                     char = sentence[idy]
                     current_word += orig_sentence[idy]
-                    if char not in self.non_word_boundaries:
+                    if KeywordProcessor.NON_WORD_CHAR_REGEX.match(char):
                         break
                     idy += 1
                 idx = idy
@@ -660,5 +650,7 @@ def replace_keywords(self, sentence):
                 if self._keyword in current_dict:
                     sequence_found = current_dict[self._keyword]
                     new_sentence += sequence_found
+                else:
+                    new_sentence += current_word
             idx += 1
         return new_sentence

setup.py

@@ -16,7 +16,7 @@ def run(self):
         raise SystemExit(errno)
 
 name = 'flashtext'
-version = '2.4'
+version = '2.7'
 
 cmdclass = {'test': PyTest}