Add pluggable regex engine support

Author: rayokota

noxfile.py

@@ -50,3 +50,21 @@ def tests(session):
 
     session.run("python", "tests/generate.py")
     session.run("py.test", "tests/", *session.posargs)
+
+
+# Exercises the optional pluggable regex_engine hook against Google's RE2
+# (tests/re2_engine_test.py). Kept as its own session so the main `tests`
+# session -- and the package itself -- stay free of a google-re2 dependency;
+# this one opts in explicitly.
+@nox.session
+def test_re2(session):
+    session.install("pytest")
+    session.install("google-re2")
+    build_and_check_dists(session)
+
+    generated_files = os.listdir("dist/")
+    generated_sdist = os.path.join("dist/", generated_files[1])
+
+    session.install(generated_sdist)
+
+    session.run("py.test", "tests/re2_engine_test.py", *session.posargs)

src/jsonata/functions.py

@@ -549,7 +549,7 @@ def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Opti
 
         if isinstance(token, str):
             result = (string.find(str(token)) != - 1)
-        elif isinstance(token, re.Pattern):
+        elif Functions.is_regex(token):
             matches = Functions.evaluate_matcher(token, string)
             # if (dbg) System.out.println("match = "+matches)
             # result = (typeof matches !== 'undefined')
@@ -647,7 +647,7 @@ def safe_replace_all(s: str, pattern: re.Pattern, replacement: Optional[Any]) ->
         r = None
         for i in range(0, 10):
             try:
-                r = re.sub(pattern, replacement, s)
+                r = pattern.sub(replacement, s)
                 break
             except Exception as e:
                 msg = str(e)
@@ -704,7 +704,7 @@ def replace_fn(t):
             else:
                 raise jexception.JException("D3012", -1)
 
-        r = re.sub(pattern, replace_fn, s)
+        r = pattern.sub(replace_fn, s)
         return r
 
     #
@@ -721,7 +721,7 @@ def safe_replace_first(s: str, pattern: re.Pattern, replacement: str) -> Optiona
         r = None
         for i in range(0, 10):
             try:
-                r = re.sub(pattern, replacement, s, count=1)
+                r = pattern.sub(replacement, s, count=1)
                 break
             except Exception as e:
                 msg = str(e)
@@ -2019,6 +2019,16 @@ def append(arg1: Optional[Any], arg2: Optional[Any]) -> Optional[Any]:
     def is_lambda(result: Optional[Any]) -> bool:
         return isinstance(result, parser.Parser.Symbol) and result._jsonata_lambda
 
+    #
+    # Tests whether a value is a compiled regex, from the stdlib re module
+    # or from a pluggable regex_engine (e.g. re2) with a compatible interface.
+    #
+    @staticmethod
+    def is_regex(value: Optional[Any]) -> bool:
+        return isinstance(value, re.Pattern) or (
+            hasattr(value, "search") and hasattr(value, "finditer") and hasattr(value, "sub")
+        )
+
     #
     # Return value from an object for a given key
     # @param {Object} input - Object/Array
@@ -2201,7 +2211,7 @@ def function_eval(expr: Optional[str], focus: Optional[Any]) -> Optional[Any]:
 
         ast = None
         try:
-            ast = jsonata.Jsonata(expr)
+            ast = jsonata.Jsonata(expr, jsonata.Jsonata.CURRENT.jsonata.regex_engine)
         except Exception as err:
             # error parsing the expression passed to $eval
             # populateMessage(err)

src/jsonata/jsonata.py

@@ -1300,7 +1300,7 @@ def evaluate_apply_expression(self, expr: Optional[parser.Parser.Symbol], input:
         return result
 
     def is_function_like(self, o: Optional[Any]) -> bool:
-        return utils.Utils.is_function(o) or functions.Functions.is_lambda(o) or (isinstance(o, re.Pattern))
+        return utils.Utils.is_function(o) or functions.Functions.is_lambda(o) or functions.Functions.is_regex(o)
 
     CURRENT = threading.local()
     MUTEX = threading.Lock()
@@ -1477,7 +1477,7 @@ def apply_inner(self, proc: Optional[Any], args: Optional[Any], input: Optional[
                 #  }
             elif isinstance(proc, Jsonata.JLambda):
                 result = proc.call(input, validated_args)
-            elif isinstance(proc, re.Pattern):
+            elif functions.Functions.is_regex(proc):
                 _res = []
                 for s in validated_args:
                     if isinstance(s, str):
@@ -1886,12 +1886,15 @@ def _static_initializer() -> None:
     #
     # JSONata
     # @param {Object} expr - JSONata expression
+    # @param {Object} regex_engine - module/object providing a `compile(pattern, flags)`
+    #     function compatible with the stdlib `re` module (e.g. `re2`), used to compile
+    #     JSONata regex literals. Defaults to the stdlib `re` module.
     # @returns Evaluated expression
     # @throws jexception.JException An exception if an error occured.
-    #      
+    #
     @staticmethod
-    def jsonata(expression: Optional[str]) -> 'Jsonata':
-        return Jsonata(expression)
+    def jsonata(expression: Optional[str], regex_engine: Any = re) -> 'Jsonata':
+        return Jsonata(expression, regex_engine)
 
     #
     # Internal constructor
@@ -1904,11 +1907,13 @@ def jsonata(expression: Optional[str]) -> 'Jsonata':
     ast: Optional[parser.Parser.Symbol]
     timestamp: int
     input: Optional[Any]
+    regex_engine: Any
 
-    def __init__(self, expr: Optional[str]) -> None:
+    def __init__(self, expr: Optional[str], regex_engine: Any = re) -> None:
+        self.regex_engine = regex_engine
         try:
             self.parser = Jsonata.get_parser()
-            self.ast = self.parser.parse(expr)  # , optionsRecover);
+            self.ast = self.parser.parse(expr, regex_engine)  # , optionsRecover);
             self.errors = self.ast.errors
             self.ast.errors = None  # delete ast.errors;
         except jexception.JException as err:
@@ -1931,13 +1936,6 @@ def __init__(self, expr: Optional[str]) -> None:
         #      return timestamp.getTime()
         #  }, "<:n>"))
 
-        # FIXED: options.RegexEngine not implemented in Java
-        #  if(options && options.RegexEngine) {
-        #      jsonata.RegexEngine = options.RegexEngine
-        #  } else {
-        #      jsonata.RegexEngine = RegExp
-        #  }
-
         # Set instance for this thread
         Jsonata.CURRENT.jsonata = self
 

src/jsonata/parser.py

@@ -25,6 +25,7 @@
 #
 
 import copy
+import re
 from typing import Any, MutableSequence, Optional, Sequence
 
 from jsonata import jexception, tokenizer, signature, utils
@@ -1413,11 +1414,11 @@ def object_parser(self, left: Optional[Symbol]) -> Symbol:
             res.type = "binary"
         return res
 
-    def parse(self, jsonata: Optional[str]) -> Symbol:
+    def parse(self, jsonata: Optional[str], regex_engine: Any = re) -> Symbol:
         self.source = jsonata
 
         # now invoke the tokenizer and the parser and return the syntax tree
-        self.lexer = tokenizer.Tokenizer(self.source)
+        self.lexer = tokenizer.Tokenizer(self.source, regex_engine)
         self.advance()
         # parse the tokens
         expr = self.expression(0)

src/jsonata/signature.py

@@ -112,7 +112,7 @@ def get_symbol(self, value: Optional[Any]) -> str:
             symbol = "m"
         else:
             # first check to see if this is a function
-            if utils.Utils.is_function(value) or functions.Functions.is_lambda(value) or isinstance(value, re.Pattern):
+            if utils.Utils.is_function(value) or functions.Functions.is_lambda(value) or functions.Functions.is_regex(value):
                 symbol = "f"
             elif isinstance(value, str):
                 symbol = "s"

src/jsonata/tokenizer.py

@@ -95,12 +95,13 @@ class Tokenizer:
     path: str
     length: int
 
-    def __init__(self, path):
+    def __init__(self, path, regex_engine=re):
         self.position = 0
         self.depth = 0
 
         self.path = path
         self.length = len(path)
+        self.regex_engine = regex_engine
 
     @dataclass
     class Token:
@@ -121,7 +122,7 @@ def is_closing_slash(self, position: int) -> bool:
                 return True
         return False
 
-    def scan_regex(self) -> re.Pattern:
+    def scan_regex(self):
         # the prefix '/' will have been previously scanned. Find the end of the regex.
         # search for closing '/' ignoring any that are escaped, or within brackets
         start = self.position
@@ -157,7 +158,7 @@ def scan_regex(self) -> re.Pattern:
                     _flags |= re.I
                 if "m" in flags:
                     _flags |= re.M
-                return re.compile(pattern, _flags)  # Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL);
+                return self.regex_engine.compile(pattern, _flags)  # Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL);
             if (current_char == '(' or current_char == '[' or current_char == '{') and self.path[self.position - 1] != '\\':
                 self.depth += 1
             if (current_char == ')' or current_char == ']' or current_char == '}') and self.path[self.position - 1] != '\\':

tests/re2_engine_test.py

@@ -0,0 +1,65 @@
+import re
+
+import jsonata
+import pytest
+
+re2 = pytest.importorskip("re2")
+
+
+class RE2Engine:
+    """
+    Adapts google-re2's Options-based compile() to the re.compile(pattern, flags)
+    interface expected by Jsonata's regex_engine hook.
+    """
+
+    @staticmethod
+    def compile(pattern, flags=0):
+        options = re2.Options()
+        options.case_sensitive = not bool(flags & re.IGNORECASE)
+        options.one_line = not bool(flags & re.MULTILINE)
+        options.log_errors = False
+        return re2.compile(pattern, options)
+
+
+class TestRE2Engine:
+
+    def test_match(self):
+        expr = jsonata.Jsonata('$match("hello world", /o w/)', RE2Engine)
+        assert expr.evaluate(None) == {"match": "o w", "index": 4, "groups": []}
+
+    def test_match_case_insensitive_flag(self):
+        expr = jsonata.Jsonata('$match("HELLO", /hello/i)', RE2Engine)
+        result = expr.evaluate(None)
+        assert result["match"] == "HELLO"
+
+    def test_contains(self):
+        expr = jsonata.Jsonata('$contains("hello", /ell/)', RE2Engine)
+        assert expr.evaluate(None) is True
+
+    def test_replace_with_string(self):
+        expr = jsonata.Jsonata('$replace("abc123def", /[0-9]+/, "#")', RE2Engine)
+        assert expr.evaluate(None) == "abc#def"
+
+    def test_replace_with_function(self):
+        expr = jsonata.Jsonata(
+            '$replace("abc123", /[0-9]+/, function($m) { $m.match & "!" })', RE2Engine
+        )
+        assert expr.evaluate(None) == "abc123!"
+
+    def test_split(self):
+        expr = jsonata.Jsonata('$split("a1b2c3", /[0-9]/)', RE2Engine)
+        assert expr.evaluate(None) == ["a", "b", "c", ""]
+
+    def test_rejects_backreferences(self):
+        # Proves RE2 is actually compiling the pattern rather than silently
+        # falling back to stdlib re: backreferences can't run in RE2's
+        # guaranteed-linear-time engine, so this must fail at parse time
+        # (regex literals are compiled while the expression is constructed).
+        with pytest.raises(re2.error):
+            jsonata.Jsonata(r'$match("abab", /(a)\1/)', RE2Engine)
+
+    def test_default_engine_still_stdlib_re(self):
+        # No regex_engine argument -> falls back to stdlib re, which *does*
+        # support backreferences. Confirms the default path is unaffected.
+        expr = jsonata.Jsonata(r'$match("xaay", /(a)\1/)')
+        assert expr.evaluate(None)["match"] == "aa"