Merge pull request #41 from rayokota/regex-option

Add pluggable regex engine support

Author: rayokota

README.md

@@ -107,6 +107,36 @@ JSONata> (a & b)
 hello world
 ```
 
+## Guardrails
+
+JSONata is Turing-complete, so it's possible to write expressions that loop forever or exhaust memory. If you evaluate
+untrusted expressions, configure these guardrails (see the JS reference implementation's
+[guardrails docs](https://docs.jsonata.org/guardrails) for more background):
+
+- **Stack overflow** — the `stack` parameter caps the depth of the eval-apply cycle. Exceeding it raises `D1011`.
+- **Excessive execution time** — the `timeout` parameter (in milliseconds) catches tail-recursive infinite loops that
+  `stack` can't. Exceeding it raises `D1012`.
+- **Rogue regular expressions** — the `regex_engine` parameter lets you swap in a linear-time engine (e.g.
+  [`google-re2`](https://pypi.org/project/google-re2/)) to protect against [ReDoS](https://en.wikipedia.org/wiki/ReDoS),
+  since the `timeout` guardrail can't interrupt a regex match in progress.
+
+```python
+import re2  # pip install google-re2
+import jsonata
+from jsonata.regex_engine import RegexFlags
+
+
+def re2_regex_engine(pattern: str, flags: RegexFlags):
+    options = re2.Options()
+    options.case_sensitive = not flags.case_insensitive
+    options.one_line = not flags.multiline
+    return re2.compile(pattern, options)
+
+
+expr = jsonata.Jsonata("<JSONata expression>", re2_regex_engine, timeout=1000, stack=500)
+result = expr.evaluate(data)
+```
+
 ## Running Tests
 
 This project uses the repository of the reference implementation as a submodule. This allows referencing the current version of the unit tests. To clone this repository, run:

noxfile.py

@@ -44,9 +44,33 @@ def tests(session):
     build_and_check_dists(session)
 
     generated_files = os.listdir("dist/")
-    generated_sdist = os.path.join("dist/", generated_files[1])
+    sdists = [f for f in generated_files if f.endswith(".tar.gz")]
+    if not sdists:
+        session.error("No sdist (.tar.gz) found in dist/")
+    generated_sdist = max((os.path.join("dist/", f) for f in sdists), key=os.path.getmtime)
 
     session.install(generated_sdist)
 
     session.run("python", "tests/generate.py")
     session.run("py.test", "tests/", *session.posargs)
+
+
+# Exercises the optional pluggable regex_engine hook against Google's RE2
+# (tests/re2_engine_test.py). Kept as its own session so the main `tests`
+# session -- and the package itself -- stay free of a google-re2 dependency;
+# this one opts in explicitly.
+@nox.session
+def test_re2(session):
+    session.install("pytest")
+    session.install("google-re2")
+    build_and_check_dists(session)
+
+    generated_files = os.listdir("dist/")
+    sdists = [f for f in generated_files if f.endswith(".tar.gz")]
+    if not sdists:
+        session.error("No sdist (.tar.gz) found in dist/")
+    generated_sdist = max((os.path.join("dist/", f) for f in sdists), key=os.path.getmtime)
+
+    session.install(generated_sdist)
+
+    session.run("py.test", "tests/re2_engine_test.py", *session.posargs)

src/jsonata/functions.py

@@ -39,6 +39,7 @@
 from typing import Any, AnyStr, Mapping, NoReturn, Optional, Sequence, Callable, Type, Union
 
 from jsonata import datetimeutils, jexception, parser, utils
+from jsonata.regex_engine import CompiledPattern
 
 
 class Functions:
@@ -514,7 +515,7 @@ class RegexpMatch:
     # @returns {object} - structure that represents the match(es)
     #     
     @staticmethod
-    def evaluate_matcher(matcher: re.Pattern, string: Optional[str]) -> list[RegexpMatch]:
+    def evaluate_matcher(matcher: CompiledPattern, string: Optional[str]) -> list[RegexpMatch]:
         res = []
         matches = matcher.finditer(string)
         for m in matches:
@@ -537,7 +538,7 @@ def evaluate_matcher(matcher: re.Pattern, string: Optional[str]) -> list[RegexpM
     # @returns {Boolean} - true if str contains token
     #     
     @staticmethod
-    def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Optional[bool]:
+    def contains(string: Optional[str], token: Union[None, str, CompiledPattern]) -> Optional[bool]:
         # undefined inputs always return undefined
         if string is None:
             return None
@@ -549,7 +550,7 @@ def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Opti
 
         if isinstance(token, str):
             result = (string.find(str(token)) != - 1)
-        elif isinstance(token, re.Pattern):
+        elif Functions.is_regex(token):
             matches = Functions.evaluate_matcher(token, string)
             # if (dbg) System.out.println("match = "+matches)
             # result = (typeof matches !== 'undefined')
@@ -568,7 +569,7 @@ def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Opti
     # @returns {Array} The array of match objects
     #     
     @staticmethod
-    def match_(string: Optional[str], regex: Optional[re.Pattern], limit: Optional[int]) -> Optional[list[dict[str, Any]]]:
+    def match_(string: Optional[str], regex: Optional[CompiledPattern], limit: Optional[int]) -> Optional[list[dict[str, Any]]]:
         # undefined inputs always return undefined
         if string is None:
             return None
@@ -636,7 +637,7 @@ def safe_replacement(in_: str) -> str:
     # @return
     #     
     @staticmethod
-    def safe_replace_all(s: str, pattern: re.Pattern, replacement: Optional[Any]) -> Optional[str]:
+    def safe_replace_all(s: str, pattern: CompiledPattern, replacement: Optional[Any]) -> Optional[str]:
 
         if not (isinstance(replacement, str)):
             return Functions.safe_replace_all_fn(s, pattern, replacement)
@@ -647,7 +648,7 @@ def safe_replace_all(s: str, pattern: re.Pattern, replacement: Optional[Any]) ->
         r = None
         for i in range(0, 10):
             try:
-                r = re.sub(pattern, replacement, s)
+                r = pattern.sub(replacement, s)
                 break
             except Exception as e:
                 msg = str(e)
@@ -696,15 +697,15 @@ def to_jsonata_match(mr: re.Match[str]) -> dict[str, list[str]]:
     # @return
     #     
     @staticmethod
-    def safe_replace_all_fn(s: str, pattern: re.Pattern, fn: Optional[Any]) -> str:
+    def safe_replace_all_fn(s: str, pattern: CompiledPattern, fn: Optional[Any]) -> str:
         def replace_fn(t):
             res = Functions.func_apply(fn, [Functions.to_jsonata_match(t)])
             if isinstance(res, str):
                 return res
             else:
                 raise jexception.JException("D3012", -1)
 
-        r = re.sub(pattern, replace_fn, s)
+        r = pattern.sub(replace_fn, s)
         return r
 
     #
@@ -716,12 +717,12 @@ def replace_fn(t):
     # @return
     #     
     @staticmethod
-    def safe_replace_first(s: str, pattern: re.Pattern, replacement: str) -> Optional[str]:
+    def safe_replace_first(s: str, pattern: CompiledPattern, replacement: str) -> Optional[str]:
         replacement = Functions.safe_replacement(replacement)
         r = None
         for i in range(0, 10):
             try:
-                r = re.sub(pattern, replacement, s, count=1)
+                r = pattern.sub(replacement, s, 1)
                 break
             except Exception as e:
                 msg = str(e)
@@ -744,7 +745,7 @@ def safe_replace_first(s: str, pattern: re.Pattern, replacement: str) -> Optiona
         return r
 
     @staticmethod
-    def replace(string: Optional[str], pattern: Union[str, re.Pattern], replacement: Optional[Any], limit: Optional[int]) -> Optional[str]:
+    def replace(string: Optional[str], pattern: Union[str, CompiledPattern], replacement: Optional[Any], limit: Optional[int]) -> Optional[str]:
         if string is None:
             return None
 
@@ -938,7 +939,7 @@ def decode_url(string: Optional[str]) -> Optional[str]:
         return urllib.parse.unquote(string, errors="strict")
 
     @staticmethod
-    def split(string: Optional[str], pattern: Union[str, Optional[re.Pattern]], limit: Optional[float]) -> Optional[list[str]]:
+    def split(string: Optional[str], pattern: Union[str, Optional[CompiledPattern]], limit: Optional[float]) -> Optional[list[str]]:
         if string is None:
             return None
 
@@ -2019,6 +2020,23 @@ def append(arg1: Optional[Any], arg2: Optional[Any]) -> Optional[Any]:
     def is_lambda(result: Optional[Any]) -> bool:
         return isinstance(result, parser.Parser.Symbol) and result._jsonata_lambda
 
+    #
+    # Tests whether a value is a compiled regex, from the stdlib re module
+    # or from a pluggable regex_engine (e.g. re2) with a compatible interface.
+    #
+    @staticmethod
+    def is_regex(value: Optional[Any]) -> bool:
+        if isinstance(value, re.Pattern):
+            return True
+        if value is None or inspect.ismodule(value) or inspect.isclass(value):
+            return False
+        return (
+            callable(getattr(value, "search", None))
+            and callable(getattr(value, "finditer", None))
+            and callable(getattr(value, "sub", None))
+            and callable(getattr(value, "split", None))
+        )
+
     #
     # Return value from an object for a given key
     # @param {Object} input - Object/Array
@@ -2191,7 +2209,15 @@ def function_eval(expr: Optional[str], focus: Optional[Any]) -> Optional[Any]:
         # undefined inputs always return undefined
         if expr is None:
             return None
-        input = jsonata.Jsonata.CURRENT.jsonata.input  # =  this.input;
+
+        # Capture the enclosing instance *before* constructing the nested
+        # ast below: Jsonata.__init__ unconditionally overwrites
+        # Jsonata.CURRENT.jsonata as a side effect, so re-reading
+        # CURRENT.jsonata afterwards would silently return the inner
+        # expression instead of the enclosing one, losing access to its
+        # environment (e.g. outer variable bindings).
+        enclosing = jsonata.Jsonata.CURRENT.jsonata
+        input = enclosing.input  # =  this.input;
         if focus is not None:
             input = focus
             # if the input is a JSON array, then wrap it in a singleton sequence so it gets treated as a single input
@@ -2201,18 +2227,47 @@ def function_eval(expr: Optional[str], focus: Optional[Any]) -> Optional[Any]:
 
         ast = None
         try:
-            ast = jsonata.Jsonata(expr)
-        except Exception as err:
-            # error parsing the expression passed to $eval
-            # populateMessage(err)
-            raise jexception.JException("D3120", -1)
-        result = None
-        try:
-            result = ast.evaluate(input, jsonata.Jsonata.CURRENT.jsonata.environment)
-        except Exception as err:
-            # error evaluating the expression passed to $eval
-            # populateMessage(err)
-            raise jexception.JException("D3121", -1)
+            try:
+                # Only used to parse expr into an AST (ast.ast below); the
+                # actual evaluation reuses the enclosing instance directly
+                # (see below), so regex_engine here just needs to be valid
+                # for parsing -- it does not need to be evaluated against.
+                ast = jsonata.Jsonata(expr, enclosing.regex_engine)
+            except Exception as err:
+                # error parsing the expression passed to $eval
+                # populateMessage(err)
+                raise jexception.JException("D3120", -1)
+
+            # Constructing `ast` overwrote Jsonata.CURRENT.jsonata as a side
+            # effect (see Jsonata.__init__); restore it to the enclosing
+            # instance before evaluating, so enclosing.eval()'s own
+            # get_per_thread_instance() lookup resolves correctly.
+            jsonata.Jsonata.CURRENT.jsonata = enclosing
+
+            result = None
+            try:
+                # Evaluate ast.ast (the parsed tree) using the *enclosing*
+                # instance's low-level eval(), reusing enclosing.environment
+                # directly rather than calling ast.evaluate(input, environment)
+                # (which copies environment's bindings one level deep into a
+                # fresh child frame rooted at ast's own static frame). This
+                # mirrors jsonata-java's Functions.functionEval, which calls
+                # Jsonata.current.get().evaluate(ast.ast, input, env) rather
+                # than constructing a second, disconnected evaluation
+                # context. Reusing the environment directly means $eval sees
+                # bindings at every level of the enclosing scope chain (not
+                # just the immediate frame) and correctly inherits any
+                # stack/timeout guardrails registered on an ancestor frame,
+                # since Frame.lookup walks the parent chain.
+                result = enclosing.eval(ast.ast, input, enclosing.environment)
+            except Exception as err:
+                # error evaluating the expression passed to $eval
+                # populateMessage(err)
+                raise jexception.JException("D3121", -1)
+        finally:
+            # Restore the enclosing instance as current now that $eval's
+            # parsing/evaluation is done.
+            jsonata.Jsonata.CURRENT.jsonata = enclosing
 
         return result
 

src/jsonata/jexception.py

@@ -175,6 +175,8 @@ def msg(error: str, location: int, arg1: Optional[Any], arg2: Optional[Any], det
         "T1007": "Attempted to partially apply a non-function. Did you mean ${{{token}}}?",
         "T1008": "Attempted to partially apply a non-function",
         "D1009": "Multiple key definitions evaluate to same key: {{value}}",
+        "D1011": "Stack overflow. Check for non-terminating recursive function.  Consider rewriting as tail-recursive",
+        "D1012": "Evaluation timeout after {{value}} milliseconds. Check for infinite loop",
         "T1010": "The matcher Object argument passed to Object {{token}} does not return the correct object structure",
         "T2001": "The left side of the {{token}} operator must evaluate to a number",
         "T2002": "The right side of the {{token}} operator must evaluate to a number",