diff --git a/README.md b/README.md index 3333b89..931c1dd 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,36 @@ JSONata> (a & b) hello world ``` +## Guardrails + +JSONata is Turing-complete, so it's possible to write expressions that loop forever or exhaust memory. If you evaluate +untrusted expressions, configure these guardrails (see the JS reference implementation's +[guardrails docs](https://docs.jsonata.org/guardrails) for more background): + +- **Stack overflow** — the `stack` parameter caps the depth of the eval-apply cycle. Exceeding it raises `D1011`. +- **Excessive execution time** — the `timeout` parameter (in milliseconds) catches tail-recursive infinite loops that + `stack` can't. Exceeding it raises `D1012`. +- **Rogue regular expressions** — the `regex_engine` parameter lets you swap in a linear-time engine (e.g. + [`google-re2`](https://pypi.org/project/google-re2/)) to protect against [ReDoS](https://en.wikipedia.org/wiki/ReDoS), + since the `timeout` guardrail can't interrupt a regex match in progress. + +```python +import re2 # pip install google-re2 +import jsonata +from jsonata.regex_engine import RegexFlags + + +def re2_regex_engine(pattern: str, flags: RegexFlags): + options = re2.Options() + options.case_sensitive = not flags.case_insensitive + options.one_line = not flags.multiline + return re2.compile(pattern, options) + + +expr = jsonata.Jsonata("", re2_regex_engine, timeout=1000, stack=500) +result = expr.evaluate(data) +``` + ## Running Tests This project uses the repository of the reference implementation as a submodule. This allows referencing the current version of the unit tests. To clone this repository, run: diff --git a/noxfile.py b/noxfile.py index fb5bc2a..e4ce33d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -44,9 +44,33 @@ def tests(session): build_and_check_dists(session) generated_files = os.listdir("dist/") - generated_sdist = os.path.join("dist/", generated_files[1]) + sdists = [f for f in generated_files if f.endswith(".tar.gz")] + if not sdists: + session.error("No sdist (.tar.gz) found in dist/") + generated_sdist = max((os.path.join("dist/", f) for f in sdists), key=os.path.getmtime) session.install(generated_sdist) session.run("python", "tests/generate.py") session.run("py.test", "tests/", *session.posargs) + + +# Exercises the optional pluggable regex_engine hook against Google's RE2 +# (tests/re2_engine_test.py). Kept as its own session so the main `tests` +# session -- and the package itself -- stay free of a google-re2 dependency; +# this one opts in explicitly. +@nox.session +def test_re2(session): + session.install("pytest") + session.install("google-re2") + build_and_check_dists(session) + + generated_files = os.listdir("dist/") + sdists = [f for f in generated_files if f.endswith(".tar.gz")] + if not sdists: + session.error("No sdist (.tar.gz) found in dist/") + generated_sdist = max((os.path.join("dist/", f) for f in sdists), key=os.path.getmtime) + + session.install(generated_sdist) + + session.run("py.test", "tests/re2_engine_test.py", *session.posargs) diff --git a/src/jsonata/functions.py b/src/jsonata/functions.py index d44b115..fa158a0 100644 --- a/src/jsonata/functions.py +++ b/src/jsonata/functions.py @@ -39,6 +39,7 @@ from typing import Any, AnyStr, Mapping, NoReturn, Optional, Sequence, Callable, Type, Union from jsonata import datetimeutils, jexception, parser, utils +from jsonata.regex_engine import CompiledPattern class Functions: @@ -514,7 +515,7 @@ class RegexpMatch: # @returns {object} - structure that represents the match(es) # @staticmethod - def evaluate_matcher(matcher: re.Pattern, string: Optional[str]) -> list[RegexpMatch]: + def evaluate_matcher(matcher: CompiledPattern, string: Optional[str]) -> list[RegexpMatch]: res = [] matches = matcher.finditer(string) for m in matches: @@ -537,7 +538,7 @@ def evaluate_matcher(matcher: re.Pattern, string: Optional[str]) -> list[RegexpM # @returns {Boolean} - true if str contains token # @staticmethod - def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Optional[bool]: + def contains(string: Optional[str], token: Union[None, str, CompiledPattern]) -> Optional[bool]: # undefined inputs always return undefined if string is None: return None @@ -549,7 +550,7 @@ def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Opti if isinstance(token, str): result = (string.find(str(token)) != - 1) - elif isinstance(token, re.Pattern): + elif Functions.is_regex(token): matches = Functions.evaluate_matcher(token, string) # if (dbg) System.out.println("match = "+matches) # result = (typeof matches !== 'undefined') @@ -568,7 +569,7 @@ def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Opti # @returns {Array} The array of match objects # @staticmethod - def match_(string: Optional[str], regex: Optional[re.Pattern], limit: Optional[int]) -> Optional[list[dict[str, Any]]]: + def match_(string: Optional[str], regex: Optional[CompiledPattern], limit: Optional[int]) -> Optional[list[dict[str, Any]]]: # undefined inputs always return undefined if string is None: return None @@ -636,7 +637,7 @@ def safe_replacement(in_: str) -> str: # @return # @staticmethod - def safe_replace_all(s: str, pattern: re.Pattern, replacement: Optional[Any]) -> Optional[str]: + def safe_replace_all(s: str, pattern: CompiledPattern, replacement: Optional[Any]) -> Optional[str]: if not (isinstance(replacement, str)): return Functions.safe_replace_all_fn(s, pattern, replacement) @@ -647,7 +648,7 @@ def safe_replace_all(s: str, pattern: re.Pattern, replacement: Optional[Any]) -> r = None for i in range(0, 10): try: - r = re.sub(pattern, replacement, s) + r = pattern.sub(replacement, s) break except Exception as e: msg = str(e) @@ -696,7 +697,7 @@ def to_jsonata_match(mr: re.Match[str]) -> dict[str, list[str]]: # @return # @staticmethod - def safe_replace_all_fn(s: str, pattern: re.Pattern, fn: Optional[Any]) -> str: + def safe_replace_all_fn(s: str, pattern: CompiledPattern, fn: Optional[Any]) -> str: def replace_fn(t): res = Functions.func_apply(fn, [Functions.to_jsonata_match(t)]) if isinstance(res, str): @@ -704,7 +705,7 @@ def replace_fn(t): else: raise jexception.JException("D3012", -1) - r = re.sub(pattern, replace_fn, s) + r = pattern.sub(replace_fn, s) return r # @@ -716,12 +717,12 @@ def replace_fn(t): # @return # @staticmethod - def safe_replace_first(s: str, pattern: re.Pattern, replacement: str) -> Optional[str]: + def safe_replace_first(s: str, pattern: CompiledPattern, replacement: str) -> Optional[str]: replacement = Functions.safe_replacement(replacement) r = None for i in range(0, 10): try: - r = re.sub(pattern, replacement, s, count=1) + r = pattern.sub(replacement, s, 1) break except Exception as e: msg = str(e) @@ -744,7 +745,7 @@ def safe_replace_first(s: str, pattern: re.Pattern, replacement: str) -> Optiona return r @staticmethod - def replace(string: Optional[str], pattern: Union[str, re.Pattern], replacement: Optional[Any], limit: Optional[int]) -> Optional[str]: + def replace(string: Optional[str], pattern: Union[str, CompiledPattern], replacement: Optional[Any], limit: Optional[int]) -> Optional[str]: if string is None: return None @@ -938,7 +939,7 @@ def decode_url(string: Optional[str]) -> Optional[str]: return urllib.parse.unquote(string, errors="strict") @staticmethod - def split(string: Optional[str], pattern: Union[str, Optional[re.Pattern]], limit: Optional[float]) -> Optional[list[str]]: + def split(string: Optional[str], pattern: Union[str, Optional[CompiledPattern]], limit: Optional[float]) -> Optional[list[str]]: if string is None: return None @@ -2019,6 +2020,23 @@ def append(arg1: Optional[Any], arg2: Optional[Any]) -> Optional[Any]: def is_lambda(result: Optional[Any]) -> bool: return isinstance(result, parser.Parser.Symbol) and result._jsonata_lambda + # + # Tests whether a value is a compiled regex, from the stdlib re module + # or from a pluggable regex_engine (e.g. re2) with a compatible interface. + # + @staticmethod + def is_regex(value: Optional[Any]) -> bool: + if isinstance(value, re.Pattern): + return True + if value is None or inspect.ismodule(value) or inspect.isclass(value): + return False + return ( + callable(getattr(value, "search", None)) + and callable(getattr(value, "finditer", None)) + and callable(getattr(value, "sub", None)) + and callable(getattr(value, "split", None)) + ) + # # Return value from an object for a given key # @param {Object} input - Object/Array @@ -2191,7 +2209,15 @@ def function_eval(expr: Optional[str], focus: Optional[Any]) -> Optional[Any]: # undefined inputs always return undefined if expr is None: return None - input = jsonata.Jsonata.CURRENT.jsonata.input # = this.input; + + # Capture the enclosing instance *before* constructing the nested + # ast below: Jsonata.__init__ unconditionally overwrites + # Jsonata.CURRENT.jsonata as a side effect, so re-reading + # CURRENT.jsonata afterwards would silently return the inner + # expression instead of the enclosing one, losing access to its + # environment (e.g. outer variable bindings). + enclosing = jsonata.Jsonata.CURRENT.jsonata + input = enclosing.input # = this.input; if focus is not None: input = focus # if the input is a JSON array, then wrap it in a singleton sequence so it gets treated as a single input @@ -2201,18 +2227,47 @@ def function_eval(expr: Optional[str], focus: Optional[Any]) -> Optional[Any]: ast = None try: - ast = jsonata.Jsonata(expr) - except Exception as err: - # error parsing the expression passed to $eval - # populateMessage(err) - raise jexception.JException("D3120", -1) - result = None - try: - result = ast.evaluate(input, jsonata.Jsonata.CURRENT.jsonata.environment) - except Exception as err: - # error evaluating the expression passed to $eval - # populateMessage(err) - raise jexception.JException("D3121", -1) + try: + # Only used to parse expr into an AST (ast.ast below); the + # actual evaluation reuses the enclosing instance directly + # (see below), so regex_engine here just needs to be valid + # for parsing -- it does not need to be evaluated against. + ast = jsonata.Jsonata(expr, enclosing.regex_engine) + except Exception as err: + # error parsing the expression passed to $eval + # populateMessage(err) + raise jexception.JException("D3120", -1) + + # Constructing `ast` overwrote Jsonata.CURRENT.jsonata as a side + # effect (see Jsonata.__init__); restore it to the enclosing + # instance before evaluating, so enclosing.eval()'s own + # get_per_thread_instance() lookup resolves correctly. + jsonata.Jsonata.CURRENT.jsonata = enclosing + + result = None + try: + # Evaluate ast.ast (the parsed tree) using the *enclosing* + # instance's low-level eval(), reusing enclosing.environment + # directly rather than calling ast.evaluate(input, environment) + # (which copies environment's bindings one level deep into a + # fresh child frame rooted at ast's own static frame). This + # mirrors jsonata-java's Functions.functionEval, which calls + # Jsonata.current.get().evaluate(ast.ast, input, env) rather + # than constructing a second, disconnected evaluation + # context. Reusing the environment directly means $eval sees + # bindings at every level of the enclosing scope chain (not + # just the immediate frame) and correctly inherits any + # stack/timeout guardrails registered on an ancestor frame, + # since Frame.lookup walks the parent chain. + result = enclosing.eval(ast.ast, input, enclosing.environment) + except Exception as err: + # error evaluating the expression passed to $eval + # populateMessage(err) + raise jexception.JException("D3121", -1) + finally: + # Restore the enclosing instance as current now that $eval's + # parsing/evaluation is done. + jsonata.Jsonata.CURRENT.jsonata = enclosing return result diff --git a/src/jsonata/jexception.py b/src/jsonata/jexception.py index ab84242..79d09bf 100644 --- a/src/jsonata/jexception.py +++ b/src/jsonata/jexception.py @@ -175,6 +175,8 @@ def msg(error: str, location: int, arg1: Optional[Any], arg2: Optional[Any], det "T1007": "Attempted to partially apply a non-function. Did you mean ${{{token}}}?", "T1008": "Attempted to partially apply a non-function", "D1009": "Multiple key definitions evaluate to same key: {{value}}", + "D1011": "Stack overflow. Check for non-terminating recursive function. Consider rewriting as tail-recursive", + "D1012": "Evaluation timeout after {{value}} milliseconds. Check for infinite loop", "T1010": "The matcher Object argument passed to Object {{token}} does not return the correct object structure", "T2001": "The left side of the {{token}} operator must evaluate to a number", "T2002": "The right side of the {{token}} operator must evaluate to a number", diff --git a/src/jsonata/jsonata.py b/src/jsonata/jsonata.py index 5707f30..8248db4 100644 --- a/src/jsonata/jsonata.py +++ b/src/jsonata/jsonata.py @@ -27,13 +27,13 @@ import copy import inspect import math -import re import sys import threading from dataclasses import dataclass from typing import Any, Callable, Mapping, MutableSequence, Optional, Sequence, Type, MutableMapping, Union from jsonata import functions, jexception, parser, signature as sig, timebox, utils +from jsonata.regex_engine import RegexEngine, default_regex_engine # @@ -72,7 +72,7 @@ def lookup(self, name: str) -> Optional[Any]: # @param timeout Timeout in millis # @param maxRecursionDepth Max recursion depth # - def set_runtime_bounds(self, timeout: int, max_recursion_depth: int) -> None: + def set_runtime_bounds(self, timeout: Optional[int], max_recursion_depth: Optional[int]) -> None: timebox.Timebox(self, timeout, max_recursion_depth) def set_evaluate_entry_callback(self, cb: Callable) -> None: @@ -231,14 +231,27 @@ def call(self, input: Optional[Any], args: Optional[Sequence]) -> Optional[Any]: def eval(self, expr: Optional[parser.Parser.Symbol], input: Optional[Any], environment: Optional[Frame]) -> Optional[Any]: # Thread safety: # Make sure each evaluate is executed on an instance per thread - return self.get_per_thread_instance()._eval(expr, input, environment) + _this = self.get_per_thread_instance() + # Save and restore the evaluation context so that nested + # evaluations (e.g. $eval()) see the correct context: without this, + # evaluating a sibling argument (e.g. $eval's own second argument) + # would leave _this.environment pointing at whatever inner scope it + # last touched, rather than the environment in effect at this call. + _input = _this.input + _environment = _this.environment + try: + return _this._eval(expr, input, environment) + finally: + _this.input = _input + _this.environment = _environment def _eval(self, expr: Optional[parser.Parser.Symbol], input: Optional[Any], environment: Optional[Frame]) -> Optional[Any]: result = None - # Store the current input - # This is required by Functions.functionEval for current $eval() input context + # Store the current input and environment + # This is required by Functions.functionEval for current $eval() context self.input = input + self.environment = environment if self.parser.dbg: print("eval expr=" + str(expr) + " type=" + expr.type) # +" input="+input); @@ -1300,7 +1313,7 @@ def evaluate_apply_expression(self, expr: Optional[parser.Parser.Symbol], input: return result def is_function_like(self, o: Optional[Any]) -> bool: - return utils.Utils.is_function(o) or functions.Functions.is_lambda(o) or (isinstance(o, re.Pattern)) + return utils.Utils.is_function(o) or functions.Functions.is_lambda(o) or functions.Functions.is_regex(o) CURRENT = threading.local() MUTEX = threading.Lock() @@ -1477,7 +1490,7 @@ def apply_inner(self, proc: Optional[Any], args: Optional[Any], input: Optional[ # } elif isinstance(proc, Jsonata.JLambda): result = proc.call(input, validated_args) - elif isinstance(proc, re.Pattern): + elif functions.Functions.is_regex(proc): _res = [] for s in validated_args: if isinstance(s, str): @@ -1886,12 +1899,22 @@ def _static_initializer() -> None: # # JSONata # @param {Object} expr - JSONata expression + # @param {Object} regex_engine - callable taking (pattern: str, flags: + # regex_engine.RegexFlags) and returning a compiled pattern, used to + # compile JSONata regex literals. Every engine, including the + # default, must translate RegexFlags into its own native + # representation -- see jsonata.regex_engine.default_regex_engine. + # @param {Integer} timeout - max evaluation time in milliseconds, or None + # for no limit. Raises D1012 if exceeded. + # @param {Integer} stack - max eval-apply recursion depth, or None for + # no limit. Raises D1011 if exceeded. # @returns Evaluated expression # @throws jexception.JException An exception if an error occured. - # + # @staticmethod - def jsonata(expression: Optional[str]) -> 'Jsonata': - return Jsonata(expression) + def jsonata(expression: Optional[str], regex_engine: RegexEngine = default_regex_engine, + timeout: Optional[int] = None, stack: Optional[int] = None) -> 'Jsonata': + return Jsonata(expression, regex_engine, timeout, stack) # # Internal constructor @@ -1904,11 +1927,18 @@ def jsonata(expression: Optional[str]) -> 'Jsonata': ast: Optional[parser.Parser.Symbol] timestamp: int input: Optional[Any] - - def __init__(self, expr: Optional[str]) -> None: + regex_engine: RegexEngine + timeout: Optional[int] + stack: Optional[int] + + def __init__(self, expr: Optional[str], regex_engine: RegexEngine = default_regex_engine, + timeout: Optional[int] = None, stack: Optional[int] = None) -> None: + self.regex_engine = regex_engine + self.timeout = timeout + self.stack = stack try: self.parser = Jsonata.get_parser() - self.ast = self.parser.parse(expr) # , optionsRecover); + self.ast = self.parser.parse(expr, regex_engine) # , optionsRecover); self.errors = self.ast.errors self.ast.errors = None # delete ast.errors; except jexception.JException as err: @@ -1916,6 +1946,8 @@ def __init__(self, expr: Optional[str]) -> None: # populateMessage(err); // possible side-effects on `err` raise err self.environment = self.create_frame(Jsonata.static_frame) + if timeout is not None or stack is not None: + self.environment.set_runtime_bounds(timeout, stack) self.timestamp = timebox.Timebox.current_milli_time() # will be overridden on each call to evalute() @@ -1931,13 +1963,6 @@ def __init__(self, expr: Optional[str]) -> None: # return timestamp.getTime() # }, "<:n>")) - # FIXED: options.RegexEngine not implemented in Java - # if(options && options.RegexEngine) { - # jsonata.RegexEngine = options.RegexEngine - # } else { - # jsonata.RegexEngine = RegExp - # } - # Set instance for this thread Jsonata.CURRENT.jsonata = self diff --git a/src/jsonata/parser.py b/src/jsonata/parser.py index 9385185..5191354 100644 --- a/src/jsonata/parser.py +++ b/src/jsonata/parser.py @@ -28,6 +28,7 @@ from typing import Any, MutableSequence, Optional, Sequence from jsonata import jexception, tokenizer, signature, utils +from jsonata.regex_engine import RegexEngine, default_regex_engine # var parseSignature = require('./signature') @@ -1413,11 +1414,11 @@ def object_parser(self, left: Optional[Symbol]) -> Symbol: res.type = "binary" return res - def parse(self, jsonata: Optional[str]) -> Symbol: + def parse(self, jsonata: Optional[str], regex_engine: RegexEngine = default_regex_engine) -> Symbol: self.source = jsonata # now invoke the tokenizer and the parser and return the syntax tree - self.lexer = tokenizer.Tokenizer(self.source) + self.lexer = tokenizer.Tokenizer(self.source, regex_engine) self.advance() # parse the tokens expr = self.expression(0) diff --git a/src/jsonata/regex_engine.py b/src/jsonata/regex_engine.py new file mode 100644 index 0000000..af37e08 --- /dev/null +++ b/src/jsonata/regex_engine.py @@ -0,0 +1,59 @@ +# +# Copyright Robert Yokota +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +from dataclasses import dataclass +from typing import Any, Callable, Optional, Protocol + + +@dataclass +class RegexFlags: + """ + Flags parsed from a JSONata regex literal's /pattern/flags suffix. + Engines translate these into their own native flag representation. + """ + + case_insensitive: bool = False + multiline: bool = False + + +class CompiledPattern(Protocol): + """ + Structural type for a compiled regex: matches stdlib `re.Pattern` + as well as whatever a pluggable regex_engine returns (e.g. a + `google-re2` pattern object). + """ + + def search(self, string: str) -> Optional[Any]: ... + def finditer(self, string: str) -> Any: ... + def sub(self, repl: Any, string: str, count: int = 0) -> str: ... + def split(self, string: str, maxsplit: int = 0) -> list[str]: ... + + +# Compiles a pattern into a CompiledPattern. Used for JSONata regex literals. +RegexEngine = Callable[[str, RegexFlags], CompiledPattern] + + +def default_regex_engine(pattern: str, flags: RegexFlags) -> re.Pattern: + """ + The built-in stdlib `re`-backed engine; this is jsonata-python's default. + """ + py_flags = 0 + if flags.case_insensitive: + py_flags |= re.IGNORECASE + if flags.multiline: + py_flags |= re.MULTILINE + return re.compile(pattern, py_flags) diff --git a/src/jsonata/signature.py b/src/jsonata/signature.py index cd58c57..8e6d60d 100644 --- a/src/jsonata/signature.py +++ b/src/jsonata/signature.py @@ -112,7 +112,7 @@ def get_symbol(self, value: Optional[Any]) -> str: symbol = "m" else: # first check to see if this is a function - if utils.Utils.is_function(value) or functions.Functions.is_lambda(value) or isinstance(value, re.Pattern): + if utils.Utils.is_function(value) or functions.Functions.is_lambda(value) or functions.Functions.is_regex(value): symbol = "f" elif isinstance(value, str): symbol = "s" diff --git a/src/jsonata/timebox.py b/src/jsonata/timebox.py index cd56228..65d8c12 100644 --- a/src/jsonata/timebox.py +++ b/src/jsonata/timebox.py @@ -21,30 +21,31 @@ # import time +from typing import Optional from jsonata import jexception # # Configure max runtime / max recursion depth. -# See Frame.setRuntimeBounds - usually not used directly -# +# See Frame.set_runtime_bounds - usually not used directly +# class Timebox: # - # Protect the process/browser from a runnaway expression + # Protect the process from a runaway expression # i.e. Infinite loop (tail recursion), or excessive stack growth # # @param {Object} expr - expression to protect - # @param {Number} timeout - max time in ms - # @param {Number} max_depth - max stack depth + # @param {Number} timeout - max time in ms, or None for no time limit + # @param {Number} max_depth - max stack depth, or None for no depth limit # - timeout: int - max_depth: int + timeout: Optional[int] + max_depth: Optional[int] time: int depth: int - def __init__(self, expr, timeout=10000, max_depth=100): + def __init__(self, expr, timeout: Optional[int] = None, max_depth: Optional[int] = None): self.timeout = timeout self.max_depth = max_depth self.time = Timebox.current_milli_time() @@ -68,21 +69,12 @@ def exit_callback(exp, input, env, res): expr.set_evaluate_exit_callback(exit_callback) def check_runaway(self) -> None: - if self.depth > self.max_depth: + if self.max_depth is not None and self.depth > self.max_depth: # stack too deep - raise jexception.JException( - "Stack overflow error: Check for non-terminating recursive function. Consider rewriting as tail-recursive. Depth=" + str( - self.depth) + " max=" + str(self.max_depth), -1) - # stack: new Error().stack, - # code: "U1001" - # } - if Timebox.current_milli_time() - self.time > self.timeout: + raise jexception.JException("D1011", -1) + if self.timeout is not None and Timebox.current_milli_time() - self.time > self.timeout: # expression has run for too long - raise jexception.JException( - "Expression evaluation timeout: " + str(self.timeout) + "ms. Check for infinite loop", -1) - # stack: new Error().stack, - # code: "U1001" - # } + raise jexception.JException("D1012", -1, self.timeout) @staticmethod def current_milli_time() -> int: diff --git a/src/jsonata/tokenizer.py b/src/jsonata/tokenizer.py index 0fad3db..15c68aa 100644 --- a/src/jsonata/tokenizer.py +++ b/src/jsonata/tokenizer.py @@ -31,6 +31,7 @@ from typing import Any, Optional from jsonata import jexception, utils +from jsonata.regex_engine import CompiledPattern, RegexEngine, RegexFlags, default_regex_engine _NUMBER_PATTERN = re.compile(r"^-?(0|([1-9][0-9]*))(\.[0-9]+)?([Ee][-+]?[0-9]+)?") @@ -95,12 +96,13 @@ class Tokenizer: path: str length: int - def __init__(self, path): + def __init__(self, path, regex_engine: RegexEngine = default_regex_engine): self.position = 0 self.depth = 0 self.path = path self.length = len(path) + self.regex_engine = regex_engine @dataclass class Token: @@ -121,7 +123,7 @@ def is_closing_slash(self, position: int) -> bool: return True return False - def scan_regex(self) -> re.Pattern: + def scan_regex(self) -> CompiledPattern: # the prefix '/' will have been previously scanned. Find the end of the regex. # search for closing '/' ignoring any that are escaped, or within brackets start = self.position @@ -151,13 +153,11 @@ def scan_regex(self) -> re.Pattern: current_char = None flags = self.path[start:self.position] + 'g' - # Convert flags to Java Pattern flags - _flags = 0 - if "i" in flags: - _flags |= re.I - if "m" in flags: - _flags |= re.M - return re.compile(pattern, _flags) # Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL); + regex_flags = RegexFlags( + case_insensitive="i" in flags, + multiline="m" in flags, + ) + return self.regex_engine(pattern, regex_flags) if (current_char == '(' or current_char == '[' or current_char == '{') and self.path[self.position - 1] != '\\': self.depth += 1 if (current_char == ')' or current_char == ']' or current_char == '}') and self.path[self.position - 1] != '\\': diff --git a/tests/re2_engine_test.py b/tests/re2_engine_test.py new file mode 100644 index 0000000..23971f1 --- /dev/null +++ b/tests/re2_engine_test.py @@ -0,0 +1,75 @@ +import jsonata +import pytest +from jsonata.regex_engine import RegexFlags + +re2 = pytest.importorskip("re2") + + +def re2_regex_engine(pattern: str, flags: RegexFlags): + """ + Adapts google-re2's Options-based compile() to the (pattern, RegexFlags) + interface expected by Jsonata's regex_engine hook. + """ + options = re2.Options() + options.case_sensitive = not flags.case_insensitive + options.one_line = not flags.multiline + options.log_errors = False + return re2.compile(pattern, options) + + +class TestRE2Engine: + + def test_match(self): + expr = jsonata.Jsonata('$match("hello world", /o w/)', re2_regex_engine) + assert expr.evaluate(None) == {"match": "o w", "index": 4, "groups": []} + + def test_match_case_insensitive_flag(self): + expr = jsonata.Jsonata('$match("HELLO", /hello/i)', re2_regex_engine) + result = expr.evaluate(None) + assert result["match"] == "HELLO" + + def test_contains(self): + expr = jsonata.Jsonata('$contains("hello", /ell/)', re2_regex_engine) + assert expr.evaluate(None) is True + + def test_replace_with_string(self): + expr = jsonata.Jsonata('$replace("abc123def", /[0-9]+/, "#")', re2_regex_engine) + assert expr.evaluate(None) == "abc#def" + + def test_replace_with_function(self): + expr = jsonata.Jsonata( + '$replace("abc123", /[0-9]+/, function($m) { $m.match & "!" })', re2_regex_engine + ) + assert expr.evaluate(None) == "abc123!" + + def test_split(self): + expr = jsonata.Jsonata('$split("a1b2c3", /[0-9]/)', re2_regex_engine) + assert expr.evaluate(None) == ["a", "b", "c", ""] + + def test_rejects_backreferences(self): + # Proves RE2 is actually compiling the pattern rather than silently + # falling back to stdlib re: backreferences can't run in RE2's + # guaranteed-linear-time engine, so this must fail at parse time + # (regex literals are compiled while the expression is constructed). + with pytest.raises(re2.error): + jsonata.Jsonata(r'$match("abab", /(a)\1/)', re2_regex_engine) + + def test_default_engine_still_stdlib_re(self): + # No regex_engine argument -> falls back to stdlib re, which *does* + # support backreferences. Confirms the default path is unaffected. + expr = jsonata.Jsonata(r'$match("xaay", /(a)\1/)') + assert expr.evaluate(None)["match"] == "aa" + + def test_eval_uses_enclosing_regex_engine(self): + # $eval dynamically parses and evaluates a nested expression + # (functions.py's function_eval); it must reuse the enclosing + # expression's regex_engine rather than silently falling back to + # stdlib re. Proven the same way as test_rejects_backreferences: + # a backreference inside the $eval'd source must fail to compile + # under RE2, surfaced as a D3120 "invalid expression" error. + expr = jsonata.Jsonata( + r"""$eval('$match("xaay", /(a)\\1/)')""", re2_regex_engine + ) + with pytest.raises(jsonata.JException) as exc_info: + expr.evaluate(None) + assert exc_info.value.error == "D3120" diff --git a/tests/string_test.py b/tests/string_test.py index 76de5cb..aa4028a 100644 --- a/tests/string_test.py +++ b/tests/string_test.py @@ -69,6 +69,30 @@ def test_eval_regex_call_next_and_check_result(self): assert result["end"] == 4 assert result["groups"] == ["l"] + def test_eval_sees_enclosing_variable_binding(self): + # $eval's dynamically-parsed expression must see variables bound in + # the enclosing scope (here, via an in-expression := assignment), + # not just the static top-level environment. + expr = jsonata.Jsonata('($x := 5; $eval("$x + 1"))') + assert expr.evaluate(None) == 6 + + def test_eval_sees_explicit_top_level_bindings(self): + # Same as above, but for bindings passed via evaluate()'s bindings + # argument rather than an in-expression assignment. + expr = jsonata.Jsonata('$eval("$x")') + assert expr.evaluate(None, {"x": 42}) == 42 + + def test_eval_unaffected_by_sibling_argument_scope(self): + # $eval's second (focus) argument is evaluated before its own body + # runs, and here contains a nested block with its own environment. + # Without saving/restoring the evaluation context around each + # nested eval() call, evaluating that sibling argument would leave + # the tracked "current" environment pointing at the inner block's + # scope, causing $eval to resolve $x (from the outer scope) as + # undefined instead of 5. + expr = jsonata.Jsonata('($x := 5; $eval("$x", (($y := 1; $y))))') + assert expr.evaluate(None) == 5 + # # Additional $split tests #