From 677ad3396823193469e5eb4267fae228d45266ae Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 3 Jul 2026 21:39:17 -0700 Subject: [PATCH 01/10] Add pluggable regex engine support --- noxfile.py | 18 +++++++++++ src/jsonata/functions.py | 20 +++++++++---- src/jsonata/jsonata.py | 26 ++++++++-------- src/jsonata/parser.py | 5 ++-- src/jsonata/signature.py | 2 +- src/jsonata/tokenizer.py | 7 +++-- tests/re2_engine_test.py | 65 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 118 insertions(+), 25 deletions(-) create mode 100644 tests/re2_engine_test.py diff --git a/noxfile.py b/noxfile.py index fb5bc2a..7b7556c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -50,3 +50,21 @@ def tests(session): session.run("python", "tests/generate.py") session.run("py.test", "tests/", *session.posargs) + + +# Exercises the optional pluggable regex_engine hook against Google's RE2 +# (tests/re2_engine_test.py). Kept as its own session so the main `tests` +# session -- and the package itself -- stay free of a google-re2 dependency; +# this one opts in explicitly. +@nox.session +def test_re2(session): + session.install("pytest") + session.install("google-re2") + build_and_check_dists(session) + + generated_files = os.listdir("dist/") + generated_sdist = os.path.join("dist/", generated_files[1]) + + session.install(generated_sdist) + + session.run("py.test", "tests/re2_engine_test.py", *session.posargs) diff --git a/src/jsonata/functions.py b/src/jsonata/functions.py index d44b115..155b9de 100644 --- a/src/jsonata/functions.py +++ b/src/jsonata/functions.py @@ -549,7 +549,7 @@ def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Opti if isinstance(token, str): result = (string.find(str(token)) != - 1) - elif isinstance(token, re.Pattern): + elif Functions.is_regex(token): matches = Functions.evaluate_matcher(token, string) # if (dbg) System.out.println("match = "+matches) # result = (typeof matches !== 'undefined') @@ -647,7 +647,7 @@ def safe_replace_all(s: str, pattern: re.Pattern, replacement: Optional[Any]) -> r = None for i in range(0, 10): try: - r = re.sub(pattern, replacement, s) + r = pattern.sub(replacement, s) break except Exception as e: msg = str(e) @@ -704,7 +704,7 @@ def replace_fn(t): else: raise jexception.JException("D3012", -1) - r = re.sub(pattern, replace_fn, s) + r = pattern.sub(replace_fn, s) return r # @@ -721,7 +721,7 @@ def safe_replace_first(s: str, pattern: re.Pattern, replacement: str) -> Optiona r = None for i in range(0, 10): try: - r = re.sub(pattern, replacement, s, count=1) + r = pattern.sub(replacement, s, count=1) break except Exception as e: msg = str(e) @@ -2019,6 +2019,16 @@ def append(arg1: Optional[Any], arg2: Optional[Any]) -> Optional[Any]: def is_lambda(result: Optional[Any]) -> bool: return isinstance(result, parser.Parser.Symbol) and result._jsonata_lambda + # + # Tests whether a value is a compiled regex, from the stdlib re module + # or from a pluggable regex_engine (e.g. re2) with a compatible interface. + # + @staticmethod + def is_regex(value: Optional[Any]) -> bool: + return isinstance(value, re.Pattern) or ( + hasattr(value, "search") and hasattr(value, "finditer") and hasattr(value, "sub") + ) + # # Return value from an object for a given key # @param {Object} input - Object/Array @@ -2201,7 +2211,7 @@ def function_eval(expr: Optional[str], focus: Optional[Any]) -> Optional[Any]: ast = None try: - ast = jsonata.Jsonata(expr) + ast = jsonata.Jsonata(expr, jsonata.Jsonata.CURRENT.jsonata.regex_engine) except Exception as err: # error parsing the expression passed to $eval # populateMessage(err) diff --git a/src/jsonata/jsonata.py b/src/jsonata/jsonata.py index 5707f30..1b3c752 100644 --- a/src/jsonata/jsonata.py +++ b/src/jsonata/jsonata.py @@ -1300,7 +1300,7 @@ def evaluate_apply_expression(self, expr: Optional[parser.Parser.Symbol], input: return result def is_function_like(self, o: Optional[Any]) -> bool: - return utils.Utils.is_function(o) or functions.Functions.is_lambda(o) or (isinstance(o, re.Pattern)) + return utils.Utils.is_function(o) or functions.Functions.is_lambda(o) or functions.Functions.is_regex(o) CURRENT = threading.local() MUTEX = threading.Lock() @@ -1477,7 +1477,7 @@ def apply_inner(self, proc: Optional[Any], args: Optional[Any], input: Optional[ # } elif isinstance(proc, Jsonata.JLambda): result = proc.call(input, validated_args) - elif isinstance(proc, re.Pattern): + elif functions.Functions.is_regex(proc): _res = [] for s in validated_args: if isinstance(s, str): @@ -1886,12 +1886,15 @@ def _static_initializer() -> None: # # JSONata # @param {Object} expr - JSONata expression + # @param {Object} regex_engine - module/object providing a `compile(pattern, flags)` + # function compatible with the stdlib `re` module (e.g. `re2`), used to compile + # JSONata regex literals. Defaults to the stdlib `re` module. # @returns Evaluated expression # @throws jexception.JException An exception if an error occured. - # + # @staticmethod - def jsonata(expression: Optional[str]) -> 'Jsonata': - return Jsonata(expression) + def jsonata(expression: Optional[str], regex_engine: Any = re) -> 'Jsonata': + return Jsonata(expression, regex_engine) # # Internal constructor @@ -1904,11 +1907,13 @@ def jsonata(expression: Optional[str]) -> 'Jsonata': ast: Optional[parser.Parser.Symbol] timestamp: int input: Optional[Any] + regex_engine: Any - def __init__(self, expr: Optional[str]) -> None: + def __init__(self, expr: Optional[str], regex_engine: Any = re) -> None: + self.regex_engine = regex_engine try: self.parser = Jsonata.get_parser() - self.ast = self.parser.parse(expr) # , optionsRecover); + self.ast = self.parser.parse(expr, regex_engine) # , optionsRecover); self.errors = self.ast.errors self.ast.errors = None # delete ast.errors; except jexception.JException as err: @@ -1931,13 +1936,6 @@ def __init__(self, expr: Optional[str]) -> None: # return timestamp.getTime() # }, "<:n>")) - # FIXED: options.RegexEngine not implemented in Java - # if(options && options.RegexEngine) { - # jsonata.RegexEngine = options.RegexEngine - # } else { - # jsonata.RegexEngine = RegExp - # } - # Set instance for this thread Jsonata.CURRENT.jsonata = self diff --git a/src/jsonata/parser.py b/src/jsonata/parser.py index 9385185..be72727 100644 --- a/src/jsonata/parser.py +++ b/src/jsonata/parser.py @@ -25,6 +25,7 @@ # import copy +import re from typing import Any, MutableSequence, Optional, Sequence from jsonata import jexception, tokenizer, signature, utils @@ -1413,11 +1414,11 @@ def object_parser(self, left: Optional[Symbol]) -> Symbol: res.type = "binary" return res - def parse(self, jsonata: Optional[str]) -> Symbol: + def parse(self, jsonata: Optional[str], regex_engine: Any = re) -> Symbol: self.source = jsonata # now invoke the tokenizer and the parser and return the syntax tree - self.lexer = tokenizer.Tokenizer(self.source) + self.lexer = tokenizer.Tokenizer(self.source, regex_engine) self.advance() # parse the tokens expr = self.expression(0) diff --git a/src/jsonata/signature.py b/src/jsonata/signature.py index cd58c57..8e6d60d 100644 --- a/src/jsonata/signature.py +++ b/src/jsonata/signature.py @@ -112,7 +112,7 @@ def get_symbol(self, value: Optional[Any]) -> str: symbol = "m" else: # first check to see if this is a function - if utils.Utils.is_function(value) or functions.Functions.is_lambda(value) or isinstance(value, re.Pattern): + if utils.Utils.is_function(value) or functions.Functions.is_lambda(value) or functions.Functions.is_regex(value): symbol = "f" elif isinstance(value, str): symbol = "s" diff --git a/src/jsonata/tokenizer.py b/src/jsonata/tokenizer.py index 0fad3db..1d73c67 100644 --- a/src/jsonata/tokenizer.py +++ b/src/jsonata/tokenizer.py @@ -95,12 +95,13 @@ class Tokenizer: path: str length: int - def __init__(self, path): + def __init__(self, path, regex_engine=re): self.position = 0 self.depth = 0 self.path = path self.length = len(path) + self.regex_engine = regex_engine @dataclass class Token: @@ -121,7 +122,7 @@ def is_closing_slash(self, position: int) -> bool: return True return False - def scan_regex(self) -> re.Pattern: + def scan_regex(self): # the prefix '/' will have been previously scanned. Find the end of the regex. # search for closing '/' ignoring any that are escaped, or within brackets start = self.position @@ -157,7 +158,7 @@ def scan_regex(self) -> re.Pattern: _flags |= re.I if "m" in flags: _flags |= re.M - return re.compile(pattern, _flags) # Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL); + return self.regex_engine.compile(pattern, _flags) # Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL); if (current_char == '(' or current_char == '[' or current_char == '{') and self.path[self.position - 1] != '\\': self.depth += 1 if (current_char == ')' or current_char == ']' or current_char == '}') and self.path[self.position - 1] != '\\': diff --git a/tests/re2_engine_test.py b/tests/re2_engine_test.py new file mode 100644 index 0000000..a31cef6 --- /dev/null +++ b/tests/re2_engine_test.py @@ -0,0 +1,65 @@ +import re + +import jsonata +import pytest + +re2 = pytest.importorskip("re2") + + +class RE2Engine: + """ + Adapts google-re2's Options-based compile() to the re.compile(pattern, flags) + interface expected by Jsonata's regex_engine hook. + """ + + @staticmethod + def compile(pattern, flags=0): + options = re2.Options() + options.case_sensitive = not bool(flags & re.IGNORECASE) + options.one_line = not bool(flags & re.MULTILINE) + options.log_errors = False + return re2.compile(pattern, options) + + +class TestRE2Engine: + + def test_match(self): + expr = jsonata.Jsonata('$match("hello world", /o w/)', RE2Engine) + assert expr.evaluate(None) == {"match": "o w", "index": 4, "groups": []} + + def test_match_case_insensitive_flag(self): + expr = jsonata.Jsonata('$match("HELLO", /hello/i)', RE2Engine) + result = expr.evaluate(None) + assert result["match"] == "HELLO" + + def test_contains(self): + expr = jsonata.Jsonata('$contains("hello", /ell/)', RE2Engine) + assert expr.evaluate(None) is True + + def test_replace_with_string(self): + expr = jsonata.Jsonata('$replace("abc123def", /[0-9]+/, "#")', RE2Engine) + assert expr.evaluate(None) == "abc#def" + + def test_replace_with_function(self): + expr = jsonata.Jsonata( + '$replace("abc123", /[0-9]+/, function($m) { $m.match & "!" })', RE2Engine + ) + assert expr.evaluate(None) == "abc123!" + + def test_split(self): + expr = jsonata.Jsonata('$split("a1b2c3", /[0-9]/)', RE2Engine) + assert expr.evaluate(None) == ["a", "b", "c", ""] + + def test_rejects_backreferences(self): + # Proves RE2 is actually compiling the pattern rather than silently + # falling back to stdlib re: backreferences can't run in RE2's + # guaranteed-linear-time engine, so this must fail at parse time + # (regex literals are compiled while the expression is constructed). + with pytest.raises(re2.error): + jsonata.Jsonata(r'$match("abab", /(a)\1/)', RE2Engine) + + def test_default_engine_still_stdlib_re(self): + # No regex_engine argument -> falls back to stdlib re, which *does* + # support backreferences. Confirms the default path is unaffected. + expr = jsonata.Jsonata(r'$match("xaay", /(a)\1/)') + assert expr.evaluate(None)["match"] == "aa" From 61395aff8262bd465b50cd365801fe04242c38e3 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 3 Jul 2026 22:01:49 -0700 Subject: [PATCH 02/10] copilot feedback --- noxfile.py | 5 ++++- src/jsonata/functions.py | 10 ++++++++-- src/jsonata/jsonata.py | 5 +++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/noxfile.py b/noxfile.py index 7b7556c..2107599 100644 --- a/noxfile.py +++ b/noxfile.py @@ -63,7 +63,10 @@ def test_re2(session): build_and_check_dists(session) generated_files = os.listdir("dist/") - generated_sdist = os.path.join("dist/", generated_files[1]) + sdists = sorted(f for f in generated_files if f.endswith(".tar.gz")) + if not sdists: + session.error("No sdist (.tar.gz) found in dist/") + generated_sdist = os.path.join("dist/", sdists[0]) session.install(generated_sdist) diff --git a/src/jsonata/functions.py b/src/jsonata/functions.py index 155b9de..90fb239 100644 --- a/src/jsonata/functions.py +++ b/src/jsonata/functions.py @@ -2025,8 +2025,14 @@ def is_lambda(result: Optional[Any]) -> bool: # @staticmethod def is_regex(value: Optional[Any]) -> bool: - return isinstance(value, re.Pattern) or ( - hasattr(value, "search") and hasattr(value, "finditer") and hasattr(value, "sub") + if isinstance(value, re.Pattern): + return True + if value is None or inspect.ismodule(value) or inspect.isclass(value): + return False + return ( + callable(getattr(value, "search", None)) + and callable(getattr(value, "finditer", None)) + and callable(getattr(value, "sub", None)) ) # diff --git a/src/jsonata/jsonata.py b/src/jsonata/jsonata.py index 1b3c752..8ec7931 100644 --- a/src/jsonata/jsonata.py +++ b/src/jsonata/jsonata.py @@ -1887,8 +1887,9 @@ def _static_initializer() -> None: # JSONata # @param {Object} expr - JSONata expression # @param {Object} regex_engine - module/object providing a `compile(pattern, flags)` - # function compatible with the stdlib `re` module (e.g. `re2`), used to compile - # JSONata regex literals. Defaults to the stdlib `re` module. + # function compatible with the stdlib `re` module (or an adapter that exposes + # this interface for engines like `google-re2`), used to compile JSONata regex + # literals. Defaults to the stdlib `re` module. # @returns Evaluated expression # @throws jexception.JException An exception if an error occured. # From 59e6eefaba9282fc1bc979274d71e33b1b445583 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 3 Jul 2026 22:24:43 -0700 Subject: [PATCH 03/10] copilot feedback --- src/jsonata/functions.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/jsonata/functions.py b/src/jsonata/functions.py index 90fb239..5f42589 100644 --- a/src/jsonata/functions.py +++ b/src/jsonata/functions.py @@ -36,7 +36,7 @@ import unicodedata import urllib.parse from dataclasses import dataclass -from typing import Any, AnyStr, Mapping, NoReturn, Optional, Sequence, Callable, Type, Union +from typing import Any, AnyStr, Mapping, NoReturn, Optional, Protocol, Sequence, Callable, Type, Union from jsonata import datetimeutils, jexception, parser, utils @@ -506,6 +506,18 @@ class RegexpMatch: index: int groups: Sequence[AnyStr] + class CompiledPattern(Protocol): + """ + Structural type for a compiled regex: matches stdlib `re.Pattern` + as well as whatever a pluggable regex_engine's compile() returns + (e.g. a `google-re2` pattern object). See Functions.is_regex. + """ + + def search(self, string: str) -> Optional[Any]: ... + def finditer(self, string: str) -> Any: ... + def sub(self, repl: Any, string: str, count: int = 0) -> str: ... + def split(self, string: str, maxsplit: int = 0) -> list[str]: ... + # # Evaluate the matcher function against the str arg # @@ -514,7 +526,7 @@ class RegexpMatch: # @returns {object} - structure that represents the match(es) # @staticmethod - def evaluate_matcher(matcher: re.Pattern, string: Optional[str]) -> list[RegexpMatch]: + def evaluate_matcher(matcher: CompiledPattern, string: Optional[str]) -> list[RegexpMatch]: res = [] matches = matcher.finditer(string) for m in matches: @@ -537,7 +549,7 @@ def evaluate_matcher(matcher: re.Pattern, string: Optional[str]) -> list[RegexpM # @returns {Boolean} - true if str contains token # @staticmethod - def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Optional[bool]: + def contains(string: Optional[str], token: Union[None, str, CompiledPattern]) -> Optional[bool]: # undefined inputs always return undefined if string is None: return None @@ -568,7 +580,7 @@ def contains(string: Optional[str], token: Union[None, str, re.Pattern]) -> Opti # @returns {Array} The array of match objects # @staticmethod - def match_(string: Optional[str], regex: Optional[re.Pattern], limit: Optional[int]) -> Optional[list[dict[str, Any]]]: + def match_(string: Optional[str], regex: Optional[CompiledPattern], limit: Optional[int]) -> Optional[list[dict[str, Any]]]: # undefined inputs always return undefined if string is None: return None @@ -636,7 +648,7 @@ def safe_replacement(in_: str) -> str: # @return # @staticmethod - def safe_replace_all(s: str, pattern: re.Pattern, replacement: Optional[Any]) -> Optional[str]: + def safe_replace_all(s: str, pattern: CompiledPattern, replacement: Optional[Any]) -> Optional[str]: if not (isinstance(replacement, str)): return Functions.safe_replace_all_fn(s, pattern, replacement) @@ -696,7 +708,7 @@ def to_jsonata_match(mr: re.Match[str]) -> dict[str, list[str]]: # @return # @staticmethod - def safe_replace_all_fn(s: str, pattern: re.Pattern, fn: Optional[Any]) -> str: + def safe_replace_all_fn(s: str, pattern: CompiledPattern, fn: Optional[Any]) -> str: def replace_fn(t): res = Functions.func_apply(fn, [Functions.to_jsonata_match(t)]) if isinstance(res, str): @@ -716,7 +728,7 @@ def replace_fn(t): # @return # @staticmethod - def safe_replace_first(s: str, pattern: re.Pattern, replacement: str) -> Optional[str]: + def safe_replace_first(s: str, pattern: CompiledPattern, replacement: str) -> Optional[str]: replacement = Functions.safe_replacement(replacement) r = None for i in range(0, 10): @@ -744,7 +756,7 @@ def safe_replace_first(s: str, pattern: re.Pattern, replacement: str) -> Optiona return r @staticmethod - def replace(string: Optional[str], pattern: Union[str, re.Pattern], replacement: Optional[Any], limit: Optional[int]) -> Optional[str]: + def replace(string: Optional[str], pattern: Union[str, CompiledPattern], replacement: Optional[Any], limit: Optional[int]) -> Optional[str]: if string is None: return None @@ -938,7 +950,7 @@ def decode_url(string: Optional[str]) -> Optional[str]: return urllib.parse.unquote(string, errors="strict") @staticmethod - def split(string: Optional[str], pattern: Union[str, Optional[re.Pattern]], limit: Optional[float]) -> Optional[list[str]]: + def split(string: Optional[str], pattern: Union[str, Optional[CompiledPattern]], limit: Optional[float]) -> Optional[list[str]]: if string is None: return None From 086926c83c5afcb7b5aaa0f61fc3d2f2826a7c62 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 3 Jul 2026 22:44:02 -0700 Subject: [PATCH 04/10] copilot feedback --- src/jsonata/functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/jsonata/functions.py b/src/jsonata/functions.py index 5f42589..d1d7be9 100644 --- a/src/jsonata/functions.py +++ b/src/jsonata/functions.py @@ -733,7 +733,7 @@ def safe_replace_first(s: str, pattern: CompiledPattern, replacement: str) -> Op r = None for i in range(0, 10): try: - r = pattern.sub(replacement, s, count=1) + r = pattern.sub(replacement, s, 1) break except Exception as e: msg = str(e) @@ -2045,6 +2045,7 @@ def is_regex(value: Optional[Any]) -> bool: callable(getattr(value, "search", None)) and callable(getattr(value, "finditer", None)) and callable(getattr(value, "sub", None)) + and callable(getattr(value, "split", None)) ) # From 1c1ef8edb1cd5a5d5d5f1fcc1159e370d623bb86 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 3 Jul 2026 22:58:09 -0700 Subject: [PATCH 05/10] copilot feedback --- noxfile.py | 9 ++++++--- tests/re2_engine_test.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 2107599..e4ce33d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -44,7 +44,10 @@ def tests(session): build_and_check_dists(session) generated_files = os.listdir("dist/") - generated_sdist = os.path.join("dist/", generated_files[1]) + sdists = [f for f in generated_files if f.endswith(".tar.gz")] + if not sdists: + session.error("No sdist (.tar.gz) found in dist/") + generated_sdist = max((os.path.join("dist/", f) for f in sdists), key=os.path.getmtime) session.install(generated_sdist) @@ -63,10 +66,10 @@ def test_re2(session): build_and_check_dists(session) generated_files = os.listdir("dist/") - sdists = sorted(f for f in generated_files if f.endswith(".tar.gz")) + sdists = [f for f in generated_files if f.endswith(".tar.gz")] if not sdists: session.error("No sdist (.tar.gz) found in dist/") - generated_sdist = os.path.join("dist/", sdists[0]) + generated_sdist = max((os.path.join("dist/", f) for f in sdists), key=os.path.getmtime) session.install(generated_sdist) diff --git a/tests/re2_engine_test.py b/tests/re2_engine_test.py index a31cef6..57b7ac4 100644 --- a/tests/re2_engine_test.py +++ b/tests/re2_engine_test.py @@ -63,3 +63,17 @@ def test_default_engine_still_stdlib_re(self): # support backreferences. Confirms the default path is unaffected. expr = jsonata.Jsonata(r'$match("xaay", /(a)\1/)') assert expr.evaluate(None)["match"] == "aa" + + def test_eval_uses_enclosing_regex_engine(self): + # $eval dynamically parses and evaluates a nested expression + # (functions.py's function_eval); it must reuse the enclosing + # expression's regex_engine rather than silently falling back to + # stdlib re. Proven the same way as test_rejects_backreferences: + # a backreference inside the $eval'd source must fail to compile + # under RE2, surfaced as a D3120 "invalid expression" error. + expr = jsonata.Jsonata( + r"""$eval('$match("xaay", /(a)\\1/)')""", RE2Engine + ) + with pytest.raises(jsonata.JException) as exc_info: + expr.evaluate(None) + assert exc_info.value.error == "D3120" From 51748bb2300d56e1091d94ef69432eee58c42b3c Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Sat, 4 Jul 2026 00:41:08 -0700 Subject: [PATCH 06/10] simplify regex engine --- src/jsonata/functions.py | 15 ++-------- src/jsonata/jsonata.py | 17 ++++++----- src/jsonata/parser.py | 4 +-- src/jsonata/regex_engine.py | 59 +++++++++++++++++++++++++++++++++++++ src/jsonata/tokenizer.py | 15 +++++----- tests/re2_engine_test.py | 36 ++++++++++------------ 6 files changed, 95 insertions(+), 51 deletions(-) create mode 100644 src/jsonata/regex_engine.py diff --git a/src/jsonata/functions.py b/src/jsonata/functions.py index d1d7be9..79a8c44 100644 --- a/src/jsonata/functions.py +++ b/src/jsonata/functions.py @@ -36,9 +36,10 @@ import unicodedata import urllib.parse from dataclasses import dataclass -from typing import Any, AnyStr, Mapping, NoReturn, Optional, Protocol, Sequence, Callable, Type, Union +from typing import Any, AnyStr, Mapping, NoReturn, Optional, Sequence, Callable, Type, Union from jsonata import datetimeutils, jexception, parser, utils +from jsonata.regex_engine import CompiledPattern class Functions: @@ -506,18 +507,6 @@ class RegexpMatch: index: int groups: Sequence[AnyStr] - class CompiledPattern(Protocol): - """ - Structural type for a compiled regex: matches stdlib `re.Pattern` - as well as whatever a pluggable regex_engine's compile() returns - (e.g. a `google-re2` pattern object). See Functions.is_regex. - """ - - def search(self, string: str) -> Optional[Any]: ... - def finditer(self, string: str) -> Any: ... - def sub(self, repl: Any, string: str, count: int = 0) -> str: ... - def split(self, string: str, maxsplit: int = 0) -> list[str]: ... - # # Evaluate the matcher function against the str arg # diff --git a/src/jsonata/jsonata.py b/src/jsonata/jsonata.py index 8ec7931..529d86e 100644 --- a/src/jsonata/jsonata.py +++ b/src/jsonata/jsonata.py @@ -27,13 +27,13 @@ import copy import inspect import math -import re import sys import threading from dataclasses import dataclass from typing import Any, Callable, Mapping, MutableSequence, Optional, Sequence, Type, MutableMapping, Union from jsonata import functions, jexception, parser, signature as sig, timebox, utils +from jsonata.regex_engine import RegexEngine, default_regex_engine # @@ -1886,15 +1886,16 @@ def _static_initializer() -> None: # # JSONata # @param {Object} expr - JSONata expression - # @param {Object} regex_engine - module/object providing a `compile(pattern, flags)` - # function compatible with the stdlib `re` module (or an adapter that exposes - # this interface for engines like `google-re2`), used to compile JSONata regex - # literals. Defaults to the stdlib `re` module. + # @param {Object} regex_engine - callable taking (pattern: str, flags: + # regex_engine.RegexFlags) and returning a compiled pattern, used to + # compile JSONata regex literals. Every engine, including the + # default, must translate RegexFlags into its own native + # representation -- see jsonata.regex_engine.default_regex_engine. # @returns Evaluated expression # @throws jexception.JException An exception if an error occured. # @staticmethod - def jsonata(expression: Optional[str], regex_engine: Any = re) -> 'Jsonata': + def jsonata(expression: Optional[str], regex_engine: RegexEngine = default_regex_engine) -> 'Jsonata': return Jsonata(expression, regex_engine) # @@ -1908,9 +1909,9 @@ def jsonata(expression: Optional[str], regex_engine: Any = re) -> 'Jsonata': ast: Optional[parser.Parser.Symbol] timestamp: int input: Optional[Any] - regex_engine: Any + regex_engine: RegexEngine - def __init__(self, expr: Optional[str], regex_engine: Any = re) -> None: + def __init__(self, expr: Optional[str], regex_engine: RegexEngine = default_regex_engine) -> None: self.regex_engine = regex_engine try: self.parser = Jsonata.get_parser() diff --git a/src/jsonata/parser.py b/src/jsonata/parser.py index be72727..5191354 100644 --- a/src/jsonata/parser.py +++ b/src/jsonata/parser.py @@ -25,10 +25,10 @@ # import copy -import re from typing import Any, MutableSequence, Optional, Sequence from jsonata import jexception, tokenizer, signature, utils +from jsonata.regex_engine import RegexEngine, default_regex_engine # var parseSignature = require('./signature') @@ -1414,7 +1414,7 @@ def object_parser(self, left: Optional[Symbol]) -> Symbol: res.type = "binary" return res - def parse(self, jsonata: Optional[str], regex_engine: Any = re) -> Symbol: + def parse(self, jsonata: Optional[str], regex_engine: RegexEngine = default_regex_engine) -> Symbol: self.source = jsonata # now invoke the tokenizer and the parser and return the syntax tree diff --git a/src/jsonata/regex_engine.py b/src/jsonata/regex_engine.py new file mode 100644 index 0000000..af37e08 --- /dev/null +++ b/src/jsonata/regex_engine.py @@ -0,0 +1,59 @@ +# +# Copyright Robert Yokota +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +from dataclasses import dataclass +from typing import Any, Callable, Optional, Protocol + + +@dataclass +class RegexFlags: + """ + Flags parsed from a JSONata regex literal's /pattern/flags suffix. + Engines translate these into their own native flag representation. + """ + + case_insensitive: bool = False + multiline: bool = False + + +class CompiledPattern(Protocol): + """ + Structural type for a compiled regex: matches stdlib `re.Pattern` + as well as whatever a pluggable regex_engine returns (e.g. a + `google-re2` pattern object). + """ + + def search(self, string: str) -> Optional[Any]: ... + def finditer(self, string: str) -> Any: ... + def sub(self, repl: Any, string: str, count: int = 0) -> str: ... + def split(self, string: str, maxsplit: int = 0) -> list[str]: ... + + +# Compiles a pattern into a CompiledPattern. Used for JSONata regex literals. +RegexEngine = Callable[[str, RegexFlags], CompiledPattern] + + +def default_regex_engine(pattern: str, flags: RegexFlags) -> re.Pattern: + """ + The built-in stdlib `re`-backed engine; this is jsonata-python's default. + """ + py_flags = 0 + if flags.case_insensitive: + py_flags |= re.IGNORECASE + if flags.multiline: + py_flags |= re.MULTILINE + return re.compile(pattern, py_flags) diff --git a/src/jsonata/tokenizer.py b/src/jsonata/tokenizer.py index 1d73c67..ff366c3 100644 --- a/src/jsonata/tokenizer.py +++ b/src/jsonata/tokenizer.py @@ -31,6 +31,7 @@ from typing import Any, Optional from jsonata import jexception, utils +from jsonata.regex_engine import RegexEngine, RegexFlags, default_regex_engine _NUMBER_PATTERN = re.compile(r"^-?(0|([1-9][0-9]*))(\.[0-9]+)?([Ee][-+]?[0-9]+)?") @@ -95,7 +96,7 @@ class Tokenizer: path: str length: int - def __init__(self, path, regex_engine=re): + def __init__(self, path, regex_engine: RegexEngine = default_regex_engine): self.position = 0 self.depth = 0 @@ -152,13 +153,11 @@ def scan_regex(self): current_char = None flags = self.path[start:self.position] + 'g' - # Convert flags to Java Pattern flags - _flags = 0 - if "i" in flags: - _flags |= re.I - if "m" in flags: - _flags |= re.M - return self.regex_engine.compile(pattern, _flags) # Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL); + regex_flags = RegexFlags( + case_insensitive="i" in flags, + multiline="m" in flags, + ) + return self.regex_engine(pattern, regex_flags) if (current_char == '(' or current_char == '[' or current_char == '{') and self.path[self.position - 1] != '\\': self.depth += 1 if (current_char == ')' or current_char == ']' or current_char == '}') and self.path[self.position - 1] != '\\': diff --git a/tests/re2_engine_test.py b/tests/re2_engine_test.py index 57b7ac4..23971f1 100644 --- a/tests/re2_engine_test.py +++ b/tests/re2_engine_test.py @@ -1,53 +1,49 @@ -import re - import jsonata import pytest +from jsonata.regex_engine import RegexFlags re2 = pytest.importorskip("re2") -class RE2Engine: +def re2_regex_engine(pattern: str, flags: RegexFlags): """ - Adapts google-re2's Options-based compile() to the re.compile(pattern, flags) + Adapts google-re2's Options-based compile() to the (pattern, RegexFlags) interface expected by Jsonata's regex_engine hook. """ - - @staticmethod - def compile(pattern, flags=0): - options = re2.Options() - options.case_sensitive = not bool(flags & re.IGNORECASE) - options.one_line = not bool(flags & re.MULTILINE) - options.log_errors = False - return re2.compile(pattern, options) + options = re2.Options() + options.case_sensitive = not flags.case_insensitive + options.one_line = not flags.multiline + options.log_errors = False + return re2.compile(pattern, options) class TestRE2Engine: def test_match(self): - expr = jsonata.Jsonata('$match("hello world", /o w/)', RE2Engine) + expr = jsonata.Jsonata('$match("hello world", /o w/)', re2_regex_engine) assert expr.evaluate(None) == {"match": "o w", "index": 4, "groups": []} def test_match_case_insensitive_flag(self): - expr = jsonata.Jsonata('$match("HELLO", /hello/i)', RE2Engine) + expr = jsonata.Jsonata('$match("HELLO", /hello/i)', re2_regex_engine) result = expr.evaluate(None) assert result["match"] == "HELLO" def test_contains(self): - expr = jsonata.Jsonata('$contains("hello", /ell/)', RE2Engine) + expr = jsonata.Jsonata('$contains("hello", /ell/)', re2_regex_engine) assert expr.evaluate(None) is True def test_replace_with_string(self): - expr = jsonata.Jsonata('$replace("abc123def", /[0-9]+/, "#")', RE2Engine) + expr = jsonata.Jsonata('$replace("abc123def", /[0-9]+/, "#")', re2_regex_engine) assert expr.evaluate(None) == "abc#def" def test_replace_with_function(self): expr = jsonata.Jsonata( - '$replace("abc123", /[0-9]+/, function($m) { $m.match & "!" })', RE2Engine + '$replace("abc123", /[0-9]+/, function($m) { $m.match & "!" })', re2_regex_engine ) assert expr.evaluate(None) == "abc123!" def test_split(self): - expr = jsonata.Jsonata('$split("a1b2c3", /[0-9]/)', RE2Engine) + expr = jsonata.Jsonata('$split("a1b2c3", /[0-9]/)', re2_regex_engine) assert expr.evaluate(None) == ["a", "b", "c", ""] def test_rejects_backreferences(self): @@ -56,7 +52,7 @@ def test_rejects_backreferences(self): # guaranteed-linear-time engine, so this must fail at parse time # (regex literals are compiled while the expression is constructed). with pytest.raises(re2.error): - jsonata.Jsonata(r'$match("abab", /(a)\1/)', RE2Engine) + jsonata.Jsonata(r'$match("abab", /(a)\1/)', re2_regex_engine) def test_default_engine_still_stdlib_re(self): # No regex_engine argument -> falls back to stdlib re, which *does* @@ -72,7 +68,7 @@ def test_eval_uses_enclosing_regex_engine(self): # a backreference inside the $eval'd source must fail to compile # under RE2, surfaced as a D3120 "invalid expression" error. expr = jsonata.Jsonata( - r"""$eval('$match("xaay", /(a)\\1/)')""", RE2Engine + r"""$eval('$match("xaay", /(a)\\1/)')""", re2_regex_engine ) with pytest.raises(jsonata.JException) as exc_info: expr.evaluate(None) From 5bd224b562d1d697910aa29597b6f35f188087a0 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Sat, 4 Jul 2026 09:38:49 -0700 Subject: [PATCH 07/10] copilot feedback --- src/jsonata/functions.py | 40 +++++++++++++++++++++++++++------------- src/jsonata/tokenizer.py | 4 ++-- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/jsonata/functions.py b/src/jsonata/functions.py index 79a8c44..462380d 100644 --- a/src/jsonata/functions.py +++ b/src/jsonata/functions.py @@ -2209,7 +2209,15 @@ def function_eval(expr: Optional[str], focus: Optional[Any]) -> Optional[Any]: # undefined inputs always return undefined if expr is None: return None - input = jsonata.Jsonata.CURRENT.jsonata.input # = this.input; + + # Capture the enclosing instance *before* constructing the nested + # ast below: Jsonata.__init__ unconditionally overwrites + # Jsonata.CURRENT.jsonata as a side effect, so re-reading + # CURRENT.jsonata afterwards would silently return the inner + # expression instead of the enclosing one, losing access to its + # environment (e.g. outer variable bindings). + enclosing = jsonata.Jsonata.CURRENT.jsonata + input = enclosing.input # = this.input; if focus is not None: input = focus # if the input is a JSON array, then wrap it in a singleton sequence so it gets treated as a single input @@ -2219,18 +2227,24 @@ def function_eval(expr: Optional[str], focus: Optional[Any]) -> Optional[Any]: ast = None try: - ast = jsonata.Jsonata(expr, jsonata.Jsonata.CURRENT.jsonata.regex_engine) - except Exception as err: - # error parsing the expression passed to $eval - # populateMessage(err) - raise jexception.JException("D3120", -1) - result = None - try: - result = ast.evaluate(input, jsonata.Jsonata.CURRENT.jsonata.environment) - except Exception as err: - # error evaluating the expression passed to $eval - # populateMessage(err) - raise jexception.JException("D3121", -1) + try: + ast = jsonata.Jsonata(expr, enclosing.regex_engine) + except Exception as err: + # error parsing the expression passed to $eval + # populateMessage(err) + raise jexception.JException("D3120", -1) + result = None + try: + result = ast.evaluate(input, enclosing.environment) + except Exception as err: + # error evaluating the expression passed to $eval + # populateMessage(err) + raise jexception.JException("D3121", -1) + finally: + # Restore the enclosing instance as current now that the nested + # expression's construction/evaluation (which needed CURRENT.jsonata + # to be the *inner* ast -- see Jsonata.get_per_thread_instance) is done. + jsonata.Jsonata.CURRENT.jsonata = enclosing return result diff --git a/src/jsonata/tokenizer.py b/src/jsonata/tokenizer.py index ff366c3..15c68aa 100644 --- a/src/jsonata/tokenizer.py +++ b/src/jsonata/tokenizer.py @@ -31,7 +31,7 @@ from typing import Any, Optional from jsonata import jexception, utils -from jsonata.regex_engine import RegexEngine, RegexFlags, default_regex_engine +from jsonata.regex_engine import CompiledPattern, RegexEngine, RegexFlags, default_regex_engine _NUMBER_PATTERN = re.compile(r"^-?(0|([1-9][0-9]*))(\.[0-9]+)?([Ee][-+]?[0-9]+)?") @@ -123,7 +123,7 @@ def is_closing_slash(self, position: int) -> bool: return True return False - def scan_regex(self): + def scan_regex(self) -> CompiledPattern: # the prefix '/' will have been previously scanned. Find the end of the regex. # search for closing '/' ignoring any that are escaped, or within brackets start = self.position From c8dd71d89e8f9cde9f624073871c916bf1998514 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Sat, 4 Jul 2026 09:49:36 -0700 Subject: [PATCH 08/10] Fix nested envs --- src/jsonata/jsonata.py | 5 +++-- tests/string_test.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/jsonata/jsonata.py b/src/jsonata/jsonata.py index 529d86e..47ce4d5 100644 --- a/src/jsonata/jsonata.py +++ b/src/jsonata/jsonata.py @@ -236,9 +236,10 @@ def eval(self, expr: Optional[parser.Parser.Symbol], input: Optional[Any], envir def _eval(self, expr: Optional[parser.Parser.Symbol], input: Optional[Any], environment: Optional[Frame]) -> Optional[Any]: result = None - # Store the current input - # This is required by Functions.functionEval for current $eval() input context + # Store the current input and environment + # This is required by Functions.functionEval for current $eval() context self.input = input + self.environment = environment if self.parser.dbg: print("eval expr=" + str(expr) + " type=" + expr.type) # +" input="+input); diff --git a/tests/string_test.py b/tests/string_test.py index 76de5cb..15e9641 100644 --- a/tests/string_test.py +++ b/tests/string_test.py @@ -69,6 +69,19 @@ def test_eval_regex_call_next_and_check_result(self): assert result["end"] == 4 assert result["groups"] == ["l"] + def test_eval_sees_enclosing_variable_binding(self): + # $eval's dynamically-parsed expression must see variables bound in + # the enclosing scope (here, via an in-expression := assignment), + # not just the static top-level environment. + expr = jsonata.Jsonata('($x := 5; $eval("$x + 1"))') + assert expr.evaluate(None) == 6 + + def test_eval_sees_explicit_top_level_bindings(self): + # Same as above, but for bindings passed via evaluate()'s bindings + # argument rather than an in-expression assignment. + expr = jsonata.Jsonata('$eval("$x")') + assert expr.evaluate(None, {"x": 42}) == 42 + # # Additional $split tests # From e722d280bb384e093326c686b1f6e7adef970d2d Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Sat, 4 Jul 2026 10:02:05 -0700 Subject: [PATCH 09/10] Fix nested envs --- src/jsonata/jsonata.py | 14 +++++++++++++- tests/string_test.py | 11 +++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/jsonata/jsonata.py b/src/jsonata/jsonata.py index 47ce4d5..cac9ece 100644 --- a/src/jsonata/jsonata.py +++ b/src/jsonata/jsonata.py @@ -231,7 +231,19 @@ def call(self, input: Optional[Any], args: Optional[Sequence]) -> Optional[Any]: def eval(self, expr: Optional[parser.Parser.Symbol], input: Optional[Any], environment: Optional[Frame]) -> Optional[Any]: # Thread safety: # Make sure each evaluate is executed on an instance per thread - return self.get_per_thread_instance()._eval(expr, input, environment) + _this = self.get_per_thread_instance() + # Save and restore the evaluation context so that nested + # evaluations (e.g. $eval()) see the correct context: without this, + # evaluating a sibling argument (e.g. $eval's own second argument) + # would leave _this.environment pointing at whatever inner scope it + # last touched, rather than the environment in effect at this call. + _input = _this.input + _environment = _this.environment + try: + return _this._eval(expr, input, environment) + finally: + _this.input = _input + _this.environment = _environment def _eval(self, expr: Optional[parser.Parser.Symbol], input: Optional[Any], environment: Optional[Frame]) -> Optional[Any]: result = None diff --git a/tests/string_test.py b/tests/string_test.py index 15e9641..aa4028a 100644 --- a/tests/string_test.py +++ b/tests/string_test.py @@ -82,6 +82,17 @@ def test_eval_sees_explicit_top_level_bindings(self): expr = jsonata.Jsonata('$eval("$x")') assert expr.evaluate(None, {"x": 42}) == 42 + def test_eval_unaffected_by_sibling_argument_scope(self): + # $eval's second (focus) argument is evaluated before its own body + # runs, and here contains a nested block with its own environment. + # Without saving/restoring the evaluation context around each + # nested eval() call, evaluating that sibling argument would leave + # the tracked "current" environment pointing at the inner block's + # scope, causing $eval to resolve $x (from the outer scope) as + # undefined instead of 5. + expr = jsonata.Jsonata('($x := 5; $eval("$x", (($y := 1; $y))))') + assert expr.evaluate(None) == 5 + # # Additional $split tests # From 66154da65d2f53b60f71841f523874d14cbd8cbc Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Sat, 4 Jul 2026 17:11:31 -0700 Subject: [PATCH 10/10] Add timeout and stack guardrails --- README.md | 30 ++++++++++++++++++++++++++++++ src/jsonata/functions.py | 31 +++++++++++++++++++++++++++---- src/jsonata/jexception.py | 2 ++ src/jsonata/jsonata.py | 20 ++++++++++++++++---- src/jsonata/timebox.py | 34 +++++++++++++--------------------- 5 files changed, 88 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 3333b89..931c1dd 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,36 @@ JSONata> (a & b) hello world ``` +## Guardrails + +JSONata is Turing-complete, so it's possible to write expressions that loop forever or exhaust memory. If you evaluate +untrusted expressions, configure these guardrails (see the JS reference implementation's +[guardrails docs](https://docs.jsonata.org/guardrails) for more background): + +- **Stack overflow** — the `stack` parameter caps the depth of the eval-apply cycle. Exceeding it raises `D1011`. +- **Excessive execution time** — the `timeout` parameter (in milliseconds) catches tail-recursive infinite loops that + `stack` can't. Exceeding it raises `D1012`. +- **Rogue regular expressions** — the `regex_engine` parameter lets you swap in a linear-time engine (e.g. + [`google-re2`](https://pypi.org/project/google-re2/)) to protect against [ReDoS](https://en.wikipedia.org/wiki/ReDoS), + since the `timeout` guardrail can't interrupt a regex match in progress. + +```python +import re2 # pip install google-re2 +import jsonata +from jsonata.regex_engine import RegexFlags + + +def re2_regex_engine(pattern: str, flags: RegexFlags): + options = re2.Options() + options.case_sensitive = not flags.case_insensitive + options.one_line = not flags.multiline + return re2.compile(pattern, options) + + +expr = jsonata.Jsonata("", re2_regex_engine, timeout=1000, stack=500) +result = expr.evaluate(data) +``` + ## Running Tests This project uses the repository of the reference implementation as a submodule. This allows referencing the current version of the unit tests. To clone this repository, run: diff --git a/src/jsonata/functions.py b/src/jsonata/functions.py index 462380d..fa158a0 100644 --- a/src/jsonata/functions.py +++ b/src/jsonata/functions.py @@ -2228,22 +2228,45 @@ def function_eval(expr: Optional[str], focus: Optional[Any]) -> Optional[Any]: ast = None try: try: + # Only used to parse expr into an AST (ast.ast below); the + # actual evaluation reuses the enclosing instance directly + # (see below), so regex_engine here just needs to be valid + # for parsing -- it does not need to be evaluated against. ast = jsonata.Jsonata(expr, enclosing.regex_engine) except Exception as err: # error parsing the expression passed to $eval # populateMessage(err) raise jexception.JException("D3120", -1) + + # Constructing `ast` overwrote Jsonata.CURRENT.jsonata as a side + # effect (see Jsonata.__init__); restore it to the enclosing + # instance before evaluating, so enclosing.eval()'s own + # get_per_thread_instance() lookup resolves correctly. + jsonata.Jsonata.CURRENT.jsonata = enclosing + result = None try: - result = ast.evaluate(input, enclosing.environment) + # Evaluate ast.ast (the parsed tree) using the *enclosing* + # instance's low-level eval(), reusing enclosing.environment + # directly rather than calling ast.evaluate(input, environment) + # (which copies environment's bindings one level deep into a + # fresh child frame rooted at ast's own static frame). This + # mirrors jsonata-java's Functions.functionEval, which calls + # Jsonata.current.get().evaluate(ast.ast, input, env) rather + # than constructing a second, disconnected evaluation + # context. Reusing the environment directly means $eval sees + # bindings at every level of the enclosing scope chain (not + # just the immediate frame) and correctly inherits any + # stack/timeout guardrails registered on an ancestor frame, + # since Frame.lookup walks the parent chain. + result = enclosing.eval(ast.ast, input, enclosing.environment) except Exception as err: # error evaluating the expression passed to $eval # populateMessage(err) raise jexception.JException("D3121", -1) finally: - # Restore the enclosing instance as current now that the nested - # expression's construction/evaluation (which needed CURRENT.jsonata - # to be the *inner* ast -- see Jsonata.get_per_thread_instance) is done. + # Restore the enclosing instance as current now that $eval's + # parsing/evaluation is done. jsonata.Jsonata.CURRENT.jsonata = enclosing return result diff --git a/src/jsonata/jexception.py b/src/jsonata/jexception.py index ab84242..79d09bf 100644 --- a/src/jsonata/jexception.py +++ b/src/jsonata/jexception.py @@ -175,6 +175,8 @@ def msg(error: str, location: int, arg1: Optional[Any], arg2: Optional[Any], det "T1007": "Attempted to partially apply a non-function. Did you mean ${{{token}}}?", "T1008": "Attempted to partially apply a non-function", "D1009": "Multiple key definitions evaluate to same key: {{value}}", + "D1011": "Stack overflow. Check for non-terminating recursive function. Consider rewriting as tail-recursive", + "D1012": "Evaluation timeout after {{value}} milliseconds. Check for infinite loop", "T1010": "The matcher Object argument passed to Object {{token}} does not return the correct object structure", "T2001": "The left side of the {{token}} operator must evaluate to a number", "T2002": "The right side of the {{token}} operator must evaluate to a number", diff --git a/src/jsonata/jsonata.py b/src/jsonata/jsonata.py index cac9ece..8248db4 100644 --- a/src/jsonata/jsonata.py +++ b/src/jsonata/jsonata.py @@ -72,7 +72,7 @@ def lookup(self, name: str) -> Optional[Any]: # @param timeout Timeout in millis # @param maxRecursionDepth Max recursion depth # - def set_runtime_bounds(self, timeout: int, max_recursion_depth: int) -> None: + def set_runtime_bounds(self, timeout: Optional[int], max_recursion_depth: Optional[int]) -> None: timebox.Timebox(self, timeout, max_recursion_depth) def set_evaluate_entry_callback(self, cb: Callable) -> None: @@ -1904,12 +1904,17 @@ def _static_initializer() -> None: # compile JSONata regex literals. Every engine, including the # default, must translate RegexFlags into its own native # representation -- see jsonata.regex_engine.default_regex_engine. + # @param {Integer} timeout - max evaluation time in milliseconds, or None + # for no limit. Raises D1012 if exceeded. + # @param {Integer} stack - max eval-apply recursion depth, or None for + # no limit. Raises D1011 if exceeded. # @returns Evaluated expression # @throws jexception.JException An exception if an error occured. # @staticmethod - def jsonata(expression: Optional[str], regex_engine: RegexEngine = default_regex_engine) -> 'Jsonata': - return Jsonata(expression, regex_engine) + def jsonata(expression: Optional[str], regex_engine: RegexEngine = default_regex_engine, + timeout: Optional[int] = None, stack: Optional[int] = None) -> 'Jsonata': + return Jsonata(expression, regex_engine, timeout, stack) # # Internal constructor @@ -1923,9 +1928,14 @@ def jsonata(expression: Optional[str], regex_engine: RegexEngine = default_regex timestamp: int input: Optional[Any] regex_engine: RegexEngine + timeout: Optional[int] + stack: Optional[int] - def __init__(self, expr: Optional[str], regex_engine: RegexEngine = default_regex_engine) -> None: + def __init__(self, expr: Optional[str], regex_engine: RegexEngine = default_regex_engine, + timeout: Optional[int] = None, stack: Optional[int] = None) -> None: self.regex_engine = regex_engine + self.timeout = timeout + self.stack = stack try: self.parser = Jsonata.get_parser() self.ast = self.parser.parse(expr, regex_engine) # , optionsRecover); @@ -1936,6 +1946,8 @@ def __init__(self, expr: Optional[str], regex_engine: RegexEngine = default_rege # populateMessage(err); // possible side-effects on `err` raise err self.environment = self.create_frame(Jsonata.static_frame) + if timeout is not None or stack is not None: + self.environment.set_runtime_bounds(timeout, stack) self.timestamp = timebox.Timebox.current_milli_time() # will be overridden on each call to evalute() diff --git a/src/jsonata/timebox.py b/src/jsonata/timebox.py index cd56228..65d8c12 100644 --- a/src/jsonata/timebox.py +++ b/src/jsonata/timebox.py @@ -21,30 +21,31 @@ # import time +from typing import Optional from jsonata import jexception # # Configure max runtime / max recursion depth. -# See Frame.setRuntimeBounds - usually not used directly -# +# See Frame.set_runtime_bounds - usually not used directly +# class Timebox: # - # Protect the process/browser from a runnaway expression + # Protect the process from a runaway expression # i.e. Infinite loop (tail recursion), or excessive stack growth # # @param {Object} expr - expression to protect - # @param {Number} timeout - max time in ms - # @param {Number} max_depth - max stack depth + # @param {Number} timeout - max time in ms, or None for no time limit + # @param {Number} max_depth - max stack depth, or None for no depth limit # - timeout: int - max_depth: int + timeout: Optional[int] + max_depth: Optional[int] time: int depth: int - def __init__(self, expr, timeout=10000, max_depth=100): + def __init__(self, expr, timeout: Optional[int] = None, max_depth: Optional[int] = None): self.timeout = timeout self.max_depth = max_depth self.time = Timebox.current_milli_time() @@ -68,21 +69,12 @@ def exit_callback(exp, input, env, res): expr.set_evaluate_exit_callback(exit_callback) def check_runaway(self) -> None: - if self.depth > self.max_depth: + if self.max_depth is not None and self.depth > self.max_depth: # stack too deep - raise jexception.JException( - "Stack overflow error: Check for non-terminating recursive function. Consider rewriting as tail-recursive. Depth=" + str( - self.depth) + " max=" + str(self.max_depth), -1) - # stack: new Error().stack, - # code: "U1001" - # } - if Timebox.current_milli_time() - self.time > self.timeout: + raise jexception.JException("D1011", -1) + if self.timeout is not None and Timebox.current_milli_time() - self.time > self.timeout: # expression has run for too long - raise jexception.JException( - "Expression evaluation timeout: " + str(self.timeout) + "ms. Check for infinite loop", -1) - # stack: new Error().stack, - # code: "U1001" - # } + raise jexception.JException("D1012", -1, self.timeout) @staticmethod def current_milli_time() -> int: