Resolve lookup plugin variables before execution (#152)

* lookup arg improvements

* Bump black version

* Additional test coverage

* Correct version number

* bump version v26.8.1 -> v26.10.0

Author: adammcdonagh

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
     hooks:
       - id: prettier
   - repo: https://github.com/psf/black
-    rev: 26.1.0
+    rev: 26.3.0
     hooks:
       - id: black
         args:

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+# v26.10.0
+
+- Resolve Jinja-templated lookup arguments before invoking lookup plugins.
+- Make lookup plugin `globals` resolution lazy so plugins can access needed variables without forcing unrelated deep variable chains to resolve.
+- Preserve lookup compatibility when lazy variable loading is enabled.
+
 # v25.8.1
 
 - Handle race condition when creating directories in logging module, triggered byu many threads trying to create the same directory at the same time.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "opentaskpy"
-version = "v26.8.1"
+version = "v26.10.0"
 authors = [{ name = "Adam McDonagh", email = "adam@elitemonkey.net" }]
 license-files = [ "LICENSE" ]
 
@@ -42,7 +42,7 @@ requires-python = ">=3.11"
 dev = [
     "types-requests >=2.28",
     "types-paramiko >=3.0",
-    "black == 26.1.0",
+    "black == 26.3.0",
     "isort",
     "pytest",
     "bumpver",
@@ -71,7 +71,7 @@ otf-batch-validator = "opentaskpy.cli.batch_validator:main"
 profile = 'black'
 
 [tool.bumpver]
-current_version = "v26.8.1"
+current_version = "v26.10.0"
 version_pattern = "vYY.WW.PATCH[-TAG]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

src/opentaskpy/config/loader.py

@@ -7,6 +7,7 @@
 import os
 import sys
 from glob import glob
+from typing import Any
 
 import jinja2
 from jinja2 import meta
@@ -21,6 +22,45 @@
 MAX_DEPTH = 5
 
 
+class LazyResolvedDict(dict):
+    """Dictionary wrapper that resolves values only when accessed."""
+
+    def __init__(
+        self,
+        values: dict[str, Any],
+        resolver: Any,
+        resolve_lookups: bool = False,
+    ) -> None:
+        """Initialise the wrapper around a dictionary of unresolved values."""
+        super().__init__(values)
+        self._resolver = resolver
+        self._resolve_lookups = resolve_lookups
+
+    def _wrap(self, value: Any) -> Any:
+        if isinstance(value, dict):
+            return LazyResolvedDict(
+                value,
+                self._resolver,
+                resolve_lookups=self._resolve_lookups,
+            )
+
+        if isinstance(value, list):
+            return [self._wrap(item) for item in value]
+
+        return self._resolver(value, resolve_lookups=self._resolve_lookups)
+
+    def __getitem__(self, key: str) -> Any:
+        """Return a lazily resolved value for the requested key."""
+        return self._wrap(super().__getitem__(key))
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Return a lazily resolved value when the key exists."""
+        if key not in self:
+            return default
+
+        return self[key]
+
+
 class ConfigLoader:
     """Class responsible for loading and validating config files."""
 
@@ -159,6 +199,28 @@ def get_global_variables(self) -> dict:
 
         return self.global_variables
 
+    def _resolve_lookup_value(self, value: Any, resolve_lookups: bool = True) -> Any:
+        """Resolve templated values before they are passed to a lookup plugin."""
+        if isinstance(value, dict):
+            return {
+                key: self._resolve_lookup_value(item, resolve_lookups=resolve_lookups)
+                for key, item in value.items()
+            }
+
+        if isinstance(value, list):
+            return [
+                self._resolve_lookup_value(item, resolve_lookups=resolve_lookups)
+                for item in value
+            ]
+
+        if isinstance(value, str):
+            if not resolve_lookups and "lookup(" in value:
+                return value
+
+            return self._resolve_templated_variables_from_string(value)
+
+        return value
+
     def template_lookup(self, plugin: str, **kwargs) -> str:  # type: ignore[no-untyped-def]
         """Lookup function used by Jinja.
 
@@ -177,8 +239,16 @@ def template_lookup(self, plugin: str, **kwargs) -> str:  # type: ignore[no-unty
             11, f"Got call to lookup function {plugin} with kwargs {kwargs}"
         )
 
+        kwargs = {
+            key: self._resolve_lookup_value(value) for key, value in kwargs.items()
+        }
+
         # Append the globals to the kwargs
-        kwargs["globals"] = self.global_variables
+        kwargs["globals"] = LazyResolvedDict(
+            self.global_variables,
+            self._resolve_lookup_value,
+            resolve_lookups=False,
+        )
 
         # Import the plugin if its not already loaded
         if f"opentaskpy.plugins.lookup.{plugin}" not in sys.modules:

test/cfg/plugins/lookup/test_plugin.py

@@ -1,6 +1,7 @@
-# ruff: noqa
 """An example plugin that simply returns the word "hello"."""
 
+import re
+
 import opentaskpy.otflogging
 
 logger = opentaskpy.otflogging.init_logging(__name__)
@@ -9,10 +10,30 @@
 
 
 def run(**kwargs) -> str:  # type: ignore[no-untyped-def]
-    """Returns hello.
+    """Test."""
+    dd = str(kwargs.get("dd"))
+    yyyy = str(kwargs.get("yyyy"))
+    globals_dict = kwargs.get("globals", {})
+    global_dd = globals_dict.get("DD")
+    global_yyyy = globals_dict.get("NESTED_VAR", {}).get("NESTED_VAR1")
+
+    # dd and YYY should be ints not strings, they should have been resolved. If not then we should error
+    # Do a regex match to check that the variables are ints
+    if (
+        not re.match(r"^\d+$", dd)
+        or not re.match(r"^\d+$", yyyy)
+        or not re.match(r"^\d+$", global_dd)
+        or not re.match(r"^\d+$", global_yyyy)
+    ):
+        raise Exception(  # pylint: disable=broad-exception-raised
+            "dd, yyyy and globals values should be resolved integers"
+        )
+
+    # dd and yyyy should be ints, so we can do some maths on them so they're not just strings when returned
+    # to prove that they're resolved
+    dd_int = int(dd)
+    yyyy_int = int(yyyy)
+
+    result = f"hello {dd_int + 1 } {yyyy_int + 1}"
 
-    Returns:
-        str: The word hello
-    """
-    # Return a random number
-    return "hello"
+    return result

tests/test_config_loader.py

@@ -10,7 +10,7 @@
 from jinja2.exceptions import UndefinedError
 from pytest_shell import fs
 
-from opentaskpy.config.loader import ConfigLoader
+from opentaskpy.config.loader import ConfigLoader, LazyResolvedDict
 from opentaskpy.exceptions import VariableResolutionTooDeepError
 
 GLOBAL_VARIABLES: str | None = None
@@ -312,7 +312,13 @@ def test_custom_plugin(tmpdir):
         [
             {
                 f"{tmpdir}/variables.json.j2": {
-                    "content": '{"test": "{{ lookup(\'test_plugin\') }}"}'
+                    "content": (
+                        "{"
+                        '"DD": "{{ now().strftime(\'%d\') }}",'
+                        '"YYYY": "{{ now().strftime(\'%Y\') }}",'
+                        '"NESTED_VAR": {"NESTED_VAR1": "{{ YYYY }}"},'
+                        '"test": "{{ lookup(\'test_plugin\', dd=DD, yyyy=NESTED_VAR.NESTED_VAR1) }}"}'
+                    )
                 }
             },
         ]
@@ -325,8 +331,120 @@ def test_custom_plugin(tmpdir):
 
     # Test that the global variables are loaded correctly
     config_loader = ConfigLoader(tmpdir)
+    global_variables = config_loader.get_global_variables()
+
+    dd = datetime.now().strftime("%d")
+    yyyy = datetime.now().strftime("%Y")
+
+    assert global_variables["DD"] == dd
+    assert global_variables["YYYY"] == yyyy
+    assert global_variables["NESTED_VAR"] == {"NESTED_VAR1": yyyy}
+    assert global_variables["test"] == f"hello {int(dd)+1} {int(yyyy)+1}"
+
+
+def test_custom_plugin_lazy_load(tmpdir):
+    os.environ["OTF_LAZY_LOAD_VARIABLES"] = "1"
+
+    fs.create_files(
+        [
+            {
+                f"{tmpdir}/variables.json.j2": {
+                    "content": (
+                        "{"
+                        '"DD": "{{ now().strftime(\'%d\') }}",'
+                        '"YYYY": "{{ now().strftime(\'%Y\') }}",'
+                        '"NESTED_VAR": {"NESTED_VAR1": "{{ YYYY }}"},'
+                        '"test": "{{ lookup(\'test_plugin\', dd=DD, yyyy=NESTED_VAR.NESTED_VAR1) }}"}'
+                    )
+                }
+            },
+            {f"{tmpdir}/task.json": {"content": '{"test": "{{ test }}"}'}},
+        ]
+    )
+
+    os.symlink(
+        os.path.join(os.path.dirname(__file__), "../test/cfg", "plugins"),
+        f"{tmpdir}/plugins",
+    )
+
+    config_loader = ConfigLoader(tmpdir)
+
+    del os.environ["OTF_LAZY_LOAD_VARIABLES"]
+
+    dd = datetime.now().strftime("%d")
+    yyyy = datetime.now().strftime("%Y")
+
+    assert config_loader.load_task_definition("task", cache=False) == {
+        "test": f"hello {int(dd)+1} {int(yyyy)+1}"
+    }
+
+
+def test_resolve_lookup_value_variants(tmpdir):
+    fs.create_files(
+        [
+            {
+                f"{tmpdir}/variables.json.j2": {
+                    "content": json.dumps(
+                        {
+                            "YYYY": "2026",
+                            "MM": "03",
+                        }
+                    )
+                }
+            },
+        ]
+    )
+
+    config_loader = ConfigLoader(tmpdir)
+
+    assert config_loader._resolve_lookup_value({"year": "{{ YYYY }}"}) == {
+        "year": "2026"
+    }
+    assert config_loader._resolve_lookup_value(["{{ YYYY }}", "{{ MM }}", 7]) == [
+        "2026",
+        "03",
+        7,
+    ]
+    assert (
+        config_loader._resolve_lookup_value(
+            "{{ lookup('file', path='/tmp/skip.txt') }}", resolve_lookups=False
+        )
+        == "{{ lookup('file', path='/tmp/skip.txt') }}"
+    )
+    assert config_loader._resolve_lookup_value(1234) == 1234
+
+
+def test_lazy_resolved_dict_accessors(tmpdir):
+    fs.create_files(
+        [
+            {
+                f"{tmpdir}/variables.json.j2": {
+                    "content": json.dumps(
+                        {
+                            "YYYY": "2026",
+                            "MM": "03",
+                        }
+                    )
+                }
+            },
+        ]
+    )
+
+    config_loader = ConfigLoader(tmpdir)
+    lazy_dict = LazyResolvedDict(
+        {
+            "nested": {"value": "{{ YYYY }}"},
+            "items": ["{{ MM }}", 2],
+            "lookup": "{{ lookup('file', path='/tmp/skip.txt') }}",
+        },
+        config_loader._resolve_lookup_value,
+        resolve_lookups=False,
+    )
 
-    assert config_loader.get_global_variables() == {"test": "hello"}
+    assert lazy_dict["nested"]["value"] == "2026"
+    assert lazy_dict["items"] == ["03", 2]
+    assert lazy_dict.get("lookup") == "{{ lookup('file', path='/tmp/skip.txt') }}"
+    assert lazy_dict.get("missing", "fallback") == "fallback"
 
 
 def test_default_filters(tmpdir):