add tests for secrets resolver logic

krisztianfekete · krisztianfekete · commit 8f951223c8d9 · 2026-06-24T14:10:02.000+02:00
diff --git a/tests/resolvers/__init__.py b/tests/resolvers/__init__.py
diff --git a/tests/resolvers/test_kubernetes.py b/tests/resolvers/test_kubernetes.py
@@ -0,0 +1,63 @@
+"""Kubernetes secret resolver tests.
+
+The kubernetes client is mocked, so these run whether or not the optional
+``kubernetes`` extra is installed.
+"""
+
+from __future__ import annotations
+
+import base64
+import sys
+from unittest.mock import MagicMock
+
+import pytest
+
+from agentevals.resolvers.kubernetes import KubernetesSecretResolver, create_kubernetes_resolver
+
+
+def _client_returning(data: dict[str, str]) -> MagicMock:
+    client = MagicMock()
+    client.read_namespaced_secret.return_value = MagicMock(data=data)
+    return client
+
+
+def _b64(value: str) -> str:
+    return base64.b64encode(value.encode()).decode()
+
+
+class TestResolve:
+    async def test_reads_and_base64_decodes_value(self):
+        client = _client_returning({"api-key": _b64("sk-secret-value")})
+        resolver = KubernetesSecretResolver(client)
+
+        value = await resolver.resolve({"namespace": "ns", "name": "creds", "key": "api-key"})
+
+        assert value == "sk-secret-value"
+        client.read_namespaced_secret.assert_called_once_with("creds", "ns")
+
+    async def test_missing_fields_raise(self):
+        resolver = KubernetesSecretResolver(MagicMock())
+        with pytest.raises(ValueError, match="namespace, name, key"):
+            await resolver.resolve({"kind": "kubernetes"})
+
+    async def test_key_not_found_lists_names_not_values(self):
+        secret_value = _b64("sk-do-not-leak")
+        client = _client_returning({"api-key": secret_value, "tls.crt": _b64("cert")})
+        resolver = KubernetesSecretResolver(client)
+
+        with pytest.raises(ValueError) as exc:
+            await resolver.resolve({"namespace": "ns", "name": "creds", "key": "wrong"})
+
+        message = str(exc.value)
+        assert "api-key" in message and "tls.crt" in message
+        # The enumeration must never echo the base64-encoded secret values.
+        assert secret_value not in message
+
+
+class TestFactory:
+    def test_requires_kubernetes_extra(self, monkeypatch):
+        # Shadow the kubernetes package so the lazy import fails regardless of
+        # whether the extra is installed in the test venv.
+        monkeypatch.setitem(sys.modules, "kubernetes", None)
+        with pytest.raises(RuntimeError, match="kubernetes"):
+            create_kubernetes_resolver({"kind": "kubernetes"})
diff --git a/tests/resolvers/test_registry.py b/tests/resolvers/test_registry.py
@@ -0,0 +1,145 @@
+"""Secret resolver framework tests.
+
+Covers entry-point discovery and the builtins < entry points < programmatic
+precedence chain (mirroring tests/run/test_sinks.py), the built-in env resolver,
+the per-run ContextVar carrier, and resolve_credential_refs.
+"""
+
+from __future__ import annotations
+
+import logging
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from agentevals.resolvers import (
+    EnvSecretResolver,
+    build_resolver,
+    clear_resolver_plugin_registry,
+    create_env_resolver,
+    get_resolved_credential,
+    register_resolver_factory,
+    registered_resolver_kinds,
+    reset_resolved_credentials,
+    resolve_credential_refs,
+    set_resolved_credentials,
+)
+
+
+@pytest.fixture
+def isolated_resolver_plugins():
+    """``register_resolver_factory`` is process-global; reset around plugin tests."""
+    clear_resolver_plugin_registry()
+    yield
+    clear_resolver_plugin_registry()
+
+
+class TestEnvResolver:
+    def test_env_resolver_discovered_via_entry_point(self):
+        assert "env" in registered_resolver_kinds()
+
+    def test_create_env_resolver_returns_env_resolver(self):
+        assert isinstance(create_env_resolver({"kind": "env"}), EnvSecretResolver)
+
+    async def test_resolves_from_environment(self, monkeypatch):
+        monkeypatch.setenv("AE_TEST_SECRET", "sk-from-env")
+        ref = {"kind": "env", "name": "AE_TEST_SECRET"}
+        assert await build_resolver(ref).resolve(ref) == "sk-from-env"
+
+    async def test_missing_name_field_raises(self):
+        with pytest.raises(ValueError, match="name"):
+            await EnvSecretResolver().resolve({"kind": "env"})
+
+    async def test_unset_variable_raises(self, monkeypatch):
+        monkeypatch.delenv("AE_TEST_MISSING", raising=False)
+        with pytest.raises(ValueError, match="not set"):
+            await EnvSecretResolver().resolve({"kind": "env", "name": "AE_TEST_MISSING"})
+
+
+class TestBuildResolver:
+    def test_missing_kind_raises(self):
+        with pytest.raises(ValueError, match="missing a 'kind'"):
+            build_resolver({"name": "X"})
+
+    def test_unknown_kind_raises_and_lists_available(self):
+        with pytest.raises(ValueError, match="unknown secret resolver kind 'nope'") as exc:
+            build_resolver({"kind": "nope"})
+        assert "env" in str(exc.value)
+
+
+class TestPluginPrecedence:
+    async def test_programmatic_registration_overrides_entry_point(self, isolated_resolver_plugins):
+        class Sentinel:
+            async def resolve(self, ref):
+                return "programmatic"
+
+        register_resolver_factory("env", lambda _ref: Sentinel())
+        resolver = build_resolver({"kind": "env"})
+        assert isinstance(resolver, Sentinel)
+        assert await resolver.resolve({"kind": "env"}) == "programmatic"
+
+    def test_register_adds_new_kind(self, isolated_resolver_plugins):
+        register_resolver_factory("custom", lambda _ref: MagicMock())
+        assert "custom" in registered_resolver_kinds()
+
+
+class TestEntryPointDiscovery:
+    """Entry-point discovery without relying on packages installed in the test venv."""
+
+    async def test_entry_point_resolver_resolves_kind(self, isolated_resolver_plugins):
+        class FromEp:
+            async def resolve(self, ref):
+                return "from-entry-point"
+
+        def factory(_ref):
+            return FromEp()
+
+        ep = MagicMock()
+        ep.name = "from_ep"
+        ep.load.return_value = factory
+
+        with patch("agentevals.resolvers.entry_points", return_value=[ep]):
+            resolver = build_resolver({"kind": "from_ep"})
+        ep.load.assert_called_once_with()
+        assert await resolver.resolve({"kind": "from_ep"}) == "from-entry-point"
+
+    def test_entry_point_load_failure_skipped(self, isolated_resolver_plugins, caplog):
+        ep = MagicMock()
+        ep.name = "broken_pkg_resolver"
+        ep.load.side_effect = ImportError("dist not installed")
+
+        with patch("agentevals.resolvers.entry_points", return_value=[ep]), caplog.at_level(logging.ERROR):
+            kinds = registered_resolver_kinds()
+        assert "broken_pkg_resolver" not in kinds
+        assert any("failed to load resolver entry point" in r.getMessage() for r in caplog.records)
+
+    def test_non_callable_entry_point_skipped(self, isolated_resolver_plugins, caplog):
+        ep = MagicMock()
+        ep.name = "bad_export"
+        ep.load.return_value = "not_callable"
+
+        with patch("agentevals.resolvers.entry_points", return_value=[ep]), caplog.at_level(logging.WARNING):
+            kinds = registered_resolver_kinds()
+        assert "bad_export" not in kinds
+        assert any("not callable" in r.getMessage() for r in caplog.records)
+
+
+class TestResolvedCredentialContext:
+    def test_get_returns_none_for_absent_key(self):
+        token = set_resolved_credentials({"present": "v"})
+        try:
+            assert get_resolved_credential("present") == "v"
+            assert get_resolved_credential("absent") is None
+        finally:
+            reset_resolved_credentials(token)
+
+    async def test_resolve_credential_refs_maps_logical_names_to_values(self, monkeypatch):
+        monkeypatch.setenv("AE_TEST_A", "val-a")
+        monkeypatch.setenv("AE_TEST_B", "val-b")
+        resolved = await resolve_credential_refs(
+            {
+                "a": {"kind": "env", "name": "AE_TEST_A"},
+                "b": {"kind": "env", "name": "AE_TEST_B"},
+            }
+        )
+        assert resolved == {"a": "val-a", "b": "val-b"}
diff --git a/tests/test_credential_injection.py b/tests/test_credential_injection.py
@@ -0,0 +1,117 @@
+"""Per-evaluation judge credential injection tests.
+
+Exercises the agentevals construction path (build_eval_metric -> get_evaluator ->
+_inject_judge_credential) without making any network calls, the fail-closed
+behavior on an unresolved credential, the concurrency isolation that motivates the
+ContextVar design, and a guard test that fails loudly if the ADK judge seam moves.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+
+from agentevals.builtin_metrics import (
+    _inject_judge_credential,
+    build_eval_metric,
+    evaluate_builtin_metric,
+    get_evaluator,
+)
+from agentevals.resolvers import (
+    get_resolved_credential,
+    reset_resolved_credentials,
+    set_resolved_credentials,
+)
+
+
+def _judge_evaluator(model_id: str = "openai/gpt-4o"):
+    return get_evaluator(build_eval_metric("final_response_match_v2", model_id, 0.5))
+
+
+class TestInjection:
+    def test_litellm_judge_uses_injected_key_not_env(self, monkeypatch):
+        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+        evaluator = _judge_evaluator()
+
+        _inject_judge_credential(evaluator, "sk-injected", base_url="https://proxy.local")
+
+        args = evaluator._judge_model._additional_args
+        assert args["api_key"] == "sk-injected"
+        assert args["base_url"] == "https://proxy.local"
+
+    def test_gemini_judge_client_replaced(self):
+        evaluator = get_evaluator(
+            build_eval_metric("rubric_based_final_response_quality_v1", "gemini-2.5-flash", 0.5, rubrics=["good?"])
+        )
+        # Seeded only by injection: an un-injected judge builds its client lazily on
+        # first access, so the cache slot is absent until we replace it.
+        assert "api_client" not in evaluator._judge_model.__dict__
+
+        _inject_judge_credential(evaluator, "gem-injected")
+
+        assert "api_client" in evaluator._judge_model.__dict__
+
+    def test_inject_is_noop_on_non_judge_evaluator(self, caplog):
+        evaluator = get_evaluator(build_eval_metric("tool_trajectory_avg_score", None, 0.5))
+        with caplog.at_level(logging.WARNING):
+            _inject_judge_credential(evaluator, "sk-x")
+        assert not hasattr(evaluator, "_judge_model")
+        assert any("not judge-backed" in r.getMessage() for r in caplog.records)
+
+
+class TestFailClosed:
+    async def test_unresolved_credential_errors_instead_of_ambient_auth(self):
+        result = await evaluate_builtin_metric(
+            metric_name="final_response_match_v2",
+            actual_invocations=[],
+            expected_invocations=[object()],
+            judge_model="openai/gpt-4o",
+            threshold=0.5,
+            credential_ref="judge-openai",
+        )
+        assert result.score is None
+        assert "judge-openai" in result.error
+        assert "credentialRefs" in result.error
+
+
+class TestConcurrencyIsolation:
+    async def test_concurrent_runs_do_not_cross_talk(self, monkeypatch):
+        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+        env_before = dict(os.environ)
+
+        async def run_one(key: str) -> str:
+            set_resolved_credentials({"judge": key})
+            # Yield after setting so the sibling task sets its own value before we
+            # read back. With a shared global instead of a ContextVar, both tasks
+            # would observe the last writer's key and this assertion would fail.
+            await asyncio.sleep(0)
+            evaluator = _judge_evaluator()
+            _inject_judge_credential(evaluator, get_resolved_credential("judge"))
+            return evaluator._judge_model._additional_args["api_key"]
+
+        first, second = await asyncio.gather(run_one("sk-AAA"), run_one("sk-BBB"))
+
+        assert first == "sk-AAA"
+        assert second == "sk-BBB"
+        assert dict(os.environ) == env_before
+
+
+class TestAdkSeamGuard:
+    """Fails loudly if the version-pinned ADK judge seam moves."""
+
+    def test_llm_as_judge_exposes_setup_auto_rater(self):
+        from google.adk.evaluation.llm_as_judge import LlmAsJudge
+
+        assert hasattr(LlmAsJudge, "_setup_auto_rater")
+
+    def test_judge_evaluator_exposes_seam_attributes(self):
+        evaluator = _judge_evaluator()
+        assert hasattr(evaluator, "_judge_model")
+        assert hasattr(evaluator, "_judge_model_options")
+        assert evaluator._judge_model_options.judge_model == "openai/gpt-4o"
+
+    def test_litellm_forwards_api_key_kwarg(self):
+        from google.adk.models.lite_llm import LiteLlm
+
+        assert LiteLlm(model="openai/gpt-4o", api_key="sentinel")._additional_args["api_key"] == "sentinel"
