build(infra): add contrib package discovery script

evaluators/builtin/tests/test_contrib_packages.py

@@ -0,0 +1,140 @@
+"""Tests for repo contrib package discovery wiring."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+SCRIPT_PATH = REPO_ROOT / "scripts" / "contrib_packages.py"
+MODULE_NAME = "agent_control_repo_contrib_packages"
+
+
+def load_contrib_packages_module() -> ModuleType:
+    """Load the repo contrib-packages script as a module for testing."""
+
+    module = sys.modules.get(MODULE_NAME)
+    if module is not None:
+        return module
+
+    spec = importlib.util.spec_from_file_location(MODULE_NAME, SCRIPT_PATH)
+    assert spec is not None
+    assert spec.loader is not None
+
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[MODULE_NAME] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_discover_contrib_packages_returns_expected_metadata() -> None:
+    """Test that real contrib packages are discovered with stable metadata."""
+
+    module = load_contrib_packages_module()
+
+    packages = module.discover_contrib_packages()
+
+    assert [(package.name, package.package, package.extra) for package in packages] == [
+        ("budget", "agent-control-evaluator-budget", "budget"),
+        ("cisco", "agent-control-evaluator-cisco", "cisco"),
+        ("galileo", "agent-control-evaluator-galileo", "galileo"),
+    ]
+
+
+def test_verify_contrib_packages_has_no_repo_wiring_drift() -> None:
+    """Test that contrib package wiring stays aligned with repo metadata."""
+
+    module = load_contrib_packages_module()
+
+    packages = module.discover_contrib_packages()
+
+    assert module.verify_contrib_packages(packages) == []
+
+
+def test_verify_contrib_packages_rejects_stale_or_incorrect_wiring(
+    monkeypatch,
+) -> None:
+    """Test that verify_contrib_packages catches stale or incorrect contrib wiring."""
+
+    module = load_contrib_packages_module()
+
+    root_pyproject_path = module.REPO_ROOT / "pyproject.toml"
+    builtin_pyproject_path = module.REPO_ROOT / "evaluators" / "builtin" / "pyproject.toml"
+
+    def fake_load_toml(path: Path) -> dict[str, object]:
+        if path == root_pyproject_path:
+            return {
+                "tool": {
+                    "semantic_release": {
+                        "version_toml": [
+                            "pyproject.toml:project.version",
+                            "evaluators/contrib/budget/pyproject.toml:project.version",
+                            "evaluators/contrib/stale/pyproject.toml:project.version",
+                        ]
+                    }
+                }
+            }
+
+        if path == builtin_pyproject_path:
+            return {
+                "project": {
+                    "optional-dependencies": {
+                        "budget": ["agent-control-evaluator-budget>=7.5.0"],
+                        "stale": ["agent-control-evaluator-stale>=7.5.0"],
+                    }
+                },
+                "tool": {
+                    "uv": {
+                        "sources": {
+                            "agent-control-evaluator-budget": {
+                                "path": "../contrib/not-budget",
+                                "editable": False,
+                            },
+                            "agent-control-evaluator-stale": {
+                                "path": "../contrib/stale",
+                                "editable": True,
+                            },
+                        }
+                    }
+                },
+            }
+
+        raise AssertionError(f"Unexpected path: {path}")
+
+    monkeypatch.setattr(module, "load_toml", fake_load_toml)
+
+    packages = [
+        module.ContribPackage(
+            name="budget",
+            directory="evaluators/contrib/budget",
+            package="agent-control-evaluator-budget",
+            extra="budget",
+        )
+    ]
+
+    errors = module.verify_contrib_packages(packages)
+
+    assert (
+        "Builtin uv source 'agent-control-evaluator-budget' in evaluators/builtin/pyproject.toml "
+        'must set path = "../contrib/budget".'
+    ) in errors
+    assert (
+        "Builtin uv source 'agent-control-evaluator-budget' in evaluators/builtin/pyproject.toml "
+        "must set editable = true."
+    ) in errors
+    assert (
+        "Unexpected semantic-release version wiring for unknown contrib package 'stale': "
+        "remove 'evaluators/contrib/stale/pyproject.toml:project.version' from "
+        "[tool.semantic_release].version_toml in pyproject.toml."
+    ) in errors
+    assert (
+        "Unexpected builtin extra 'stale' in evaluators/builtin/pyproject.toml: "
+        "no discovered contrib package matches this extra."
+    ) in errors
+    assert (
+        "Unexpected uv source for unknown contrib package 'stale': remove "
+        "[tool.uv.sources].agent-control-evaluator-stale from "
+        "evaluators/builtin/pyproject.toml."
+    ) in errors

pyproject.toml

@@ -73,6 +73,8 @@ version_toml = [
     "telemetry/pyproject.toml:project.version",
     "server/pyproject.toml:project.version",
     "evaluators/builtin/pyproject.toml:project.version",
+    "evaluators/contrib/budget/pyproject.toml:project.version",
+    "evaluators/contrib/cisco/pyproject.toml:project.version",
     "evaluators/contrib/galileo/pyproject.toml:project.version",
 ]
 version_source = "tag"