diff --git a/evaluators/builtin/tests/test_contrib_packages.py b/evaluators/builtin/tests/test_contrib_packages.py new file mode 100644 index 00000000..a59dd070 --- /dev/null +++ b/evaluators/builtin/tests/test_contrib_packages.py @@ -0,0 +1,140 @@ +"""Tests for repo contrib package discovery wiring.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + +REPO_ROOT = Path(__file__).resolve().parents[3] +SCRIPT_PATH = REPO_ROOT / "scripts" / "contrib_packages.py" +MODULE_NAME = "agent_control_repo_contrib_packages" + + +def load_contrib_packages_module() -> ModuleType: + """Load the repo contrib-packages script as a module for testing.""" + + module = sys.modules.get(MODULE_NAME) + if module is not None: + return module + + spec = importlib.util.spec_from_file_location(MODULE_NAME, SCRIPT_PATH) + assert spec is not None + assert spec.loader is not None + + module = importlib.util.module_from_spec(spec) + sys.modules[MODULE_NAME] = module + spec.loader.exec_module(module) + return module + + +def test_discover_contrib_packages_returns_expected_metadata() -> None: + """Test that real contrib packages are discovered with stable metadata.""" + + module = load_contrib_packages_module() + + packages = module.discover_contrib_packages() + + assert [(package.name, package.package, package.extra) for package in packages] == [ + ("budget", "agent-control-evaluator-budget", "budget"), + ("cisco", "agent-control-evaluator-cisco", "cisco"), + ("galileo", "agent-control-evaluator-galileo", "galileo"), + ] + + +def test_verify_contrib_packages_has_no_repo_wiring_drift() -> None: + """Test that contrib package wiring stays aligned with repo metadata.""" + + module = load_contrib_packages_module() + + packages = module.discover_contrib_packages() + + assert module.verify_contrib_packages(packages) == [] + + +def test_verify_contrib_packages_rejects_stale_or_incorrect_wiring( + monkeypatch, +) -> None: + """Test that verify_contrib_packages catches stale or incorrect contrib wiring.""" + + module = load_contrib_packages_module() + + root_pyproject_path = module.REPO_ROOT / "pyproject.toml" + builtin_pyproject_path = module.REPO_ROOT / "evaluators" / "builtin" / "pyproject.toml" + + def fake_load_toml(path: Path) -> dict[str, object]: + if path == root_pyproject_path: + return { + "tool": { + "semantic_release": { + "version_toml": [ + "pyproject.toml:project.version", + "evaluators/contrib/budget/pyproject.toml:project.version", + "evaluators/contrib/stale/pyproject.toml:project.version", + ] + } + } + } + + if path == builtin_pyproject_path: + return { + "project": { + "optional-dependencies": { + "budget": ["agent-control-evaluator-budget>=7.5.0"], + "stale": ["agent-control-evaluator-stale>=7.5.0"], + } + }, + "tool": { + "uv": { + "sources": { + "agent-control-evaluator-budget": { + "path": "../contrib/not-budget", + "editable": False, + }, + "agent-control-evaluator-stale": { + "path": "../contrib/stale", + "editable": True, + }, + } + } + }, + } + + raise AssertionError(f"Unexpected path: {path}") + + monkeypatch.setattr(module, "load_toml", fake_load_toml) + + packages = [ + module.ContribPackage( + name="budget", + directory="evaluators/contrib/budget", + package="agent-control-evaluator-budget", + extra="budget", + ) + ] + + errors = module.verify_contrib_packages(packages) + + assert ( + "Builtin uv source 'agent-control-evaluator-budget' in evaluators/builtin/pyproject.toml " + 'must set path = "../contrib/budget".' + ) in errors + assert ( + "Builtin uv source 'agent-control-evaluator-budget' in evaluators/builtin/pyproject.toml " + "must set editable = true." + ) in errors + assert ( + "Unexpected semantic-release version wiring for unknown contrib package 'stale': " + "remove 'evaluators/contrib/stale/pyproject.toml:project.version' from " + "[tool.semantic_release].version_toml in pyproject.toml." + ) in errors + assert ( + "Unexpected builtin extra 'stale' in evaluators/builtin/pyproject.toml: " + "no discovered contrib package matches this extra." + ) in errors + assert ( + "Unexpected uv source for unknown contrib package 'stale': remove " + "[tool.uv.sources].agent-control-evaluator-stale from " + "evaluators/builtin/pyproject.toml." + ) in errors diff --git a/pyproject.toml b/pyproject.toml index a3272db6..89dcbfd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,8 @@ version_toml = [ "telemetry/pyproject.toml:project.version", "server/pyproject.toml:project.version", "evaluators/builtin/pyproject.toml:project.version", + "evaluators/contrib/budget/pyproject.toml:project.version", + "evaluators/contrib/cisco/pyproject.toml:project.version", "evaluators/contrib/galileo/pyproject.toml:project.version", ] version_source = "tag" diff --git a/scripts/contrib_packages.py b/scripts/contrib_packages.py new file mode 100644 index 00000000..394e1b69 --- /dev/null +++ b/scripts/contrib_packages.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +"""Discover and verify real contrib evaluator packages.""" + +from __future__ import annotations + +import argparse +import json +import sys +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +EVALUATOR_ENTRY_GROUP = "agent_control.evaluators" +CONTRIB_PACKAGE_PREFIX = "agent-control-evaluator-" +REPO_ROOT = Path(__file__).resolve().parent.parent +CONTRIB_ROOT = REPO_ROOT / "evaluators" / "contrib" + + +class ContribPackagesError(Exception): + """Raised when contrib package discovery or verification cannot proceed.""" + + +@dataclass(frozen=True) +class ContribPackage: + """Normalized metadata for a real contrib evaluator package.""" + + name: str + directory: str + package: str + extra: str + entry_group: str = EVALUATOR_ENTRY_GROUP + + @property + def version_toml_entry(self) -> str: + return f"{self.directory}/pyproject.toml:project.version" + + @property + def builtin_uv_source_path(self) -> str: + return f"../contrib/{self.name}" + + def to_matrix_entry(self) -> dict[str, str]: + return { + "name": self.name, + "dir": self.directory, + "package": self.package, + "extra": self.extra, + "entry_group": self.entry_group, + } + + +def load_toml(path: Path) -> dict[str, Any]: + """Load a TOML file with contextual parse errors.""" + + try: + with path.open("rb") as handle: + data = tomllib.load(handle) + except FileNotFoundError as exc: + raise ContribPackagesError( + f"Required file is missing: {display_path(path)}." + ) from exc + except tomllib.TOMLDecodeError as exc: + raise ContribPackagesError( + f"Failed to parse {display_path(path)}: {exc}." + ) from exc + + if not isinstance(data, dict): + raise ContribPackagesError( + f"{display_path(path)} did not parse to a TOML table." + ) + + return data + + +def display_path(path: Path) -> str: + """Render a path relative to the repo root when possible.""" + + try: + return path.relative_to(REPO_ROOT).as_posix() + except ValueError: + return path.as_posix() + + +def require_table( + data: dict[str, Any], key: str, *, path: Path, parent_description: str +) -> dict[str, Any]: + """Return a TOML table or raise a targeted error.""" + + value = data.get(key) + if not isinstance(value, dict): + table_name = f"{parent_description}.{key}" if parent_description else key + raise ContribPackagesError( + f"{display_path(path)} must define [{table_name}] as a TOML table." + ) + return value + + +def require_string(value: Any, *, path: Path, description: str) -> str: + """Return a non-empty string or raise a targeted error.""" + + if not isinstance(value, str) or not value: + raise ContribPackagesError( + f"{display_path(path)} must define {description} as a non-empty string." + ) + return value + + +def dependency_name(requirement: str) -> str: + """Extract the distribution name from a PEP 508 requirement string.""" + + end = len(requirement) + for index, character in enumerate(requirement): + if character in " [<>=!~;": + end = index + break + return requirement[:end].strip().lower() + + +def contrib_name_from_distribution(distribution: str) -> str | None: + """Return the contrib package name when the distribution matches repo naming.""" + + if distribution.startswith(CONTRIB_PACKAGE_PREFIX): + return distribution.removeprefix(CONTRIB_PACKAGE_PREFIX) + return None + + +def contrib_name_from_version_toml_entry(entry: str) -> str | None: + """Return the contrib package name when a version_toml entry targets a contrib package.""" + + path_text, separator, version_field = entry.partition(":") + if separator != ":" or version_field != "project.version": + return None + + entry_path = Path(path_text) + if entry_path.parts[:2] != ("evaluators", "contrib"): + return None + if len(entry_path.parts) != 4 or entry_path.name != "pyproject.toml": + return None + + return entry_path.parts[2] + + +def discover_contrib_packages() -> list[ContribPackage]: + """Discover real contrib evaluator packages under evaluators/contrib.""" + + packages: list[ContribPackage] = [] + + if not CONTRIB_ROOT.is_dir(): + raise ContribPackagesError( + f"Expected contrib root at {display_path(CONTRIB_ROOT)}, but it does not exist." + ) + + for candidate in sorted(CONTRIB_ROOT.iterdir(), key=lambda path: path.name): + if not candidate.is_dir() or candidate.name == "template": + continue + + manifest_path = candidate / "pyproject.toml" + if not manifest_path.is_file(): + continue + + manifest = load_toml(manifest_path) + project = require_table(manifest, "project", path=manifest_path, parent_description="") + project_name = require_string( + project.get("name"), + path=manifest_path, + description='[project].name', + ) + + entry_points = require_table( + project, + "entry-points", + path=manifest_path, + parent_description="project", + ) + evaluator_entries = entry_points.get(EVALUATOR_ENTRY_GROUP) + if not isinstance(evaluator_entries, dict) or not evaluator_entries: + raise ContribPackagesError( + f"{display_path(manifest_path)} must define at least one " + f'[project.entry-points."{EVALUATOR_ENTRY_GROUP}"] entry.' + ) + + expected_package_name = f"agent-control-evaluator-{candidate.name}" + if project_name != expected_package_name: + raise ContribPackagesError( + f"{display_path(manifest_path)} declares project.name = {project_name!r}, " + f"but contrib package {candidate.name!r} must use {expected_package_name!r}." + ) + + packages.append( + ContribPackage( + name=candidate.name, + directory=display_path(candidate), + package=expected_package_name, + extra=candidate.name, + ) + ) + + return packages + + +def verify_contrib_packages(packages: list[ContribPackage]) -> list[str]: + """Return human-readable verification errors for contrib wiring drift.""" + + root_pyproject_path = REPO_ROOT / "pyproject.toml" + builtin_pyproject_path = REPO_ROOT / "evaluators" / "builtin" / "pyproject.toml" + + root_pyproject = load_toml(root_pyproject_path) + builtin_pyproject = load_toml(builtin_pyproject_path) + + tool_table = require_table( + root_pyproject, + "tool", + path=root_pyproject_path, + parent_description="", + ) + semantic_release = require_table( + tool_table, + "semantic_release", + path=root_pyproject_path, + parent_description="tool", + ) + version_toml = semantic_release.get("version_toml") + if not isinstance(version_toml, list) or not all( + isinstance(item, str) for item in version_toml + ): + raise ContribPackagesError( + f"{display_path(root_pyproject_path)} must define [tool.semantic_release].version_toml " + "as a list of strings." + ) + + builtin_project = require_table( + builtin_pyproject, + "project", + path=builtin_pyproject_path, + parent_description="", + ) + optional_dependencies = require_table( + builtin_project, + "optional-dependencies", + path=builtin_pyproject_path, + parent_description="project", + ) + + builtin_tool = require_table( + builtin_pyproject, + "tool", + path=builtin_pyproject_path, + parent_description="", + ) + builtin_uv = require_table( + builtin_tool, + "uv", + path=builtin_pyproject_path, + parent_description="tool", + ) + builtin_sources = require_table( + builtin_uv, + "sources", + path=builtin_pyproject_path, + parent_description="tool.uv", + ) + + errors: list[str] = [] + discovered_names = {package.name for package in packages} + for package in packages: + if package.version_toml_entry not in version_toml: + errors.append( + f"Missing semantic-release version wiring for contrib package {package.name!r}: " + f"add {package.version_toml_entry!r} to [tool.semantic_release].version_toml " + f"in {display_path(root_pyproject_path)}." + ) + + extra_dependencies = optional_dependencies.get(package.extra) + if extra_dependencies is None: + errors.append( + f"Missing builtin extra for contrib package {package.name!r}: " + f"add [project.optional-dependencies].{package.extra} = " + f"[\"{package.package}>=\"] in " + f"{display_path(builtin_pyproject_path)}." + ) + elif not isinstance(extra_dependencies, list) or not all( + isinstance(item, str) for item in extra_dependencies + ): + errors.append( + f"Builtin extra {package.extra!r} in " + f"{display_path(builtin_pyproject_path)} must be " + "a list of dependency strings." + ) + else: + dependency_names = {dependency_name(item) for item in extra_dependencies} + if package.package not in dependency_names: + errors.append( + f"Builtin extra {package.extra!r} in {display_path(builtin_pyproject_path)} " + f"does not reference {package.package!r}: update it to include " + f"\"{package.package}>=\"." + ) + + source_entry = builtin_sources.get(package.package) + if source_entry is None: + errors.append( + f"Missing uv source for contrib package {package.name!r}: " + f"add [tool.uv.sources].{package.package} = " + f'{{ path = "{package.builtin_uv_source_path}", editable = true }} ' + f"in {display_path(builtin_pyproject_path)}." + ) + elif not isinstance(source_entry, dict): + errors.append( + f"Builtin uv source {package.package!r} in {display_path(builtin_pyproject_path)} " + "must be a TOML table." + ) + else: + source_path = source_entry.get("path") + if source_path != package.builtin_uv_source_path: + errors.append( + f"Builtin uv source {package.package!r} in " + f"{display_path(builtin_pyproject_path)} must set " + f'path = "{package.builtin_uv_source_path}".' + ) + + if source_entry.get("editable") is not True: + errors.append( + f"Builtin uv source {package.package!r} in " + f"{display_path(builtin_pyproject_path)} must set editable = true." + ) + + for entry in version_toml: + contrib_name = contrib_name_from_version_toml_entry(entry) + if contrib_name is not None and contrib_name not in discovered_names: + errors.append( + f"Unexpected semantic-release version wiring for unknown contrib package " + f"{contrib_name!r}: remove {entry!r} from [tool.semantic_release].version_toml " + f"in {display_path(root_pyproject_path)}." + ) + + for extra_name, extra_dependencies in optional_dependencies.items(): + if not isinstance(extra_dependencies, list) or not all( + isinstance(item, str) for item in extra_dependencies + ): + continue + + dependency_names = {dependency_name(item) for item in extra_dependencies} + has_contrib_dependency = any( + contrib_name_from_distribution(name) is not None for name in dependency_names + ) + if has_contrib_dependency and extra_name not in discovered_names: + errors.append( + f"Unexpected builtin extra {extra_name!r} in " + f"{display_path(builtin_pyproject_path)}: " + "no discovered contrib package matches this extra." + ) + + for source_name in builtin_sources: + contrib_name = contrib_name_from_distribution(source_name) + if contrib_name is not None and contrib_name not in discovered_names: + errors.append( + f"Unexpected uv source for unknown contrib package {contrib_name!r}: " + f"remove [tool.uv.sources].{source_name} from " + f"{display_path(builtin_pyproject_path)}." + ) + + return errors + + +def run_list(packages: list[ContribPackage]) -> int: + """Print a human-readable contrib package summary.""" + + for package in packages: + print( + f"{package.name}: dir={package.directory} package={package.package} " + f"extra={package.extra} entry_group={package.entry_group}" + ) + return 0 + + +def run_names(packages: list[ContribPackage]) -> int: + """Print newline-separated contrib package names.""" + + for package in packages: + print(package.name) + return 0 + + +def run_matrix(packages: list[ContribPackage]) -> int: + """Print a JSON matrix for GitHub Actions or other automation.""" + + print(json.dumps([package.to_matrix_entry() for package in packages], separators=(",", ":"))) + return 0 + + +def run_verify(packages: list[ContribPackage]) -> int: + """Verify root contrib wiring and print actionable drift errors.""" + + errors = verify_contrib_packages(packages) + if errors: + print("Contrib package wiring verification failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + discovered = ", ".join(package.name for package in packages) or "(none)" + print(f"Verified contrib package wiring for: {discovered}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + """Build the CLI parser.""" + + parser = argparse.ArgumentParser( + description="Discover and verify real contrib evaluator packages." + ) + parser.add_argument( + "command", + choices=("list", "names", "matrix", "verify"), + help="Command to run.", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Program entry point.""" + + parser = build_parser() + args = parser.parse_args(argv) + + try: + packages = discover_contrib_packages() + if args.command == "list": + return run_list(packages) + if args.command == "names": + return run_names(packages) + if args.command == "matrix": + return run_matrix(packages) + if args.command == "verify": + return run_verify(packages) + except ContribPackagesError as error: + print(f"Error: {error}", file=sys.stderr) + return 1 + + parser.error(f"Unsupported command: {args.command}") + return 2 + + +if __name__ == "__main__": + sys.exit(main())