diff --git a/evaluators/contrib/template/Makefile b/evaluators/contrib/template/Makefile new file mode 100644 index 00000000..d6944fc5 --- /dev/null +++ b/evaluators/contrib/template/Makefile @@ -0,0 +1,31 @@ +.PHONY: help test lint lint-fix typecheck check build + +PACKAGE_DIR := $(notdir $(abspath $(CURDIR))) +COVERAGE_XML := ../../../coverage-evaluators-$(PACKAGE_DIR).xml + +help: + @echo "Agent Control contrib evaluator template" + @echo "" + @echo " make test - run pytest" + @echo " make lint - run ruff check" + @echo " make lint-fix - run ruff check --fix" + @echo " make typecheck - run mypy" + @echo " make check - run lint, typecheck, and tests" + @echo " make build - build package" + +test: + uv run --with pytest --with pytest-asyncio --with pytest-cov pytest tests --cov=src --cov-report=xml:$(COVERAGE_XML) -q + +lint: + uv run --with ruff ruff check --config ../../../pyproject.toml src/ + +lint-fix: + uv run --with ruff ruff check --config ../../../pyproject.toml --fix src/ + +typecheck: + uv run --with mypy mypy --config-file ../../../pyproject.toml src/ + +check: lint typecheck test + +build: + uv build diff --git a/evaluators/contrib/template/README.md b/evaluators/contrib/template/README.md index f3090a02..3f1b1771 100644 --- a/evaluators/contrib/template/README.md +++ b/evaluators/contrib/template/README.md @@ -1,5 +1,121 @@ -# Evaluator Template +# Contrib Evaluator Template -Starter template for creating a custom evaluator package. +This directory is scaffolding for a new contrib evaluator package. + +It is intentionally excluded from repo automation until you convert it into a real package. In +particular, `template/` does not participate in root `make check`, CI, semantic-release, or +publishing because it ships a `pyproject.toml.template` placeholder instead of a real +`pyproject.toml`. + +## Naming contract + +Pick `` as a short lowercase single-word identifier such as `galileo`, `cisco`, or +`budget`. That same value should appear in the steady-state package shape: + +- directory: `evaluators/contrib//` +- pip package: `agent-control-evaluator-` +- Python module: `agent_control_evaluator_` +- extra name: `agent-control-evaluators[]` + +The template uses `{{NAME}}` for that package identifier. It does not use `{{ORG}}`. + +Keep the public evaluator reference separate from the package identifier: + +- `{{ENTRY_POINT}}` is the user-facing evaluator name and should match + `EvaluatorMetadata.name` in your package code. +- Single-evaluator packages can keep that public name flat, such as `budget`. +- Packages that expose a family of evaluator ids should namespace it, such as + `cisco.ai_defense` or `galileo.luna2`. + +## Scaffold a new contrib package + +1. Copy the template and rename the manifest: + + ```bash + cp -r evaluators/contrib/template evaluators/contrib/ + mv evaluators/contrib//pyproject.toml.template \ + evaluators/contrib//pyproject.toml + ``` + +2. Replace placeholders in `pyproject.toml`: + + - `{{NAME}}` -> contrib package identifier + - `{{ENTRY_POINT}}` -> public evaluator reference / `EvaluatorMetadata.name` + - `{{EVALUATOR}}` -> evaluator module path segment (for example `budget` or `ai_defense`) + - `{{CLASS}}` -> evaluator class name + - `{{AUTHOR}}` -> authoring team + + For a package with one primary evaluator, `{{ENTRY_POINT}}` is often just ``. For a + package that groups provider-specific evaluators, use `.`. + + The template starts new packages at `0.1.0`; change that if your release plan differs. + Also replace the copied `README.md` with package-specific install, configuration, and usage + docs before your first build or publish. Then confirm the package `version` reflects your + release plan and that the `agent-control-evaluators` / `agent-control-models` dependency + floors match the compatibility floor you intend to support. Keep those dependency floors + aligned with the builtin extra you add below before you commit the new package. + +3. Add package code and tests: + + - `src/agent_control_evaluator_/` + - `tests/` + +4. Validate the package locally: + + ```bash + make lint + make lint-fix + make typecheck + make test + make check + make build + ``` + +## Canonical install docs + +Contributor-facing and user-facing package docs should treat this as the canonical install path: + +```bash +pip install "agent-control-evaluators[]" +``` + +Direct wheel installs such as `pip install agent-control-evaluator-` can still be +documented, but they are secondary to the extra on `agent-control-evaluators`. + +## Expected repo wiring + +After the new package exists as a real contrib package, wire it into the repo contract: + +1. Add the extra to `evaluators/builtin/pyproject.toml`: + + ```toml + [project.optional-dependencies] + = ["agent-control-evaluator->="] + ``` + + Keep this extra on the current monorepo release line. The release build rewrites builtin + dependency floors to the active release version before publishing + `agent-control-evaluators`, so a lower source floor here would not survive into the + published extra metadata. + +2. Add the workspace source pin to `evaluators/builtin/pyproject.toml`: + + ```toml + [tool.uv.sources] + agent-control-evaluator- = { path = "../contrib/", editable = true } + ``` + +3. Add the package to `tool.semantic_release.version_toml` in the root `pyproject.toml`: + + ```toml + "evaluators/contrib//pyproject.toml:project.version", + ``` + + The repo's release automation discovers real contrib packages automatically via + `scripts/contrib_packages.py`, so once the package has a real `pyproject.toml` and the + builtin extra / uv source wiring above is in place, `scripts/build.py` and + `.github/workflows/release.yaml` will pick it up without additional manual edits. + +Until those steps are done, the package is still scaffolding rather than a real contrib package. Docs: https://docs.agentcontrol.dev/concepts/evaluators/contributing-evaluator diff --git a/evaluators/contrib/template/pyproject.toml.template b/evaluators/contrib/template/pyproject.toml.template index 484864bb..aeca4891 100644 --- a/evaluators/contrib/template/pyproject.toml.template +++ b/evaluators/contrib/template/pyproject.toml.template @@ -1,7 +1,8 @@ [project] -name = "agent-control-evaluator-{{ORG}}" -version = "3.0.0" -description = "{{ORG}} evaluators for agent-control" +name = "agent-control-evaluator-{{NAME}}" +version = "0.1.0" +description = "{{NAME}} evaluators for agent-control" +readme = "README.md" requires-python = ">=3.12" license = { text = "Apache-2.0" } authors = [{ name = "{{AUTHOR}}" }] @@ -15,20 +16,22 @@ dependencies = [ dev = [ "pytest>=8.0.0", "pytest-asyncio>=0.23.0", + "pytest-cov>=4.0.0", "ruff>=0.1.0", "mypy>=1.8.0", ] [project.entry-points."agent_control.evaluators"] -# Format: "org.evaluator_name" = "package.module:Class" -"{{ORG}}.{{EVALUATOR}}" = "agent_control_evaluator_{{ORG}}.{{EVALUATOR}}:{{CLASS}}" +# Keep this aligned with EvaluatorMetadata.name (for example "budget" or +# "cisco.ai_defense"). +"{{ENTRY_POINT}}" = "agent_control_evaluator_{{NAME}}.{{EVALUATOR}}:{{CLASS}}" [build-system] requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["src/agent_control_evaluator_{{ORG}}"] +packages = ["src/agent_control_evaluator_{{NAME}}"] [tool.uv.sources] agent-control-evaluators = { path = "../../builtin", editable = true }