chore: refresh contrib template scaffolding

Author: lan17

evaluators/contrib/template/Makefile

@@ -0,0 +1,31 @@
+.PHONY: help test lint lint-fix typecheck check build
+
+PACKAGE_DIR := $(notdir $(abspath $(CURDIR)))
+COVERAGE_XML := ../../../coverage-evaluators-$(PACKAGE_DIR).xml
+
+help:
+	@echo "Agent Control contrib evaluator template"
+	@echo ""
+	@echo "  make test       - run pytest"
+	@echo "  make lint       - run ruff check"
+	@echo "  make lint-fix   - run ruff check --fix"
+	@echo "  make typecheck  - run mypy"
+	@echo "  make check      - run lint, typecheck, and tests"
+	@echo "  make build      - build package"
+
+test:
+	uv run --with pytest --with pytest-asyncio --with pytest-cov pytest tests --cov=src --cov-report=xml:$(COVERAGE_XML) -q
+
+lint:
+	uv run --with ruff ruff check --config ../../../pyproject.toml src/
+
+lint-fix:
+	uv run --with ruff ruff check --config ../../../pyproject.toml --fix src/
+
+typecheck:
+	uv run --with mypy mypy --config-file ../../../pyproject.toml src/
+
+check: lint typecheck test
+
+build:
+	uv build

evaluators/contrib/template/README.md

@@ -1,5 +1,93 @@
-# Evaluator Template
+# Contrib Evaluator Template
 
-Starter template for creating a custom evaluator package.
+This directory is scaffolding for a new contrib evaluator package.
+
+It is intentionally excluded from repo automation until you convert it into a real package. In
+particular, `template/` does not participate in root `make check`, CI, semantic-release, or
+publishing because it ships a `pyproject.toml.template` placeholder instead of a real
+`pyproject.toml`.
+
+## Naming contract
+
+Pick `<name>` as a short lowercase single-word identifier such as `galileo`, `cisco`, or
+`budget`. That same value should appear in the steady-state package shape:
+
+- directory: `evaluators/contrib/<name>/`
+- pip package: `agent-control-evaluator-<name>`
+- Python module: `agent_control_evaluator_<name>`
+- extra name: `agent-control-evaluators[<name>]`
+- entry-point namespace: `<name>.<evaluator_id>`
+
+The template uses `{{NAME}}` for that package identifier. It does not use `{{ORG}}`.
+
+## Scaffold a new contrib package
+
+1. Copy the template and rename the manifest:
+
+   ```bash
+   cp -r evaluators/contrib/template evaluators/contrib/<name>
+   mv evaluators/contrib/<name>/pyproject.toml.template \
+     evaluators/contrib/<name>/pyproject.toml
+   ```
+
+2. Replace placeholders in `pyproject.toml`:
+
+   - `{{NAME}}` -> contrib package identifier
+   - `{{EVALUATOR}}` -> evaluator snake_case id
+   - `{{CLASS}}` -> evaluator class name
+   - `{{AUTHOR}}` -> authoring team
+
+3. Add package code and tests:
+
+   - `src/agent_control_evaluator_<name>/`
+   - `tests/`
+
+4. Validate the package locally:
+
+   ```bash
+   make lint
+   make lint-fix
+   make typecheck
+   make test
+   make check
+   make build
+   ```
+
+## Canonical install docs
+
+Contributor-facing and user-facing package docs should treat this as the canonical install path:
+
+```bash
+pip install "agent-control-evaluators[<name>]"
+```
+
+Direct wheel installs such as `pip install agent-control-evaluator-<name>` can still be
+documented, but they are secondary to the extra on `agent-control-evaluators`.
+
+## Expected repo wiring
+
+After the new package exists as a real contrib package, wire it into the repo contract:
+
+1. Add the extra to `evaluators/builtin/pyproject.toml`:
+
+   ```toml
+   [project.optional-dependencies]
+   <name> = ["agent-control-evaluator-<name>>=<current-version>"]
+   ```
+
+2. Add the workspace source pin to `evaluators/builtin/pyproject.toml`:
+
+   ```toml
+   [tool.uv.sources]
+   agent-control-evaluator-<name> = { path = "../contrib/<name>", editable = true }
+   ```
+
+3. Add the package to the root semantic-release version train in `pyproject.toml`:
+
+   ```toml
+   "evaluators/contrib/<name>/pyproject.toml:project.version",
+   ```
+
+Until those steps are done, the package is still scaffolding rather than a real contrib package.
 
 Docs: https://docs.agentcontrol.dev/concepts/evaluators/contributing-evaluator

evaluators/contrib/template/pyproject.toml.template

@@ -1,34 +1,36 @@
 [project]
-name = "agent-control-evaluator-{{ORG}}"
-version = "3.0.0"
-description = "{{ORG}} evaluators for agent-control"
+name = "agent-control-evaluator-{{NAME}}"
+version = "7.5.0"
+description = "{{NAME}} evaluators for agent-control"
+readme = "README.md"
 requires-python = ">=3.12"
 license = { text = "Apache-2.0" }
 authors = [{ name = "{{AUTHOR}}" }]
 dependencies = [
-    "agent-control-evaluators>=3.0.0",
-    "agent-control-models>=3.0.0",
+    "agent-control-evaluators>=7.5.0",
+    "agent-control-models>=7.5.0",
     # Add your package-specific dependencies here
 ]
 
 [project.optional-dependencies]
 dev = [
     "pytest>=8.0.0",
     "pytest-asyncio>=0.23.0",
+    "pytest-cov>=4.0.0",
     "ruff>=0.1.0",
     "mypy>=1.8.0",
 ]
 
 [project.entry-points."agent_control.evaluators"]
-# Format: "org.evaluator_name" = "package.module:Class"
-"{{ORG}}.{{EVALUATOR}}" = "agent_control_evaluator_{{ORG}}.{{EVALUATOR}}:{{CLASS}}"
+# Format: "<name>.<evaluator_name>" = "package.module:Class"
+"{{NAME}}.{{EVALUATOR}}" = "agent_control_evaluator_{{NAME}}.{{EVALUATOR}}:{{CLASS}}"
 
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/agent_control_evaluator_{{ORG}}"]
+packages = ["src/agent_control_evaluator_{{NAME}}"]
 
 [tool.uv.sources]
 agent-control-evaluators = { path = "../../builtin", editable = true }