build(evaluators): standardize contrib packaging and release flow

.github/workflows/ci.yml

@@ -47,22 +47,36 @@ jobs:
           uv --version
           python --version
 
+      - name: Verify contrib packaging contract
+        run: make contrib-verify
+
       - name: Sync dependencies
         run: make sync
 
-      - name: Lint
-        run: make lint
+      - name: Run Python checks
+        run: make check
 
-      - name: Type check
-        run: make typecheck
-
-      - name: Test with coverage
-        run: make test
+      - name: Collect coverage files
+        id: coverage_files
+        run: |
+          python - <<'PY' >> "$GITHUB_OUTPUT"
+          from pathlib import Path
+
+          base_reports = [
+              "coverage-models.xml",
+              "coverage-engine.xml",
+              "coverage-telemetry.xml",
+              "coverage-server.xml",
+              "coverage-sdk.xml",
+          ]
+          contrib_reports = sorted(path.name for path in Path(".").glob("coverage-evaluators-*.xml"))
+          print(f"files={','.join([*base_reports, *contrib_reports])}")
+          PY
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
-          files: coverage-models.xml,coverage-engine.xml,coverage-telemetry.xml,coverage-server.xml,coverage-sdk.xml
+          files: ${{ steps.coverage_files.outputs.files }}
           fail_ci_if_error: false
           token: ${{ secrets.CODECOV_TOKEN }}
 

.github/workflows/release.yaml

@@ -8,6 +8,9 @@ jobs:
     permissions:
       contents: write
       id-token: write
+    outputs:
+      released: ${{ steps.semantic_release.outputs.released }}
+      tag: ${{ steps.semantic_release.outputs.tag }}
 
     steps:
       - name: Checkout
@@ -25,8 +28,11 @@ jobs:
       - name: Setup UV
         uses: astral-sh/setup-uv@v4
 
+      - name: Verify contrib wiring
+        run: python scripts/contrib_packages.py verify
+
       - name: Python Semantic Release
-        id: release
+        id: semantic_release
         uses: python-semantic-release/python-semantic-release@v10.5.3
         with:
           git_committer_name: galileo-automation
@@ -37,54 +43,132 @@ jobs:
           root_options: "-vv"
 
       - name: Build Packages
-        if: steps.release.outputs.released == 'true'
+        if: steps.semantic_release.outputs.released == 'true'
         run: |
           uv sync
           uv run python scripts/build.py all
 
-      # Publish in dependency order: models -> evaluators -> sdk -> evaluator-galileo
+      - name: Stage built distributions
+        if: steps.semantic_release.outputs.released == 'true'
+        run: |
+          rm -rf release-dists
+          mkdir -p release-dists/models
+          mkdir -p release-dists/evaluators/builtin
+          mkdir -p release-dists/evaluators/contrib
+          mkdir -p release-dists/pypi/evaluators
+          mkdir -p release-dists/sdks/python
+          mkdir -p release-dists/server
+
+          cp -R models/dist release-dists/models/
+          cp -R evaluators/builtin/dist release-dists/evaluators/builtin/
+          cp evaluators/builtin/dist/* release-dists/pypi/evaluators/
+          cp -R sdks/python/dist release-dists/sdks/python/
+          cp -R server/dist release-dists/server/
+
+          for contrib_dir in evaluators/contrib/*/dist; do
+            contrib_name="$(basename "$(dirname "$contrib_dir")")"
+            mkdir -p "release-dists/evaluators/contrib/$contrib_name"
+            cp -R "$contrib_dir" "release-dists/evaluators/contrib/$contrib_name/"
+            cp "$contrib_dir"/* release-dists/pypi/evaluators/
+          done
+
+      - name: Upload built distributions
+        if: steps.semantic_release.outputs.released == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-dists
+          if-no-files-found: error
+          path: release-dists/
+
+  publish-models:
+    runs-on: ubuntu-latest
+    needs: release
+    if: needs.release.outputs.released == 'true'
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download built distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: release-dists
+          path: .
+
       - name: Publish agent-control-models to PyPI
-        if: steps.release.outputs.released == 'true'
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          packages-dir: models/dist/
+          packages-dir: release-dists/models/dist/
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
 
-      - name: Publish agent-control-evaluators to PyPI
-        if: steps.release.outputs.released == 'true'
+  publish-evaluators:
+    runs-on: ubuntu-latest
+    needs: [release, publish-models]
+    if: needs.release.outputs.released == 'true'
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download built distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: release-dists
+          path: .
+
+      - name: Publish evaluator distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          packages-dir: evaluators/builtin/dist/
+          packages-dir: release-dists/pypi/evaluators/
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
 
+  publish-sdk:
+    runs-on: ubuntu-latest
+    needs: [release, publish-evaluators]
+    if: >-
+      always() &&
+      needs.release.outputs.released == 'true' &&
+      needs.publish-evaluators.result == 'success'
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download built distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: release-dists
+          path: .
+
       - name: Publish agent-control-sdk to PyPI
-        if: steps.release.outputs.released == 'true'
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          packages-dir: sdks/python/dist/
+          packages-dir: release-dists/sdks/python/dist/
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
 
-      - name: Publish agent-control-evaluator-galileo to PyPI
-        if: steps.release.outputs.released == 'true'
-        uses: pypa/gh-action-pypi-publish@release/v1
+  upload-release-assets:
+    runs-on: ubuntu-latest
+    needs: [release, publish-sdk]
+    if: needs.release.outputs.released == 'true'
+    permissions:
+      contents: write
+
+    steps:
+      - name: Download built distributions
+        uses: actions/download-artifact@v4
         with:
-          packages-dir: evaluators/contrib/galileo/dist/
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
+          name: release-dists
+          path: .
 
       - name: Upload to GitHub Release
-        if: steps.release.outputs.released == 'true'
         uses: python-semantic-release/upload-to-gh-release@main
         with:
           github_token: ${{ secrets.GALILEO_AUTOMATION_GITHUB_TOKEN || github.token }}
-          tag: ${{ steps.release.outputs.tag }}
+          tag: ${{ needs.release.outputs.tag }}
           root_options: "-vv"
           dist_glob: |
-            models/dist/*
-            evaluators/builtin/dist/*
-            sdks/python/dist/*
-            server/dist/*
-            evaluators/contrib/galileo/dist/*
+            release-dists/models/dist/*
+            release-dists/evaluators/builtin/dist/*
+            release-dists/sdks/python/dist/*
+            release-dists/server/dist/*
+            release-dists/evaluators/contrib/*/dist/*

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help sync openapi-spec openapi-spec-check test test-extras test-all models-test test-models test-sdk lint lint-fix typecheck check build build-models build-server build-sdk publish publish-models publish-server publish-sdk hooks-install hooks-uninstall prepush evaluators-test evaluators-lint evaluators-lint-fix evaluators-typecheck evaluators-build galileo-test galileo-lint galileo-lint-fix galileo-typecheck galileo-build sdk-ts-generate sdk-ts-overlay-test sdk-ts-name-check sdk-ts-generate-check sdk-ts-build sdk-ts-test sdk-ts-lint sdk-ts-typecheck sdk-ts-release-check sdk-ts-publish-dry-run sdk-ts-publish telemetry-test telemetry-lint telemetry-lint-fix telemetry-typecheck telemetry-build telemetry-publish
+.PHONY: help sync openapi-spec openapi-spec-check test test-extras test-all contrib-verify scripts-test models-test test-models test-sdk lint lint-fix typecheck check build build-models build-server build-sdk publish publish-models publish-server publish-sdk hooks-install hooks-uninstall prepush evaluators-test evaluators-lint evaluators-lint-fix evaluators-typecheck evaluators-build contrib-test contrib-lint contrib-lint-fix contrib-typecheck contrib-build sdk-ts-generate sdk-ts-overlay-test sdk-ts-name-check sdk-ts-generate-check sdk-ts-build sdk-ts-test sdk-ts-lint sdk-ts-typecheck sdk-ts-release-check sdk-ts-publish-dry-run sdk-ts-publish telemetry-test telemetry-lint telemetry-lint-fix telemetry-typecheck telemetry-build telemetry-publish
 
 # Workspace package names
 PACK_MODELS := agent-control-models
@@ -17,9 +17,17 @@ TS_SDK_DIR := sdks/typescript
 ENGINE_DIR := engine
 TELEMETRY_DIR := telemetry
 EVALUATORS_DIR := evaluators/builtin
-GALILEO_DIR := evaluators/contrib/galileo
+CONTRIB_DIR := evaluators/contrib
 UI_DIR := ui
 
+define run-contrib-target
+	@set -e; \
+	packages=$$(uv run python scripts/contrib_packages.py names); \
+	for package in $$packages; do \
+		$(MAKE) -C $(CONTRIB_DIR)/$$package $(1); \
+	done
+endef
+
 help:
 	@echo "Agent Control - Makefile commands"
 	@echo ""
@@ -33,10 +41,12 @@ help:
 	@echo "  make openapi-spec-check - verify OpenAPI generation succeeds"
 	@echo ""
 	@echo "Test:"
-	@echo "  make test            - run tests for core packages (models, telemetry, server, engine, sdk, evaluators)"
+	@echo "  make test            - run tests for core packages and all discovered contrib evaluators"
+	@echo "  make contrib-verify  - verify root contrib packaging contract wiring"
+	@echo "  make scripts-test    - run root contrib packaging contract tests"
 	@echo "  make models-test     - run shared model tests with coverage"
-	@echo "  make test-extras     - run tests for contrib evaluators (galileo, etc.)"
-	@echo "  make test-all        - run all tests (core + extras)"
+	@echo "  make test-extras     - run tests for all discovered contrib evaluators"
+	@echo "  make test-all        - alias for make test"
 	@echo "  make sdk-ts-test     - run TypeScript SDK tests"
 	@echo ""
 	@echo "Quality:"
@@ -84,7 +94,13 @@ openapi-spec-check: openapi-spec
 # Test
 # ---------------------------
 
-test: models-test telemetry-test server-test engine-test sdk-test evaluators-test
+test: contrib-verify scripts-test models-test telemetry-test server-test engine-test sdk-test evaluators-test contrib-test
+
+contrib-verify:
+	uv run python scripts/contrib_packages.py verify
+
+scripts-test:
+	uv run --with pytest pytest scripts/tests -q
 
 models-test:
 	cd $(MODELS_DIR) && uv run pytest --cov=src --cov-report=xml:../coverage-models.xml -q
@@ -94,11 +110,11 @@ test-models: models-test
 telemetry-test:
 	$(MAKE) -C $(TELEMETRY_DIR) test
 
-# Run tests for contrib evaluators (not included in default test target)
-test-extras: galileo-test
+# Run tests for discovered contrib evaluators
+test-extras: contrib-test
 
-# Run all tests (core + extras)
-test-all: test test-extras
+# Run all tests (alias for test)
+test-all: test
 
 # Run tests, lint, and typecheck
 check: test lint typecheck
@@ -107,17 +123,17 @@ check: test lint typecheck
 # Quality
 # ---------------------------
 
-lint: engine-lint telemetry-lint evaluators-lint
+lint: engine-lint telemetry-lint evaluators-lint contrib-lint
 	uv run --package $(PACK_MODELS) ruff check --config pyproject.toml models/src
 	uv run --package $(PACK_SERVER) ruff check --config pyproject.toml server/src
 	uv run --package $(PACK_SDK) ruff check --config pyproject.toml sdks/python/src
 
-lint-fix: engine-lint-fix telemetry-lint-fix evaluators-lint-fix
+lint-fix: engine-lint-fix telemetry-lint-fix evaluators-lint-fix contrib-lint-fix
 	uv run --package $(PACK_MODELS) ruff check --config pyproject.toml --fix models/src
 	uv run --package $(PACK_SERVER) ruff check --config pyproject.toml --fix server/src
 	uv run --package $(PACK_SDK) ruff check --config pyproject.toml --fix sdks/python/src
 
-typecheck: engine-typecheck telemetry-typecheck evaluators-typecheck
+typecheck: engine-typecheck telemetry-typecheck evaluators-typecheck contrib-typecheck
 	uv run --package $(PACK_MODELS) mypy --config-file pyproject.toml models/src
 	uv run --package $(PACK_SERVER) mypy --config-file pyproject.toml server/src
 	uv run --package $(PACK_SDK) mypy --config-file pyproject.toml sdks/python/src
@@ -135,7 +151,7 @@ telemetry-typecheck:
 # Build / Publish
 # ---------------------------
 
-build: build-models build-server build-sdk engine-build telemetry-build evaluators-build
+build: build-models build-server build-sdk engine-build telemetry-build evaluators-build contrib-build
 
 build-models:
 	cd $(MODELS_DIR) && uv build
@@ -246,21 +262,17 @@ server-%:
 ui-%:
 	$(MAKE) -C $(UI_DIR) $(patsubst ui-%,%,$@)
 
-# ---------------------------
-# Contrib Evaluators (Galileo)
-# ---------------------------
-
-galileo-test:
-	$(MAKE) -C $(GALILEO_DIR) test
+contrib-test:
+	$(call run-contrib-target,test)
 
-galileo-lint:
-	$(MAKE) -C $(GALILEO_DIR) lint
+contrib-lint:
+	$(call run-contrib-target,lint)
 
-galileo-lint-fix:
-	$(MAKE) -C $(GALILEO_DIR) lint-fix
+contrib-lint-fix:
+	$(call run-contrib-target,lint-fix)
 
-galileo-typecheck:
-	$(MAKE) -C $(GALILEO_DIR) typecheck
+contrib-typecheck:
+	$(call run-contrib-target,typecheck)
 
-galileo-build:
-	$(MAKE) -C $(GALILEO_DIR) build
+contrib-build:
+	$(call run-contrib-target,build)

evaluators/builtin/pyproject.toml

@@ -7,7 +7,7 @@ requires-python = ">=3.12"
 license = { text = "Apache-2.0" }
 authors = [{ name = "Agent Control Team" }]
 dependencies = [
-    "agent-control-models",
+    "agent-control-models>=7.5.0",
     "pydantic>=2.12.4",
     "google-re2>=1.1",
     "jsonschema>=4.0.0",
@@ -16,8 +16,9 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-galileo = ["agent-control-evaluator-galileo>=3.0.0"]
-cisco = ["agent-control-evaluator-cisco>=0.1.0"]
+galileo = ["agent-control-evaluator-galileo>=7.5.0"]
+budget = ["agent-control-evaluator-budget>=7.5.0"]
+cisco = ["agent-control-evaluator-cisco>=7.5.0"]
 dev = ["pytest>=8.0.0", "pytest-asyncio>=0.23.0"]
 
 [project.entry-points."agent_control.evaluators"]
@@ -35,6 +36,7 @@ packages = ["src/agent_control_evaluators"]
 
 [tool.uv.sources]
 agent-control-models = { workspace = true }
-# For local dev: use local galileo package instead of PyPI
+# For local dev: use local contrib packages instead of PyPI
 agent-control-evaluator-galileo = { path = "../contrib/galileo", editable = true }
+agent-control-evaluator-budget = { path = "../contrib/budget", editable = true }
 agent-control-evaluator-cisco = { path = "../contrib/cisco", editable = true }