Revert "fix(packaging): wire contrib evaluator releases"

This reverts commit ce2a8da.

Author: lan17

.github/workflows/ci.yml

@@ -59,13 +59,10 @@ jobs:
       - name: Test with coverage
         run: make test
 
-      - name: Test contrib evaluators
-        run: make test-extras
-
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
-          files: coverage-models.xml,coverage-engine.xml,coverage-telemetry.xml,coverage-server.xml,coverage-sdk.xml,coverage-evaluators-budget.xml,coverage-evaluators-cisco.xml,coverage-evaluators-galileo.xml
+          files: coverage-models.xml,coverage-engine.xml,coverage-telemetry.xml,coverage-server.xml,coverage-sdk.xml
           fail_ci_if_error: false
           token: ${{ secrets.CODECOV_TOKEN }}
 

.github/workflows/release.yaml

@@ -42,7 +42,7 @@ jobs:
           uv sync
           uv run python scripts/build.py all
 
-      # Publish in dependency order: models -> evaluators -> sdk -> contrib evaluators
+      # Publish in dependency order: models -> evaluators -> sdk -> evaluator-galileo
       - name: Publish agent-control-models to PyPI
         if: steps.release.outputs.released == 'true'
         uses: pypa/gh-action-pypi-publish@release/v1
@@ -67,22 +67,6 @@ jobs:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
 
-      - name: Publish agent-control-evaluator-budget to PyPI
-        if: steps.release.outputs.released == 'true'
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: evaluators/contrib/budget/dist/
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
-
-      - name: Publish agent-control-evaluator-cisco to PyPI
-        if: steps.release.outputs.released == 'true'
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: evaluators/contrib/cisco/dist/
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
-
       - name: Publish agent-control-evaluator-galileo to PyPI
         if: steps.release.outputs.released == 'true'
         uses: pypa/gh-action-pypi-publish@release/v1
@@ -103,6 +87,4 @@ jobs:
             evaluators/builtin/dist/*
             sdks/python/dist/*
             server/dist/*
-            evaluators/contrib/budget/dist/*
-            evaluators/contrib/cisco/dist/*
             evaluators/contrib/galileo/dist/*

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help sync openapi-spec openapi-spec-check test test-extras test-all models-test test-models test-sdk lint lint-fix typecheck check build build-models build-server build-sdk publish publish-models publish-server publish-sdk hooks-install hooks-uninstall prepush evaluators-test evaluators-lint evaluators-lint-fix evaluators-typecheck evaluators-build budget-test budget-lint budget-lint-fix budget-typecheck budget-build cisco-test cisco-lint cisco-lint-fix cisco-typecheck cisco-build galileo-test galileo-lint galileo-lint-fix galileo-typecheck galileo-build sdk-ts-generate sdk-ts-overlay-test sdk-ts-name-check sdk-ts-generate-check sdk-ts-build sdk-ts-test sdk-ts-lint sdk-ts-typecheck sdk-ts-release-check sdk-ts-publish-dry-run sdk-ts-publish telemetry-test telemetry-lint telemetry-lint-fix telemetry-typecheck telemetry-build telemetry-publish
+.PHONY: help sync openapi-spec openapi-spec-check test test-extras test-all models-test test-models test-sdk lint lint-fix typecheck check build build-models build-server build-sdk publish publish-models publish-server publish-sdk hooks-install hooks-uninstall prepush evaluators-test evaluators-lint evaluators-lint-fix evaluators-typecheck evaluators-build galileo-test galileo-lint galileo-lint-fix galileo-typecheck galileo-build sdk-ts-generate sdk-ts-overlay-test sdk-ts-name-check sdk-ts-generate-check sdk-ts-build sdk-ts-test sdk-ts-lint sdk-ts-typecheck sdk-ts-release-check sdk-ts-publish-dry-run sdk-ts-publish telemetry-test telemetry-lint telemetry-lint-fix telemetry-typecheck telemetry-build telemetry-publish
 
 # Workspace package names
 PACK_MODELS := agent-control-models
@@ -17,8 +17,6 @@ TS_SDK_DIR := sdks/typescript
 ENGINE_DIR := engine
 TELEMETRY_DIR := telemetry
 EVALUATORS_DIR := evaluators/builtin
-BUDGET_DIR := evaluators/contrib/budget
-CISCO_DIR := evaluators/contrib/cisco
 GALILEO_DIR := evaluators/contrib/galileo
 UI_DIR := ui
 
@@ -37,7 +35,7 @@ help:
 	@echo "Test:"
 	@echo "  make test            - run tests for core packages (models, telemetry, server, engine, sdk, evaluators)"
 	@echo "  make models-test     - run shared model tests with coverage"
-	@echo "  make test-extras     - run tests for contrib evaluators (budget, cisco, galileo)"
+	@echo "  make test-extras     - run tests for contrib evaluators (galileo, etc.)"
 	@echo "  make test-all        - run all tests (core + extras)"
 	@echo "  make sdk-ts-test     - run TypeScript SDK tests"
 	@echo ""
@@ -97,7 +95,7 @@ telemetry-test:
 	$(MAKE) -C $(TELEMETRY_DIR) test
 
 # Run tests for contrib evaluators (not included in default test target)
-test-extras: budget-test cisco-test galileo-test
+test-extras: galileo-test
 
 # Run all tests (core + extras)
 test-all: test test-extras
@@ -248,44 +246,6 @@ server-%:
 ui-%:
 	$(MAKE) -C $(UI_DIR) $(patsubst ui-%,%,$@)
 
-# ---------------------------
-# Contrib Evaluators (Budget)
-# ---------------------------
-
-budget-test:
-	$(MAKE) -C $(BUDGET_DIR) test
-
-budget-lint:
-	$(MAKE) -C $(BUDGET_DIR) lint
-
-budget-lint-fix:
-	$(MAKE) -C $(BUDGET_DIR) lint-fix
-
-budget-typecheck:
-	$(MAKE) -C $(BUDGET_DIR) typecheck
-
-budget-build:
-	$(MAKE) -C $(BUDGET_DIR) build
-
-# ---------------------------
-# Contrib Evaluators (Cisco)
-# ---------------------------
-
-cisco-test:
-	$(MAKE) -C $(CISCO_DIR) test
-
-cisco-lint:
-	$(MAKE) -C $(CISCO_DIR) lint
-
-cisco-lint-fix:
-	$(MAKE) -C $(CISCO_DIR) lint-fix
-
-cisco-typecheck:
-	$(MAKE) -C $(CISCO_DIR) typecheck
-
-cisco-build:
-	$(MAKE) -C $(CISCO_DIR) build
-
 # ---------------------------
 # Contrib Evaluators (Galileo)
 # ---------------------------

evaluators/contrib/budget/src/agent_control_evaluator_budget/budget/config.py

@@ -109,7 +109,7 @@ class BudgetEvaluatorConfig(EvaluatorConfig):
     metadata_paths: dict[str, str] = Field(default_factory=dict)
 
     @model_validator(mode="after")
-    def require_pricing_for_cost_rules(self) -> BudgetEvaluatorConfig:
+    def require_pricing_for_cost_rules(self) -> "BudgetEvaluatorConfig":
         has_cost_rule = any(rule.limit_unit == "usd_cents" for rule in self.limits)
         if has_cost_rule and self.pricing is None:
             raise ValueError('pricing is required when any rule uses limit_unit="usd_cents"')

evaluators/contrib/budget/src/agent_control_evaluator_budget/budget/evaluator.py

@@ -15,7 +15,7 @@
 import logging
 import math
 import threading
-from typing import Any, cast
+from typing import Any
 
 from agent_control_evaluators._base import Evaluator, EvaluatorMetadata
 from agent_control_evaluators._registry import register_evaluator
@@ -114,7 +114,7 @@ def _extract_tokens(data: Any, token_path: str | None) -> tuple[int, int]:
             inp_ok = isinstance(inp, int) and not isinstance(inp, bool)
             out_ok = isinstance(out, int) and not isinstance(out, bool)
             if inp_ok and out_ok:
-                return max(0, cast(int, inp)), max(0, cast(int, out))
+                return max(0, inp), max(0, out)
             total = usage.get("total_tokens")
             if isinstance(total, int) and not isinstance(total, bool) and total > 0:
                 return 0, max(0, total)
@@ -190,17 +190,17 @@ async def evaluate(self, data: Any) -> EvaluatorResult:
         input_tokens, output_tokens = _extract_tokens(data, self.config.token_path)
 
         model: str | None = None
-        model_path = self.config.model_path
-        if model_path:
-            val = _extract_by_path(data, model_path)
+        model_path_configured = bool(self.config.model_path)
+        if model_path_configured:
+            val = _extract_by_path(data, self.config.model_path)
             if val is not None:
                 model = str(val)
 
         cost = _estimate_cost(model, input_tokens, output_tokens, self.config.pricing)
 
         step_metadata = _extract_metadata(data, self.config.metadata_paths)
 
-        if model_path and model is None:
+        if model_path_configured and model is None:
             model_known = False
         else:
             model_known = (

pyproject.toml

@@ -73,8 +73,6 @@ version_toml = [
     "telemetry/pyproject.toml:project.version",
     "server/pyproject.toml:project.version",
     "evaluators/builtin/pyproject.toml:project.version",
-    "evaluators/contrib/budget/pyproject.toml:project.version",
-    "evaluators/contrib/cisco/pyproject.toml:project.version",
     "evaluators/contrib/galileo/pyproject.toml:project.version",
 ]
 version_source = "tag"

scripts/build.py

@@ -6,7 +6,7 @@
 then cleans up afterward. This allows the published wheels to be self-contained.
 
 Usage:
-    python scripts/build.py [models|evaluators|sdk|server|budget|cisco|galileo|all]
+    python scripts/build.py [models|evaluators|sdk|server|galileo|all]
 """
 
 import shutil
@@ -240,53 +240,13 @@ def build_evaluator_galileo() -> None:
     print(f"  Built agent-control-evaluator-galileo v{version}")
 
 
-def build_evaluator_budget() -> None:
-    """Build agent-control-evaluator-budget (standalone, no vendoring needed)."""
-    version = get_global_version()
-    budget_dir = ROOT / "evaluators" / "contrib" / "budget"
-
-    print(f"Building agent-control-evaluator-budget v{version}")
-
-    # Clean previous builds
-    dist_dir = budget_dir / "dist"
-    if dist_dir.exists():
-        shutil.rmtree(dist_dir)
-
-    # Set version
-    set_package_version(budget_dir / "pyproject.toml", version)
-
-    subprocess.run(["uv", "build", "-o", str(dist_dir)], cwd=budget_dir, check=True)
-    print(f"  Built agent-control-evaluator-budget v{version}")
-
-
-def build_evaluator_cisco() -> None:
-    """Build agent-control-evaluator-cisco (standalone, no vendoring needed)."""
-    version = get_global_version()
-    cisco_dir = ROOT / "evaluators" / "contrib" / "cisco"
-
-    print(f"Building agent-control-evaluator-cisco v{version}")
-
-    # Clean previous builds
-    dist_dir = cisco_dir / "dist"
-    if dist_dir.exists():
-        shutil.rmtree(dist_dir)
-
-    # Set version
-    set_package_version(cisco_dir / "pyproject.toml", version)
-
-    subprocess.run(["uv", "build", "-o", str(dist_dir)], cwd=cisco_dir, check=True)
-    print(f"  Built agent-control-evaluator-cisco v{version}")
-
-
 def build_all() -> None:
     """Build all packages."""
     print(f"Building all packages (version {get_global_version()})\n")
     build_models()
     build_evaluators()
     build_sdk()
     build_server()
-    build_evaluator_budget()
-    build_evaluator_cisco()
     build_evaluator_galileo()
     print("\nAll packages built successfully!")
 
@@ -304,14 +264,10 @@ def build_all() -> None:
         build_sdk()
     elif target == "server":
         build_server()
-    elif target == "budget":
-        build_evaluator_budget()
-    elif target == "cisco":
-        build_evaluator_cisco()
     elif target == "galileo":
         build_evaluator_galileo()
     elif target == "all":
         build_all()
     else:
-        print("Usage: python scripts/build.py [models|evaluators|sdk|server|budget|cisco|galileo|all]")
+        print("Usage: python scripts/build.py [models|evaluators|sdk|server|galileo|all]")
         sys.exit(1)