feat(evaluators): add yelp.detect_secrets contrib evaluator

evaluators/builtin/pyproject.toml

@@ -19,6 +19,7 @@ dependencies = [
 galileo = ["agent-control-evaluator-galileo>=7.5.0"]
 budget = ["agent-control-evaluator-budget>=7.5.0"]
 cisco = ["agent-control-evaluator-cisco>=7.5.0"]
+detect_secrets = ["agent-control-evaluator-detect_secrets>=7.5.0"]
 dev = ["pytest>=8.0.0", "pytest-asyncio>=0.23.0"]
 
 [project.entry-points."agent_control.evaluators"]
@@ -40,3 +41,4 @@ agent-control-models = { workspace = true }
 agent-control-evaluator-galileo = { path = "../contrib/galileo", editable = true }
 agent-control-evaluator-budget = { path = "../contrib/budget", editable = true }
 agent-control-evaluator-cisco = { path = "../contrib/cisco", editable = true }
+agent-control-evaluator-detect_secrets = { path = "../contrib/detect_secrets", editable = true }

evaluators/builtin/tests/test_contrib_packages.py

@@ -39,6 +39,7 @@ def test_discover_contrib_packages_returns_expected_metadata() -> None:
     assert [(package.name, package.package, package.extra) for package in packages] == [
         ("budget", "agent-control-evaluator-budget", "budget"),
         ("cisco", "agent-control-evaluator-cisco", "cisco"),
+        ("detect_secrets", "agent-control-evaluator-detect_secrets", "detect_secrets"),
         ("galileo", "agent-control-evaluator-galileo", "galileo"),
     ]
 

evaluators/contrib/README.md

@@ -3,6 +3,7 @@
 Contributed evaluators and templates for extending Agent Control.
 
 - `galileo/` — Luna-2 evaluator integration
+- `detect_secrets/` — detect-secrets runtime scanner integration
 - `template/` — Starter template for adding new evaluators
 
 Full guide: https://docs.agentcontrol.dev/concepts/evaluators/custom-evaluators

evaluators/contrib/detect_secrets/Makefile

@@ -0,0 +1,19 @@
+.PHONY: sync test lint typecheck check build
+
+sync:
+	uv sync --group dev
+
+test:
+	uv run --group dev pytest --cov=src --cov-report=xml:../../../coverage-evaluators-detect-secrets.xml -q
+
+lint:
+	uv run --group dev ruff check .
+	uv run --group dev ruff format --check .
+
+typecheck:
+	uv run --group dev mypy .
+
+check: sync lint typecheck test build
+
+build:
+	uv build

evaluators/contrib/detect_secrets/README.md

@@ -0,0 +1,91 @@
+# Agent Control Evaluator - detect-secrets
+
+External evaluator that scans selector-selected payloads for likely secrets using
+[`detect-secrets-async`](https://pypi.org/project/detect-secrets-async/), which wraps Yelp
+`detect-secrets` in a bounded subprocess runtime.
+
+- Entry point name: `yelp.detect_secrets`
+- Transport/runtime: `detect-secrets-async`
+
+## Installation
+
+Install the evaluator package:
+
+```bash
+pip install agent-control-evaluator-detect_secrets
+```
+
+For local development from this repo:
+
+```bash
+uv pip install -e evaluators/contrib/detect_secrets
+```
+
+## Configuration
+
+Evaluator config fields:
+
+- `timeout_ms: int = 10000`
+- `on_error: "allow" | "deny" = "allow"`
+- `max_bytes: int = 1048576`
+- `enabled_plugins: list[str] | None = None`
+- `exclude_lines_regex: list[str] = []`
+
+Notes:
+
+- `enabled_plugins` takes upstream `detect-secrets` plugin class names such as
+  `GitHubTokenDetector`.
+- If `enabled_plugins` is omitted, the evaluator uses the pinned upstream default plugin set from
+  `detect-secrets-async`.
+- `exclude_lines_regex` uses RE2 syntax and blanks matching lines before scan submission so line
+  numbering stays stable for plain string payloads.
+
+## Behavior
+
+- selector-selected `str` payloads are scanned directly
+- selector-selected `dict` / `list` payloads are normalized to deterministic pretty JSON before
+  scanning
+- scalar numbers / booleans are normalized to JSON scalar text
+- `None` produces `matched=False`
+
+Safe metadata:
+
+- `findings_count`
+- `findings[]` with `type`, plus:
+  - `line_number` for plain selected strings
+  - `json_pointer` for normalized `dict` / `list` payloads when a finding maps back to a structural
+    location; pointers are conservatively truncated to the nearest safe ancestor when a key segment
+    looks secret-like
+- `normalized_payload_type`
+- `detect_secrets_version`
+- `failure_mode` on evaluator failures
+- `fallback_action` on fail-closed paths
+
+Plaintext secrets, snippets, matching lines, and upstream `hashed_secret` are never surfaced.
+
+## Usage
+
+Once installed, the evaluator is auto-discovered:
+
+```python
+from agent_control_evaluators import discover_evaluators, get_evaluator
+
+discover_evaluators()
+DetectSecretsEvaluator = get_evaluator("yelp.detect_secrets")
+```
+
+Example control fragment:
+
+```json
+{
+  "selector": { "path": "output" },
+  "evaluator": {
+    "name": "yelp.detect_secrets",
+    "config": {
+      "timeout_ms": 10000,
+      "on_error": "allow",
+      "enabled_plugins": ["GitHubTokenDetector"]
+    }
+  }
+}
+```

evaluators/contrib/detect_secrets/pyproject.toml

@@ -0,0 +1,60 @@
+[project]
+name = "agent-control-evaluator-detect_secrets"
+version = "7.6.0"
+description = "detect-secrets evaluator for agent-control"
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "Apache-2.0" }
+authors = [{ name = "Agent Control Team" }]
+dependencies = [
+    "agent-control-evaluators>=7.5.0",
+    "agent-control-models>=7.5.0",
+    "detect-secrets-async>=0.2.0,<0.3.0",
+    "google-re2>=1.1",
+    "pydantic>=2.12.4",
+]
+
+[project.entry-points."agent_control.evaluators"]
+"yelp.detect_secrets" = "agent_control_evaluator_detect_secrets.detect_secrets:DetectSecretsEvaluator"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/agent_control_evaluator_detect_secrets"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP"]
+
+[tool.mypy]
+python_version = "3.12"
+strict = true
+files = ["src", "tests"]
+
+[[tool.mypy.overrides]]
+module = "re2"
+ignore_missing_imports = true
+
+[tool.uv.sources]
+agent-control-evaluators = { path = "../../builtin", editable = true }
+agent-control-models = { path = "../../../models", editable = true }
+
+[tool.uv]
+default-groups = ["dev"]
+
+[dependency-groups]
+dev = [
+    "mypy>=1.8.0",
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.23.0",
+    "pytest-cov>=4.0.0",
+    "ruff>=0.1.0",
+]

evaluators/contrib/detect_secrets/src/agent_control_evaluator_detect_secrets/__init__.py

@@ -0,0 +1,18 @@
+"""Agent Control evaluator package for detect-secrets."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("agent-control-evaluator-detect_secrets")
+except PackageNotFoundError:
+    __version__ = "0.0.0.dev"
+
+from agent_control_evaluator_detect_secrets.detect_secrets import (
+    DetectSecretsEvaluator,
+    DetectSecretsEvaluatorConfig,
+)
+
+__all__ = [
+    "DetectSecretsEvaluator",
+    "DetectSecretsEvaluatorConfig",
+]

evaluators/contrib/detect_secrets/src/agent_control_evaluator_detect_secrets/detect_secrets/__init__.py

@@ -0,0 +1,13 @@
+"""detect-secrets evaluator exports."""
+
+from agent_control_evaluator_detect_secrets.detect_secrets.config import (
+    DetectSecretsEvaluatorConfig,
+)
+from agent_control_evaluator_detect_secrets.detect_secrets.evaluator import (
+    DetectSecretsEvaluator,
+)
+
+__all__ = [
+    "DetectSecretsEvaluator",
+    "DetectSecretsEvaluatorConfig",
+]

evaluators/contrib/detect_secrets/src/agent_control_evaluator_detect_secrets/detect_secrets/config.py

@@ -0,0 +1,81 @@
+"""Configuration for the detect-secrets evaluator."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+import re2
+from agent_control_evaluators import EvaluatorConfig
+from detect_secrets_async import get_runtime_info
+from pydantic import Field, field_validator
+
+DEFAULT_TIMEOUT_MS = 10_000
+DEFAULT_MAX_BYTES = 1_048_576
+
+
+class DetectSecretsEvaluatorConfig(EvaluatorConfig):
+    """Typed configuration for the detect-secrets evaluator."""
+
+    timeout_ms: int = Field(
+        default=DEFAULT_TIMEOUT_MS,
+        gt=0,
+        description="End-to-end timeout in milliseconds for queue wait and scan execution.",
+    )
+    on_error: Literal["allow", "deny"] = Field(
+        default="allow",
+        description="Whether evaluator failures should fail open or fail closed.",
+    )
+    max_bytes: int = Field(
+        default=DEFAULT_MAX_BYTES,
+        gt=0,
+        description="Maximum UTF-8 payload size after normalization and line filtering.",
+    )
+    enabled_plugins: list[str] | None = Field(
+        default=None,
+        description="Optional explicit upstream detect-secrets plugin class names.",
+    )
+    exclude_lines_regex: list[str] = Field(
+        default_factory=list,
+        description="RE2 patterns for lines that should be blanked before scanning.",
+    )
+
+    @field_validator("enabled_plugins")
+    @classmethod
+    def validate_enabled_plugins(cls, value: list[str] | None) -> list[str] | None:
+        """Validate explicit upstream plugin names against detect-secrets-async introspection."""
+        if value is None:
+            return None
+
+        try:
+            available = set(get_runtime_info().available_plugin_names)
+        except Exception as exc:
+            raise ValueError(
+                "Unable to validate detect-secrets plugins because runtime introspection failed"
+            ) from exc
+        normalized: list[str] = []
+        seen: set[str] = set()
+
+        for plugin_name in value:
+            candidate = plugin_name.strip()
+            if not candidate:
+                raise ValueError("enabled_plugins entries must be non-empty")
+            if candidate not in available:
+                raise ValueError(f"Unknown detect-secrets plugin: {candidate}")
+            if candidate not in seen:
+                normalized.append(candidate)
+                seen.add(candidate)
+
+        return normalized
+
+    @field_validator("exclude_lines_regex")
+    @classmethod
+    def validate_exclude_lines_regex(cls, value: list[str]) -> list[str]:
+        """Validate each configured exclude pattern as a RE2 regex."""
+        for pattern in value:
+            if pattern == "":
+                raise ValueError("exclude_lines_regex entries must be non-empty")
+            try:
+                re2.compile(pattern)
+            except re2.error as exc:
+                raise ValueError(f"Invalid RE2 pattern '{pattern}': {exc}") from exc
+        return value