feat: add TaskVerifierRegistry for custom task verification (#89)

Add a registry pattern for custom task verifiers that can inspect VM
state after task execution. This enables GoTo IT Autopilot (and other
integrators) to register domain-specific verification functions without
subclassing BenchmarkAdapter.

- TaskVerifierRegistry with decorator and programmatic registration
- VerificationResult dataclass with success/score/details
- WAALiveAdapter.run_powershell() for executing PowerShell on the VM
- Built-in clear_browsing_data reference verifier
- 33 tests covering registry operations and built-in verifiers
- Exports from evaluation package and main package __init__

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: abrichr

openadapt_evals/__init__.py

@@ -104,6 +104,10 @@
     LiveEvaluationTracker,
     save_execution_trace,
 )
+from openadapt_evals.evaluation.verifier_registry import (
+    TaskVerifierRegistry,
+    VerificationResult,
+)
 
 # Lazy imports for optional dependencies
 def __getattr__(name: str):
@@ -140,6 +144,9 @@ def __getattr__(name: str):
     "evaluate_agent_on_benchmark",
     "compute_metrics",
     "compute_domain_metrics",
+    # Task verification
+    "TaskVerifierRegistry",
+    "VerificationResult",
     # WAA adapters
     "WAAAdapter",
     "WAAConfig",

openadapt_evals/adapters/waa/live.py

@@ -678,6 +678,63 @@ def _recover_failsafe(self) -> bool:
             logger.error(f"Fail-safe recovery request failed: {e}")
             return False
 
+    def run_powershell(self, script: str) -> str:
+        """Execute a PowerShell command on the Windows VM and return stdout.
+
+        Sends the script to the WAA server's ``/execute`` endpoint wrapped
+        in a ``python -c "..."`` command that invokes PowerShell via
+        ``subprocess.run``.
+
+        This is primarily intended for use by task verifiers that need to
+        inspect VM state (e.g., checking file counts, registry values, etc.).
+
+        Args:
+            script: PowerShell command or script to execute. Multi-line
+                scripts are supported but should be kept simple.
+
+        Returns:
+            The stdout output from the PowerShell command as a string.
+
+        Raises:
+            RuntimeError: If the command execution fails or the server
+                is unreachable.
+        """
+        import requests
+
+        # Build a python -c command that runs PowerShell via subprocess.
+        # The /execute endpoint requires the "python -c ..." format.
+        # We use repr() for safe escaping of the PowerShell script.
+        python_code = (
+            "import subprocess; "
+            f"r = subprocess.run(['powershell', '-Command', {repr(script)}], "
+            "capture_output=True, text=True, timeout=30); "
+            "print(r.stdout)"
+        )
+        command = f'python -c "{python_code}"'
+
+        try:
+            resp = requests.post(
+                f"{self.config.server_url}/execute",
+                json={"command": command},
+                timeout=self.config.timeout,
+            )
+            if resp.status_code == 200:
+                result = resp.json()
+                stdout = result.get("stdout", "")
+                stderr = result.get("stderr", "")
+                if stderr:
+                    logger.warning("PowerShell stderr: %s", stderr)
+                return stdout
+            else:
+                raise RuntimeError(
+                    f"PowerShell execution failed (HTTP {resp.status_code}): "
+                    f"{resp.text}"
+                )
+        except requests.RequestException as e:
+            raise RuntimeError(
+                f"Failed to connect to WAA server for PowerShell execution: {e}"
+            ) from e
+
     def evaluate(self, task: BenchmarkTask) -> BenchmarkResult:
         """Evaluate current state against task success criteria.
 

openadapt_evals/evaluation/__init__.py

@@ -3,6 +3,9 @@
 This module provides client-side evaluation without requiring a sidecar service.
 The evaluators run locally, making HTTP calls to the WAA server's /execute endpoint.
 
+It also provides a task verifier registry for registering custom verification
+functions that can inspect the VM state after task execution.
+
 Example usage:
     from openadapt_evals.evaluation import EvaluatorClient, discover_vm_ip
 
@@ -16,15 +19,34 @@
     # Evaluate a task
     result = client.evaluate(task_config)
     print(f"Success: {result.success}, Score: {result.score}")
+
+    # Custom task verification
+    from openadapt_evals.evaluation import register, registry, VerificationResult
+
+    @register("my_task")
+    def verify_my_task(adapter):
+        return VerificationResult(success=True, score=1.0)
+
+    result = registry.verify("my_task", adapter)
 """
 
 from .discovery import VMIPDiscovery, DiscoveryMethod, discover_vm_ip
 from .client import EvaluatorClient, EvaluationResult
+from .verifier_registry import (
+    TaskVerifierRegistry,
+    VerificationResult,
+    register,
+    registry,
+)
 
 __all__ = [
     "VMIPDiscovery",
     "DiscoveryMethod",
     "discover_vm_ip",
     "EvaluatorClient",
     "EvaluationResult",
+    "TaskVerifierRegistry",
+    "VerificationResult",
+    "register",
+    "registry",
 ]

openadapt_evals/evaluation/builtin_verifiers.py

@@ -0,0 +1,68 @@
+"""Built-in task verifiers for common verification scenarios.
+
+This module contains reference verifier implementations that demonstrate the
+verifier registry pattern. Import this module to register the built-in
+verifiers with the global registry.
+
+Example:
+    # Import to register built-in verifiers
+    import openadapt_evals.evaluation.builtin_verifiers
+
+    from openadapt_evals.evaluation.verifier_registry import registry
+    result = registry.verify("clear_browsing_data", adapter)
+"""
+
+from __future__ import annotations
+
+import logging
+
+from openadapt_evals.evaluation.verifier_registry import (
+    VerificationResult,
+    register,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@register("clear_browsing_data")
+def verify_clear_browsing_data(adapter) -> VerificationResult:
+    """Verify Chrome browsing data has been cleared.
+
+    Checks that Chrome's cache directory is empty by running a PowerShell
+    command on the Windows VM via the adapter's run_powershell() method.
+
+    Args:
+        adapter: A BenchmarkAdapter with run_powershell() support
+            (e.g., WAALiveAdapter).
+
+    Returns:
+        VerificationResult indicating whether the cache was cleared.
+    """
+    try:
+        result = adapter.run_powershell(
+            "(Get-ChildItem -Path "
+            "$env:LOCALAPPDATA\\Google\\Chrome\\"
+            "'User Data'\\Default\\Cache "
+            "-Recurse -ErrorAction SilentlyContinue "
+            "| Measure-Object).Count"
+        )
+        count = int(result.strip())
+        success = count == 0
+        return VerificationResult(
+            success=success,
+            score=1.0 if success else 0.0,
+            details={
+                "cache_file_count": count,
+                "cache_path": (
+                    "%LOCALAPPDATA%\\Google\\Chrome\\"
+                    "User Data\\Default\\Cache"
+                ),
+            },
+        )
+    except Exception as e:
+        logger.error("Failed to verify clear_browsing_data: %s", e)
+        return VerificationResult(
+            success=False,
+            score=0.0,
+            details={"error": str(e)},
+        )