Merge branch 'main' into fix/harden-failsafe-detection

Author: abrichr

CHANGELOG.md

@@ -1,6 +1,25 @@
 # CHANGELOG
 
 
+## v0.23.0 (2026-03-03)
+
+### Features
+
+- Add 4-layer WAA probe for per-layer diagnostics
+  ([#75](https://github.com/OpenAdaptAI/openadapt-evals/pull/75),
+  [`7a22ec1`](https://github.com/OpenAdaptAI/openadapt-evals/commit/7a22ec1e3f1c069a5dc89159c73da1474325c9d2))
+
+Add multi-layer probe that tests screenshot (PNG capture), accessibility (a11y tree), action
+  (pyautogui pipeline), and score (evaluate endpoint) layers individually using existing WAA
+  endpoints. No server-side changes.
+
+- New probe.py module with ProbeLayerResult/MultiLayerProbeResult dataclasses - CLI: --detailed,
+  --json, --layers, --evaluate-url args on probe command - VMMonitor: check_waa_detailed() method
+  and waa_detailed_probe field - 41 tests covering all layers, orchestrator, and helpers
+
+Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
+
+
 ## v0.22.0 (2026-03-03)
 
 ### Features

openadapt_evals/benchmarks/cli.py

@@ -890,7 +890,42 @@ def cmd_probe(args: argparse.Namespace) -> int:
         return 1
 
     server_url = args.server
+    detailed = args.detailed or args.json_output
+    layers = args.layers.split(",") if args.layers else None
+    if layers:
+        detailed = True
+
+    if detailed:
+        from openadapt_evals.infrastructure.probe import (
+            multi_layer_probe,
+            print_probe_results,
+        )
+
+        max_attempts = args.wait_attempts if args.wait else 1
+        attempt = 0
+
+        while attempt < max_attempts:
+            attempt += 1
+            result = multi_layer_probe(
+                server_url,
+                layers=layers,
+                evaluate_url=args.evaluate_url,
+            )
+            if args.json_output:
+                print(result.to_json())
+            else:
+                print_probe_results(result)
+
+            if result.overall_ready:
+                return 0
+
+            if args.wait and attempt < max_attempts:
+                print(f"Attempt {attempt}/{max_attempts}: not ready, retrying in {args.wait_interval}s...")
+                time.sleep(args.wait_interval)
+
+        return 0 if result.overall_ready else 1
 
+    # Default binary probe (unchanged)
     print(f"Probing WAA server at {server_url}...")
 
     max_attempts = args.wait_attempts if args.wait else 1
@@ -2264,6 +2299,14 @@ def main() -> int:
                              help="Max attempts when waiting")
     probe_parser.add_argument("--wait-interval", type=int, default=5,
                              help="Seconds between attempts")
+    probe_parser.add_argument("--detailed", action="store_true",
+                             help="Run 4-layer probe (screenshot, a11y, action, score)")
+    probe_parser.add_argument("--json", dest="json_output", action="store_true",
+                             help="Output JSON (implies --detailed)")
+    probe_parser.add_argument("--layers", type=str, default=None,
+                             help="Comma-separated layer subset (e.g. screenshot,a11y)")
+    probe_parser.add_argument("--evaluate-url", type=str, default=None,
+                             help="Separate URL for score layer (e.g. http://localhost:5051)")
 
     # Generate viewer
     view_parser = subparsers.add_parser("view", help="Generate HTML viewer for results")

openadapt_evals/infrastructure/__init__.py

@@ -47,6 +47,12 @@
     wait_for_stable_screen,
 )
 from openadapt_evals.infrastructure.ssh_tunnel import SSHTunnelManager, get_tunnel_manager
+from openadapt_evals.infrastructure.probe import (
+    MultiLayerProbeResult,
+    ProbeLayerResult,
+    multi_layer_probe,
+    print_probe_results,
+)
 from openadapt_evals.infrastructure.vm_ip import resolve_vm_ip
 from openadapt_evals.infrastructure.vm_monitor import VMMonitor, VMConfig
 from openadapt_evals.infrastructure.vm_provider import VMProvider
@@ -60,15 +66,19 @@
     "AWSVMManager",
     "AzureOpsTracker",
     "AzureVMManager",
+    "MultiLayerProbeResult",
     "PoolManager",
     "PoolRunResult",
+    "ProbeLayerResult",
     "QEMUResetManager",
     "VMMonitor",
     "VMConfig",
     "VMProvider",
     "SSHTunnelManager",
     "compare_screenshots",
     "get_tunnel_manager",
+    "multi_layer_probe",
+    "print_probe_results",
     "resolve_vm_ip",
     "wait_for_stable_screen",
 ]