fix(cli): fix --task flag concatenation bug and three other issues (#31)

* fix(cli): fix --task flag concatenation bug and three other issues

Bug 1 (Critical): --task flag produced `find_task.pycd` due to missing
`&&` separator between pre_cmd and `cd /client`. Every `run --task`
invocation since v0.4.2 silently failed. Fixed by adding `&&`.

Bug 2: --num-tasks defaulted to 1, silently limiting runs. Changed
default to None (all tasks).

Bug 3: probe --wait timeout of 1200s was too short for first boot
(OOBE takes 18-22 min). Increased to 1800s.

Bug 4: Default VM size (D4ds_v4, 16GB) OOMs with navi agent's
GroundingDINO + SoM models. Changed default to D8ds_v5 (32GB).
Added warning when standard mode is used explicitly.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* refactor: remove --fast flag, standardize on D8ds_v5 (32GB) VM

D4ds_v4 (16GB) OOMs with navi agent's GroundingDINO + SoM models.
Standardize on D8ds_v5 across all commands — no more --fast/--standard flags.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: abrichr

openadapt_evals/benchmarks/vm_cli.py

@@ -46,19 +46,16 @@
 # Constants (single source of truth)
 # =============================================================================
 
-# VM sizes with nested virtualization support
-# Standard: $0.19/hr, 4 vCPU, 16GB RAM - baseline
-# Fast: $0.38/hr, 8 vCPU, 32GB RAM - ~30% faster install, ~40% faster eval
-VM_SIZE_STANDARD = "Standard_D4ds_v4"
-VM_SIZE_FAST = "Standard_D8ds_v5"
-VM_SIZE = VM_SIZE_STANDARD  # Default, can be overridden by --fast flag
-
-# Fallback sizes for --fast mode (in order of preference)
+# VM size: D8ds_v5 ($0.38/hr, 8 vCPU, 32GB RAM)
+# D4ds_v4 (16GB) OOMs with navi agent's GroundingDINO + SoM models — do not use.
+VM_SIZE = "Standard_D8ds_v5"
+
+# Fallback VM sizes (in order of preference, all 8 vCPU / 32GB)
 # D8ds_v5: First choice (v5 with local SSD)
 # D8s_v5: v5 without local SSD
 # D8ds_v4: v4 with local SSD
 # D8as_v5: AMD version
-VM_SIZE_FAST_FALLBACKS = [
+VM_SIZE_FALLBACKS = [
     ("Standard_D8ds_v5", 0.38),
     ("Standard_D8s_v5", 0.36),
     ("Standard_D8ds_v4", 0.38),
@@ -258,19 +255,12 @@ def cmd_create(args):
         log("CREATE", "Use 'delete' first if you want to recreate")
         return 0
 
-    # Determine which sizes to try
-    use_fast = getattr(args, "fast", False)
-    if use_fast:
-        # Try multiple fast sizes with fallbacks
-        sizes_to_try = VM_SIZE_FAST_FALLBACKS
-        log(
-            "CREATE",
-            f"Creating VM '{VM_NAME}' with --fast (trying multiple D8 sizes)...",
-        )
-    else:
-        # Standard mode: single size
-        sizes_to_try = [(VM_SIZE_STANDARD, 0.19)]
-        log("CREATE", f"Creating VM '{VM_NAME}' ({VM_SIZE_STANDARD}, $0.19/hr)...")
+    # Try multiple D8 sizes with fallbacks (all 32GB, required for navi agent)
+    sizes_to_try = VM_SIZE_FALLBACKS
+    log(
+        "CREATE",
+        f"Creating VM '{VM_NAME}' (trying D8 sizes with fallbacks)...",
+    )
 
     # Try size+region combinations until one works
     vm_created = False
@@ -627,7 +617,6 @@ def cmd_pool_create(args):
     from openadapt_evals.infrastructure.pool import PoolManager
 
     num_workers = getattr(args, "workers", 3)
-    use_standard = getattr(args, "standard", False)
     auto_shutdown_hours = getattr(args, "auto_shutdown_hours", 4)
 
     vm_manager = AzureVMManager(resource_group=RESOURCE_GROUP)
@@ -636,7 +625,6 @@ def cmd_pool_create(args):
     try:
         manager.create(
             workers=num_workers,
-            fast=not use_standard,
             auto_shutdown_hours=auto_shutdown_hours,
         )
         return 0
@@ -1428,18 +1416,10 @@ def cmd_start(args):
     # - Downloads Windows 11 Enterprise if not present
     # - Boots QEMU VM
     # - Runs WAA server automatically via FirstLogonCommands
-    # QEMU resource allocation (--fast uses more resources on D8ds_v5)
-    if getattr(args, "fast", False):
-        ram_size = "16G"
-        cpu_cores = 6
-        log(
-            "START",
-            "Starting container with VERSION=11e (FAST mode: 6 cores, 16GB RAM)...",
-        )
-    else:
-        ram_size = "8G"
-        cpu_cores = 4
-        log("START", "Starting container with VERSION=11e...")
+    # QEMU resource allocation — always use 16G/6 cores (D8ds_v5, 32GB host)
+    ram_size = "16G"
+    cpu_cores = 6
+    log("START", "Starting container with VERSION=11e (6 cores, 16GB RAM)...")
 
     # Get agent and model from args (defaults match WAA defaults)
     getattr(args, "agent", "navi")
@@ -1568,11 +1548,10 @@ def cmd_test_golden_image(args):
     # Start container from golden image (NOT fresh)
     log("TEST", "Starting container from golden image...")
 
-    # Use fast mode for quicker boot
-    ram_size = "16G" if args.fast else "8G"
-    cpu_cores = 6 if args.fast else 4
-    mode_str = "FAST mode" if args.fast else "standard mode"
-    log("TEST", f"  Using {mode_str}: {cpu_cores} cores, {ram_size} RAM")
+    # 16GB RAM / 6 cores for D8ds_v5 VM
+    ram_size = "16G"
+    cpu_cores = 6
+    log("TEST", f"  Using {cpu_cores} cores, {ram_size} RAM")
 
     docker_cmd = f"""docker run -d \\
   --name winarena \\
@@ -1923,7 +1902,6 @@ class FakeArgs:
     log("TEST-ALL", "-" * 30)
 
     class FakeArgs2:
-        fast = getattr(args, "fast", False)
         timeout = 120
 
     results["golden_image"] = cmd_test_golden_image(FakeArgs2()) == 0
@@ -2091,8 +2069,10 @@ def cmd_run(args):
         task_info.append(f"task={task}")
     elif domain != "all":
         task_info.append(f"domain={domain}")
-    else:
+    elif args.num_tasks:
         task_info.append(f"{args.num_tasks} task(s)")
+    else:
+        task_info.append("all tasks")
 
     log("RUN", f"Starting benchmark: {', '.join(task_info)}, model={model}")
 
@@ -2127,7 +2107,7 @@ def cmd_run(args):
 print(f"ERROR: Task {{task_id}} not found in test_all.json")
 sys.exit(1)
 FINDEOF
-python3 /tmp/find_task.py"""
+python3 /tmp/find_task.py && """
         run_args.append("--test_all_meta_path evaluation_examples_windows/test_custom.json")
         pre_cmd = create_custom_test_cmd
     elif args.num_tasks and args.num_tasks < 154:
@@ -4341,13 +4321,11 @@ def cmd_run_azure_ml_auto(args):
     probe_timeout = getattr(args, "probe_timeout", 1800)  # 30 min for WAA server
     skip_upload = getattr(args, "skip_upload", False)
     skip_benchmark = getattr(args, "skip_benchmark", False)
-    fast_vm = getattr(args, "fast", False)
 
     log("AUTO", "Configuration:")
     log("AUTO", f"  Workers: {num_workers}")
     log("AUTO", f"  Setup timeout: {timeout_minutes} min")
     log("AUTO", f"  Probe timeout: {probe_timeout} sec")
-    log("AUTO", f"  Fast VM: {fast_vm}")
     log("AUTO", "")
 
     # =========================================================================
@@ -4390,7 +4368,6 @@ def cmd_run_azure_ml_auto(args):
 
         # Build args for cmd_create
         class CreateArgs:
-            fast = fast_vm
             workers = 1
 
         result = cmd_create(CreateArgs())
@@ -4447,8 +4424,8 @@ class CreateArgs:
         ssh_run(ip, "docker stop winarena 2>/dev/null; docker rm -f winarena 2>/dev/null")
 
         # Start container with VERSION=11e
-        ram_size = "16G" if fast_vm else "8G"
-        cpu_cores = 6 if fast_vm else 4
+        ram_size = "16G"
+        cpu_cores = 6
 
         docker_cmd = f"""docker run -d \\
   --name winarena \\
@@ -7340,11 +7317,6 @@ def main():
 
     # create
     p_create = subparsers.add_parser("create", help="Create Azure VM")
-    p_create.add_argument(
-        "--fast",
-        action="store_true",
-        help="Use larger VM (D8ds_v5, $0.38/hr) for ~30%% faster install, ~40%% faster eval",
-    )
     p_create.add_argument(
         "--workers",
         type=int,
@@ -7390,15 +7362,6 @@ def main():
         default=3,
         help="Number of worker VMs to create (default: 3)",
     )
-    p_pool_create.add_argument(
-        "--fast",
-        action="store_true",
-        default=True,
-        help="Use D8 (8 vCPU) VMs for faster evaluation (default: True)",
-    )
-    p_pool_create.add_argument(
-        "--standard", action="store_true", help="Use D4 (4 vCPU) VMs to save costs"
-    )
     p_pool_create.add_argument(
         "--auto-shutdown-hours",
         type=int,
@@ -7527,11 +7490,6 @@ def main():
         "--fresh", action="store_true", help="Clean storage for fresh Windows install"
     )
     p_start.add_argument("--no-vnc", action="store_true", help="Don't auto-launch VNC viewer")
-    p_start.add_argument(
-        "--fast",
-        action="store_true",
-        help="Allocate more CPU/RAM to QEMU (use with D8ds_v5 VM)",
-    )
     p_start.set_defaults(func=cmd_start)
 
     # stop
@@ -7543,7 +7501,7 @@ def main():
     p_probe = subparsers.add_parser("probe", help="Check if WAA server is ready")
     p_probe.add_argument("--wait", action="store_true", help="Wait until ready")
     p_probe.add_argument(
-        "--timeout", type=int, default=1200, help="Timeout in seconds (default: 1200)"
+        "--timeout", type=int, default=1800, help="Timeout in seconds (default: 1800)"
     )
     p_probe.set_defaults(func=cmd_probe)
 
@@ -7557,11 +7515,6 @@ def main():
         default=180,
         help="Max wait time in seconds (default: 180)",
     )
-    p_test_golden.add_argument(
-        "--fast",
-        action="store_true",
-        help="Use more CPU/RAM for faster boot (requires D8ds_v5 VM)",
-    )
     p_test_golden.set_defaults(func=cmd_test_golden_image)
 
     # test-blob-access
@@ -7584,20 +7537,15 @@ def main():
         "test-all", help="Run all pre-flight tests before Azure ML benchmark"
     )
     p_test_all.add_argument("--api-key", help="OpenAI API key (or set OPENAI_API_KEY in .env)")
-    p_test_all.add_argument(
-        "--fast",
-        action="store_true",
-        help="Use more CPU/RAM for faster boot (requires D8ds_v5 VM)",
-    )
     p_test_all.set_defaults(func=cmd_test_all)
 
     # run
     p_run = subparsers.add_parser("run", help="Run benchmark tasks (uses vanilla WAA navi agent)")
     p_run.add_argument(
         "--num-tasks",
         type=int,
-        default=1,
-        help="Number of tasks to run (ignored if --task specified)",
+        default=None,
+        help="Number of tasks to run (default: all; ignored if --task specified)",
     )
     p_run.add_argument("--task", help="Specific task ID to run")
     p_run.add_argument(
@@ -7812,11 +7760,6 @@ def main():
         default=1800,
         help="WAA server probe timeout in seconds (default: 1800 = 30 min)",
     )
-    p_azure_ml_auto.add_argument(
-        "--fast",
-        action="store_true",
-        help="Use larger VM (D8ds_v5, $0.38/hr) for faster setup",
-    )
     p_azure_ml_auto.add_argument(
         "--skip-upload",
         action="store_true",

openadapt_evals/infrastructure/azure_vm.py

@@ -56,10 +56,10 @@
     "ServerAliveCountMax=10",
 ]
 
-# VM size constants
-VM_SIZE_STANDARD = "Standard_D4ds_v4"
-VM_SIZE_FAST = "Standard_D8ds_v5"
-VM_SIZE_FAST_FALLBACKS = [
+# VM size: D8ds_v5 ($0.38/hr, 8 vCPU, 32GB RAM)
+# D4ds_v4 (16GB) OOMs with navi agent's GroundingDINO + SoM models — do not use.
+VM_SIZE = "Standard_D8ds_v5"
+VM_SIZE_FALLBACKS = [
     ("Standard_D8ds_v5", 0.38),
     ("Standard_D8s_v5", 0.36),
     ("Standard_D8ds_v4", 0.38),
@@ -367,28 +367,19 @@ def _cli_set_auto_shutdown(self, name: str, hours: int) -> bool:
         )
         return result.returncode == 0
 
-    def find_available_size_and_region(
-        self,
-        fast: bool = True,
-    ) -> tuple[str, str, float]:
+    def find_available_size_and_region(self) -> tuple[str, str, float]:
         """Find a working VM size and region by creating a test VM.
 
         Tries size/region combinations until one succeeds, then cleans up
         the test VM.
 
-        Args:
-            fast: If True, try D8 sizes first. If False, use standard D4.
-
         Returns:
             Tuple of (vm_size, region, cost_per_hour).
 
         Raises:
             RuntimeError: If no available size/region found.
         """
-        if fast:
-            sizes_to_try = VM_SIZE_FAST_FALLBACKS
-        else:
-            sizes_to_try = [(VM_SIZE_STANDARD, 0.19)]
+        sizes_to_try = VM_SIZE_FALLBACKS
 
         test_vm_to_cleanup = None
         try:

openadapt_evals/infrastructure/pool.py

@@ -173,7 +173,6 @@ def _log(self, step: str, message: str, end: str = "\n") -> None:
     def create(
         self,
         workers: int = 3,
-        fast: bool = True,
         auto_shutdown_hours: int = 4,
     ) -> VMPool:
         """Create a pool of VMs for parallel WAA evaluation.
@@ -182,7 +181,6 @@ def create(
 
         Args:
             workers: Number of worker VMs to create.
-            fast: If True, use D8 VM sizes. If False, use standard D4.
             auto_shutdown_hours: Hours until auto-shutdown (safety net).
 
         Returns:
@@ -199,9 +197,7 @@ def create(
 
         # Find available size/region
         self._log("POOL", "Finding available region and VM size...")
-        vm_size, region, cost = self.vm_manager.find_available_size_and_region(
-            fast=fast,
-        )
+        vm_size, region, cost = self.vm_manager.find_available_size_and_region()
         self._log("POOL", f"Using {vm_size} (${cost:.2f}/hr) in {region}")
 
         if auto_shutdown_hours > 0: