feat(AUTO-1452): perf-only cached rows, spread, --no-cache

Cached benchmark rows now report measured perf only. A cached row's $/hr
would have to come from a different machine than the one benchmarked, so
perf/$ is omitted until benchmark rows carry the machine's price
(AUTO-1130 follow-up). The score-biased current-offers price median is
removed.

- show median + range (min-max) + n on the cached line, since perf
  variance is real and workload-dependent
- a cache hit with no current offers is flagged "no offers available to
  rent right now" instead of showing a result you can't act on
- rename --fresh to --no-cache; drop --max_age (30d is internal)
- sdk: search_benchmarks accepts str | dict

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: vastzuby

tests/cli/test_benchmarks_commands.py

@@ -219,15 +219,10 @@ def test_delete_failure_does_not_raise(self):
 
 def _run_cli(parse_argv, argv, *, create_resp=None, workers_seq=None,
              rental_dph=None, template=None, preflight_offers=1,
-             create_workergroup_raises=None, benchmark_rows=None,
-             offer_dphs=None):
+             create_workergroup_raises=None, benchmark_rows=None):
     """Parse argv and invoke the command with a mocked VastAI."""
     template = template if template is not None else _FAKE_TEMPLATE
-    if offer_dphs is not None:
-        fake_offer_list = [{"id": i, "dph_total": d}
-                           for i, d in enumerate(offer_dphs)]
-    else:
-        fake_offer_list = [{"id": i} for i in range(preflight_offers)]
+    fake_offer_list = [{"id": i} for i in range(preflight_offers)]
     instance_resp = {"dph_total": rental_dph} if rental_dph is not None else {}
 
     vast = MagicMock()
@@ -365,23 +360,38 @@ def test_cache_hit_skips_rental_and_uses_median(self, parse_argv):
         vast.create_endpoint.assert_not_called()
         vast.create_workergroup.assert_not_called()
 
-    def test_cached_price_is_current_market_median(self, parse_argv):
+    def test_cached_rows_have_no_price(self, parse_argv):
+        # Cached rows report perf only; $/hr would come from a different
+        # machine than the one benchmarked, so it is omitted.
         rows, _ = _run_cli(
             parse_argv,
             ["run", "benchmarks", "--template_id", "99999",
              "--gpus", "RTX_3060", "-y", "--raw"],
             benchmark_rows=[_bench_row(20.0)],
-            offer_dphs=[0.2, 0.6, 0.4],
         )
-        assert rows[0]["rental_dph"] == 0.4
-        assert rows[0]["perf_per_dollar"] == 50.0
+        assert rows[0]["rental_dph"] is None
+        assert rows[0]["perf_per_dollar"] is None
+
+    def test_cached_unrentable_is_flagged(self, parse_argv, capsys):
+        rows, vast = _run_cli(
+            parse_argv,
+            ["run", "benchmarks", "--template_id", "99999",
+             "--gpus", "RTX_3060", "-y", "--raw"],
+            benchmark_rows=[_bench_row(10.0)],
+            preflight_offers=0,
+        )
+        assert rows[0]["status"] == "cached"
+        assert rows[0]["measured_perf"] == 10.0
+        vast.create_endpoint.assert_not_called()
+        err = " ".join(capsys.readouterr().err.split())  # rich may wrap lines
+        assert "no offers available to rent" in err.lower()
 
-    def test_fresh_bypasses_cache(self, parse_argv):
+    def test_no_cache_bypasses_cache(self, parse_argv):
         rows, vast = _run_cli(
             parse_argv,
             ["run", "benchmarks", "--template_id", "99999",
              "--gpus", "RTX_3060", "--timeout", "60", "-y", "--raw",
-             "--fresh"],
+             "--no-cache"],
             benchmark_rows=[_bench_row(10.0)],
             workers_seq=[[{"id": 1, "measured_perf": 5.0, "status": "idle"}]],
             rental_dph=0.5,
@@ -417,16 +427,18 @@ def test_row_matching_by_template_id_only(self, parse_argv):
 
     def test_cache_query_shape(self, parse_argv):
         import time
+        from vastai.cli.commands.benchmarks import _DEFAULT_CACHE_MAX_AGE_DAYS
         _, vast = _run_cli(
             parse_argv,
             ["run", "benchmarks", "--template_id", "99999",
-             "--gpus", "RTX_3060", "--max_age", "7", "-y", "--raw"],
+             "--gpus", "RTX_3060", "-y", "--raw"],
             benchmark_rows=[_bench_row(20.0)],
         )
         query = vast.search_benchmarks.call_args.kwargs["query"]
         assert query["type"] == {"eq": "perf"}
         assert query["gpu_name"] == {"eq": "RTX 3060"}
         assert query["num_gpus"] == {"eq": 1}
         cutoff = query["last_update"]["gte"]
-        assert abs((time.time() - cutoff) - 7 * 86400) < 60
+        assert abs((time.time() - cutoff)
+                   - _DEFAULT_CACHE_MAX_AGE_DAYS * 86400) < 60
 

vastai/cli/commands/benchmarks.py

@@ -272,19 +272,15 @@ def _pick_num_gpus(gpu_name, extra_filters):
     return _get_gpu_chunk_size(raw_min)
 
 
-# ------------------------------------------------------------------------------------
-# cached-benchmark pre-flight
-# every run reports its measured perf to the benchmarks table (autoscaler PUT
-# /benchmarks, type="perf"), so the table doubles as a cross-user cache: specs
-# benchmarked recently can be served from it instead of rented again
-# ------------------------------------------------------------------------------------
+# Cache pre-flight: runs report measured perf to the benchmarks table, so a
+# recent matching row lets us serve this spec instead of renting it again.
 
 _DEFAULT_CACHE_MAX_AGE_DAYS = 30
 
 
 def _lookup_cached_benchmark(vast, *, gpu_name, num_gpus, template_hash,
                              template_id, max_age_days):
-    """Median of recent reported benchmarks for this exact spec, or None.
+    """Recent reported benchmarks for this exact spec (median + spread), or None.
 
     Template is matched client-side: rows carry template_hash or template_id
     depending on how the benchmarked workergroup was created.
@@ -307,26 +303,17 @@ def _lookup_cached_benchmark(vast, *, gpu_name, num_gpus, template_hash,
     ]
     if not matched:
         return None
+    values = [r["value"] for r in matched]
     newest = max((r.get("last_update") or 0) for r in matched)
     return {
-        "median": statistics.median(r["value"] for r in matched),
+        "median": statistics.median(values),
+        "low": min(values),
+        "high": max(values),
         "n": len(matched),
         "age_days": max(0.0, (time.time() - newest) / 86400),
     }
 
 
-def _current_median_dph(vast, *, gpu_name, num_gpus, extra_filters):
-    """Median $/hr of the offers a run would rent from today. Cached rows
-    carry no price of their own; perf/$ should reflect renting now."""
-    query = dict(extra_filters or {})
-    query["gpu_name"] = {"eq": gpu_name}
-    query["num_gpus"] = {"eq": num_gpus}
-    offers = vast.search_offers(query=query, limit=100)
-    prices = [o.get("dph_total") for o in (offers or []) if isinstance(o, dict)]
-    prices = [p for p in prices if isinstance(p, (int, float)) and p > 0]
-    return statistics.median(prices) if prices else None
-
-
 def _update_worker_states(worker_states, current_workers, gpu_name):
     """Update worker_states with current poll, and print only on worker
     rotation (the live rich table already shows current status / elapsed).
@@ -620,10 +607,8 @@ def _benchmark_gpu(vast, *, gpu_name, num_gpus, timeout,
              help="GPUs per instance for tokens without an Nx prefix (default 1); overridden by inline Nx in --gpus"),
     argument("--timeout", type=int, default=_DEFAULT_BENCHMARK_TIMEOUT,
              help=f"max seconds to wait for a benchmark before giving up (default {_DEFAULT_BENCHMARK_TIMEOUT})"),
-    argument("--fresh", action="store_true",
-             help="re-measure even when cached benchmark results exist"),
-    argument("--max_age", type=float, default=_DEFAULT_CACHE_MAX_AGE_DAYS,
-             help=f"reuse cached benchmark results up to this many days old (default {_DEFAULT_CACHE_MAX_AGE_DAYS}); ignored with --fresh"),
+    argument("--no-cache", dest="no_cache", action="store_true",
+             help="re-measure instead of reusing recent cached benchmark results"),
     argument("-y", "--yes", action="store_true",
              help="Skip confirmation prompt"),
     argument("--auto_instance", type=str, default=None, help=argparse.SUPPRESS),
@@ -634,10 +619,9 @@ def _benchmark_gpu(vast, *, gpu_name, num_gpus, timeout,
         Rents one instance per GPU in parallel, measures perf, tears down.
         Each rental runs for up to --timeout seconds and costs real money.
 
-        Specs already benchmarked within the last --max_age days (same
-        template, GPU, and count, by any user) are served from the benchmarks
-        table instead of rented; their $/hr is the current market median.
-        Pass --fresh to re-measure.
+        Specs benchmarked recently (same template, GPU, and count, by any
+        user) are served from the benchmarks table as an estimated perf
+        instead of being rented again. Pass --no-cache to re-measure.
 
         Examples:
             # auto-sweep the default GPUs against TGI
@@ -656,7 +640,7 @@ def _benchmark_gpu(vast, *, gpu_name, num_gpus, timeout,
             vastai run benchmarks --template_hash 393fa8572e6c73c927c8275fe4dffd53 --timeout 1800 -y
 
             # ignore cached results and re-measure everything
-            vastai run benchmarks --template_hash 79ebdd2ebfb9d42cedf7a221c42d37a5 --fresh
+            vastai run benchmarks --template_hash 79ebdd2ebfb9d42cedf7a221c42d37a5 --no-cache
 
             # raw JSON output for piping into another tool
             vastai run benchmarks --template_hash 40ef49becc953aa910ee05bd4653b9b3 --raw
@@ -720,30 +704,38 @@ def run__benchmarks(args):
         deduped.append((g, n))
     gpu_specs = deduped
 
-    # Pre-flight: serve specs from cached benchmark results unless --fresh,
+    # Pre-flight: serve specs from cached benchmark results unless --no-cache,
     # then skip specs that have 0 matching offers before prompting,
     # so the user sees skip reasons before approving the rentals.
     compatible_specs = []
     skipped_results = []
     cached_results = []
     for g, n in gpu_specs:
-        if not args.fresh:
+        if not args.no_cache:
             hit = _lookup_cached_benchmark(
                 vast, gpu_name=g, num_gpus=n,
                 template_hash=template.get("hash_id"),
                 template_id=template.get("id"),
-                max_age_days=args.max_age,
+                max_age_days=_DEFAULT_CACHE_MAX_AGE_DAYS,
             )
             if hit:
-                dph = _current_median_dph(vast, gpu_name=g, num_gpus=n,
-                                          extra_filters=extra_filters)
+                # Perf only: a cached row's $/hr would come from a different
+                # machine than the one benchmarked. TODO: serve real perf/$
+                # once benchmark rows carry the machine's price (AUTO-1130).
+                rentable = _has_matching_offer(
+                    vast, gpu_name=g, num_gpus=n, extra_filters=extra_filters)
                 age = (f"{hit['age_days']:.0f}d" if hit["age_days"] >= 1
                        else "<1d")
-                console.print(
-                    f"[green][{n}x {g}] cached:[/green] median perf "
-                    f"{hit['median']:.1f} (n={hit['n']}, newest {age} ago); "
-                    f"pass --fresh to re-measure", highlight=False)
-                cached_results.append((g, n, "cached", hit["median"], None, dph))
+                note = None if rentable else "no offers available to rent right now"
+                msg = (f"[green][{n}x {g}] cached:[/green] median perf "
+                       f"{hit['median']:.1f} "
+                       f"(n={hit['n']}, range {hit['low']:.1f}-{hit['high']:.1f}, "
+                       f"newest {age} ago)")
+                if note:
+                    msg += f"; [yellow]{note}[/yellow]"
+                console.print(msg + "; pass --no-cache to re-measure",
+                              highlight=False)
+                cached_results.append((g, n, "cached", hit["median"], note, None))
                 continue
         if not _has_matching_offer(
             vast, gpu_name=g, num_gpus=n,

vastai/sdk.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import warnings
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
 
 from vastai._base import _resolve_api_key, _APIKEY_SENTINEL
 from vastai.api.client import VastClient
@@ -276,7 +276,7 @@ def search_templates(self, query: Optional[str] = None) -> list[dict]:
         """Search for templates."""
         return offers.search_templates(self.client, query=query)
 
-    def search_benchmarks(self, query: Optional[str] = None) -> list[dict]:
+    def search_benchmarks(self, query: Optional[Union[str, dict]] = None) -> list[dict]:
         """Search for benchmarks."""
         return offers.search_benchmarks(self.client, query=query)