Merge pull request #57 from scaleapi/fix/wandb-base-url-normalization

fix: repair a scheme-less WANDB_BASE_URL and stop losing W&B sink failures

Author: shehabyasser-scale

vero/src/vero/harbor/deployment.py

@@ -25,6 +25,7 @@
 from vero.evaluation.engine import EvaluationEngine
 from vero.harbor.backend import HarborBackend, HarborBackendConfig
 from vero.models import StrictModel
+from vero.runtime.artifacts import ArtifactStore
 from vero.sandbox import LocalSandbox
 from vero.sidecar.serve import SidecarComponents
 from vero.sidecar.session import initialize_harbor_session_manifest
@@ -323,13 +324,28 @@ async def build_harbor_components(config: dict) -> SidecarComponents:
                 log_traces=parsed.wandb.log_traces,
             )
             engine.listeners.append(wandb_sink)
-        except Exception:
+        except Exception as error:
             logger.warning(
                 "W&B reporting disabled: the sidecar sink could not be "
                 "initialized; the evaluation sidecar continues without it",
                 exc_info=True,
             )
             wandb_sink = None
+            # The warning above goes to the sidecar container's stderr, which no
+            # run artifact captures, so a silently disabled sink looks exactly
+            # like a healthy run that logged nothing. Record why in the session
+            # artifacts, which are archived into the exported run record.
+            try:
+                ArtifactStore(session_dir / "artifacts").write_json(
+                    "wandb/init-error.json",
+                    {
+                        "project": parsed.wandb.project,
+                        "error_type": type(error).__name__,
+                        "error": str(error),
+                    },
+                )
+            except OSError:
+                logger.warning("could not record the W&B init failure", exc_info=True)
 
     usage_path = (
         Path(parsed.inference_usage_path)

vero/src/vero/runtime/wandb.py

@@ -6,6 +6,7 @@
 import hashlib
 import json
 import logging
+import os
 import tempfile
 from pathlib import Path
 from typing import Any
@@ -19,6 +20,31 @@
 logger = logging.getLogger(__name__)
 
 
+def normalize_wandb_base_url(environment: dict[str, str] | None = None) -> str | None:
+    """Give a scheme-less ``WANDB_BASE_URL`` the ``https://`` it needs.
+
+    A self-hosted host is naturally written ``scaleai.wandb.io``, but W&B's
+    settings model parses ``base_url`` as a URL and rejects that with
+    ``Input should be a valid URL, relative URL without a base``. The error
+    surfaces out of ``wandb.init()``, which callers treat as "W&B unavailable",
+    so one missing scheme silently costs a run all of its reporting.
+
+    Returns the value in effect, or None when the variable is unset.
+    """
+    environ = os.environ if environment is None else environment
+    value = (environ.get("WANDB_BASE_URL") or "").strip()
+    if not value or "://" in value:
+        return value or None
+    corrected = f"https://{value}"
+    environ["WANDB_BASE_URL"] = corrected
+    logger.warning(
+        "WANDB_BASE_URL %r has no scheme and would be rejected by W&B; using %r",
+        value,
+        corrected,
+    )
+    return corrected
+
+
 def _open_wandb_run(
     *,
     project: str,
@@ -43,6 +69,7 @@ def _open_wandb_run(
             raise RuntimeError(
                 "W&B reporting requires `pip install scale-vero[wandb]`"
             ) from error
+    normalize_wandb_base_url()
     wandb_dir.mkdir(parents=True, exist_ok=True)
     stable_id = run_id or ("vero-" + hashlib.sha256(session_id.encode()).hexdigest()[:16])
     init_kwargs: dict[str, Any] = {
@@ -96,6 +123,7 @@ def __init__(
                     "W&B reporting requires `pip install scale-vero[wandb]`"
                 ) from error
 
+        normalize_wandb_base_url()
         wandb_dir = session_dir / "artifacts" / "wandb"
         wandb_dir.mkdir(parents=True, exist_ok=True)
         self.artifacts = ArtifactStore(session_dir / "artifacts")

vero/tests/test_v05_harbor_deployment.py

@@ -270,3 +270,76 @@ class Result:
     assert (
         exported / "requests/requests-00001.jsonl"
     ).read_text() == '{"scope":"producer"}\n'
+
+
+@pytest.mark.asyncio
+async def test_wandb_init_failure_is_recorded_in_the_session_artifacts(tmp_path):
+    # A sink that cannot start only logs to the sidecar container's stderr, which
+    # no run artifact captures, so a disabled sink is indistinguishable from a
+    # healthy run that logged nothing. Leave the reason behind in the session.
+    trusted = tmp_path / "trusted"
+    agent = tmp_path / "agent"
+    _repo(trusted, "VALUE = 1\n")
+    _repo(agent, "VALUE = 1\n")
+    cases = tmp_path / "cases.json"
+    cases.write_text('[{"id":"task","task_name":"org/task"}]')
+    evaluation_set = EvaluationSet(name="benchmark")
+    objective = ObjectiveSpec(
+        selector=MetricSelector(metric="score"),
+        direction="maximize",
+    )
+    session_dir = tmp_path / "state/session"
+    config = {
+        "repo_path": str(trusted),
+        "agent_repo_path": str(agent),
+        "session_dir": str(session_dir),
+        "backends": {
+            "backend": {
+                "task_source": "org/benchmark@1.0",
+                "agent_import_path": "program:Agent",
+                "cases_path": str(cases),
+                "harbor_requirement": "harbor==0.1.17",
+                "uv_executable": sys.executable,
+            }
+        },
+        "access_policies": [],
+        "budgets": [],
+        "selection": {
+            "mode": "auto_best",
+            "backend_id": "backend",
+            "evaluation_set": evaluation_set.model_dump(mode="json"),
+            "objective": objective.model_dump(mode="json"),
+            "baseline_version": "HEAD",
+        },
+        "targets": [
+            {
+                "reward_key": "reward",
+                "backend_id": "backend",
+                "evaluation_set": evaluation_set.model_dump(mode="json"),
+                "objective": objective.model_dump(mode="json"),
+            }
+        ],
+        "admin_volume": str(tmp_path / "state/admin"),
+        "wandb": {"project": "vero-tests"},
+    }
+
+    import vero.runtime.wandb as wandb_module
+
+    def _explode(**kwargs):
+        raise ValueError("Input should be a valid URL, relative URL without a base")
+
+    original = wandb_module.SidecarWandbSink
+    wandb_module.SidecarWandbSink = _explode
+    try:
+        components = await build_harbor_components(config)
+    finally:
+        wandb_module.SidecarWandbSink = original
+
+    # The eval path survives: observability never takes the sidecar down.
+    assert components.telemetry is None
+    recorded = json.loads(
+        (session_dir / "artifacts/wandb/init-error.json").read_text()
+    )
+    assert recorded["project"] == "vero-tests"
+    assert recorded["error_type"] == "ValueError"
+    assert "valid URL" in recorded["error"]

vero/tests/test_v05_wandb.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import os
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
@@ -402,3 +403,31 @@ def test_sidecar_wandb_telemetry_is_best_effort(tmp_path: Path):
     poller.poll_once()  # must not raise
     assert client.run.logged == []
     assert client.run.artifacts == []
+
+
+def test_scheme_less_wandb_base_url_is_repaired_before_init(tmp_path: Path, monkeypatch):
+    """A self-hosted host written without a scheme must not silently kill reporting.
+
+    W&B parses `base_url` as a URL, so `WANDB_BASE_URL=scaleai.wandb.io` raises
+    out of `wandb.init()` and the sidecar disables W&B for the whole run.
+    """
+    from vero.runtime.wandb import normalize_wandb_base_url
+
+    monkeypatch.setenv("WANDB_BASE_URL", "scaleai.wandb.io")
+    assert normalize_wandb_base_url() == "https://scaleai.wandb.io"
+    assert os.environ["WANDB_BASE_URL"] == "https://scaleai.wandb.io"
+
+    # Already-qualified values, including plain http, are left exactly as given.
+    monkeypatch.setenv("WANDB_BASE_URL", "http://localhost:8080")
+    assert normalize_wandb_base_url() == "http://localhost:8080"
+    assert os.environ["WANDB_BASE_URL"] == "http://localhost:8080"
+
+    monkeypatch.delenv("WANDB_BASE_URL", raising=False)
+    assert normalize_wandb_base_url() is None
+
+    # And the repair happens before a sink opens its run.
+    monkeypatch.setenv("WANDB_BASE_URL", "scaleai.wandb.io")
+    SidecarWandbSink(
+        project="v", session_id="s", session_dir=tmp_path / "session", client=FakeWandb()
+    )
+    assert os.environ["WANDB_BASE_URL"] == "https://scaleai.wandb.io"