Fix eval and dashboard for FujitsuResearch/OneCompressionn PR 23

Author: FKKimura

dashboard/README.md

@@ -116,6 +116,9 @@ mkdir -p tmp/redis
 
 . .venv/bin/activate
 export ONECOMP_DEVICE=cuda
+# Local model root — required on worker **and** API when using short local names
+# instead of a Hugging Face repo id. See Environment variables.
+export LOCAL_MODEL_ROOT="$(pwd)/models"
 # Required on nodes without nvcc (CUDA toolkit); see Environment variables below
 export VLLM_USE_FLASHINFER_SAMPLER=0
 nohup uv run python start_worker.py > /tmp/worker.log 2>&1 &
@@ -130,6 +133,7 @@ srun --jobid=41229 --pty bash      # replace 41229 with your JOBID
 cd backend/
 . .venv/bin/activate
 export ONECOMP_DEVICE=cuda
+export LOCAL_MODEL_ROOT="$(pwd)/models"
 uv run python start_backend.py --reload --port 8001
 ```
 
@@ -230,7 +234,33 @@ tail -80 /tmp/vllm-<job-id>.log    # when vLLM fails
 
 ## Environment variables
 
-All use the `ONECOMP_` prefix. HPC defaults live in `backend/app/core/config.py`.
+`ONECOMP_*` settings default in `backend/app/core/config.py`. Other variables
+below are read directly from the process environment.
+
+### `LOCAL_MODEL_ROOT` (local model directory)
+
+Set this **before starting both the Celery worker and the API** when jobs use
+short local directory names (e.g. `gemma-2-2b-it`) rather than a Hugging Face
+repo id (`org/model`). The server maps `model_name` to
+`{LOCAL_MODEL_ROOT}/<model_name>` for job validation and quantization.
+
+| Variable | Default | Description |
+|---|---|---|
+| `LOCAL_MODEL_ROOT` | `/models` | Root directory of pre-downloaded models on shared storage |
+
+Example (run from `backend/`; models live in `backend/models/`):
+
+```bash
+export LOCAL_MODEL_ROOT="$(pwd)/models"
+```
+
+Use the **same value** in Terminal A (worker) and Terminal B (API). If only
+one process has it, jobs may pass validation but fail during quantization with
+“not a local folder” / Hugging Face Hub errors.
+
+After changing `LOCAL_MODEL_ROOT`, restart **both** processes (step **3**).
+
+### `ONECOMP_*` (application settings)
 
 | Variable | Default (HPC) | Description |
 |---|---|---|
@@ -261,6 +291,7 @@ works but the CUDA toolkit is not installed.
 pkill -f start_worker.py
 cd backend && . .venv/bin/activate
 export ONECOMP_DEVICE=cuda
+export LOCAL_MODEL_ROOT="$(pwd)/models"
 export VLLM_USE_FLASHINFER_SAMPLER=0
 nohup uv run python start_worker.py > /tmp/worker.log 2>&1 &
 ```
@@ -273,7 +304,9 @@ does not propagate to the worker.
 ```bash
 pkill -f start_worker.py
 cd backend && . .venv/bin/activate
-ONECOMP_DEVICE=cuda nohup uv run python start_worker.py > /tmp/worker.log 2>&1 &
+export ONECOMP_DEVICE=cuda
+export LOCAL_MODEL_ROOT="$(pwd)/models"
+nohup uv run python start_worker.py > /tmp/worker.log 2>&1 &
 ```
 
 ---
@@ -310,6 +343,7 @@ See [troubleshooting in docs/setup-hpc.md](docs/setup-hpc.md#5-troubleshooting)
 | SSH tunnel established but API unreachable | Point `LocalForward` at the GPU node name |
 | Chat is slow / keeps polling | `ONECOMP_DEVICE=cuda`, then Stop → Deploy |
 | Deploy fails: `Could not find nvcc` | `VLLM_USE_FLASHINFER_SAMPLER=0` on the **worker**, restart worker, Stop → Deploy ([#10](docs/setup-hpc.md#10-vllm-deploy-fails--could-not-find-nvcc)) |
+| Local model name fails / “not a local folder” on HF | Set the same `LOCAL_MODEL_ROOT` on **worker and API**, restart both; model dir must be `{LOCAL_MODEL_ROOT}/<name>/` |
 
 ### Separating the vLLM venv
 

dashboard/backend/app/api/jobs.py

@@ -15,7 +15,7 @@
     JobListResponse,
     JobResponse,
 )
-from app.services.huggingface import check_model_exists
+from app.services.huggingface import check_model_exists, resolve_model_identifier
 from app.services.inference import chat_vllm, stop_inference
 from app.worker.celery_app import celery_app
 from app.worker.tasks import chat_with_model, deploy_model, run_quantization
@@ -98,7 +98,7 @@ def estimate_wbits(body: EstimateWbitsRequest) -> EstimateWbitsResponse:
 
     try:
         result = estimate_wbits_from_vram(
-            body.model_name,
+            resolve_model_identifier(body.model_name),
             vram_ratio=body.vram_ratio,
             total_vram_gb=body.total_vram_gb,
             group_size=body.group_size,

dashboard/backend/app/services/huggingface.py

@@ -7,29 +7,67 @@
 
 import logging
 import os
+from pathlib import Path
 
 import httpx
 
 logger = logging.getLogger(__name__)
 
 _HF_API_BASE = "https://huggingface.co/api/models"
 _TIMEOUT = httpx.Timeout(10.0, connect=5.0)
+_LOCAL_MODEL_ROOT_ENV = "LOCAL_MODEL_ROOT"
+_DEFAULT_LOCAL_MODEL_ROOT = "/models"
 
 
-def check_model_exists(model_id: str) -> None:
-    """Raise ``ValueError`` if ``model_id`` is not resolvable on HuggingFace Hub.
+def _local_model_root() -> Path:
+    return Path(
+        os.environ.get(_LOCAL_MODEL_ROOT_ENV, _DEFAULT_LOCAL_MODEL_ROOT),
+    ).resolve()
+
+
+def _local_model_dir(model_id: str) -> Path | None:
+    """Return the resolved local model directory when it exists under the allowed root."""
+    root = _local_model_root()
+    candidate = (root / model_id).resolve()
+    try:
+        candidate.relative_to(root)
+    except ValueError:
+        return None
+    return candidate if candidate.is_dir() else None
+
+
+def resolve_model_identifier(model_id: str) -> str:
+    """Normalize *model_id* and map local names to an absolute filesystem path.
 
-    A local path that exists on disk is treated as a valid model identifier,
-    matching ``transformers.AutoConfig.from_pretrained`` behaviour. This is
-    important when callers point at an already-downloaded model directory.
+    Short identifiers such as ``gemma-2-2b-it`` that exist under
+    ``LOCAL_MODEL_ROOT`` are returned as absolute paths so downstream loaders
+    (``ModelConfig``, ``transformers``) use the local snapshot instead of
+    HuggingFace Hub.
     """
-    if not model_id or not model_id.strip():
+    normalized = model_id.strip()
+    if not normalized:
         raise ValueError("Model name must not be empty.")
 
-    model_id = model_id.strip()
+    absolute = Path(normalized)
+    if absolute.is_absolute() and absolute.is_dir():
+        return str(absolute.resolve())
+
+    local_dir = _local_model_dir(normalized)
+    if local_dir is not None:
+        return str(local_dir)
+    return normalized
+
+
+def check_model_exists(model_id: str) -> None:
+    """Raise ``ValueError`` if ``model_id`` is not resolvable on HuggingFace Hub.
+
+    A model directory under ``LOCAL_MODEL_ROOT`` (default ``/models``) is
+    treated as a valid local identifier. This matches the deployment layout
+    where already-downloaded models live on shared storage.
+    """
+    model_id = resolve_model_identifier(model_id)
 
-    # Local path takes precedence (matches transformers' resolution order).
-    if os.path.isdir(model_id):
+    if Path(model_id).is_dir():
         return
 
     token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")

dashboard/backend/app/worker/tasks.py

@@ -12,6 +12,7 @@
 from app.core.config import settings
 from app.core.database import SessionLocal
 from app.models.job import Job
+from app.services.huggingface import resolve_model_identifier
 from app.services.job_store import update_job
 from app.worker.celery_app import celery_app
 
@@ -136,7 +137,8 @@ def _run_real_quantization(job_id: str, model_name: str, quant_method: str, para
     update_job(job_id, progress=5)
 
     quant_device = "cuda" if settings.device == "cuda" else "cpu"
-    model_config = ModelConfig(model_id=model_name, device=quant_device)
+    resolved_model = resolve_model_identifier(model_name)
+    model_config = ModelConfig(model_id=resolved_model, device=quant_device)
 
     try:
         quantizer_name, force_qep = QUANTIZER_MAP[quant_method]
@@ -164,7 +166,7 @@ def _run_real_quantization(job_id: str, model_name: str, quant_method: str, para
 
         groupsize = 128
         est = estimate_wbits_from_vram(
-            model_name,
+            resolved_model,
             total_vram_gb=params.get("total_vram_gb"),
             group_size=groupsize,
             logger=logger,

onecomp/eval/evals/mt_bench/judge.py

@@ -104,12 +104,8 @@ def judge_answers(
             f"{probed}"
         )
 
-    client_kwargs: dict = {
-        "api_key": api_key,
-        "base_url": _resolve_judge_base_url(judge_api_base),
-    }
-    logger.info("[JUDGE] Using base_url=%s", client_kwargs["base_url"])
-    client = OpenAI(**client_kwargs)
+    logger.info("[JUDGE] Initializing judge client")
+    client = OpenAI(api_key=api_key, base_url=_resolve_judge_base_url(judge_api_base))
 
     question_file = data_dir / "question.jsonl"
     questions = {q["question_id"]: q for q in load_questions(question_file)}

onecomp/eval/orchestrator/server.py

@@ -234,8 +234,6 @@ def _build_argv(self) -> list[str]:
             str(self.cfg.max_model_len),
             "--dtype",
             dtype,
-            "--api-key",
-            self.cfg.api_key,
         ]
         if self.cfg.trust_remote_code:
             argv.append("--trust-remote-code")