Merge branch 'lab/mt-bench' into 'export/v1-2-0'

add mt-bench and throughput evaluator

See merge request onecomp/onecomp-lab!80

Author: aki916f

.gitignore

@@ -29,6 +29,9 @@ tests/onecomp/quantizer/jointq/data/model_layers_0_self_attn_k_proj.pth
 .uv-sync-vllm.lock
 .uv-sync-vllm.done
 .venv-vllm/
+# MT-Bench eval data (download via onecomp/eval/scripts/download_mt_bench_data*.sh)
+onecomp/eval/data/
+onecomp/eval/data_backup/
 
 # ── dashboard/ (HPC LLM quantization dashboard) ──────────────
 # Re-include the app/models package shadowed by the global `models` rule above

CHANGELOG.md

@@ -3,6 +3,10 @@
 
 ## [v1.2.0] 2026-06-04 (WIP)
 
+### Evaluation:
+- Added `onecomp.eval` and the `onecomp-eval` CLI: one vLLM server, subprocess evaluators, aggregated `summary.json` / `summary.csv`
+- Added `mt_bench` (Japanese MT-Bench) and opt-in `throughput` (TTFT / decode tok/s) evaluators
+
 ## New Feature : Dashboard
 
 - Added `dashboard/`, a browser-based web app for OneCompression on **SLURM-managed HPC GPU nodes without Docker**: pick a Hugging Face model and quantization settings in the UI, run jobs on the GPU, deploy the quantized checkpoint, and validate inference via chat

dashboard/backend/app/api/jobs.py

@@ -1,6 +1,3 @@
-from fastapi import APIRouter, Depends, HTTPException
-from sqlalchemy.orm import Session
-
 from app.constants import ChatTaskStatus, InferenceStatus, JobStatus
 from app.core.config import settings
 from app.core.database import get_db
@@ -20,6 +17,8 @@
 from app.services.inference import chat_vllm, stop_inference
 from app.worker.celery_app import celery_app
 from app.worker.tasks import chat_with_model, deploy_model, run_quantization
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy.orm import Session
 
 router = APIRouter(prefix="/jobs", tags=["jobs"])
 
@@ -200,9 +199,7 @@ def get_chat_result(task_id: str) -> ChatTaskResult:
 
 
 @router.get("")
-def list_jobs(
-    limit: int = 20, offset: int = 0, db: Session = Depends(get_db)
-) -> JobListResponse:
+def list_jobs(limit: int = 20, offset: int = 0, db: Session = Depends(get_db)) -> JobListResponse:
     total = db.query(Job).count()
     jobs = db.query(Job).order_by(Job.created_at.desc()).offset(offset).limit(limit).all()
     return JobListResponse(

dashboard/backend/app/core/database.py

@@ -1,8 +1,7 @@
+from app.core.config import settings
 from sqlalchemy import create_engine
 from sqlalchemy.orm import DeclarativeBase, sessionmaker
 
-from app.core.config import settings
-
 _connect_args = {"check_same_thread": False} if settings.database_url.startswith("sqlite") else {}
 engine = create_engine(settings.database_url, connect_args=_connect_args)
 SessionLocal = sessionmaker(bind=engine)

dashboard/backend/app/main.py

@@ -1,12 +1,11 @@
 from contextlib import asynccontextmanager
 
+from app.api.jobs import router as jobs_router
+from app.core.database import Base, engine
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from sqlalchemy import inspect, text
 
-from app.api.jobs import router as jobs_router
-from app.core.database import Base, engine
-
 
 def _sync_schema():
     """Create missing tables and add missing columns to existing tables."""
@@ -24,7 +23,9 @@ def _sync_schema():
                 col_type = col.type.compile(dialect=engine.dialect)
                 nullable = "NULL" if col.nullable else "NOT NULL"
                 conn.execute(
-                    text(f'ALTER TABLE "{table_name}" ADD COLUMN "{col.name}" {col_type} {nullable}')
+                    text(
+                        f'ALTER TABLE "{table_name}" ADD COLUMN "{col.name}" {col_type} {nullable}'
+                    )
                 )
 
 

dashboard/backend/app/models/job.py

@@ -1,22 +1,19 @@
 import uuid
 from datetime import datetime, timezone
 
+from app.constants import InferenceStatus, JobStatus
+from app.core.database import Base
 from sqlalchemy import JSON, DateTime, Integer, String, Text
 from sqlalchemy.orm import Mapped, mapped_column
 
-from app.core.database import Base
-from app.constants import InferenceStatus, JobStatus
-
 
 class Job(Base):
     __tablename__ = "jobs"
 
     id: Mapped[str] = mapped_column(
         String(36), primary_key=True, default=lambda: str(uuid.uuid4())
     )
-    status: Mapped[str] = mapped_column(
-        String(20), default=JobStatus.PENDING, index=True
-    )
+    status: Mapped[str] = mapped_column(String(20), default=JobStatus.PENDING, index=True)
     progress: Mapped[int] = mapped_column(Integer, default=0)
 
     model_name: Mapped[str] = mapped_column(String(256))

dashboard/backend/app/schemas/job.py

@@ -1,8 +1,7 @@
 from datetime import datetime
 
-from pydantic import BaseModel, Field
-
 from app.constants import ChatTaskStatus, InferenceStatus, JobStatus
+from pydantic import BaseModel, Field
 
 
 class QuantParams(BaseModel):

dashboard/backend/app/services/inference.py

@@ -7,9 +7,8 @@
 
 import httpx
 import torch
-
-from app.core.config import settings
 from app.constants import InferenceStatus
+from app.core.config import settings
 from app.core.database import SessionLocal
 from app.models.job import Job
 from app.schemas.job import ChatMessage
@@ -28,6 +27,7 @@
 
 # ── Mock implementations ──────────────────────────────────────
 
+
 def deploy_mock(job_id: str) -> None:
     update_job(job_id, inference_status=InferenceStatus.DEPLOYING)
     time.sleep(1)
@@ -36,12 +36,14 @@ def deploy_mock(job_id: str) -> None:
 
 # ── OneComp implementations (CPU / MPS) ──────────────────────
 
+
 def deploy_onecomp(job_id: str, model_name: str, model_dir: str) -> None:
     """Load quantized model directly from local path."""
     update_job(job_id, inference_status=InferenceStatus.DEPLOYING)
 
     try:
         from onecomp import load_quantized_model
+
         model, tokenizer = load_quantized_model(model_dir)
 
         if settings.device == "mps" and torch.backends.mps.is_available():
@@ -87,7 +89,7 @@ def chat_onecomp(
             temperature=temperature if temperature > 0 else None,
         )
     generated = tokenizer.decode(
-        output_ids[0][inputs["input_ids"].shape[1]:],
+        output_ids[0][inputs["input_ids"].shape[1] :],
         skip_special_tokens=True,
     )
 
@@ -139,8 +141,9 @@ def _pick_gpu_memory_utilization() -> float:
     free, total = torch.cuda.mem_get_info()
     free_gb, total_gb = free / (1 << 30), total / (1 << 30)
     util = round(min(max((free_gb * 0.9) / total_gb, 0.1), 0.9), 2)
-    logger.info("GPU memory: %.1f/%.1f GiB free → gpu-memory-utilization=%.2f",
-                free_gb, total_gb, util)
+    logger.info(
+        "GPU memory: %.1f/%.1f GiB free → gpu-memory-utilization=%.2f", free_gb, total_gb, util
+    )
     return util
 
 
@@ -170,13 +173,20 @@ def deploy_vllm(job_id: str, model_name: str, model_dir: str) -> None:
         env.setdefault("NCCL_SOCKET_FAMILY", "AF_INET")
 
         cmd = [
-            settings.vllm_python, "-m", "vllm.entrypoints.openai.api_server",
-            "--model", model_dir,
-            "--served-model-name", model_name,
-            "--port", str(port),
-            "--host", "0.0.0.0",
+            settings.vllm_python,
+            "-m",
+            "vllm.entrypoints.openai.api_server",
+            "--model",
+            model_dir,
+            "--served-model-name",
+            model_name,
+            "--port",
+            str(port),
+            "--host",
+            "0.0.0.0",
             "--trust-remote-code",
-            "--gpu-memory-utilization", str(gpu_util),
+            "--gpu-memory-utilization",
+            str(gpu_util),
             "--no-enable-log-requests",
             "--enforce-eager",
         ]
@@ -200,13 +210,16 @@ def deploy_vllm(job_id: str, model_name: str, model_dir: str) -> None:
                 with open(log_path) as f:
                     lines = f.readlines()
                 error_lines = [
-                    l.rstrip() for l in lines
+                    l.rstrip()
+                    for l in lines
                     if "ERROR" in l or "ValueError" in l or "RuntimeError" in l
                 ]
                 tail = "".join(lines[-40:])
                 raise RuntimeError(
                     f"vLLM exited with code {proc.returncode}:\n"
-                    + "\n".join(error_lines[-20:]) + "\n---\n" + tail
+                    + "\n".join(error_lines[-20:])
+                    + "\n---\n"
+                    + tail
                 )
             try:
                 resp = httpx.get(health_url, timeout=5)
@@ -217,9 +230,7 @@ def deploy_vllm(job_id: str, model_name: str, model_dir: str) -> None:
 
         else:
             proc.terminate()
-            raise RuntimeError(
-                f"vLLM failed to become healthy within {_VLLM_HEALTH_TIMEOUT}s"
-            )
+            raise RuntimeError(f"vLLM failed to become healthy within {_VLLM_HEALTH_TIMEOUT}s")
 
         inference_url = f"http://{host}:{port}"
         update_job(
@@ -266,6 +277,7 @@ def chat_vllm(
 
 # ── Lifecycle ─────────────────────────────────────────────────
 
+
 def stop_inference(job_id: str) -> None:
     _loaded_models.pop(job_id, None)
 

dashboard/backend/app/worker/celery_app.py

@@ -1,6 +1,5 @@
-from celery import Celery
-
 from app.core.config import settings
+from celery import Celery
 
 celery_app = Celery(
     "onecomp_worker",

dashboard/backend/app/worker/tasks.py

@@ -6,8 +6,8 @@
 import threading
 import time
 
-from app.core.config import settings
 from app.constants import InferenceStatus, JobStatus
+from app.core.config import settings
 from app.core.database import SessionLocal
 from app.models.job import Job
 from app.services.job_store import update_job
@@ -171,9 +171,7 @@ def _run_real_quantization(job_id: str, model_name: str, quant_method: str, para
         # VRAM budget. Matches onecomp.Runner.auto_run.
         wbits = math.floor(est.target_bitwidth * 100) / 100
         if wbits <= 0:
-            raise RuntimeError(
-                f"VRAM-based wbits estimation returned non-positive value: {wbits}"
-            )
+            raise RuntimeError(f"VRAM-based wbits estimation returned non-positive value: {wbits}")
         resolved_params = {
             **params,
             "bits": wbits,
@@ -247,6 +245,7 @@ def run_quantization(self, job_id: str):
         gc.collect()
         try:
             import torch
+
             if torch.cuda.is_available():
                 torch.cuda.empty_cache()
                 logger.info("Released CUDA cache after quantization")