Merge branch 'export/v1-1-0' into lab/docs-v1-1-0

Author: aki916f

CHANGELOG.md

@@ -105,6 +105,8 @@
 
 ### Bug Fix
 
+- Fixed `model_config.py`: `load_model()` VLM fallback did not trigger for models raising `"Unrecognized configuration class"` (e.g. Cohere2VisionForConditionalGeneration). Added the error pattern to `_vlm_hints`
+- Fixed `gptq/_gptq.py`: Cholesky decomposition in `run_gptq` could fail with `LinAlgError` on ill-conditioned Hessians (observed on large VLMs at deeper layers). Extracted `_compute_inverse_hessian()` with progressive damping fallback (up to 5 retries, 10x damping increase per retry). No impact on normal operation
 - Fixed `TypeError` in `QuantLinear.forward` when `S_qk` scaling was applied to MLP layers (`onecomp/pre_process/quant_models.py`)
 - Fixed silent weight corruption in `GPTQLinear` when `qzero=0` was stored through the GPTQ v1 zero-point path (`onecomp/quantizer/gptq/gptq_layer.py`)
   - Root cause: AutoGPTQ v1 stores `raw_zero - 1`, so `qzero=0` becomes `-1`; without masking, its sign-extended bits corrupted neighboring packed slots
@@ -113,6 +115,14 @@
   - Added regression tests for per-slot pack corruption, packed/unpacked forward paths, the `gptq_v2` branch, and the `from_saved_state` path for GPTQ v1 tensors
   - **NOTE**: If you have GPTQ models quantized with previous versions, please re-quantize them with this release, as they may contain corrupted internal data.
 
+### Packaging
+
+- Bumped minimum `transformers` requirement from `>=5.3.0` to `>=5.5.0` (`pyproject.toml`)
+- Added `cu130` optional-dependency extra and the `pytorch-cu130` wheel index (`https://download.pytorch.org/whl/cu130`) for CUDA 13 hosts (e.g. NVIDIA B200) (`pyproject.toml`)
+- Pinned the `vllm` extra to `vllm>=0.10` to prevent uv from falling back to legacy versions whose source build requires `CUDA_HOME` (`pyproject.toml`)
+- Added uv `conflicts` declarations between the `vllm` extra and the `cpu` / `cu118` / `cu121` / `cu124` / `cu126` / `cu128` extras: vLLM `>=0.20` requires `torch>=2.10`, which is only published for `cu130`. This forces `vllm` to be installed only with `--extra cu130` and prevents silent fallback to a `vllm` version incompatible with `transformers>=5` at runtime (`pyproject.toml`)
+- Restricted `tool.uv.environments` to `sys_platform == 'linux'` and `python_full_version >= '3.12', < '3.14'` to skip lock splits for unused Windows and out-of-range Python versions (`pyproject.toml`)
+
 ### Examples
 
 - Added `example/example_custom_calibration.py`: Demonstrates `CalibrationConfig` with a custom calibration dataset (Python code snippets in `example/data/python_calibration.txt`).  Quantizes TinyLlama with GPTQ 3-bit using both default C4 and custom Python-code calibration, then compares inference outputs across multiple prompts to show how calibration data choice affects quantization quality.
@@ -142,6 +152,7 @@
 - Added `docs/api/quantizers/onebit.md` (OneBit API reference)
 - Updated `mkdocs.yml` nav: added AutoBit/JointQ algorithm pages, OneBit API page; renamed Post-Process nav title to include Block-wise PTQ
 - Added example script links to `docs/user-guide/pre-process.md`
+- Added a Troubleshooting section to `docs/user-guide/vllm-inference.md` describing how to bypass the unconditional DeepGEMM (FP8) kernel warmup for non-FP8 quantization (GPTQ / DBF / Mixed-GPTQ) by setting `VLLM_USE_DEEP_GEMM=0` and `VLLM_DEEP_GEMM_WARMUP=skip`
 
 ### Tests
 

README.md

@@ -79,6 +79,7 @@ Choose the appropriate CUDA version for your system:
 | CUDA 12.4    | `pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124` |
 | CUDA 12.6    | `pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu126` |
 | CUDA 12.8    | `pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128` |
+| CUDA 13.0    | `pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu130` |
 
 Check your CUDA version:
 ```bash
@@ -124,18 +125,20 @@ uv sync --extra cu128 --extra dev --extra visualize
 The `uv sync` command creates a Python virtual environment and installs all dependent libraries.
 
 The `--extra cu128` option installs the CUDA-enabled version of PyTorch (along with `torchvision` from the same CUDA index).
-Replace `cu128` with the appropriate variant for your environment: `cpu`, `cu118`, `cu121`, `cu124`, `cu126`, or `cu128`.
+Replace `cu128` with the appropriate variant for your environment: `cpu`, `cu118`, `cu121`, `cu124`, `cu126`, `cu128`, or `cu130`.
 PyTorch will be automatically downloaded by `uv`, so you do not need to install it beforehand.
 
 Adding `--extra dev` installs development tools (black, pytest, pylint).
 Adding `--extra visualize` installs matplotlib for visualization features.
 
-To use vLLM for serving quantized models, add `--extra vllm`:
+To use vLLM for serving quantized models, add `--extra vllm` together with `--extra cu130`:
 
 ```bash
-uv sync --extra cu128 --extra dev --extra visualize --extra vllm
+uv sync --extra cu130 --extra dev --extra visualize --extra vllm
 ```
 
+> **Note:** `--extra vllm` is only compatible with `--extra cu130`. Recent vLLM releases require `torch>=2.10`, whose wheels are only published for the `cu130` index. Combining `--extra vllm` with `cpu` / `cu118` / `cu121` / `cu124` / `cu126` / `cu128` is rejected by `uv` at lock time.
+
 > **Note:** `--extra vllm` may take a long time on the first run if a pre-built `xformers` wheel is not available for your Python/CUDA combination (e.g. Python 3.13). Using Python 3.12 typically avoids this.
 
 #### Running commands (uv environment)
@@ -171,7 +174,7 @@ pip install torch --index-url https://download.pytorch.org/whl/cu128
 pip install -e ".[dev]"
 ```
 
-Replace `cu128` with the appropriate variant for your environment: `cpu`, `cu118`, `cu121`, `cu124`, `cu126`, or `cu128`.
+Replace `cu128` with the appropriate variant for your environment: `cpu`, `cu118`, `cu121`, `cu124`, `cu126`, `cu128`, or `cu130`.
 
 
 ### Building Documentation Locally
@@ -209,8 +212,8 @@ OneComp-quantized models can be served with [vLLM](https://docs.vllm.ai/) via bu
 Combined with [Open WebUI](https://github.com/open-webui/open-webui), you can chat with your quantized model through a ChatGPT-like browser interface — entirely on your local machine.
 
 ```bash
-# uv users
-uv sync --extra cu128 --extra vllm
+# uv users (vLLM requires cu130; see Installation for details)
+uv sync --extra cu130 --extra vllm
 
 # pip users
 pip install vllm

benchmark/llama3-8b-jointq/.gitignore

@@ -54,3 +54,6 @@ llama3-8b-fixed-lam0.3/
 llama3-8b-fixed-lam0.5/
 llama3-8b-incremental/
 llama3-8b-incremental-actorder-mse/
+
+run_llama3-8b_diagonal_lam*_actorder.sh
+llama3-8b-diagonal-lam*-actorder/

benchmark/qwen3-14b-jointq/.gitignore

@@ -55,3 +55,6 @@ qwen3-14b-fixed-lam0.3/
 qwen3-14b-fixed-lam0.5/
 qwen3-14b-incremental/
 qwen3-14b-incremental-actorder-mse/
+
+run_qwen3-14b_diagonal_lam*_actorder.sh
+qwen3-14b-diagonal-lam*-actorder/

benchmark/qwen3-8b-jointq/.gitignore

@@ -55,3 +55,6 @@ qwen3-8b-fixed-lam0.3/
 qwen3-8b-fixed-lam0.5/
 qwen3-8b-incremental/
 qwen3-8b-incremental-actorder-mse/
+
+run_qwen3-8b_diagonal_lam*_actorder.sh
+qwen3-8b-diagonal-lam*-actorder/

docs/getting-started/installation.md

@@ -49,6 +49,12 @@ Install the appropriate version of PyTorch for your system.
     pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128
     ```
 
+=== "CUDA 13.0"
+
+    ```bash
+    pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu130
+    ```
+
 Check your CUDA version:
 
 ```bash
@@ -92,17 +98,20 @@ uv sync --extra cu128 --extra dev --extra visualize
 ```
 
 The `uv sync` command creates a virtual environment and installs all dependencies (including `torchvision` from the same CUDA index as PyTorch).
-Replace `cu128` with the appropriate CUDA variant for your system: `cpu`, `cu118`, `cu121`, `cu124`, `cu126`, or `cu128`.
+Replace `cu128` with the appropriate CUDA variant for your system: `cpu`, `cu118`, `cu121`, `cu124`, `cu126`, `cu128`, or `cu130`.
 
 Adding `--extra dev` installs development tools (black, pytest, pylint).
 Adding `--extra visualize` installs matplotlib for visualization features.
 
-To use vLLM for serving quantized models, add `--extra vllm`:
+To use vLLM for serving quantized models, add `--extra vllm` together with `--extra cu130`:
 
 ```bash
-uv sync --extra cu128 --extra dev --extra visualize --extra vllm
+uv sync --extra cu130 --extra dev --extra visualize --extra vllm
 ```
 
+!!! note "vLLM requires the `cu130` extra"
+    Recent vLLM releases depend on `torch>=2.10`, whose wheels are only published for the `cu130` index. The `--extra vllm` declaration in `pyproject.toml` therefore conflicts with `cpu`, `cu118`, `cu121`, `cu124`, `cu126`, and `cu128`; combining any of these with `--extra vllm` is rejected by `uv` at lock time.
+
 !!! warning
     Do **not** install vLLM with `uv pip install vllm` after `uv sync`. Packages installed via `uv pip` are not tracked by the lockfile and will be removed or overwritten by subsequent `uv sync` or `uv run` commands. Always use `--extra vllm` instead.
 

docs/user-guide/vllm-inference.md

@@ -20,10 +20,11 @@ vLLM is available as an optional dependency:
 === "uv (recommended)"
 
     ```bash
-    uv sync --extra cu128 --extra vllm
+    uv sync --extra cu130 --extra vllm
     ```
 
-    Replace `cu128` with your CUDA variant (`cu118`, `cu121`, `cu124`, `cu126`, or `cu128`).
+    !!! note "Use `cu130`; older CUDA extras are rejected"
+        Recent vLLM releases depend on `torch>=2.10`, whose wheels are only published for the `cu130` PyTorch index. `pyproject.toml` therefore declares `--extra vllm` as conflicting with `cpu`, `cu118`, `cu121`, `cu124`, `cu126`, and `cu128`; combining any of those with `--extra vllm` will fail at lock time. Use `--extra cu130` for vLLM workflows.
 
 === "pip"
 
@@ -218,3 +219,27 @@ Select the model from the dropdown at the top of the chat screen and start a con
 | Variable | Default | Description |
 |----------|---------|-------------|
 | `ONECOMP_DBF_NAIVE_LINEAR` | `0` | Set to `1` to force the naive (non-GemLite) kernel for DBF inference. Useful for debugging or when GemLite is unavailable. |
+
+## Troubleshooting
+
+### `RuntimeError: DeepGEMM backend is not available or outdated`
+
+vLLM unconditionally runs a DeepGEMM (FP8) kernel warmup at engine startup, even for non-FP8 quantization such as GPTQ, DBF, or Mixed-GPTQ. When the optional [`deep_gemm`](https://github.com/deepseek-ai/DeepGEMM) package is not installed, the warmup fails with:
+
+```
+RuntimeError: DeepGEMM backend is not available or outdated. Please install or update the `deep_gemm` to a newer version to enable FP8 kernels.
+```
+
+OneComp-quantized models do not require DeepGEMM. Disable the FP8 kernel path before launching vLLM:
+
+```bash
+export VLLM_USE_DEEP_GEMM=0
+export VLLM_DEEP_GEMM_WARMUP=skip
+
+# Then launch vllm as usual
+vllm serve ./your-quantized-model
+# or
+python your_vllm_script.py
+```
+
+Both variables are read directly by vLLM; OneComp does not interpret them.

onecomp/model_config.py

@@ -83,7 +83,7 @@ def load_model(self, device_map=None):
         try:
             model = AutoModelForCausalLM.from_pretrained(self.get_model_id_or_path(), **kwargs)
         except ValueError as e:
-            _vlm_hints = ("Unrecognized model", "is not supported")
+            _vlm_hints = ("Unrecognized configuration class", "Unrecognized model", "is not supported")
             if not _HAS_VLM_AUTO or not any(h in str(e) for h in _vlm_hints):
                 raise
             self.logger.info("AutoModelForCausalLM failed; trying AutoModelForImageTextToText.")

onecomp/quantizer/gptq/_gptq.py

@@ -463,6 +463,53 @@ def create_inference_layer(self, result, linear_module, **kwargs):
         )
 
 
+def _compute_inverse_hessian(
+    hessian: torch.Tensor,
+    percdamp: float,
+    max_retries: int = 5,
+) -> torch.Tensor:
+    """Compute the upper-triangular Cholesky factor of the inverse Hessian.
+
+    Applies damping to the diagonal for numerical stability.  If the
+    Cholesky decomposition fails (non-positive-definite), progressively
+    increases damping and retries up to *max_retries* times.
+
+    Args:
+        hessian: Square Hessian matrix (modified in-place).
+        percdamp: Base damping as a fraction of the mean diagonal.
+        max_retries: Maximum number of retry attempts with increased damping.
+
+    Returns:
+        Upper-triangular Cholesky factor of the inverse Hessian.
+    """
+    damp = percdamp * torch.mean(torch.diag(hessian))
+    diag = torch.arange(hessian.shape[0], device=hessian.device)
+    hessian[diag, diag] += damp
+
+    damp_scale = 1.0
+    for attempt in range(max_retries):
+        try:
+            cholesky_lower = torch.linalg.cholesky(hessian)
+            break
+        except torch._C._LinAlgError:
+            damp_scale *= 10.0
+            extra = damp_scale * damp
+            hessian[diag, diag] += extra
+            logger.warning(
+                "Cholesky failed (attempt %d/%d); adding extra damping %.2e",
+                attempt + 1,
+                max_retries,
+                extra,
+            )
+    else:
+        raise RuntimeError(
+            "Cholesky decomposition failed after %d damping attempts. "
+            "The Hessian may be severely ill-conditioned." % max_retries
+        )
+    hessian = torch.cholesky_inverse(cholesky_lower)
+    return torch.linalg.cholesky(hessian, upper=True)
+
+
 def run_gptq(  # pylint: disable=too-many-positional-arguments
     hessian: torch.Tensor,
     layer: torch.nn.Module,
@@ -518,13 +565,7 @@ def run_gptq(  # pylint: disable=too-many-positional-arguments
 
     Q_int = torch.zeros_like(matrix_W, dtype=torch.int32)
 
-    damp = percdamp * torch.mean(torch.diag(hessian))
-    diag = torch.arange(hessian.shape[0], device=hessian.device)
-    hessian[diag, diag] += damp
-    hessian = torch.linalg.cholesky(hessian)
-    hessian = torch.cholesky_inverse(hessian)
-    hessian = torch.linalg.cholesky(hessian, upper=True)
-    Hinv = hessian
+    Hinv = _compute_inverse_hessian(hessian, percdamp)
 
     # Accumulate per-group scale/zero for grouped quantization
     if groupsize != -1:

pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
     "Intended Audience :: Science/Research",
 ]
 dependencies = [
-    "transformers >= 5.3.0",
+    "transformers >= 5.5.0",
     "accelerate",
     "datasets",
     "lm-eval",
@@ -52,6 +52,7 @@ cu121 = ["torch", "torchvision"]
 cu124 = ["torch", "torchvision"]
 cu126 = ["torch", "torchvision"]
 cu128 = ["torch", "torchvision"]
+cu130 = ["torch", "torchvision"]
 dev = [
     "black",
     "pylint",
@@ -60,7 +61,7 @@ dev = [
 visualize = [
     "matplotlib>=3.10.8",
 ]
-vllm = ["vllm"]
+vllm = ["vllm>=0.10"]
 docs = [
     "mkdocs-material",
     "mkdocstrings[python]",
@@ -74,7 +75,21 @@ mixed_gptq = "vllm_plugins.gptq.vllm_plugin:register_vllm_plugin"
 
 [tool.uv]
 conflicts = [
-    [{extra = "cpu"}, {extra = "cu118"}, {extra = "cu121"}, {extra = "cu124"}, {extra = "cu126"}, {extra = "cu128"}],
+    [{extra = "cpu"}, {extra = "cu118"}, {extra = "cu121"}, {extra = "cu124"}, {extra = "cu126"}, {extra = "cu128"}, {extra = "cu130"}],
+    # vLLM (latest, >=0.20) requires torch 2.10/2.11, whose wheels are only
+    # published for cu130. Forbid using vllm together with the older CUDA
+    # wheel indices so that uv either picks vllm>=0.20 on cu130 or refuses
+    # the combination, instead of silently falling back to a vllm version
+    # that is incompatible with transformers>=5 at runtime.
+    [{extra = "cpu"}, {extra = "vllm"}],
+    [{extra = "cu118"}, {extra = "vllm"}],
+    [{extra = "cu121"}, {extra = "vllm"}],
+    [{extra = "cu124"}, {extra = "vllm"}],
+    [{extra = "cu126"}, {extra = "vllm"}],
+    [{extra = "cu128"}, {extra = "vllm"}],
+]
+environments = [
+    "sys_platform == 'linux' and python_full_version >= '3.12' and python_full_version < '3.14'",
 ]
 
 [tool.uv.sources]
@@ -85,6 +100,7 @@ torch = [
     {index = "pytorch-cu124", extra = "cu124"},
     {index = "pytorch-cu126", extra = "cu126"},
     {index = "pytorch-cu128", extra = "cu128"},
+    {index = "pytorch-cu130", extra = "cu130"},
 ]
 torchvision = [
     {index = "pytorch-cpu", extra = "cpu"},
@@ -93,6 +109,7 @@ torchvision = [
     {index = "pytorch-cu124", extra = "cu124"},
     {index = "pytorch-cu126", extra = "cu126"},
     {index = "pytorch-cu128", extra = "cu128"},
+    {index = "pytorch-cu130", extra = "cu130"},
 ]
 
 [[tool.uv.index]]
@@ -125,6 +142,11 @@ name = "pytorch-cu128"
 url = "https://download.pytorch.org/whl/cu128"
 explicit = true
 
+[[tool.uv.index]]
+name = "pytorch-cu130"
+url = "https://download.pytorch.org/whl/cu130"
+explicit = true
+
 [tool.uv.extra-build-dependencies]
 xformers = ["torch"]