fix: Correct dequant inverse for CUTLASS GEMM convention

The CUTLASS fused quantize GEMM computes A @ R (no transpose on the
rotation matrix R). The dequant inverse must therefore apply R^T, not R.
This was masked with the plain Hadamard (which is symmetric, H = H^T)
but broke with the randomized Hadamard (R ≠ R^T).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: TimDettmers

bitsandbytes/backends/cuda/ops.py

@@ -915,9 +915,10 @@ def _(A: torch.Tensor, tensor_scale: Optional[float] = None) -> tuple[torch.Tens
 def _get_rotation_matrix(device: torch.device) -> torch.Tensor:
     """Get cached 16x16 randomized Hadamard matrix for fused quantize.
 
-    Builds H * D where H is the 16x16 normalized Hadamard matrix and D is a
-    diagonal sign-flip matrix (±1 per column) from a fixed seed. The same
-    matrix must be used for both weight and activation quantization.
+    Builds R = H * D where H is the 16x16 normalized Hadamard matrix and D is
+    a diagonal sign-flip matrix (±1 per column) from a fixed seed. The CUTLASS
+    GEMM computes ``A @ R`` (no transpose), so dequant must apply ``@ R^T``.
+    The same matrix must be used for both weight and activation quantization.
     """
     if device not in _rotation_matrices:
         # Build normalized 16x16 Hadamard via Sylvester construction

bitsandbytes/functional.py

@@ -1237,12 +1237,12 @@ def dequantize_nvfp4(
     )
 
     if quant_state.rotated:
-        # Undo rotation: data was quantized as x @ B^T, so recover x = out @ B.
-        # B is the cached randomized Hadamard matrix (orthogonal, so B^T·B = I).
+        # Undo rotation: the CUTLASS GEMM computes x @ R (no transpose on R),
+        # so dequant gives approx x @ R. To recover x, multiply by R^{-1} = R^T.
         from bitsandbytes.backends.cuda.ops import _get_rotation_matrix
 
-        B = _get_rotation_matrix(out.device)
-        out = (out.view(-1, 16) @ B).view(-1)
+        R = _get_rotation_matrix(out.device)
+        out = (out.view(-1, 16) @ R.T).view(-1)
 
     return out.reshape(quant_state.shape)