fix: correct FP8 support check on Ada+ GPUs by using compressed-tensors (#1110)

* fix: fp8 support check for dynamic fp8

* bump compressed-tensors

Author: AlpinDale

aphrodite/quantization/compressed_tensors/compressed_tensors.py

@@ -1,6 +1,10 @@
 from typing import Any, Dict, List, Optional
 
 import torch
+from compressed_tensors.config import CompressionFormat
+from compressed_tensors.quantization import (QuantizationArgs,
+                                             QuantizationStrategy,
+                                             QuantizationType)
 from pydantic import BaseModel
 
 from aphrodite.modeling.layers.fused_moe import FusedMoE
@@ -17,8 +21,7 @@
     CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
     CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
 from aphrodite.quantization.compressed_tensors.utils import (
-    CompressionFormat, QuantizationArgs, QuantizationStrategy,
-    QuantizationType, find_matched_target, is_activation_quantization_format,
+    find_matched_target, is_activation_quantization_format,
     should_ignore_layer)
 from aphrodite.quantization.kv_cache import BaseKVCacheMethod
 

aphrodite/quantization/compressed_tensors/compressed_tensors_moe.py

@@ -3,13 +3,13 @@
 from typing import Callable, List, Optional
 
 import torch
+from compressed_tensors import CompressionFormat
 
 from aphrodite import _custom_ops as ops
 from aphrodite.modeling.layers.fused_moe import FusedMoEMethodBase
 from aphrodite.modeling.utils import set_weight_attrs
 from aphrodite.quantization.compressed_tensors.schemes import (
     WNA16_SUPPORTED_BITS)
-from aphrodite.quantization.compressed_tensors.utils import CompressionFormat
 
 
 class GPTQMarlinState(Enum):

aphrodite/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py

@@ -1,14 +1,13 @@
 from typing import Callable, List, Optional
 
 import torch
+from compressed_tensors.quantization import QuantizationStrategy
 
 from aphrodite.modeling.parameter import (ChannelQuantScaleParameter,
                                           ModelWeightParameter,
                                           PerTensorScaleParameter)
 from aphrodite.quantization.compressed_tensors.schemes import (
     CompressedTensorsScheme)
-from aphrodite.quantization.compressed_tensors.utils import (
-    QuantizationStrategy)
 from aphrodite.quantization.utils.marlin_utils_fp8 import (
     apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
 from aphrodite.quantization.utils.w8a8_utils import convert_to_channelwise

aphrodite/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py

@@ -1,6 +1,7 @@
 from typing import Callable, List, Optional
 
 import torch
+from compressed_tensors.quantization import QuantizationStrategy
 from torch.nn import Parameter
 
 from aphrodite.common.utils import is_hip
@@ -9,8 +10,6 @@
                                           PerTensorScaleParameter)
 from aphrodite.quantization.compressed_tensors.schemes import (
     CompressedTensorsScheme)
-from aphrodite.quantization.compressed_tensors.utils import (
-    QuantizationStrategy)
 from aphrodite.quantization.utils.w8a8_utils import (
     apply_fp8_linear, cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz,
     requantize_with_max_scale)

aphrodite/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py

@@ -1,6 +1,7 @@
 from typing import Callable, List, Optional
 
 import torch
+from compressed_tensors.quantization import QuantizationStrategy
 from torch.nn import Parameter
 
 from aphrodite.modeling.parameter import (BaseAphroditeParameter,
@@ -9,8 +10,6 @@
                                           PerTensorScaleParameter)
 from aphrodite.quantization.compressed_tensors.schemes import (
     CompressedTensorsScheme)
-from aphrodite.quantization.compressed_tensors.utils import (
-    QuantizationStrategy)
 from aphrodite.quantization.utils.w8a8_utils import (apply_int8_linear,
                                                      convert_to_channelwise)
 

aphrodite/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py

@@ -1,6 +1,7 @@
 from typing import Callable, List, Optional, Set
 
 import torch
+from compressed_tensors.quantization import ActivationOrdering
 from loguru import logger
 
 from aphrodite.modeling.parameter import (BaseAphroditeParameter,
@@ -10,7 +11,6 @@
                                           RowAphroditeParameter)
 from aphrodite.quantization.compressed_tensors.schemes import (
     CompressedTensorsScheme)
-from aphrodite.quantization.compressed_tensors.utils import ActivationOrdering
 from aphrodite.quantization.kernels import (MPLinearLayerConfig,
                                             choose_mp_linear_kernel)
 from aphrodite.quantization.utils.marlin_utils import (

aphrodite/quantization/compressed_tensors/utils.py

@@ -1,106 +1,12 @@
 import re
-from enum import Enum
-from typing import Any, Dict, Iterable, Optional, Union
+from typing import Iterable, Optional
 
-from pydantic import BaseModel, Field, field_validator
+from compressed_tensors import CompressionFormat
 from torch.nn import Module
 
 from aphrodite.quantization.utils.quant_utils import FUSED_LAYER_NAME_MAPPING
 
 
-class CompressionFormat(Enum):
-    dense = "dense"
-    sparse_bitmask = "sparse-bitmask"
-    naive_quantized = "naive-quantized"
-    float_quantized = "float-quantized"
-    int_quantized = "int-quantized"
-    pack_quantized = "pack-quantized"
-    marlin_24 = "marlin-24"
-
-
-class QuantizationType(str, Enum):
-    """
-    Enum storing quantization type options
-    """
-
-    INT = "int"
-    FLOAT = "float"
-
-
-class QuantizationStrategy(str, Enum):
-    """
-    Enum storing quantization strategy options
-    """
-
-    TENSOR = "tensor"
-    CHANNEL = "channel"
-    GROUP = "group"
-    BLOCK = "block"
-    TOKEN = "token"
-
-
-class ActivationOrdering(str, Enum):
-    """
-    Enum storing strategies for activation ordering
-    Group: reorder groups and weight\n
-    Weight: only reorder weight, not groups. Slightly lower latency and
-    accuracy compared to group actorder\n
-    """
-    GROUP = "group"
-    WEIGHT = "weight"
-
-
-class QuantizationArgs(BaseModel):
-    """
-    User facing arguments used to define a quantization config 
-    for weights or activations
-
-    :param num_bits: quantization bit depth
-    :param type: dtype to quantized to, either int or float
-    :param symmetric: whether or not quantization scale is symmetric
-    :param strategy: string determining the scope of scale/zero-point to apply
-    :param group_size: group length to use for the group strategy
-    :param block_structure: 2d block structure to use for the block 
-    strategy, must be of the format "2x4", "8x16", etc.
-    :param dynamic: set True to perform dynamic quantization -
-        values will not be calibrated during calibration phase, 
-        instead during inference new quantization ranges will be 
-        observed with every sample. Defaults to False for static
-        quantization. Note that enabling dynamic quantization 
-        will change the default observer to a memoryless one
-    :param actorder: whether to apply group quantization in decreasing order of
-        activation. Defaults to None for arbitrary ordering
-    """
-
-    num_bits: int = 8
-    type: QuantizationType = QuantizationType.INT
-    symmetric: bool = True
-    group_size: Optional[int] = None
-    strategy: Optional[QuantizationStrategy] = None
-    block_structure: Optional[str] = None
-    dynamic: bool = False
-    actorder: Union[ActivationOrdering, bool, None] = None
-    observer: str = Field(
-        default="minmax",
-        description=("The class to use to compute the quantization param - "
-                     "scale and zero-point'"),
-    )
-    observer_kwargs: Dict[str, Any] = Field(
-        default_factory=dict,
-        description=
-        ("optional dict of kwargs to be passed directly to torch quantization "
-         "Observers constructor excluding quantization range or symmetry"),
-    )
-
-    @field_validator("actorder", mode="before")
-    def validate_actorder(cls, value) -> Optional[ActivationOrdering]:
-        if isinstance(value, bool):
-            return ActivationOrdering.GROUP if value else None
-        if isinstance(value, str):
-            return ActivationOrdering(value.lower())
-        return value
-
-
 def is_activation_quantization_format(format: str) -> bool:
     _ACTIVATION_QUANTIZATION_FORMATS = [
         CompressionFormat.naive_quantized.value,

requirements-common.txt

@@ -36,3 +36,4 @@ python-multipart
 partial-json-parser
 opencv-python-headless
 einops
+compressed-tensors == 0.8.0