|
1 | | -"""Quantization utilities for Parakeet model export.""" |
| 1 | +"""Quantization utilities for Parakeet model export. |
2 | 2 |
|
3 | | -from typing import Optional |
| 3 | +Re-exports quantize_model_ from the shared ExecuTorch LLM export library. |
| 4 | +""" |
4 | 5 |
|
5 | | -import torch |
| 6 | +from executorch.extension.llm.export.quantize import quantize_model_ |
6 | 7 |
|
7 | | - |
8 | | -def quantize_model_( # noqa: C901 |
9 | | - module: torch.nn.Module, |
10 | | - qlinear_config: Optional[str] = None, |
11 | | - qlinear_group_size: int = 32, |
12 | | - qlinear_packing_format: Optional[str] = None, |
13 | | - qembedding_config: Optional[str] = None, |
14 | | - qembedding_group_size: int = 0, |
15 | | -) -> None: |
16 | | - """Quantize linear and embedding layers in a module in-place. |
17 | | -
|
18 | | - Args: |
19 | | - module: The PyTorch module to quantize. |
20 | | - qlinear_config: Quantization config for linear layers ("4w", "8w", "8da4w", "8da8w", "fpa4w"). |
21 | | - qlinear_group_size: Group size for linear quantization (default: 32). |
22 | | - qlinear_packing_format: Packing format for linear layers (e.g., "tile_packed_to_4d"). |
23 | | - qembedding_config: Quantization config for embedding layers ("4w", "8w"). |
24 | | - qembedding_group_size: Group size for embedding quantization (default: 0 = per-axis). |
25 | | - """ |
26 | | - if not qlinear_config and not qembedding_config: |
27 | | - return |
28 | | - |
29 | | - from torchao.quantization.quant_api import quantize_ |
30 | | - |
31 | | - # Metal (MPS) quantization uses different API |
32 | | - if qlinear_config == "fpa4w": |
33 | | - # Load MPS ops |
34 | | - import torchao.experimental.ops.mps # noqa: F401 |
35 | | - from torchao.experimental.quant_api import UIntxWeightOnlyConfig |
36 | | - |
37 | | - config = UIntxWeightOnlyConfig( |
38 | | - group_size=qlinear_group_size, |
39 | | - bitwidth=4, |
40 | | - uintx_choose_qparams_algorithm="hqq", |
41 | | - ) |
42 | | - |
43 | | - def linear_filter(m, fqn): |
44 | | - if isinstance(m, torch.nn.Linear): |
45 | | - if m.weight.shape[1] % qlinear_group_size != 0: |
46 | | - raise ValueError( |
47 | | - f"Metal int4 quantization requires weight dimension (K) to be multiple of group_size. " |
48 | | - f"Layer {fqn} has weight shape {m.weight.shape} (K={m.weight.shape[1]}, group_size={qlinear_group_size})" # noqa: E501 |
49 | | - ) |
50 | | - return True |
51 | | - return False |
52 | | - |
53 | | - print( |
54 | | - f" Applying {qlinear_config} linear quantization " |
55 | | - f"(group_size={qlinear_group_size})..." |
56 | | - ) |
57 | | - quantize_(module, config, filter_fn=linear_filter) |
58 | | - return |
59 | | - |
60 | | - from torchao.quantization.granularity import PerAxis, PerGroup |
61 | | - from torchao.quantization.quant_api import ( |
62 | | - Int4WeightOnlyConfig, |
63 | | - Int8DynamicActivationIntxWeightConfig, |
64 | | - IntxWeightOnlyConfig, |
65 | | - ) |
66 | | - |
67 | | - # Quantize embedding layers first |
68 | | - if qembedding_config: |
69 | | - if qembedding_group_size == 0: |
70 | | - embedding_granularity = PerAxis(0) |
71 | | - else: |
72 | | - assert ( |
73 | | - qembedding_group_size % 2 == 0 |
74 | | - ), "Embedding group size must be a multiple of 2." |
75 | | - embedding_granularity = PerGroup(qembedding_group_size) |
76 | | - |
77 | | - embedding_config = IntxWeightOnlyConfig( |
78 | | - weight_dtype=torch.int4 if qembedding_config == "4w" else torch.int8, |
79 | | - granularity=embedding_granularity, |
80 | | - ) |
81 | | - |
82 | | - print( |
83 | | - f" Applying {qembedding_config} embedding quantization " |
84 | | - f"(group_size={qembedding_group_size})..." |
85 | | - ) |
86 | | - quantize_( |
87 | | - module, |
88 | | - embedding_config, |
89 | | - lambda m, fqn: isinstance(m, torch.nn.Embedding), |
90 | | - ) |
91 | | - |
92 | | - # Quantize linear layers |
93 | | - if qlinear_config: |
94 | | - # Determine granularity |
95 | | - if qlinear_group_size == 0: |
96 | | - granularity = PerAxis(0) |
97 | | - else: |
98 | | - granularity = PerGroup(qlinear_group_size) |
99 | | - |
100 | | - # Build quantization config |
101 | | - if qlinear_config == "4w": |
102 | | - if qlinear_packing_format: |
103 | | - config = Int4WeightOnlyConfig( |
104 | | - group_size=qlinear_group_size, |
105 | | - int4_packing_format=qlinear_packing_format, |
106 | | - int4_choose_qparams_algorithm="hqq", |
107 | | - ) |
108 | | - else: |
109 | | - config = IntxWeightOnlyConfig( |
110 | | - weight_dtype=torch.int4, |
111 | | - granularity=granularity, |
112 | | - ) |
113 | | - elif qlinear_config == "8w": |
114 | | - config = IntxWeightOnlyConfig( |
115 | | - weight_dtype=torch.int8, |
116 | | - granularity=granularity, |
117 | | - ) |
118 | | - elif qlinear_config == "8da4w": |
119 | | - config = Int8DynamicActivationIntxWeightConfig( |
120 | | - weight_dtype=torch.int4, |
121 | | - weight_granularity=granularity, |
122 | | - intx_choose_qparams_algorithm="hqq_scale_only", |
123 | | - ) |
124 | | - elif qlinear_config == "8da8w": |
125 | | - config = Int8DynamicActivationIntxWeightConfig( |
126 | | - weight_dtype=torch.int8, |
127 | | - weight_granularity=PerAxis(0), |
128 | | - ) |
129 | | - else: |
130 | | - raise ValueError(f"Unsupported qlinear_config: {qlinear_config}") |
131 | | - |
132 | | - # Filter: only quantize Linear layers with compatible dimensions |
133 | | - def linear_filter(m, fqn): |
134 | | - if isinstance(m, torch.nn.Linear): |
135 | | - if qlinear_group_size == 0: |
136 | | - return True |
137 | | - if m.weight.shape[1] % qlinear_group_size != 0: |
138 | | - print( |
139 | | - f" Skipping {fqn}: weight shape {m.weight.shape} " |
140 | | - f"incompatible with group_size={qlinear_group_size}" |
141 | | - ) |
142 | | - return False |
143 | | - return True |
144 | | - return False |
145 | | - |
146 | | - print( |
147 | | - f" Applying {qlinear_config} linear quantization " |
148 | | - f"(group_size={qlinear_group_size}, packing={qlinear_packing_format})..." |
149 | | - ) |
150 | | - quantize_(module, config, filter_fn=linear_filter) |
| 8 | +__all__ = ["quantize_model_"] |
0 commit comments