From 4af45b91392adcc8526289a8b15139f09e136ccc Mon Sep 17 00:00:00 2001 From: Qubitium Date: Sun, 26 Jul 2026 21:10:58 +0000 Subject: [PATCH] fix(qlinear): size GPTQ buffers by bits/pack_dtype_bits instead of pack_factor or 32 The number of packed words is ceil(dim * bits / pack_dtype_bits), not ceil(dim / pack_factor). - In _register_gptq_buffers, use in_features*bits/pack_dtype_bits and out_features*bits/pack_dtype_bits for qweight/qzeros rows/cols. - In pack_original, use dim*bits/pack_dtype_bits for both qweight and qzeros. This is equivalent to the previous pack_factor formula for 2/4/8-bit, and correct for 3-bit and non-32-bit pack dtypes (int8/int16). --- gptqmodel/nn_modules/qlinear/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gptqmodel/nn_modules/qlinear/__init__.py b/gptqmodel/nn_modules/qlinear/__init__.py index 6c6c6ed91..932c42d83 100644 --- a/gptqmodel/nn_modules/qlinear/__init__.py +++ b/gptqmodel/nn_modules/qlinear/__init__.py @@ -750,14 +750,14 @@ def _register_gptq_buffers( self.register_buffer( "qweight", - t.zeros((math.ceil(in_features / self.pack_factor), out_features), dtype=self.pack_dtype), + t.zeros((math.ceil(in_features * self.bits / self.pack_dtype_bits), out_features), dtype=self.pack_dtype), ) self.register_buffer( "qzeros", t.zeros( ( math.ceil(in_features / self.group_size), - math.ceil(out_features / self.pack_factor), + math.ceil(out_features * self.bits / self.pack_dtype_bits), ), dtype=self.pack_dtype, ), @@ -1395,7 +1395,7 @@ def pack_original(self, linear: nn.Module, scales: t.Tensor, zeros: t.Tensor, g_ int_weight = int_weight.to(t.int32).T.contiguous() int_weight = int_weight.numpy().astype(self.pack_np_math_dtype) - qweight = np.zeros((math.ceil(int_weight.shape[0] * self.bits / 32), int_weight.shape[1]), + qweight = np.zeros((math.ceil(int_weight.shape[0] * self.bits / self.pack_dtype_bits), int_weight.shape[1]), dtype=self.pack_np_math_dtype) if self.bits in [2, 4, 8]: for row in range(qweight.shape[0]): @@ -1428,7 +1428,7 @@ def pack_original(self, linear: nn.Module, scales: t.Tensor, zeros: t.Tensor, g_ self.register_buffer("qweight", t.from_numpy(qweight.astype(self.pack_np_dtype))) zeros = zeros.numpy().astype(self.pack_np_math_dtype) - qzeros = np.zeros((zeros.shape[0], math.ceil(zeros.shape[1] * self.bits / 32)), dtype=self.pack_np_math_dtype) + qzeros = np.zeros((zeros.shape[0], math.ceil(zeros.shape[1] * self.bits / self.pack_dtype_bits)), dtype=self.pack_np_math_dtype) if self.bits in [2, 4, 8]: for col in range(qzeros.shape[1]): for j in range(self.pack_factor):