Enable transpose_a support for LoRA Correction

src/nncf/quantization/algorithms/weight_compression/algorithm.py

@@ -1187,11 +1187,6 @@ def apply_with_parameters(
                 )
 
             if self._lora_correction:
-                for wc_params in all_weight_params:
-                    if self._backend_entity.matmul_has_transposed_activations(wc_params.node_with_weight, graph):
-                        msg = "Transposed activations are not supported yet for the LoRa correction algorithm"
-                        raise nncf.UnsupportedModelError(msg)
-
                 lora_correction_params = self._advanced_parameters.lora_correction_params
                 lora_correction_algo = LoraCorrectionAlgorithm(statistics, lora_correction_params)
                 description += " with correction of low-rank adapters"

src/nncf/quantization/algorithms/weight_compression/lora_correction.py

@@ -106,14 +106,19 @@ def is_applicable(self, wc_params: WeightCompressionParameters):
         return wc_params.compression_config.num_bits == 4
 
     def calculate_adapters(
-        self, weight: Tensor, compressed_weight: CompressedWeight, wc_params: WeightCompressionParameters
+        self,
+        weight: Tensor,
+        compressed_weight: CompressedWeight,
+        wc_params: WeightCompressionParameters,
+        act_ch_axis: int,
     ) -> tuple[Tensor, Tensor, list[float]]:
         """
         Calculates low rank matrices for a given original and compressed weights.
 
         :param weight: original floating-point weight matrix.
         :param compressed_weight: compressed weight matrix.
         :param wc_params: parameters of weight compression.
+        :param act_ch_axis: axis number of the activation tensor which correspond to it channel.
         :return: two low rank matrices in the order of execution of corresponding linear layers.
         """
         layer_name = wc_params.node_with_weight.node_name
@@ -126,6 +131,7 @@ def calculate_adapters(
             wc_params.reduction_axes,
             self._lora_correction_params,
             layer_statistics,
+            act_ch_axis,
             is_debug,
         )
         if is_debug:
@@ -140,6 +146,7 @@ def calculate_low_rank_matrices(
         reduction_axes: tuple[int, ...],
         lora_correction_params: AdvancedLoraCorrectionParameters,
         layer_statistics: WCTensorStatistic,
+        act_ch_axis: int,
         is_debug: bool | None = False,
     ):
         """
@@ -155,6 +162,7 @@ def calculate_low_rank_matrices(
         :param reduction_axes: axes along which different statistics reduced.
         :param lora_correction_params: parameters to configure the algorithm.
         :param layer_statistics: an object containing statistics for the layer.
+        :param act_ch_axis: axis number of the activation tensor which correspond to it channel.
         :param is_debug: whether to collect debug information, defaults to False.
         :return: two low rank matrices in the order of execution of corresponding linear layers and list of mean noises.
             Noises are collected from each step of the algorithm if debug was enabled.
@@ -190,7 +198,7 @@ def calculate_low_rank_matrices(
             svd_residual = fns.transpose(svd_residual)
         residual = svd_residual.clone()  # [H, O]
 
-        s, X = process_stats(layer_statistics, subset_size)  # [H], [H, SS]
+        s, X = process_stats(layer_statistics, subset_size, act_ch_axis)  # [H], [H, SS]
         X = fns.transpose(X)  # [SS, H]
         if compression_config.group_size > 0:
             # Multiply residual of weights by maximum channel magnitude of activations normalized per quantization

src/nncf/quantization/algorithms/weight_compression/openvino_backend.py

@@ -208,7 +208,8 @@ def insert_adapters(
             A_W = opset.constant(lora_A.data)
             B_W = opset.constant(lora_B.data)
 
-        A_MM = opset.matmul(input_node, A_W, transpose_a=False, transpose_b=True)
+        transpose_a = self.matmul_has_transposed_activations(wc_params.node_with_weight, graph=None)
+        A_MM = opset.matmul(input_node, A_W, transpose_a=transpose_a, transpose_b=True)
         B_MM = opset.matmul(A_MM, B_W, transpose_a=False, transpose_b=True)
 
         node_output_port = mm_node.output(0)
@@ -365,7 +366,9 @@ def transform_model(
                 compressed_weight.tensor = compressed_weight.tensor.as_numpy_tensor()
                 if compressed_weight.zero_point is not None:
                     compressed_weight.zero_point = compressed_weight.zero_point.as_numpy_tensor()
-                adapters = lora_correction_algo.calculate_adapters(weight, compressed_weight, wc_params)
+
+                act_ch_axis, _ = self.get_activation_channel_axis_and_shape(graph, wc_params.node_with_weight)
+                adapters = lora_correction_algo.calculate_adapters(weight, compressed_weight, wc_params, act_ch_axis)
                 self.insert_adapters(wc_params, *adapters, int8_lora=lora_correction_algo.use_int8_adapters)
         self.name_to_node_mapping = None
 

tests/cross_fw/test_templates/template_test_weights_compression.py

@@ -1061,7 +1061,6 @@ def get_transposable_awq_model(
     @pytest.mark.parametrize(
         "kwargs",
         [
-            dict(lora_correction=True),
             dict(
                 gptq=True,
                 advanced_parameters=CompressionParams(gptq_params=GPTQParams(subset_size=2)),
@@ -1071,8 +1070,6 @@ def get_transposable_awq_model(
     def test_compression_skipped_with_transposed_activations(self, kwargs):
         if kwargs.get("gptq", False) and "gptq" in self.get_not_supported_algorithms():
             pytest.skip("GPTQ is not supported")
-        if kwargs.get("lora_correction", False) and "lora_correction" in self.get_not_supported_algorithms():
-            pytest.skip("lora_correction is not supported")
 
         INPUT_SHAPE = (2, 4)
         model = self.get_transposable_awq_model(transpose_a=True, transpose_b=True, input_shape=INPUT_SHAPE)

tests/openvino/native/quantization/test_weights_compression.py

@@ -1720,12 +1720,17 @@ def test_call_max_var_criterion_with_dataset_gptq_neg_group_size(mode):
 
 
 @pytest.mark.parametrize(
-    "params, transpose_b",
-    ((None, True), (LoraParams(adapter_rank=4, use_int8_adapters=False), False)),
+    "params, transpose_a, transpose_b",
+    (
+        (None, False, True),
+        (LoraParams(adapter_rank=4, use_int8_adapters=False), False, False),
+        (LoraParams(adapter_rank=4, use_int8_adapters=False), True, False),
+        (LoraParams(adapter_rank=8, use_int8_adapters=True), True, True),
+    ),
 )
-def test_lora_adapters_in_the_graph(params, transpose_b):
+def test_lora_adapters_in_the_graph(params, transpose_a, transpose_b):
     advanced_parameters = CompressionParams() if params is None else CompressionParams(lora_correction_params=params)
-    model = LMLinearModel(transpose_b=transpose_b)
+    model = LMLinearModel(transpose_a=transpose_a, transpose_b=transpose_b)
     ov_model = model.ov_model
     dataset = Dataset(np.ones(inp.shape) for inp in ov_model.inputs)