[QNN EP] Add Case-2 LPBQ pattern support for Gemm and Matmul nodes (microsoft#25865)

quic-tirupath · tirupath-qti · web-flow · commit c1da27c2d573 · 2026-01-09T09:57:14.000-08:00
### Description

 - Case-2 LPBQ pattern omits QuantizeLinear node in LPBQ packing pattern
- Modify LPBQ fusion logic in QNN EP implemented for Gemma and MatMul
nodes to gracefully handle the optional QuantizeLinear node in LPBQ
packing pattern.
- Add unit tests to verify Case-2 LPBQ pattern fusion for Gemm and
MatMul nodes.



### Motivation and Context
- QuantizeLinear node in LowPowerBlockQuantization encoding packing
pattern can be optional as it helps to keep the weights in INT datatype
and further helps to reduce the size of model.

---------

Co-authored-by: tirupath-qti &lt;tirupath@qti.qualcomm.com&gt;
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_node_group/lpbqgemm_fusion.cc b/onnxruntime/core/providers/qnn/builder/qnn_node_group/lpbqgemm_fusion.cc
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_node_group/lpbqgemm_fusion.h b/onnxruntime/core/providers/qnn/builder/qnn_node_group/lpbqgemm_fusion.h
@@ -24,7 +24,7 @@ class QnnModelWrapper;
 class LowPowerBlockQuantizedGemmFusion : public IQnnNodeGroup {
  public:
   LowPowerBlockQuantizedGemmFusion(const NodeUnit& Scale_DQL_node_unit,
-                                   const NodeUnit& W_QL_node_unit,
+                                   const NodeUnit* W_QL_node_unit,
                                    const NodeUnit& W_DQL_node_unit,
                                    const NodeUnit& Act_DQL_node_unit,
                                    const NodeUnit& Gemm_node_unit,
@@ -46,7 +46,8 @@ class LowPowerBlockQuantizedGemmFusion : public IQnnNodeGroup {
 
  private:
   std::array<const NodeUnit*, 6> node_units_;
+  std::vector<const NodeUnit*> filtered_node_units_;
 };
 
 }  // namespace qnn
-}  // namespace onnxruntime
+}  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_node_group/lpbqmatmul_fusion.cc b/onnxruntime/core/providers/qnn/builder/qnn_node_group/lpbqmatmul_fusion.cc
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_node_group/lpbqmatmul_fusion.h b/onnxruntime/core/providers/qnn/builder/qnn_node_group/lpbqmatmul_fusion.h
@@ -24,7 +24,7 @@ class QnnModelWrapper;
 class LowPowerBlockQuantizedMatMulFusion : public IQnnNodeGroup {
  public:
   LowPowerBlockQuantizedMatMulFusion(const NodeUnit& Scale_DQL_node_unit,
-                                     const NodeUnit& W_QL_node_unit,
+                                     const NodeUnit* W_QL_node_unit,
                                      const NodeUnit& MatMul_node_unit);
   ORT_DISALLOW_COPY_AND_ASSIGNMENT(LowPowerBlockQuantizedMatMulFusion);
 
@@ -43,7 +43,8 @@ class LowPowerBlockQuantizedMatMulFusion : public IQnnNodeGroup {
 
  private:
   std::array<const NodeUnit*, 3> node_units_;
+  std::vector<const NodeUnit*> filtered_node_units_;
 };
 
 }  // namespace qnn
-}  // namespace onnxruntime
+}  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_node_group/utils.cc b/onnxruntime/core/providers/qnn/builder/qnn_node_group/utils.cc
@@ -314,5 +314,62 @@ const NodeUnit* GetOnlyChildOfOutput(const GraphViewer& graph_viewer,
   return (child_count == 1) ? p_child_node_unit : nullptr;
 }
 
+const NodeUnit* GetParentOfInputByName(const GraphViewer& graph_viewer,
+                                       const NodeUnit& node_unit,
+                                       const std::string& input_name,
+                                       const std::unordered_map<const Node*, const NodeUnit*>& node_unit_map,
+                                       const std::unordered_map<const NodeUnit*, const IQnnNodeGroup*>& qnn_node_group_map) {
+  // Iterate through all nodes in the group
+  for (auto node : node_unit.GetAllNodesInGroup()) {
+    // Check if this node has the input we're looking for
+    bool has_input = std::any_of(node->InputDefs().begin(), node->InputDefs().end(),
+                                 [&](const NodeArg* node_input) {
+                                   return node_input->Name() == input_name;
+                                 });
+    if (!has_input) {
+      continue;
+    }
+
+    const Node& child_node = *node;
+
+    for (auto edge = child_node.InputEdgesBegin(); edge != child_node.InputEdgesEnd(); ++edge) {
+      const Node& parent_node = edge->GetNode();
+      if (parent_node.OutputDefs()[0]->Name() != input_name) {
+        continue;
+      }
+
+      if (graph_viewer.GetNode(parent_node.Index()) == nullptr) {
+        // Node is not in this GraphViewer
+        return nullptr;
+      }
+
+      if (graph_viewer.NodeProducesGraphOutput(parent_node)) {
+        // Node is producing a graph output
+        return nullptr;
+      }
+
+      const auto parent_node_unit_it = node_unit_map.find(&parent_node);
+      if (parent_node_unit_it == node_unit_map.end()) {
+        return nullptr;
+      }
+      const NodeUnit* p_parent_node_unit = parent_node_unit_it->second;
+
+      // Check if parent node has already been handled. Should not be the case if the calling
+      // fusion function has been called in topological order, but check to be safe.
+      if (qnn_node_group_map.count(p_parent_node_unit) != 0) {
+        return nullptr;
+      }
+
+      // parent must not already be part of a QDQ NodeUnit (i.e., be standalone).
+      if (p_parent_node_unit->UnitType() != NodeUnit::Type::SingleNode) {
+        return nullptr;
+      }
+
+      return p_parent_node_unit;
+    }
+  }
+  return nullptr;
+}
+
 }  // namespace qnn
 }  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_node_group/utils.h b/onnxruntime/core/providers/qnn/builder/qnn_node_group/utils.h
@@ -57,5 +57,11 @@ const NodeUnit* GetOnlyChildOfOutput(const GraphViewer& graph_viewer,
                                      const std::unordered_map<const Node*, const NodeUnit*>& node_unit_map,
                                      const std::unordered_map<const NodeUnit*, const IQnnNodeGroup*>& qnn_node_group_map);
 
+const NodeUnit* GetParentOfInputByName(const GraphViewer& graph_viewer,
+                                       const NodeUnit& node_unit,
+                                       const std::string& input_name,
+                                       const std::unordered_map<const Node*, const NodeUnit*>& node_unit_map,
+                                       const std::unordered_map<const NodeUnit*, const IQnnNodeGroup*>& qnn_node_group_map);
+
 }  // namespace qnn
 }  // namespace onnxruntime
diff --git a/onnxruntime/test/providers/qnn/qnn_node_group/lpbqgemm_fusion_without_ql_test.cc b/onnxruntime/test/providers/qnn/qnn_node_group/lpbqgemm_fusion_without_ql_test.cc
@@ -0,0 +1,135 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#if !defined(ORT_MINIMAL_BUILD)
+
+#include <string>
+#include <vector>
+#include <cmath>
+#include <optional>
+#include <utility>
+#include <array>
+#include <memory>
+#include <unordered_map>
+
+#include "core/graph/graph.h"
+#include "core/graph/node_attr_utils.h"
+#include "test/unittest_util/qdq_test_utils.h"
+#include "test/providers/qnn/qnn_test_utils.h"
+#include "gtest/gtest.h"
+
+namespace onnxruntime {
+namespace test {
+
+#if defined(__aarch64__) || defined(_M_ARM64)
+
+namespace {
+
+GetQDQTestCaseFn BuildLPBQGemmWithoutQLTestCase() {
+  return [](ModelTestBuilder& builder) -> void {
+    // Define the test case for LPBQGemm fusion without QuantizeLinear node
+    const int64_t input_channels = 16;
+    const int64_t output_channels = 16;
+    const int64_t blocks_per_axis = 4;
+    const std::vector<int64_t> input_shape{1, input_channels};
+    auto input_def = TestInputDef<float>(input_shape, false, -0.5f, 0.5f);
+    NodeArg* input = MakeTestInput<float>(builder, input_def);
+
+    // QuantizeLinear for Activation
+    NodeArg* act_ql_output = builder.MakeIntermediate();
+    NodeArg* act_ql_scale = builder.MakeScalarInitializer<float>(0.00005509183756657876f);
+    NodeArg* act_ql_zero_point = builder.MakeScalarInitializer<uint16_t>(23715);
+    builder.AddNode("QuantizeLinear", {input, act_ql_scale, act_ql_zero_point}, {act_ql_output});
+
+    // DequantizeLinear for Activation
+    NodeArg* act_dql_output = builder.MakeIntermediate();
+    NodeArg* act_dql_scale = builder.MakeScalarInitializer<float>(0.00005509183756657876f);
+    NodeArg* act_dql_zero_point = builder.MakeScalarInitializer<uint16_t>(23715);
+    builder.AddNode("DequantizeLinear", {act_ql_output, act_dql_scale, act_dql_zero_point}, {act_dql_output});
+
+    // DequantizeLinear for Scale
+    NodeArg* scale_dql_input = builder.MakeInitializer<uint8_t>({blocks_per_axis, output_channels}, 1, 15);
+    NodeArg* scale_dql_scale = builder.MakeInitializer<float>({output_channels}, 0.01f, 0.02f);
+    std::vector<uint8_t> dql_zero_points_data = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    NodeArg* scale_dql_zero_point = builder.Make1DInitializer<uint8_t>(dql_zero_points_data);
+    NodeArg* scale_dql_output = builder.MakeIntermediate();
+    Node& scale_dql = builder.AddNode("DequantizeLinear", {scale_dql_input, scale_dql_scale, scale_dql_zero_point}, {scale_dql_output});
+    scale_dql.AddAttribute("axis", static_cast<int64_t>(1));
+
+    // Create quantized weight directly (skip QuantizeLinear)
+    // We're creating a pre-quantized weight tensor that would normally be the output of QuantizeLinear
+    std::vector<Int4x2> quantized_weight_data;
+    size_t num_storage_elems = input_channels * output_channels;
+    quantized_weight_data.resize(Int4x2::CalcNumInt4Pairs(num_storage_elems));
+    for (size_t i = 0; i < num_storage_elems / 2; ++i) {
+      // Set some pattern in the quantized weights
+      quantized_weight_data[i].SetElem(0, i % 16);
+      quantized_weight_data[i].SetElem(1, (i + 8) % 16);
+    }
+    NodeArg* w_quantized = builder.MakeInitializer<Int4x2>({input_channels, output_channels}, quantized_weight_data);
+
+    // DequantizeLinear for Weight (directly using the quantized weight)
+    std::vector<Int4x2> zero_points_data;
+    size_t num_zp_elems = blocks_per_axis * output_channels;
+    zero_points_data.resize(Int4x2::CalcNumInt4Pairs(num_zp_elems));
+    for (size_t i = 0; i < num_zp_elems; ++i) {
+      size_t r = i >> 1;
+      size_t c = i & 0x1;
+      zero_points_data[r].SetElem(c, 0);
+    }
+    NodeArg* w_dql_zero_point = builder.MakeInitializer<Int4x2>({blocks_per_axis, output_channels}, zero_points_data);
+    NodeArg* w_dql_output = builder.MakeIntermediate();
+    Node& w_dql = builder.AddNode("DequantizeLinear", {w_quantized, scale_dql_output, w_dql_zero_point}, {w_dql_output});
+    w_dql.AddAttribute("axis", static_cast<int64_t>(0));
+    w_dql.AddAttribute("block_size", static_cast<int64_t>(4));
+
+    // Gemm
+    NodeArg* gemm_bias = builder.MakeInitializer<float>({output_channels}, -1.0f, 1.0f);
+    NodeArg* gemm_output = builder.MakeIntermediate();
+    builder.AddNode("Gemm", {act_dql_output, w_dql_output, gemm_bias}, {gemm_output});
+
+    // QuantizeLinear for Output
+    NodeArg* output_ql_scale = builder.MakeScalarInitializer<float>(0.00019595865160226822f);
+    NodeArg* output_ql_zero_point = builder.MakeScalarInitializer<uint16_t>(31693);
+    NodeArg* output_ql_output = builder.MakeIntermediate();
+    builder.AddNode("QuantizeLinear", {gemm_output, output_ql_scale, output_ql_zero_point}, {output_ql_output});
+
+    // DequantizeLinear for Output
+    NodeArg* output_dql_scale = builder.MakeScalarInitializer<float>(0.00019595865160226822f);
+    NodeArg* output_dql_zero_point = builder.MakeScalarInitializer<uint16_t>(31693);
+    NodeArg* output_dql_output = builder.MakeOutput();
+    builder.AddNode("DequantizeLinear", {output_ql_output, output_dql_scale, output_dql_zero_point}, {output_dql_output});
+  };
+}
+
+ProviderOptions GetProviderOptions() {
+  ProviderOptions provider_options;
+  provider_options["backend_type"] = "htp";
+  provider_options["offload_graph_io_quantization"] = "0";
+  return provider_options;
+}
+
+}  // namespace
+
+#if defined(_WIN32)
+// Graph fails to compose on ARM64 Windows since QNN 2.37.0
+TEST_F(QnnHTPBackendTests, DISABLED_LPBQGemmFusionWithoutQL) {
+#else
+TEST_F(QnnHTPBackendTests, LPBQGemmFusionWithoutQL) {
+#endif
+  ProviderOptions provider_options = GetProviderOptions();
+  RunQnnModelTest(BuildLPBQGemmWithoutQLTestCase(),
+                  provider_options,
+                  /*opset_version=*/21,
+                  /*expected_ep_assignment=*/ExpectedEPNodeAssignment::Some,
+                  /*fp32_abs_err=*/1e-2f,
+                  /*log_severity =*/logging::Severity::kERROR,
+                  /*verify_outputs=*/false);
+}
+
+#endif  // defined(__aarch64__) || defined(_M_ARM64)
+
+}  // namespace test
+}  // namespace onnxruntime
+
+#endif  // !defined(ORT_MINIMAL_BUILD)
diff --git a/onnxruntime/test/providers/qnn/qnn_node_group/lpbqmatmul_fusion_without_ql_test.cc b/onnxruntime/test/providers/qnn/qnn_node_group/lpbqmatmul_fusion_without_ql_test.cc
@@ -0,0 +1,134 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#if !defined(ORT_MINIMAL_BUILD)
+
+#include <string>
+#include <vector>
+#include <cmath>
+#include <optional>
+#include <utility>
+#include <array>
+#include <memory>
+#include <unordered_map>
+
+#include "core/graph/graph.h"
+#include "core/graph/node_attr_utils.h"
+#include "test/unittest_util/qdq_test_utils.h"
+#include "test/providers/qnn/qnn_test_utils.h"
+#include "gtest/gtest.h"
+
+namespace onnxruntime {
+namespace test {
+
+#if defined(__aarch64__) || defined(_M_ARM64)
+
+namespace {
+
+GetQDQTestCaseFn BuildLPBQMatMulWithoutQLTestCase() {
+  return [](ModelTestBuilder& builder) -> void {
+    // Define the test case for LPBQMatMul fusion without QuantizeLinear node
+    const int64_t input_channels = 16;
+    const int64_t output_channels = 16;
+    const int64_t blocks_per_axis = 4;
+    const std::vector<int64_t> input_shape{1, input_channels};
+    auto input_def = TestInputDef<float>(input_shape, false, -0.5f, 0.5f);
+    NodeArg* input = MakeTestInput<float>(builder, input_def);
+
+    // QuantizeLinear for Activation
+    NodeArg* act_ql_output = builder.MakeIntermediate();
+    NodeArg* act_ql_scale = builder.MakeScalarInitializer<float>(0.00005509183756657876f);
+    NodeArg* act_ql_zero_point = builder.MakeScalarInitializer<uint16_t>(23715);
+    builder.AddNode("QuantizeLinear", {input, act_ql_scale, act_ql_zero_point}, {act_ql_output});
+
+    // DequantizeLinear for Activation
+    NodeArg* act_dql_output = builder.MakeIntermediate();
+    NodeArg* act_dql_scale = builder.MakeScalarInitializer<float>(0.00005509183756657876f);
+    NodeArg* act_dql_zero_point = builder.MakeScalarInitializer<uint16_t>(23715);
+    builder.AddNode("DequantizeLinear", {act_ql_output, act_dql_scale, act_dql_zero_point}, {act_dql_output});
+
+    // DequantizeLinear for Scale
+    NodeArg* scale_dql_input = builder.MakeInitializer<uint8_t>({blocks_per_axis, output_channels}, 1, 15);
+    NodeArg* scale_dql_scale = builder.MakeInitializer<float>({output_channels}, 0.01f, 0.02f);
+    std::vector<uint8_t> dql_zero_points_data = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    NodeArg* scale_dql_zero_point = builder.Make1DInitializer<uint8_t>(dql_zero_points_data);
+    NodeArg* scale_dql_output = builder.MakeIntermediate();
+    Node& scale_dql = builder.AddNode("DequantizeLinear", {scale_dql_input, scale_dql_scale, scale_dql_zero_point}, {scale_dql_output});
+    scale_dql.AddAttribute("axis", static_cast<int64_t>(1));
+
+    // Create quantized weight directly (skip QuantizeLinear)
+    // We're creating a pre-quantized weight tensor that would normally be the output of QuantizeLinear
+    std::vector<Int4x2> quantized_weight_data;
+    size_t num_storage_elems = input_channels * output_channels;
+    quantized_weight_data.resize(Int4x2::CalcNumInt4Pairs(num_storage_elems));
+    for (size_t i = 0; i < num_storage_elems / 2; ++i) {
+      // Set some pattern in the quantized weights
+      quantized_weight_data[i].SetElem(0, i % 16);
+      quantized_weight_data[i].SetElem(1, (i + 8) % 16);
+    }
+    NodeArg* w_quantized = builder.MakeInitializer<Int4x2>({input_channels, output_channels}, quantized_weight_data);
+
+    // DequantizeLinear for Weight
+    std::vector<Int4x2> zero_points_data;
+    size_t num_zp_elems = blocks_per_axis * output_channels;
+    zero_points_data.resize(Int4x2::CalcNumInt4Pairs(num_zp_elems));
+    for (size_t i = 0; i < num_zp_elems; ++i) {
+      size_t r = i >> 1;
+      size_t c = i & 0x1;
+      zero_points_data[r].SetElem(c, 0);
+    }
+    NodeArg* w_dql_zero_point = builder.MakeInitializer<Int4x2>({blocks_per_axis, output_channels}, zero_points_data);
+    NodeArg* w_dql_output = builder.MakeIntermediate();
+    Node& w_dql = builder.AddNode("DequantizeLinear", {w_quantized, scale_dql_output, w_dql_zero_point}, {w_dql_output});
+    w_dql.AddAttribute("axis", static_cast<int64_t>(0));
+    w_dql.AddAttribute("block_size", static_cast<int64_t>(4));
+
+    // MatMul
+    NodeArg* matmul_output = builder.MakeIntermediate();
+    builder.AddNode("MatMul", {act_dql_output, w_dql_output}, {matmul_output});
+
+    // QuantizeLinear for Output
+    NodeArg* output_ql_scale = builder.MakeScalarInitializer<float>(0.00019595865160226822f);
+    NodeArg* output_ql_zero_point = builder.MakeScalarInitializer<uint16_t>(31693);
+    NodeArg* output_ql_output = builder.MakeIntermediate();
+    builder.AddNode("QuantizeLinear", {matmul_output, output_ql_scale, output_ql_zero_point}, {output_ql_output});
+
+    // DequantizeLinear for Output
+    NodeArg* output_dql_scale = builder.MakeScalarInitializer<float>(0.00019595865160226822f);
+    NodeArg* output_dql_zero_point = builder.MakeScalarInitializer<uint16_t>(31693);
+    NodeArg* output_dql_output = builder.MakeOutput();
+    builder.AddNode("DequantizeLinear", {output_ql_output, output_dql_scale, output_dql_zero_point}, {output_dql_output});
+  };
+}
+
+ProviderOptions GetProviderOptions() {
+  ProviderOptions provider_options;
+  provider_options["backend_type"] = "htp";
+  provider_options["offload_graph_io_quantization"] = "0";
+  return provider_options;
+}
+
+}  // namespace
+
+#if defined(_WIN32)
+// Graph fails to compose on ARM64 Windows since QNN 2.37.0
+TEST_F(QnnHTPBackendTests, DISABLED_LPBQMatMulFusionWithoutQL) {
+#else
+TEST_F(QnnHTPBackendTests, LPBQMatMulFusionWithoutQL) {
+#endif
+  ProviderOptions provider_options = GetProviderOptions();
+  RunQnnModelTest(BuildLPBQMatMulWithoutQLTestCase(),
+                  provider_options,
+                  /*opset_version=*/21,
+                  /*expected_ep_assignment=*/ExpectedEPNodeAssignment::Some,
+                  /*fp32_abs_err=*/1e-2f,
+                  /*log_severity =*/logging::Severity::kERROR,
+                  /*verify_outputs=*/false);
+}
+
+#endif  // defined(__aarch64__) || defined(_M_ARM64)
+
+}  // namespace test
+}  // namespace onnxruntime
+
+#endif  // !defined(ORT_MINIMAL_BUILD)
