Skip to content

Commit 2a9cccf

Browse files
intbfintbf
committed
block the fc horizontal fusion for weights that are not constants
1 parent b8a1c7e commit 2a9cccf

2 files changed

Lines changed: 134 additions & 1 deletion

File tree

‎src/plugins/intel_gpu/src/plugin/transformations/fc_horizontal_fusion.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,15 @@ FullyConnectedHorizontalFusion::FullyConnectedHorizontalFusion(bool fuse_mlp_swi
7070
const auto& fc_user = ov::as_type_ptr<op::FullyConnectedCompressed>(u);
7171
if (!fc_user)
7272
continue;
73+
74+
// Skip horizontal fusion when the weight is not a constant. The fused-weight Concat
75+
// created during fusion relies on constant-folding at compile time; with a non-constant
76+
// weight (e.g. weights provided as runtime inputs) it cannot fold, survives to program
77+
// build, and may hit formats/types without a Concat implementation. Even when a Concat
78+
// impl exists, concatenating weights on every inference is pure overhead with no benefit.
79+
if (!is_constant(fc_user->get_input_node_shared_ptr(1)))
80+
return false;
81+
7382
auto num_inputs = fc_user->inputs().size();
7483
if (num_inputs >= 5)
7584
nodes_with_zp++;

‎src/plugins/intel_gpu/tests/unit/transformations/horizontal_fc_fusion_test.cpp‎

Lines changed: 125 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "openvino/op/reshape.hpp"
2020
#include "openvino/op/add.hpp"
2121
#include "openvino/op/multiply.hpp"
22+
#include "openvino/op/convert.hpp"
2223
#include "openvino/pass/manager.hpp"
2324

2425
#include <transformations/utils/utils.hpp>
@@ -96,7 +97,130 @@ TEST_F(TransformationTestsF, FullyConnectedHorizontalFusion_no_bias_no_zp) {
9697
}
9798
}
9899

99-
TEST_F(TransformationTestsF, FullyConnectedHorizontalFusion_bias_zp) {
100+
TEST_F(TransformationTestsF, FullyConnectedHorizontalFusion_parameter_weights_no_fusion) {
101+
// Weights-as-inputs / share_weights rewrites weight Constants into Parameters. The fused-weight
102+
// Concat would be unfoldable and reach GPU program build (no i4/u4 concat kernel). Horizontal
103+
// fusion must NOT fire in this case, so model_ref is identical to model (3 separate FCs).
104+
std::vector<int64_t> pattern = {7, -1};
105+
{
106+
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{-1, 7, 4096});
107+
auto weight1 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{1024, 4096});
108+
weight1->set_friendly_name("weight1_1");
109+
auto weight2 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{512, 4096});
110+
weight2->set_friendly_name("weight1_2");
111+
auto weight3 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{128, 4096});
112+
weight3->set_friendly_name("weight1_3");
113+
auto bias1 = std::make_shared<ov::intel_gpu::op::Placeholder>();
114+
auto bias2 = std::make_shared<ov::intel_gpu::op::Placeholder>();
115+
auto bias3 = std::make_shared<ov::intel_gpu::op::Placeholder>();
116+
auto scale1 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{1024, 32});
117+
auto scale2 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{512, 32});
118+
auto scale3 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{128, 32});
119+
auto fc1 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight1, bias1, scale1);
120+
fc1->set_friendly_name("fc1");
121+
auto fc2 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight2, bias2, scale2);
122+
auto fc3 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight3, bias3, scale3);
123+
auto reshape_pattern = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{2}, pattern);
124+
auto reshape1 = std::make_shared<ov::op::v1::Reshape>(fc1, reshape_pattern, true);
125+
auto reshape2 = std::make_shared<ov::op::v1::Reshape>(fc2, reshape_pattern, true);
126+
auto reshape3 = std::make_shared<ov::op::v1::Reshape>(fc3, reshape_pattern, true);
127+
auto result1 = std::make_shared<ov::op::v0::Result>(reshape1);
128+
auto result2 = std::make_shared<ov::op::v0::Result>(reshape2);
129+
auto result3 = std::make_shared<ov::op::v0::Result>(reshape3);
130+
model = std::make_shared<ov::Model>(ov::ResultVector{result1, result2, result3}, ov::ParameterVector{input, weight1, weight2, weight3});
131+
manager.register_pass<FullyConnectedHorizontalFusion>();
132+
}
133+
{
134+
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{-1, 7, 4096});
135+
auto weight1 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{1024, 4096});
136+
weight1->set_friendly_name("weight1_1");
137+
auto weight2 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{512, 4096});
138+
weight2->set_friendly_name("weight1_2");
139+
auto weight3 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{128, 4096});
140+
weight3->set_friendly_name("weight1_3");
141+
auto bias1 = std::make_shared<ov::intel_gpu::op::Placeholder>();
142+
auto bias2 = std::make_shared<ov::intel_gpu::op::Placeholder>();
143+
auto bias3 = std::make_shared<ov::intel_gpu::op::Placeholder>();
144+
auto scale1 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{1024, 32});
145+
auto scale2 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{512, 32});
146+
auto scale3 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{128, 32});
147+
auto fc1 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight1, bias1, scale1);
148+
fc1->set_friendly_name("fc1");
149+
auto fc2 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight2, bias2, scale2);
150+
auto fc3 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight3, bias3, scale3);
151+
auto reshape_pattern = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{2}, pattern);
152+
auto reshape1 = std::make_shared<ov::op::v1::Reshape>(fc1, reshape_pattern, true);
153+
auto reshape2 = std::make_shared<ov::op::v1::Reshape>(fc2, reshape_pattern, true);
154+
auto reshape3 = std::make_shared<ov::op::v1::Reshape>(fc3, reshape_pattern, true);
155+
auto result1 = std::make_shared<ov::op::v0::Result>(reshape1);
156+
auto result2 = std::make_shared<ov::op::v0::Result>(reshape2);
157+
auto result3 = std::make_shared<ov::op::v0::Result>(reshape3);
158+
model_ref = std::make_shared<ov::Model>(ov::ResultVector{result1, result2, result3}, ov::ParameterVector{input, weight1, weight2, weight3});
159+
comparator.enable(FunctionsComparator::ATTRIBUTES);
160+
}
161+
}
162+
163+
TEST_F(TransformationTestsF, FullyConnectedHorizontalFusion_convert_parameter_weights_no_fusion) {
164+
// Convert(Parameter) weight form must also block horizontal fusion (is_constant returns false
165+
// because the Convert input is a Parameter, not a Constant).
166+
std::vector<int64_t> pattern = {7, -1};
167+
{
168+
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{-1, 7, 4096});
169+
auto weight1_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{1024, 4096});
170+
auto weight2_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{512, 4096});
171+
auto weight3_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{128, 4096});
172+
auto weight1 = std::make_shared<ov::op::v0::Convert>(weight1_p, ov::element::f16);
173+
auto weight2 = std::make_shared<ov::op::v0::Convert>(weight2_p, ov::element::f16);
174+
auto weight3 = std::make_shared<ov::op::v0::Convert>(weight3_p, ov::element::f16);
175+
auto bias1 = std::make_shared<ov::intel_gpu::op::Placeholder>();
176+
auto bias2 = std::make_shared<ov::intel_gpu::op::Placeholder>();
177+
auto bias3 = std::make_shared<ov::intel_gpu::op::Placeholder>();
178+
auto scale1 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{1024, 32});
179+
auto scale2 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{512, 32});
180+
auto scale3 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{128, 32});
181+
auto fc1 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight1, bias1, scale1);
182+
auto fc2 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight2, bias2, scale2);
183+
auto fc3 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight3, bias3, scale3);
184+
auto reshape_pattern = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{2}, pattern);
185+
auto reshape1 = std::make_shared<ov::op::v1::Reshape>(fc1, reshape_pattern, true);
186+
auto reshape2 = std::make_shared<ov::op::v1::Reshape>(fc2, reshape_pattern, true);
187+
auto reshape3 = std::make_shared<ov::op::v1::Reshape>(fc3, reshape_pattern, true);
188+
auto result1 = std::make_shared<ov::op::v0::Result>(reshape1);
189+
auto result2 = std::make_shared<ov::op::v0::Result>(reshape2);
190+
auto result3 = std::make_shared<ov::op::v0::Result>(reshape3);
191+
model = std::make_shared<ov::Model>(ov::ResultVector{result1, result2, result3}, ov::ParameterVector{input, weight1_p, weight2_p, weight3_p});
192+
manager.register_pass<FullyConnectedHorizontalFusion>();
193+
}
194+
{
195+
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{-1, 7, 4096});
196+
auto weight1_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{1024, 4096});
197+
auto weight2_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{512, 4096});
198+
auto weight3_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{128, 4096});
199+
auto weight1 = std::make_shared<ov::op::v0::Convert>(weight1_p, ov::element::f16);
200+
auto weight2 = std::make_shared<ov::op::v0::Convert>(weight2_p, ov::element::f16);
201+
auto weight3 = std::make_shared<ov::op::v0::Convert>(weight3_p, ov::element::f16);
202+
auto bias1 = std::make_shared<ov::intel_gpu::op::Placeholder>();
203+
auto bias2 = std::make_shared<ov::intel_gpu::op::Placeholder>();
204+
auto bias3 = std::make_shared<ov::intel_gpu::op::Placeholder>();
205+
auto scale1 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{1024, 32});
206+
auto scale2 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{512, 32});
207+
auto scale3 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{128, 32});
208+
auto fc1 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight1, bias1, scale1);
209+
auto fc2 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight2, bias2, scale2);
210+
auto fc3 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight3, bias3, scale3);
211+
auto reshape_pattern = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{2}, pattern);
212+
auto reshape1 = std::make_shared<ov::op::v1::Reshape>(fc1, reshape_pattern, true);
213+
auto reshape2 = std::make_shared<ov::op::v1::Reshape>(fc2, reshape_pattern, true);
214+
auto reshape3 = std::make_shared<ov::op::v1::Reshape>(fc3, reshape_pattern, true);
215+
auto result1 = std::make_shared<ov::op::v0::Result>(reshape1);
216+
auto result2 = std::make_shared<ov::op::v0::Result>(reshape2);
217+
auto result3 = std::make_shared<ov::op::v0::Result>(reshape3);
218+
model_ref = std::make_shared<ov::Model>(ov::ResultVector{result1, result2, result3}, ov::ParameterVector{input, weight1_p, weight2_p, weight3_p});
219+
comparator.enable(FunctionsComparator::ATTRIBUTES);
220+
}
221+
}
222+
223+
TEST_F(TransformationTestsF, FullyConnectedHorizontalFusion_bias_no_zp) {
100224
std::vector<int64_t> pattern = {7, -1};
101225
{
102226
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{-1, 7, 4096});

0 commit comments

Comments
 (0)