Skip to content

Commit 2744594

Browse files
mpaulitschmpaulitsch
committed
clang-format-18 -- formatting changes according to style template
1 parent fde2926 commit 2744594

6 files changed

Lines changed: 318 additions & 57 deletions

File tree

‎src/common/transformations/src/transformations/common_optimizations/glu_fusion.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ GLUFusion::GLUFusion() {
5050

5151
// Mul(Xw, Xv) = Swish(Xw) * Xv
5252
auto glu_m = std::make_shared<pattern::op::Or>(OutputVector{swish_m, gelu_m});
53-
auto mul_m = pattern::wrap_type<v1::Multiply>({glu_m, variadic_split_m->output(1)});
53+
auto mul_m = pattern::wrap_type<v1::Multiply>({glu_m, variadic_split_m->output(1)});
5454

5555
ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](pattern::Matcher& m) {
5656
const auto& pattern_map = m.get_pattern_value_map();

‎src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,31 @@ static bool should_skip_execution(const dynamic_quantize_node& node, const layou
2424
if (!node.is_runtime_skippable() || !act_layout.is_static())
2525
return false;
2626

27+
// The 2nd-token skip optimization assumes a single fully-connected consumer that
28+
// takes all dynamic-quantize outputs via duplicated connections
29+
// (get_users().size() == get_outputs_count()). A dynamic-quantize that is shared
30+
// across several fully-connected nodes (e.g. VAE mid-block attention to_q/to_k/to_v,
31+
// where SharedOpOptimization dedups the identical per-FC quantizers into one) has more
32+
// user connections than outputs. That pattern does not match the skip heuristic, so
33+
// execute dynamic quantization normally instead of asserting. This mismatch only
34+
// surfaces at runtime for dynamically-shaped activations, since statically-shaped
35+
// shared quantizers get their outputs reconciled at compile time.
36+
if (node.get_users().size() != node.get_outputs_count())
37+
return false;
38+
39+
// The 2nd-token skip optimization assumes a single fully-connected consumer that
40+
// takes all dynamic-quantize outputs via duplicated connections
41+
// (get_users().size() == get_outputs_count()). A dynamic-quantize that is shared
42+
// across several fully-connected nodes (e.g. VAE mid-block attention to_q/to_k/to_v,
43+
// where SharedOpOptimization dedups the identical per-FC quantizers into one) has more
44+
// user connections than outputs. That pattern does not match the skip heuristic, so
45+
// execute dynamic quantization normally instead of asserting. This mismatch only
46+
// surfaces at runtime for dynamically-shaped activations, since statically-shaped
47+
// shared quantizers get their outputs reconciled at compile time.
48+
if (node.get_users().size() != node.get_outputs_count())
49+
return false;
50+
2751
// Do not skip dynamic quantization if next node is not fully connected.(such as SDPA)
28-
OPENVINO_ASSERT(node.get_users().size() == node.get_outputs_count(),
29-
"Dynamic quantization is supposed to have only one user-node with duplicated connection: ",
30-
node.id());
3152
if (!(*node.get_users().begin())->is_type<fully_connected>())
3253
return false;
3354

‎src/plugins/intel_gpu/src/plugin/transformations/disable_fp16_comp_flux2_rope.cpp‎

Lines changed: 54 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,15 @@
55
#include "disable_fp16_comp_flux2_rope.hpp"
66

77
#include "openvino/core/graph_util.hpp"
8+
#include "openvino/op/add.hpp"
89
#include "openvino/op/concat.hpp"
9-
#include "openvino/op/cos.hpp"
10-
#include "openvino/op/matmul.hpp"
11-
#include "openvino/op/sin.hpp"
12-
#include "openvino/op/transpose.hpp"
10+
#include "openvino/op/multiply.hpp"
11+
#include "openvino/op/reshape.hpp"
12+
#include "openvino/op/split.hpp"
13+
#include "openvino/op/squeeze.hpp"
14+
#include "openvino/op/unsqueeze.hpp"
1315
#include "openvino/op/util/shape_of_base.hpp"
16+
#include "openvino/pass/pattern/op/optional.hpp"
1417
#include "openvino/pass/pattern/op/wrap_type.hpp"
1518
#include "transformations/rt_info/disable_precision_conversion.hpp"
1619
#include "transformations/utils/utils.hpp"
@@ -23,9 +26,7 @@ namespace ov::intel_gpu {
2326

2427
namespace {
2528

26-
void mark_path_from_output(ov::Node* root,
27-
std::unordered_set<ov::Node*>& visited,
28-
const std::function<bool(ov::Node*)>& skip_node_predicate) {
29+
void mark_path_from_output(ov::Node* root, std::unordered_set<ov::Node*>& visited, const std::function<bool(ov::Node*)>& skip_node_predicate) {
2930
if (!root || visited.count(root))
3031
return;
3132
auto visit_func = [](ov::Node* node) {
@@ -36,8 +37,7 @@ void mark_path_from_output(ov::Node* root,
3637

3738
auto skip_node_predicate() {
3839
return [](ov::Node* node) -> bool {
39-
return ov::is_type<v0::Constant>(node) || ov::is_type<v0::Parameter>(node) ||
40-
ov::is_type<op_util::ShapeOfBase>(node);
40+
return ov::is_type<v0::Constant>(node) || ov::is_type<v0::Parameter>(node) || ov::is_type<op_util::ShapeOfBase>(node);
4141
};
4242
}
4343

@@ -46,40 +46,53 @@ auto skip_node_predicate() {
4646
DisableFP16CompFlux2RoPEPattern::DisableFP16CompFlux2RoPEPattern() {
4747
using namespace ov::pass::pattern;
4848

49-
// FLUX.2 pos_embed frequency tables: outer-product MatMul -> Cos / Sin.
50-
auto outer_matmul = wrap_type<v0::MatMul>({any_input(), any_input()});
51-
auto pos_cos = wrap_type<v0::Cos>({outer_matmul});
52-
auto pos_sin = wrap_type<v0::Sin>({outer_matmul});
53-
54-
// rotary_emb / LLM-style tables: Concat -> Cos or Sin (e.g. text encoder rotary_emb).
55-
auto rot_concat = wrap_type<v0::Concat>({any_input(), any_input()});
56-
auto table_cos = wrap_type<v0::Cos>({rot_concat});
57-
auto table_sin = wrap_type<v0::Sin>({rot_concat});
58-
59-
// MatMul -> Transpose -> Concat -> Cos/Sin (MarkRopeInputsToKeepInMixedPrecision reference pattern).
60-
auto table_matmul = wrap_type<v0::MatMul>({any_input(), any_input()});
61-
auto table_transpose = wrap_type<v1::Transpose>({table_matmul, any_input()});
62-
auto table_concat = wrap_type<v0::Concat>({table_transpose, table_transpose}, {{"axis", -1}});
63-
auto ref_cos = wrap_type<v0::Cos>({table_concat});
64-
auto ref_sin = wrap_type<v0::Sin>({table_concat});
65-
66-
auto register_mark_root = [this](const std::shared_ptr<ov::Node>& pattern_root, const char* name) {
67-
ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](Matcher& m) {
68-
if (transformation_callback(m.get_match_root()))
69-
return false;
70-
const auto skip_pred = skip_node_predicate();
71-
mark_path_from_output(m.get_pattern_value_map().at(pattern_root).get_node(), m_visited, skip_pred);
72-
return true;
73-
};
74-
register_matcher(std::make_shared<Matcher>(pattern_root, name), callback);
49+
// This pass runs *before* RoPEFusion, so the RoPE is still decomposed and the
50+
// fused ov::op::internal::RoPE op does not exist yet. We therefore anchor on the
51+
// decomposed application graph, mirroring ov::pass::RoPEFusionFlux but without the
52+
// symbolic shape constraints (there is no symbolic inference context here):
53+
//
54+
// x1 = Reshape(x)
55+
// x1_0,x1_1 = Split(x1, axis=-1, num_splits=2)
56+
// x2 = Concat(-x1_1, x1_0, axis=-1) // rotate_half(x)
57+
// x3 = Reshape(x2)
58+
// y = x * t_cos + x3 * t_sin // <-- match root
59+
//
60+
// Only t_cos / t_sin captured from this structure are genuine rotary tables, so we
61+
// walk backward from exactly those two tensors. This avoids pulling unrelated
62+
// MatMul->Cos / Concat->Cos subgraphs into FP32.
63+
auto x = any_input();
64+
auto t_cos = any_input();
65+
auto t_sin = any_input();
66+
67+
auto x1 = wrap_type<v1::Reshape>({x, any_input()});
68+
auto split = wrap_type<v1::Split>({x1, -1}, {{"num_splits", 2}});
69+
split->set_output_size(2);
70+
71+
// The negation of one split half is optionally wrapped in Squeeze/Unsqueeze
72+
// depending on which transformations ran before this pass (see RoPEFusionFlux).
73+
auto opt_squeeze = optional<v0::Squeeze>({split->output(1), -1});
74+
auto x1_1_neg = wrap_type<v1::Multiply>({opt_squeeze, -1}, {{"auto_broadcast", "numpy"}});
75+
auto opt_squeeze_1 = optional<v0::Squeeze>({x1_1_neg, -1});
76+
auto opt_unsqueeze = optional<v0::Unsqueeze>({opt_squeeze_1, -1});
77+
78+
auto x2 = wrap_type<v0::Concat>({opt_unsqueeze, split->output(0)}, {{"axis", -1}});
79+
auto x3 = wrap_type<v1::Reshape>({x2, any_input()});
80+
81+
auto y1 = wrap_type<v1::Multiply>({x, t_cos}, {{"auto_broadcast", "numpy"}});
82+
auto y2 = wrap_type<v1::Multiply>({x3, t_sin}, {{"auto_broadcast", "numpy"}});
83+
auto result = wrap_type<v1::Add>({y1, y2}, {{"auto_broadcast", "numpy"}});
84+
85+
ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](Matcher& m) {
86+
if (transformation_callback(m.get_match_root()))
87+
return false;
88+
const auto& pattern_map = m.get_pattern_value_map();
89+
const auto skip_pred = skip_node_predicate();
90+
mark_path_from_output(pattern_map.at(t_cos).get_node(), m_visited, skip_pred);
91+
mark_path_from_output(pattern_map.at(t_sin).get_node(), m_visited, skip_pred);
92+
return false;
7593
};
7694

77-
register_mark_root(pos_cos, "DisableFP16CompFlux2RoPEPattern_PosCos");
78-
register_mark_root(pos_sin, "DisableFP16CompFlux2RoPEPattern_PosSin");
79-
register_mark_root(table_cos, "DisableFP16CompFlux2RoPEPattern_TableCos");
80-
register_mark_root(table_sin, "DisableFP16CompFlux2RoPEPattern_TableSin");
81-
register_mark_root(ref_cos, "DisableFP16CompFlux2RoPEPattern_RefCos");
82-
register_mark_root(ref_sin, "DisableFP16CompFlux2RoPEPattern_RefSin");
95+
register_matcher(std::make_shared<Matcher>(result, "DisableFP16CompFlux2RoPEPattern"), callback);
8396
}
8497

8598
} // namespace ov::intel_gpu

‎src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@
77
#include <algorithm>
88
#include <cctype>
99
#include <cmath>
10-
#include <cstdlib>
11-
#include <cstring>
1210
#include <deque>
1311
#include <limits>
1412
#include <memory>
@@ -97,8 +95,8 @@
9795
#include "plugin/transformations/convert_matmul_to_fc.hpp"
9896
#include "plugin/transformations/convert_stridedslices_to_variadicsplit.hpp"
9997
#include "plugin/transformations/decompose_reduce_scalar_output.hpp"
100-
#include "plugin/transformations/disable_fp16_comp_flux2_rope.hpp"
10198
#include "plugin/transformations/disable_fp16_comp_cumsum_sin_gen.hpp"
99+
#include "plugin/transformations/disable_fp16_comp_flux2_rope.hpp"
102100
#include "plugin/transformations/disable_fp16_comp_rms.hpp"
103101
#include "plugin/transformations/disable_fp16_comp_sin_gen.hpp"
104102
#include "plugin/transformations/dynamic_quantize_fully_connected.hpp"
@@ -237,12 +235,6 @@ static typename std::enable_if<std::is_integral<T>::value, T>::type align_to(T s
237235
return static_cast<T>((size % align == 0) ? size : size - size % align + align);
238236
}
239237

240-
// Set OV_GPU_FLUX2_ROPE_FP16_MARK=0 to skip pre-ConvertPrecision FLUX.2 RoPE FP32 marking (A/B testing).
241-
static bool flux2_rope_fp16_mark_enabled() {
242-
const char* env = std::getenv("OV_GPU_FLUX2_ROPE_FP16_MARK");
243-
return env == nullptr || env[0] == '\0' || std::strcmp(env, "0") != 0;
244-
}
245-
246238
static bool is_decompression_multiply(const std::shared_ptr<const ov::Node> node, bool supports_immad) {
247239
std::vector<ov::DiscreteTypeInfo> target_consumers = {ov::opset1::MatMul::get_type_info_static(),
248240
ov::op::internal::MOE::get_type_info_static(),
@@ -730,9 +722,7 @@ void TransformationsPipeline::apply(std::shared_ptr<ov::Model> func) {
730722
return ov::is_conversion_disabled(node, ov::element::f16);
731723
});
732724
manager.register_pass<DisableFP16ComSinGenPatternForHiFiGAN>();
733-
if (flux2_rope_fp16_mark_enabled()) {
734-
manager.register_pass<DisableFP16CompFlux2RoPEPattern>();
735-
}
725+
manager.register_pass<DisableFP16CompFlux2RoPEPattern>();
736726
const bool keep_precision_sensitive_in_fp32_1 = true;
737727
const bool convert_input_output_precision = false;
738728
const bool store_original_precision_as_rt_attribute = true;

‎src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -541,4 +541,93 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_input_4d,
541541
::testing::Values(0),
542542
::testing::Values(1.0f)),
543543
MatmulWeightsDecompression::get_test_case_name);
544+
545+
/*
546+
* Regression for the intel_gpu DynamicQuantize fanout assert (dynamic_quantize.cpp).
547+
*
548+
* A single activation feeding several FullyConnected nodes (e.g. VAE mid-block
549+
* self-attention to_q / to_k / to_v in FLUX.2-klein) yields, after
550+
* DynamicQuantizeFullyConnected + SharedOpOptimization, one shared DynamicQuantize
551+
* with multiple FC users (get_users().size() > get_outputs_count()). For a
552+
* dynamically-shaped activation this used to trip
553+
* OPENVINO_ASSERT(get_users().size() == get_outputs_count()) at the first infer.
554+
* This verifies compile + inference succeeds with dynamic quantization enabled.
555+
*
556+
* Data (dynamic [-1,-1,512])
557+
* / | \
558+
* MatMul_q MatMul_k MatMul_v (each with decompressed int8 weights)
559+
*
560+
* The three projections use distinct output channel counts so GPU horizontal FC
561+
* fusion keeps them separate, guaranteeing the shared DynamicQuantize retains
562+
* multiple FC users (which is what triggered the assert).
563+
*/
564+
class MatmulSharedDynQuantMultipleFC : public testing::WithParamInterface<uint64_t>,
565+
virtual public ov::test::SubgraphBaseTest {
566+
public:
567+
static std::string get_test_case_name(testing::TestParamInfo<uint64_t> obj) {
568+
std::ostringstream result;
569+
result << "dyn_quan_group_size=" << obj.param;
570+
return result.str();
571+
}
572+
573+
protected:
574+
std::shared_ptr<ov::Node> make_compressed_weights(size_t input_channels,
575+
size_t output_channels,
576+
const ov::element::Type& data_precision) {
577+
auto weights_tensor = ov::test::utils::create_and_fill_tensor(ov::element::u8, ov::Shape{input_channels, output_channels});
578+
auto weights = std::make_shared<ov::op::v0::Constant>(weights_tensor);
579+
weights->set_friendly_name("Compressed_weights");
580+
auto weights_convert = std::make_shared<ov::op::v0::Convert>(weights, data_precision);
581+
582+
ov::test::utils::InputGenerateData in_data;
583+
in_data.start_from = -0.5;
584+
in_data.range = 1;
585+
in_data.resolution = 30000;
586+
auto scale_tensor = ov::test::utils::create_and_fill_tensor(data_precision, ov::Shape{1, output_channels}, in_data);
587+
for (size_t i = 0; i < scale_tensor.get_size(); i++) {
588+
if (data_precision == ov::element::f16)
589+
scale_tensor.data<ov::float16>()[i] /= ov::float16(16.f);
590+
else if (data_precision == ov::element::f32)
591+
scale_tensor.data<float>()[i] /= 16.f;
592+
}
593+
auto scale_const = std::make_shared<ov::op::v0::Constant>(scale_tensor);
594+
return std::make_shared<ov::op::v1::Multiply>(weights_convert, scale_const);
595+
}
596+
597+
void SetUp() override {
598+
targetDevice = ov::test::utils::DEVICE_GPU;
599+
const uint64_t dyn_quan_group_size = GetParam();
600+
const size_t hidden = 512;
601+
602+
// Dynamic 3D activation, matching the VAE mid-block attention input [-1, -1, 512].
603+
InputShape data_shape = {{-1, -1, static_cast<ov::Dimension::value_type>(hidden)},
604+
{{1, 64, hidden}, {1, 256, hidden}}};
605+
init_input_shapes({data_shape});
606+
inType = outType = ov::element::f16;
607+
608+
auto param = std::make_shared<ov::op::v0::Parameter>(ov::element::f16, inputDynamicShapes[0]);
609+
// Distinct output sizes for to_q / to_k / to_v so horizontal FC fusion keeps them
610+
// separate, while they still share one DynamicQuantize on the common activation.
611+
const std::vector<size_t> output_channels = {hidden, hidden - 128, hidden - 256};
612+
ov::OutputVector matmuls;
613+
for (auto out_ch : output_channels) {
614+
auto weights = make_compressed_weights(hidden, out_ch, ov::element::f16);
615+
matmuls.push_back(std::make_shared<ov::op::v0::MatMul>(param, weights));
616+
}
617+
function = std::make_shared<ov::Model>(matmuls, ov::ParameterVector{param}, "SharedDynQuantMultipleFC");
618+
619+
abs_threshold = 2.0f; // dynamic quantization accuracy tolerance (see other dyn_quan cases)
620+
configuration.insert({ov::hint::dynamic_quantization_group_size(dyn_quan_group_size)});
621+
}
622+
};
623+
624+
TEST_P(MatmulSharedDynQuantMultipleFC, Inference) {
625+
SKIP_IF_CURRENT_TEST_IS_DISABLED();
626+
run();
627+
}
628+
629+
INSTANTIATE_TEST_SUITE_P(smoke_MatMulSharedDynQuantMultipleFC,
630+
MatmulSharedDynQuantMultipleFC,
631+
::testing::Values<uint64_t>(32, 128, std::numeric_limits<uint64_t>::max()),
632+
MatmulSharedDynQuantMultipleFC::get_test_case_name);
544633
} // namespace

0 commit comments

Comments
 (0)