Skip to content

Commit 337f0f6

Browse files
[NPUW]MoE optimization - RoPE cache and unfold infer request. (#36887)
### Details: - Extend to support RoPE caching for GPT-OSS-20B-A3B and Qwen3-30B-A3B - Use unfold infer for DEVICE_ROUTED mode. ### Tickets: - *[EISW-222432](https://jira.devtools.intel.com/browse/EISW-222432)* - *[EISW-222430](https://jira.devtools.intel.com/browse/EISW-222430)* ### AI Assistance: - *AI assistance used: no / yes* - *If yes, summarize how AI was used and what human validation was performed (build/tests/manual checks).* --------- Signed-off-by: intelgaoxiong <xiong.gao@intel.com>
1 parent 3902b4a commit 337f0f6

4 files changed

Lines changed: 154 additions & 66 deletions

File tree

src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp

Lines changed: 28 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -502,7 +502,7 @@ void apply_moe_config(ov::AnyMap& stage_config,
502502
"DEVICE_ROUTED mode uses in-graph gather-based expert selection which is only "
503503
"optimized for GENERATE stage. Please use HOST_ROUTED or DENSE for PREFILL.");
504504
}
505-
stage_config["NPUW_UNFOLD_IREQS"] = "NO";
505+
stage_config["NPUW_UNFOLD_IREQS"] = "YES";
506506
} else if (moe_hint == ::intel_npu::npuw::llm::MoEHint::DENSE) {
507507
LOG_INFO("MoE config for " << stage_name << " stage: DENSE (all experts active)");
508508
// DENSE mode requires CPU-only device due to extremely long NPU compilation time and high resource consumption
@@ -736,14 +736,6 @@ ov::npuw::LLMCompiledModel::LLMCompiledModel(const std::shared_ptr<ov::Model>& m
736736
// Auto-detect MoE model by scanning for router/expert nodes
737737
const bool is_moe = is_moe_model(model);
738738
if (is_moe) {
739-
// Only apply MoE defaults if not explicitly set in external config
740-
if (npuw_llm_props.find("NPUW_LLM_SHARED_HEAD") == npuw_llm_props.end()) {
741-
m_cfg.update({{"NPUW_LLM_SHARED_HEAD", "NO"}});
742-
}
743-
if (npuw_llm_props.find("NPUW_LLM_GENERATE_HINT") == npuw_llm_props.end()) {
744-
m_cfg.update({{"NPUW_LLM_GENERATE_HINT", "BEST_PERF"}});
745-
}
746-
747739
// Enable DEVICE_ROUTED mode by default for MoE models on newer compiler versions, as it's more efficient than
748740
// HOST_ROUTED
749741
if (npuw_llm_props.find("NPUW_LLM_GENERATE_MOE_HINT") == npuw_llm_props.end() && npudesc->arch == "5010" &&
@@ -1069,32 +1061,6 @@ ov::npuw::LLMCompiledModel::LLMCompiledModel(const std::shared_ptr<ov::Model>& m
10691061
generate_config["NPUW_FALLBACK_EXEC"] = "NO";
10701062
}
10711063

1072-
if (is_moe) {
1073-
// Apply MoE configuration for prefill stage
1074-
const auto prefill_moe_hint = m_cfg.get<::intel_npu::NPUW_LLM_PREFILL_MOE_HINT>();
1075-
apply_moe_config(prefill_config, prefill_moe_hint, "PREFILL");
1076-
1077-
// Apply MoE configuration for generate stage
1078-
const auto generate_moe_hint = m_cfg.get<::intel_npu::NPUW_LLM_GENERATE_MOE_HINT>();
1079-
apply_moe_config(generate_config, generate_moe_hint, "GENERATE");
1080-
1081-
// Fold shape-compute chains (ShapeOf→Gather→Concat etc.) in the prefill model before
1082-
// online partitioning runs pattern matching (e.g. GPTOSSRouter). Must run after
1083-
// ReshapeToStatic has made all shapes static so that ShapeOf bounds are resolvable.
1084-
ov::npuw::patterns::util::FoldShapeComputeChain().run_on_model(prefill_model);
1085-
for (auto&& model_variant : generate_model_variants) {
1086-
ov::npuw::patterns::util::FoldShapeComputeChain().run_on_model(model_variant);
1087-
}
1088-
1089-
if (generate_moe_hint == ::intel_npu::npuw::llm::MoEHint::DEVICE_ROUTED) {
1090-
// Apply model transformations only to GENERATE stage (PREFILL doesn't support DEVICE_ROUTED
1091-
// transformations)
1092-
for (auto&& model_variant : generate_model_variants) {
1093-
ov::npuw::ApplyMoEDeviceRoutedTransforms().run_on_model(model_variant);
1094-
}
1095-
}
1096-
}
1097-
10981064
if (m_is_whisper) {
10991065
update_config_for_whisper(prefill_config);
11001066
if (is_int8_compressed(model)) {
@@ -1140,6 +1106,33 @@ ov::npuw::LLMCompiledModel::LLMCompiledModel(const std::shared_ptr<ov::Model>& m
11401106
}
11411107
}
11421108
}
1109+
1110+
if (is_moe) {
1111+
// Apply MoE configuration for prefill stage
1112+
const auto prefill_moe_hint = m_cfg.get<::intel_npu::NPUW_LLM_PREFILL_MOE_HINT>();
1113+
apply_moe_config(prefill_config, prefill_moe_hint, "PREFILL");
1114+
1115+
// Apply MoE configuration for generate stage
1116+
const auto generate_moe_hint = m_cfg.get<::intel_npu::NPUW_LLM_GENERATE_MOE_HINT>();
1117+
apply_moe_config(generate_config, generate_moe_hint, "GENERATE");
1118+
1119+
// Fold shape-compute chains (ShapeOf→Gather→Concat etc.) in the prefill model before
1120+
// online partitioning runs pattern matching (e.g. GPTOSSRouter). Must run after
1121+
// ReshapeToStatic has made all shapes static so that ShapeOf bounds are resolvable.
1122+
ov::npuw::patterns::util::FoldShapeComputeChain().run_on_model(prefill_model);
1123+
for (auto&& model_variant : generate_model_variants) {
1124+
ov::npuw::patterns::util::FoldShapeComputeChain().run_on_model(model_variant);
1125+
}
1126+
1127+
if (generate_moe_hint == ::intel_npu::npuw::llm::MoEHint::DEVICE_ROUTED) {
1128+
// Apply model transformations only to GENERATE stage (PREFILL doesn't support DEVICE_ROUTED
1129+
// transformations)
1130+
for (auto&& model_variant : generate_model_variants) {
1131+
ov::npuw::ApplyMoEDeviceRoutedTransforms().run_on_model(model_variant);
1132+
}
1133+
}
1134+
}
1135+
11431136
// Regularize models for the better partitioning assuming it is a transformer
11441137
// Apply these transformations to all variant models
11451138
{

src/plugins/intel_npu/src/plugin/npuw/partitioning/patterns/pre_compute.cpp

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,16 @@ namespace pre_compute = ov::npuw::patterns::pre_compute;
1717

1818
namespace {
1919
// TODO: copied from common tests
20+
// Builds a [1, max_position_embeddings, rotary_ndims] sin/cos LUT.
21+
// When duplicate=true (LLama2-style rotate_half), rotary_ndims = inv_freq_size*2
22+
// and the second half mirrors the first (torch.cat([freqs, freqs], dim=-1)).
23+
// When duplicate=false (GPT-style), rotary_ndims = inv_freq_size with no mirroring.
2024
static ov::OutputVector makeCosSinCache(const size_t max_position_embeddings,
21-
const std::shared_ptr<ov::Node> inverse_frequencies) {
25+
const std::shared_ptr<ov::Node> inverse_frequencies,
26+
bool duplicate = true) {
2227
const auto inverse_freq_fp32 = ov::as_type_ptr<ov::op::v0::Constant>(inverse_frequencies)->cast_vector<float>();
23-
const size_t rotary_ndims = ov::shape_size(inverse_frequencies->get_shape()) * 2;
28+
const size_t inv_freq_size = ov::shape_size(inverse_frequencies->get_shape());
29+
const size_t rotary_ndims = duplicate ? inv_freq_size * 2 : inv_freq_size;
2430

2531
std::vector<ov::float16> lut_sin(max_position_embeddings * rotary_ndims, 0.0f);
2632
std::vector<ov::float16> lut_cos(max_position_embeddings * rotary_ndims, 0.0f);
@@ -29,18 +35,17 @@ static ov::OutputVector makeCosSinCache(const size_t max_position_embeddings,
2935
// y1 = cos(m*xita_i) * x1 - sin(m*xita_i) * x2
3036
// y2 = cos(m*xita_i) * x2 + sin(m*xita_i) * x1
3137
//
32-
for (size_t i = 0, k = 0; i < rotary_ndims; i += 2, k++) {
33-
auto xita_i = inverse_freq_fp32[i >> 1];
38+
for (size_t k = 0; k < inv_freq_size; k++) {
39+
auto xita_i = inverse_freq_fp32[k];
3440
ov::float16* psin = lut_sin.data();
3541
ov::float16* pcos = lut_cos.data();
3642
for (size_t m = 0; m < max_position_embeddings; m++, psin += rotary_ndims, pcos += rotary_ndims) {
37-
auto vsin = std::sin(xita_i * m);
38-
auto vcos = std::cos(xita_i * m);
39-
pcos[k] = ov::float16{vcos};
40-
pcos[k + rotary_ndims / 2] = pcos[k];
41-
42-
psin[k] = ov::float16{vsin};
43-
psin[k + rotary_ndims / 2] = psin[k];
43+
pcos[k] = ov::float16{std::cos(xita_i * static_cast<float>(m))};
44+
psin[k] = ov::float16{std::sin(xita_i * static_cast<float>(m))};
45+
if (duplicate) {
46+
pcos[k + inv_freq_size] = pcos[k];
47+
psin[k + inv_freq_size] = psin[k];
48+
}
4449
}
4550
}
4651

@@ -120,6 +125,7 @@ void replaceSinCosByCache(int max_prompt_len, const ov::OutputVector& cache, con
120125
auto gather_input_to_concat = rpe->matched_concat->input(0);
121126
gather_input_to_concat.get_source_output().remove_target_input(gather_input_to_concat);
122127
}
128+
123129
} // namespace
124130

125131
ov::npuw::patterns::pre_compute::RopePatternLLama2::RopePatternLLama2() : matcher("sin-cos-matcher") {
@@ -135,7 +141,9 @@ ov::npuw::patterns::pre_compute::RopePatternLLama2::RopePatternLLama2() : matche
135141
auto convert = opp::wrap_type<ov::op::v0::Convert>({unsqueeze});
136142
auto matmul = opp::wrap_type<ov::op::v0::MatMul>({broadcast, convert});
137143
auto transpose = opp::wrap_type<ov::op::v1::Transpose>({matmul, opp::any_input()});
138-
auto concat_2 = opp::wrap_type<ov::op::v0::Concat>({transpose, opp::any_input()});
144+
// Optional Concat between Transpose and Sin/Cos: present in LLama2-style RoPE
145+
// (torch.cat([freqs, freqs], dim=-1)), absent in GPT-style RoPE (inv_freq shape [1, n, 1]).
146+
auto concat_2 = opp::optional<ov::op::v0::Concat>({transpose->output(0), opp::any_input()});
139147
auto output_sin = opp::wrap_type<ov::op::v0::Sin>({concat_2});
140148
auto output_cos = opp::wrap_type<ov::op::v0::Cos>({concat_2});
141149

@@ -150,7 +158,13 @@ ov::npuw::patterns::pre_compute::RopePatternLLama2::RopePatternLLama2() : matche
150158
this->matched_cos = map_cos.at(output_cos).get_node_shared_ptr();
151159
this->matched_sin = map_sin.at(output_sin).get_node_shared_ptr();
152160

153-
LOG_VERB("Rope found : sin=" << matched_sin->get_name() << ", cos=" << matched_cos->get_name());
161+
// Determine if freq duplication was applied by inspecting the actual graph:
162+
// sin's direct input is Concat (LLama2-style) or Transpose (GPT-style, no dup).
163+
auto sin_input = this->matched_sin->input_value(0).get_node_shared_ptr();
164+
this->duplicate_freqs = ov::is_type<ov::op::v0::Concat>(sin_input);
165+
166+
LOG_VERB("Rope found : sin=" << matched_sin->get_name() << ", cos=" << matched_cos->get_name()
167+
<< " (duplicate_freqs=" << duplicate_freqs << ")");
154168

155169
return true;
156170
};
@@ -341,7 +355,7 @@ ov::npuw::patterns::pre_compute::RopeCacheMatcher::RopeCacheMatcher(const uint32
341355
auto rpe = std::make_shared<RopePatternLLama2>();
342356

343357
rpe->transform_cb = [&]() {
344-
auto cache = makeCosSinCache(max_prompt_len, rpe->matched_inv_freq);
358+
auto cache = makeCosSinCache(max_prompt_len, rpe->matched_inv_freq, rpe->duplicate_freqs);
345359
replaceSinCosByCache(max_prompt_len, cache, rpe.get());
346360
};
347361
rpe->run_on_model(model);

src/plugins/intel_npu/src/plugin/npuw/partitioning/patterns/pre_compute.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ class RopePatternDesc {
2222
std::shared_ptr<ov::Node> matched_sin;
2323
std::shared_ptr<ov::Node> matched_cos;
2424
std::shared_ptr<ov::Node> matched_concat;
25+
bool duplicate_freqs = false;
2526

2627
std::function<void()> transform_cb;
2728

src/plugins/intel_npu/tests/unit/npuw/pre_compute_test.cpp

Lines changed: 97 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
// SPDX-License-Identifier: Apache-2.0
33
//
44

5+
#include "partitioning/patterns/pre_compute.hpp"
6+
57
#include <gtest/gtest.h>
68

79
#include <algorithm>
@@ -11,26 +13,22 @@
1113

1214
#include "openvino/core/except.hpp"
1315
#include "openvino/op/ops.hpp"
14-
#include "partitioning/patterns/pre_compute.hpp"
1516

1617
namespace {
1718

1819
std::shared_ptr<ov::Model> make_longrope_v5_model(const std::vector<float>& short_factor_values,
19-
const std::vector<float>& long_factor_values,
20-
const std::vector<float>& multiply_values,
21-
const std::vector<float>& power_values) {
20+
const std::vector<float>& long_factor_values,
21+
const std::vector<float>& multiply_values,
22+
const std::vector<float>& power_values) {
2223
auto data = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::Shape{1, 4, 2});
2324
auto position_ids = std::make_shared<ov::op::v0::Parameter>(ov::element::i32, ov::Shape{2, 1});
2425

25-
auto short_factor = ov::op::v0::Constant::create(ov::element::f32,
26-
ov::Shape{short_factor_values.size()},
27-
short_factor_values);
28-
auto long_factor = ov::op::v0::Constant::create(ov::element::f32,
29-
ov::Shape{long_factor_values.size()},
30-
long_factor_values);
31-
auto multiply_const = ov::op::v0::Constant::create(ov::element::f32,
32-
ov::Shape{multiply_values.size()},
33-
multiply_values);
26+
auto short_factor =
27+
ov::op::v0::Constant::create(ov::element::f32, ov::Shape{short_factor_values.size()}, short_factor_values);
28+
auto long_factor =
29+
ov::op::v0::Constant::create(ov::element::f32, ov::Shape{long_factor_values.size()}, long_factor_values);
30+
auto multiply_const =
31+
ov::op::v0::Constant::create(ov::element::f32, ov::Shape{multiply_values.size()}, multiply_values);
3432
auto power_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{power_values.size()}, power_values);
3533

3634
auto reduce_axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {0, 1});
@@ -79,6 +77,50 @@ std::shared_ptr<ov::Model> make_longrope_v5_model(const std::vector<float>& shor
7977
"longrope_v5_test_model");
8078
}
8179

80+
// Builds a minimal RoPE model matching RopePatternLLama2.
81+
// When with_concat2=true (LLama2 style): Transpose → Concat_2 → Sin/Cos, duplicate_freqs=true.
82+
// When with_concat2=false (GPT style): Transpose → Sin/Cos directly, duplicate_freqs=false.
83+
std::shared_ptr<ov::Model> make_rope_model(bool with_concat2) {
84+
auto data = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::Shape{1, 4, 4});
85+
auto position_ids = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::Shape{1, 4});
86+
87+
// inv_freq: constant [1, half_dim=2, 1]
88+
auto inv_freq = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1, 2, 1}, {0.5f, 0.1f});
89+
90+
// ShapeOf → Gather(batch dim) → Concat_1 (broadcast target shape)
91+
auto shape_of = std::make_shared<ov::op::v3::ShapeOf>(data);
92+
auto gather_idx = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {0});
93+
auto gather_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0});
94+
auto gather = std::make_shared<ov::op::v8::Gather>(shape_of, gather_idx, gather_axis);
95+
auto ndims_const = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {2});
96+
auto one_const = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {1});
97+
auto concat_1 = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{gather, ndims_const, one_const}, 0);
98+
99+
// Broadcast inv_freq to [1,2,1], MatMul with position_ids → Transpose
100+
auto broadcast = std::make_shared<ov::op::v3::Broadcast>(inv_freq, concat_1);
101+
auto unsq_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {1});
102+
auto unsqueeze = std::make_shared<ov::op::v0::Unsqueeze>(position_ids, unsq_axis);
103+
auto convert = std::make_shared<ov::op::v0::Convert>(unsqueeze, ov::element::f32);
104+
auto matmul = std::make_shared<ov::op::v0::MatMul>(broadcast, convert); // [1,2,4]
105+
auto perm = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{3}, std::vector<int64_t>{0, 2, 1});
106+
auto transpose = std::make_shared<ov::op::v1::Transpose>(matmul, perm); // [1,4,2]
107+
108+
// Sin/Cos either via Concat_2 (LLama2) or directly (GPT)
109+
ov::Output<ov::Node> sin_cos_input = transpose->output(0);
110+
if (with_concat2) {
111+
auto zeros = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1, 4, 2}, std::vector<float>(8, 0.f));
112+
sin_cos_input = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{transpose, zeros}, -1)->output(0);
113+
}
114+
115+
auto sin = std::make_shared<ov::op::v0::Sin>(sin_cos_input);
116+
auto cos = std::make_shared<ov::op::v0::Cos>(sin_cos_input);
117+
118+
return std::make_shared<ov::Model>(
119+
ov::ResultVector{std::make_shared<ov::op::v0::Result>(sin), std::make_shared<ov::op::v0::Result>(cos)},
120+
ov::ParameterVector{data, position_ids},
121+
with_concat2 ? "llama2_rope_test_model" : "gpt_rope_test_model");
122+
}
123+
82124
bool has_input_name(const std::shared_ptr<ov::Model>& model, const std::string& name) {
83125
const auto inputs = model->inputs();
84126
return std::any_of(inputs.begin(), inputs.end(), [&name](const auto& input) {
@@ -113,16 +155,54 @@ TEST(PreComputeTest, RopeCacheThrowsOnMismatchedFactorSizesInLongRopeV5) {
113155
auto model = make_longrope_v5_model({1.0f, 2.0f}, {4.0f, 5.0f}, {1.0f}, {1.0f});
114156
ov::npuw::patterns::pre_compute::RopeCache pass(/*max_prompt_len=*/16, "longrope_input");
115157

116-
EXPECT_THROW(pass.run_on_model(model),
117-
ov::AssertFailure);
158+
EXPECT_THROW(pass.run_on_model(model), ov::AssertFailure);
118159
}
119160

120161
TEST(PreComputeTest, RopeCacheThrowsOnNonScalarPowerInLongRopeV5) {
121162
auto model = make_longrope_v5_model({1.0f, 2.0f}, {4.0f, 5.0f}, {1.0f, 2.0f}, {1.0f, 2.0f});
122163
ov::npuw::patterns::pre_compute::RopeCache pass(/*max_prompt_len=*/16, "longrope_input");
123164

124-
EXPECT_THROW(pass.run_on_model(model),
125-
ov::AssertFailure);
165+
EXPECT_THROW(pass.run_on_model(model), ov::AssertFailure);
166+
}
167+
168+
// Verifies that the merged RopePatternLLama2 correctly detects and removes the
169+
// LLama2-style sin/cos subgraph (with Concat_2 present, duplicate_freqs=true).
170+
TEST(PreComputeTest, RopeCacheTransformsLLama2Pattern) {
171+
auto model = make_rope_model(/*with_concat2=*/true);
172+
173+
ov::npuw::patterns::pre_compute::RopeCache pass(/*max_prompt_len=*/16, /*longrope_input_name=*/{});
174+
ASSERT_NO_THROW(pass.run_on_model(model));
175+
176+
const auto& ops = model->get_ops();
177+
const auto sin_count = std::count_if(ops.begin(), ops.end(), [](const auto& op) {
178+
return ov::is_type<ov::op::v0::Sin>(op);
179+
});
180+
const auto cos_count = std::count_if(ops.begin(), ops.end(), [](const auto& op) {
181+
return ov::is_type<ov::op::v0::Cos>(op);
182+
});
183+
184+
EXPECT_EQ(sin_count, 0) << "Sin should be replaced by Gather from the duplicated LUT";
185+
EXPECT_EQ(cos_count, 0) << "Cos should be replaced by Gather from the duplicated LUT";
186+
}
187+
188+
// Verifies that the merged RopePatternLLama2 correctly detects and removes the
189+
// GPT-style sin/cos subgraph (Concat_2 absent, duplicate_freqs=false).
190+
TEST(PreComputeTest, RopeCacheTransformsGPTPattern) {
191+
auto model = make_rope_model(/*with_concat2=*/false);
192+
193+
ov::npuw::patterns::pre_compute::RopeCache pass(/*max_prompt_len=*/16, /*longrope_input_name=*/{});
194+
ASSERT_NO_THROW(pass.run_on_model(model));
195+
196+
const auto& ops = model->get_ops();
197+
const auto sin_count = std::count_if(ops.begin(), ops.end(), [](const auto& op) {
198+
return ov::is_type<ov::op::v0::Sin>(op);
199+
});
200+
const auto cos_count = std::count_if(ops.begin(), ops.end(), [](const auto& op) {
201+
return ov::is_type<ov::op::v0::Cos>(op);
202+
});
203+
204+
EXPECT_EQ(sin_count, 0) << "Sin should be replaced by Gather from the non-duplicated LUT";
205+
EXPECT_EQ(cos_count, 0) << "Cos should be replaced by Gather from the non-duplicated LUT";
126206
}
127207

128208
} // namespace

0 commit comments

Comments
 (0)