diff --git a/src/common/snippets/include/snippets/op/subgraph.hpp b/src/common/snippets/include/snippets/op/subgraph.hpp index 814a3ff9965f3f..b1d68188cc99b3 100644 --- a/src/common/snippets/include/snippets/op/subgraph.hpp +++ b/src/common/snippets/include/snippets/op/subgraph.hpp @@ -266,6 +266,13 @@ static inline auto build_subgraph(const std::shared_ptr& node, const std::string& name = "") -> std::shared_ptr { auto subgraph = std::make_shared(inputs, body); copy_runtime_info(node, subgraph); + // copy_runtime_info drops the non-copyable DisablePrecisionConversion; propagate it by rt_info key + // (snippets cannot depend on transformations) so precision markup survives tokenization. + for (const auto& item : node->get_rt_info()) { + if (item.first.rfind("DisablePrecisionConversion", 0) == 0) { + subgraph->get_rt_info()[item.first] = item.second; + } + } subgraph->set_friendly_name(name.empty() ? node->get_friendly_name() : name); return subgraph; } diff --git a/src/common/snippets/src/utils/tokenization_utils.cpp b/src/common/snippets/src/utils/tokenization_utils.cpp index 420534a627baa5..c26cb633456b5c 100644 --- a/src/common/snippets/src/utils/tokenization_utils.cpp +++ b/src/common/snippets/src/utils/tokenization_utils.cpp @@ -50,6 +50,20 @@ namespace ov::snippets::utils { namespace { +// copy_runtime_info drops the non-copyable DisablePrecisionConversion attribute; propagate it +// explicitly from the tokenized nodes to the Subgraph so precision markup (e.g. kept-in-fp32 sin/cos +// inputs) survives tokenization. Copied by rt_info key because snippets cannot depend on the +// transformations library. +void propagate_precision_conversion_disable(const ov::NodeVector& from, const std::shared_ptr& to) { + for (const auto& node : from) { + for (const auto& item : node->get_rt_info()) { + if (item.first.rfind("DisablePrecisionConversion", 0) == 0) { + to->get_rt_info()[item.first] = item.second; + } + } + } +} + auto has_result_child(const std::shared_ptr& node) -> bool { const auto& users = node->get_users(); return std::any_of(users.begin(), users.end(), [](const std::shared_ptr& child) { @@ -446,6 +460,7 @@ bool tokenize_node(const std::shared_ptr& node, const ov::snippets::pa } auto subgraph = op::build_subgraph(node, external_inputs, body, subgraph_name); copy_runtime_info(replaced_nodes, subgraph); + propagate_precision_conversion_disable(replaced_nodes, subgraph); const auto& act_body = subgraph->body(); for (size_t i = 0; i < act_body.get_parameters().size(); i++) { act_body.get_parameters()[i]->set_friendly_name(body_parameters[i]->get_friendly_name()); @@ -556,6 +571,7 @@ std::shared_ptr tokenize_ordered_nodes(const ov::Nod auto body = op::create_body(last_node->get_friendly_name(), body_results, body_parameters); auto subgraph = std::make_shared(subgraph_inputs, body); copy_runtime_info(last_node, subgraph); + propagate_precision_conversion_disable({last_node}, subgraph); subgraph->set_friendly_name(last_node->get_friendly_name()); for (size_t i = 0; i < subgraph->get_output_size(); ++i) { diff --git a/src/common/transformations/include/transformations/fp16_compression/disable_fp16_comp_ltx_rope.hpp b/src/common/transformations/include/transformations/fp16_compression/disable_fp16_comp_ltx_rope.hpp new file mode 100644 index 00000000000000..fc9b61ac62229a --- /dev/null +++ b/src/common/transformations/include/transformations/fp16_compression/disable_fp16_comp_ltx_rope.hpp @@ -0,0 +1,27 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/matcher_pass.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace pass { + +/** + * @ingroup ov_transformation_common_api + * @brief Keeps the decomposed LTX-Video RoPE angle chain (Multiply -> Add -> Transpose -> Reshape -> + * Sin/Cos) in f32: the angles reach ~1.6e4 rad, which f16/bf16 quantize in steps larger than 2*pi. + * + * The fused RoPE variant is handled on GPU by IncreasePositionIdsPrecisionForLtxVideo. + */ +class TRANSFORMATIONS_API DisableFP16CompForLtxVideoRopePattern : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("DisableFP16CompForLtxVideoRopePattern"); + DisableFP16CompForLtxVideoRopePattern(); +}; + +} // namespace pass +} // namespace ov diff --git a/src/common/transformations/src/transformations/fp16_compression/disable_fp16_comp_ltx_rope.cpp b/src/common/transformations/src/transformations/fp16_compression/disable_fp16_comp_ltx_rope.cpp new file mode 100644 index 00000000000000..5a987eb74eb5b9 --- /dev/null +++ b/src/common/transformations/src/transformations/fp16_compression/disable_fp16_comp_ltx_rope.cpp @@ -0,0 +1,51 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/fp16_compression/disable_fp16_comp_ltx_rope.hpp" + +#include "itt.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/cos.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/sin.hpp" +#include "openvino/op/transpose.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/rt_info/disable_precision_conversion.hpp" + +namespace v0 = ov::op::v0; +namespace v1 = ov::op::v1; + +namespace ov::pass { + +DisableFP16CompForLtxVideoRopePattern::DisableFP16CompForLtxVideoRopePattern() { + MATCHER_SCOPE(DisableFP16CompForLtxVideoRopePattern); + using namespace ov::pass::pattern; + + // grid values are small; the large magnitude appears only from the frequency Multiply onwards + auto mul = wrap_type({any_input(), any_input()}); + auto add_constant = wrap_type(); + auto add = wrap_type({mul, add_constant}); + auto transpose = wrap_type({add, any_input()}); + auto reshape = wrap_type({transpose, any_input()}); + auto sin = wrap_type({reshape}); + auto cos = wrap_type({reshape}); + auto sin_or_cos = std::make_shared(OutputVector{sin, cos}); + + matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](Matcher& m) { + const auto& pattern_map = m.get_pattern_value_map(); + for (const auto& node : {mul, add_constant, add, transpose, reshape}) { + ov::disable_conversion(pattern_map.at(node).get_node_shared_ptr(), element::f16); + } + ov::disable_conversion(m.get_match_root(), element::f16); + return false; + }; + + auto m = std::make_shared(sin_or_cos, matcher_name); + this->register_matcher(m, callback); +} + +} // namespace ov::pass diff --git a/src/common/transformations/tests/fp16_compression/disable_fp16_comp_ltx_rope_test.cpp b/src/common/transformations/tests/fp16_compression/disable_fp16_comp_ltx_rope_test.cpp new file mode 100644 index 00000000000000..513157c113f539 --- /dev/null +++ b/src/common/transformations/tests/fp16_compression/disable_fp16_comp_ltx_rope_test.cpp @@ -0,0 +1,78 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "transformations/fp16_compression/disable_fp16_comp_ltx_rope.hpp" + +#include "common_test_utils/ov_test_utils.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/cos.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/sin.hpp" +#include "openvino/op/transpose.hpp" +#include "transformations/rt_info/disable_precision_conversion.hpp" + +namespace v0 = ov::op::v0; +namespace v1 = ov::op::v1; + +namespace { +// Multiply -> Add(Constant) -> Transpose -> Reshape -> Sin/Cos, the decomposed LTX-Video rope angles. +struct RopeChain { + std::shared_ptr grid; + std::shared_ptr mul; + std::shared_ptr add_const; + std::shared_ptr add; + std::shared_ptr transpose; + std::shared_ptr reshape; + std::shared_ptr sin; + std::shared_ptr cos; + std::shared_ptr model(const std::string& name) const { + return std::make_shared(ov::OutputVector{sin, cos}, ov::ParameterVector{grid}, name); + } +}; + +RopeChain make_rope_chain() { + RopeChain c; + c.grid = std::make_shared(ov::element::f32, ov::Shape{1, 8, 1}); + auto freqs = v0::Constant::create(ov::element::f32, ov::Shape{1, 1, 16}, std::vector(16, 1000.0f)); + c.mul = std::make_shared(c.grid, freqs); + c.add_const = v0::Constant::create(ov::element::f32, ov::Shape{}, {-1.0f}); + c.add = std::make_shared(c.mul, c.add_const); + auto order = v0::Constant::create(ov::element::i32, ov::Shape{3}, {0, 2, 1}); + c.transpose = std::make_shared(c.add, order); + auto shape = v0::Constant::create(ov::element::i32, ov::Shape{2}, {1, 128}); + c.reshape = std::make_shared(c.transpose, shape, false); + c.sin = std::make_shared(c.reshape); + c.cos = std::make_shared(c.reshape); + return c; +} +} // namespace + +// The whole angle chain from the frequency Multiply up to Sin/Cos is kept in f32. +TEST_F(TransformationTestsF, DisableFP16CompForLtxVideoRopeMarksAngleChain) { + model = make_rope_chain().model("model"); + + manager.register_pass(); + + { + auto c = make_rope_chain(); + for (const auto& node : + std::vector>{c.mul, c.add_const, c.add, c.transpose, c.reshape, c.sin, c.cos}) { + disable_conversion(node, ov::element::f16); + } + model_ref = c.model("model_ref"); + } + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); +} + +// Sin/Cos outside the rope pattern are left untouched, so unrelated subgraphs keep low precision. +TEST_F(TransformationTestsF, DisableFP16CompForLtxVideoRopeSkipsUnrelatedSinCos) { + auto x = std::make_shared(ov::element::f32, ov::Shape{1, 8}); + auto sin = std::make_shared(x); + model = std::make_shared(ov::OutputVector{sin}, ov::ParameterVector{x}, "model"); + + manager.register_pass(); +} diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_power_static.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_power_static.cpp index 6c505b3d0695b4..1ff830b75dd7ac 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_power_static.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_power_static.cpp @@ -36,6 +36,7 @@ #include "ov_ops/fully_connected.hpp" #include "transformations/cpu_opset/common/op/power_static.hpp" #include "transformations/rt_info/dequantization_node.hpp" +#include "transformations/rt_info/disable_precision_conversion.hpp" #include "utils/general_utils.h" namespace { @@ -184,6 +185,12 @@ ov::intel_cpu::ConvertToPowerStatic::ConvertToPowerStatic() { } toReplace->set_friendly_name(node->get_friendly_name()); ov::copy_runtime_info(node, toReplace); + // DisablePrecisionConversion is non-copyable and dropped by copy_runtime_info, re-apply it + const auto& rt_info = node->get_rt_info(); + const auto dpc_it = rt_info.find(ov::DisablePrecisionConversion::get_type_info_static()); + if (dpc_it != rt_info.end()) { + toReplace->get_rt_info().insert(*dpc_it); + } ov::replace_node(node, toReplace); return true; }; diff --git a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp index 5a43dc09306dee..04cfa928378f40 100644 --- a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp +++ b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp @@ -79,6 +79,7 @@ #include "transformations/common_optimizations/wrap_interpolate_into_transposes.hpp" #include "transformations/convert_precision.hpp" #include "transformations/fp16_compression/convert_compression_only_to_legacy.hpp" +#include "transformations/fp16_compression/disable_fp16_comp_ltx_rope.hpp" #include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/fp16_compression/mark_floatpoint_range.hpp" #include "transformations/init_node_info.hpp" @@ -557,6 +558,8 @@ void Transformations::PreLpt(const std::vector& defaultPrecis #else type_to_fuse_map fuse_map = {{ov::op::PagedAttentionExtension::get_type_info_static(), fuse_type_to_pa}}; #endif + // mark rope angles before they are converted to f16 + CPU_REGISTER_PASS_COMMON(manager, ov::pass::DisableFP16CompForLtxVideoRopePattern); const bool keep_precision_sensitive_in_fp32 = true; CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertPrecision, @@ -1192,6 +1195,10 @@ void Transformations::PostLpt() { CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::MarkRopeInputsToKeepInMixedPrecision); CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::MarkFloatingPointRange); } + // f16 is marked before ConvertPrecision (PreLpt); bf16 is enforced per node after transformations + if (config.inferencePrecision == ov::element::bf16) { + CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::DisableFP16CompForLtxVideoRopePattern); + } // Should be before Snippets pipeline because Ngram pattern contains eltwise nodes that can be tokenized by // Snippets. diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/x64/rope_table_precision.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/x64/rope_table_precision.cpp new file mode 100644 index 00000000000000..a9123c7ecf8ef8 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/x64/rope_table_precision.cpp @@ -0,0 +1,114 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "common_test_utils/node_builders/constant.hpp" +#include "common_test_utils/ov_tensor_utils.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/cos.hpp" +#include "openvino/op/matmul.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/sin.hpp" +#include "openvino/op/transpose.hpp" +#include "openvino/runtime/system_conf.hpp" +#include "shared_test_classes/base/ov_subgraph.hpp" + +namespace ov::test { + +// Decomposed LTX-Video rope table applied to a projection: the angle chain must stay f32 under bf16 +// enforcement, MatMuls stay bf16. The static case also exercises Snippets tokenization. +class RopeTablePrecisionCPUTest : public testing::WithParamInterface, public SubgraphBaseTest { +public: + static std::string getTestCaseName(const testing::TestParamInfo& obj) { + std::ostringstream result; + result << (obj.param.first.is_dynamic() ? "dynamic" : "static"); + result << "_IS=" << ov::test::utils::partialShape2str({obj.param.first}); + return result.str(); + } + +protected: + void SetUp() override { + targetDevice = utils::DEVICE_CPU; + configuration.insert({ov::hint::inference_precision.name(), ov::element::bf16}); + // bf16-computed angles give O(1) errors; honest bf16 rounding stays well under this + rel_threshold = 0.05; + abs_threshold = 0.5; + + // the rope table size is fixed; only the hidden states shape varies + init_input_shapes({InputShape{ov::PartialShape{1, GRID, 1}, {ov::Shape{1, GRID, 1}}}, GetParam()}); + + ov::ParameterVector params{std::make_shared(ov::element::f32, inputDynamicShapes[0]), + std::make_shared(ov::element::f32, inputDynamicShapes[1])}; + + // ~500 rad max after the freq multiply: fatal for bf16 (~2 rad steps), exact for f32 sin/cos + auto freqs = utils::make_constant(ov::element::f32, ov::Shape{1, 1, BANDS}, std::vector{1, 2, 4, 8}); + auto angles = std::make_shared(params[0], freqs); + auto shifted = std::make_shared( + angles, + utils::make_constant(ov::element::f32, ov::Shape{}, std::vector{-1.0F})); + auto order = utils::make_constant(ov::element::i32, ov::Shape{3}, std::vector{0, 2, 1}); + auto transpose = std::make_shared(shifted, order); + auto target_shape = + utils::make_constant(ov::element::i32, ov::Shape{2}, std::vector{1, static_cast(HIDDEN_SIZE)}); + auto reshape = std::make_shared(transpose, target_shape, false); + auto cos = std::make_shared(reshape); + auto sin = std::make_shared(reshape); + + ov::test::utils::InputGenerateData weights_data(-1, 2, 256); + auto proj_weights = utils::make_constant(ov::element::f32, ov::Shape{HIDDEN_SIZE, HIDDEN_SIZE}, weights_data); + auto q = std::make_shared(params[1], proj_weights); + + auto q_cos = std::make_shared(q, cos); + auto q_sin = std::make_shared(q, sin); + auto rotated = std::make_shared(q_cos, q_sin); + + auto out_weights = utils::make_constant(ov::element::f32, ov::Shape{HIDDEN_SIZE, HIDDEN_SIZE}, weights_data); + auto matmul = std::make_shared(rotated, out_weights); + + function = std::make_shared(ov::OutputVector{std::make_shared(matmul)}, + params, + "RopeTablePrecision"); + } + + void generate_inputs(const std::vector& targetInputStaticShapes) override { + inputs.clear(); + const auto& funcInputs = function->inputs(); + + ov::Tensor grid{ov::element::f32, targetInputStaticShapes[0]}; + auto* grid_data = grid.data(); + for (size_t i = 0; i < grid.get_size(); ++i) { + grid_data[i] = static_cast(i); + } + inputs.insert({funcInputs[0].get_node_shared_ptr(), grid}); + + utils::InputGenerateData in_data; + in_data.start_from = -1; + in_data.range = 2; + in_data.resolution = 256; + auto hidden = + utils::create_and_fill_tensor(funcInputs[1].get_element_type(), targetInputStaticShapes[1], in_data); + inputs.insert({funcInputs[1].get_node_shared_ptr(), hidden}); + } + + static constexpr size_t GRID = 64; + static constexpr size_t BANDS = 4; + static constexpr size_t HIDDEN_SIZE = GRID * BANDS; +}; + +TEST_P(RopeTablePrecisionCPUTest, CompareWithRefs) { + if (!ov::with_cpu_x86_bfloat16()) { + GTEST_SKIP() << "No bf16 hardware support"; + } + run(); +} + +INSTANTIATE_TEST_SUITE_P(smoke_RopeTablePrecision, + RopeTablePrecisionCPUTest, + testing::Values(InputShape{ov::PartialShape{-1, 256}, + {ov::Shape{64, 256}, ov::Shape{32, 256}}}, + InputShape{ov::PartialShape{64, 256}, {ov::Shape{64, 256}}}), + RopeTablePrecisionCPUTest::getTestCaseName); + +} // namespace ov::test