diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index 127d9dfc0bd0bd..4f71b0af8b0afb 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -31,6 +31,7 @@ #include "openvino/pass/constant_folding.hpp" #include "openvino/pass/manager.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" +#include "openvino/pass/pattern/op/pattern.hpp" #include "openvino/util/log.hpp" #include "ov_ops/type_relaxed.hpp" #include "transformations/common_optimizations/lin_op_sequence_fusion.hpp" diff --git a/src/common/transformations/include/openvino/decompositions/low_precision_dequantize.hpp b/src/common/transformations/include/openvino/decompositions/low_precision_dequantize.hpp index 84a1a4e23f04a9..85c27dee7433be 100644 --- a/src/common/transformations/include/openvino/decompositions/low_precision_dequantize.hpp +++ b/src/common/transformations/include/openvino/decompositions/low_precision_dequantize.hpp @@ -56,19 +56,26 @@ namespace decomposition { /// \param output_shape Optional shape constant. When provided a Reshape with /// special_zero=false is appended after the Multiply /// (skipped if the Multiply output already has that shape). -ov::Output TRANSFORMATIONS_API low_precision_dequantize(ov::pass::NodeRegistry& reg, - const ov::Output& x, - const ov::Output& scale, - const ov::Output& zero_point = {}, - const ov::Output& output_shape = {}); +/// \param scale_decompression_precision Optional element type for the Convert node +/// that converts the quantized scale and zp to the output element type. If not provided, the +/// Convert node is omitted and the scale is used as-is. +ov::Output TRANSFORMATIONS_API +low_precision_dequantize(ov::pass::NodeRegistry& reg, + const ov::Output& x, + const ov::Output& scale, + const ov::Output& zero_point = {}, + const ov::Output& output_shape = {}, + const ov::element::Type& decompression_precision = ov::element::dynamic); /// \brief Convenience overload for callers that do not need access to the /// intermediate nodes. Internally allocates a NodeRegistry and /// forwards to the registry-based overload. -ov::Output TRANSFORMATIONS_API low_precision_dequantize(const ov::Output& x, - const ov::Output& scale, - const ov::Output& zero_point = {}, - const ov::Output& output_shape = {}); +ov::Output TRANSFORMATIONS_API +low_precision_dequantize(const ov::Output& x, + const ov::Output& scale, + const ov::Output& zero_point = {}, + const ov::Output& output_shape = {}, + const ov::element::Type& decompression_precision = ov::element::dynamic); } // namespace decomposition } // namespace ov diff --git a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp index e760f23c83ffed..ef1cf24d6ba01c 100644 --- a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp +++ b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp @@ -4,6 +4,7 @@ #pragma once +#include "openvino/core/attribute_adapter.hpp" #include "openvino/op/op.hpp" #include "transformations_visibility.hpp" @@ -56,6 +57,8 @@ class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; + const Attributes& get_attrs() const { return m_attrs; } @@ -89,4 +92,29 @@ class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { } // namespace internal } // namespace op + +std::ostream& operator<<(std::ostream& s, const ov::op::internal::DynamicQuantize::QuantizationType& quantization_type); +std::ostream& operator<<(std::ostream& s, + const ov::op::internal::DynamicQuantize::OutputStorageType& output_storage_type); + +template <> +class OPENVINO_API AttributeAdapter + : public EnumAttributeAdapterBase { +public: + AttributeAdapter(ov::op::internal::DynamicQuantize::QuantizationType& value) + : EnumAttributeAdapterBase(value) {} + + OPENVINO_RTTI("AttributeAdapter"); +}; + +template <> +class OPENVINO_API AttributeAdapter + : public EnumAttributeAdapterBase { +public: + AttributeAdapter(ov::op::internal::DynamicQuantize::OutputStorageType& value) + : EnumAttributeAdapterBase(value) {} + + OPENVINO_RTTI("AttributeAdapter"); +}; + } // namespace ov diff --git a/src/common/transformations/src/decompositions/low_precision_dequantize.cpp b/src/common/transformations/src/decompositions/low_precision_dequantize.cpp index fa5b4dc9bec07e..ef9550ad2da91e 100644 --- a/src/common/transformations/src/decompositions/low_precision_dequantize.cpp +++ b/src/common/transformations/src/decompositions/low_precision_dequantize.cpp @@ -43,12 +43,14 @@ ov::Output low_precision_dequantize(ov::pass::NodeRegistry& reg, const ov::Output& x, const ov::Output& scale, const ov::Output& zero_point, - const ov::Output& output_shape) { + const ov::Output& output_shape, + const ov::element::Type& decompression_precision) { // Decomposition shape (matches ov::pass::MarkDequantization): // Multiply(Subtract(Convert(x), zp), scale) [-> Reshape] // or, when zero_point is empty: // Multiply(Convert(x), scale) [-> Reshape] - const auto& dst_type = scale.get_element_type(); + const auto& dst_type = + decompression_precision == ov::element::dynamic ? scale.get_element_type() : decompression_precision; ov::Output result = reg.make(x, dst_type); if (zero_point.get_node_shared_ptr()) { @@ -59,7 +61,11 @@ ov::Output low_precision_dequantize(ov::pass::NodeRegistry& reg, result = reg.make(result, zp); } - result = reg.make(result, scale); + ov::Output scale_convert = scale; + if (scale.get_element_type() != dst_type) { + scale_convert = reg.make(scale, dst_type); + } + result = reg.make(result, scale_convert); if (output_shape.get_node_shared_ptr() && !reshape_is_noop(result, output_shape)) { result = reg.make(result, output_shape, /*special_zero=*/false); @@ -71,9 +77,10 @@ ov::Output low_precision_dequantize(ov::pass::NodeRegistry& reg, ov::Output low_precision_dequantize(const ov::Output& x, const ov::Output& scale, const ov::Output& zero_point, - const ov::Output& output_shape) { + const ov::Output& output_shape, + const ov::element::Type& decompression_precision) { ov::pass::NodeRegistry reg; - return low_precision_dequantize(reg, x, scale, zero_point, output_shape); + return low_precision_dequantize(reg, x, scale, zero_point, output_shape, decompression_precision); } } // namespace decomposition diff --git a/src/common/transformations/src/ov_ops/dynamic_quantize.cpp b/src/common/transformations/src/ov_ops/dynamic_quantize.cpp index 247bfb374fd2aa..6798b4c2d0668d 100644 --- a/src/common/transformations/src/ov_ops/dynamic_quantize.cpp +++ b/src/common/transformations/src/ov_ops/dynamic_quantize.cpp @@ -4,6 +4,7 @@ #include "ov_ops/dynamic_quantize.hpp" +#include "itt.hpp" #include "openvino/core/partial_shape.hpp" #include "openvino/core/validation_util.hpp" #include "openvino/op/variadic_split.hpp" @@ -134,6 +135,55 @@ std::vector DynamicQuantize::shape_infer(const DynamicQuantize return out_shapes; } +bool DynamicQuantize::visit_attributes(AttributeVisitor& visitor) { + INTERNAL_OP_SCOPE(internal_DynamicQuantize_visit_attributes); + visitor.on_attribute("quantization_type", m_attrs.quantization_type); + visitor.on_attribute("quantization_dt", m_attrs.quantization_dt); + visitor.on_attribute("scale_dt", m_attrs.scale_dt); + visitor.on_attribute("zp_dt", m_attrs.zp_dt); + visitor.on_attribute("precomputed_reduction_dt", m_attrs.precomputed_reduction_dt); + visitor.on_attribute("precomputed_reduction", m_attrs.precomputed_reduction); + visitor.on_attribute("group_sizes", m_attrs.group_sizes); + visitor.on_attribute("scales_zp_output_order", m_attrs.scales_zp_output_order); + visitor.on_attribute("output_storage_type", m_attrs.output_storage_type); + return true; +} + } // namespace internal } // namespace op + +std::ostream& operator<<(std::ostream& s, + const ov::op::internal::DynamicQuantize::QuantizationType& quantization_type) { + return s << ov::as_string(quantization_type); +} + +std::ostream& operator<<(std::ostream& s, + const ov::op::internal::DynamicQuantize::OutputStorageType& output_storage_type) { + return s << ov::as_string(output_storage_type); +} + +template <> +OPENVINO_API EnumNames& +EnumNames::get() { + static auto enum_names = EnumNames( + "ov::op::internal::DynamicQuantize::QuantizationType", + { + {"symmetric", ov::op::internal::DynamicQuantize::QuantizationType::Symmetric}, + {"asymmetric", ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric}, + }); + return enum_names; +} + +template <> +OPENVINO_API EnumNames& +EnumNames::get() { + static auto enum_names = EnumNames( + "ov::op::internal::DynamicQuantize::OutputStorageType", + { + {"planar", ov::op::internal::DynamicQuantize::OutputStorageType::Planar}, + {"interleaved_scales_zp", ov::op::internal::DynamicQuantize::OutputStorageType::InterleavedScalesZP}, + }); + return enum_names; +} + } // namespace ov diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index c1effbbda757fb..4520c65664c015 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -7,6 +7,7 @@ #include #include "itt.hpp" +#include "low_precision/low_precision.hpp" #include "openvino/pass/constant_folding.hpp" #include "openvino/pass/manager.hpp" #include "transformations/common_optimizations/adaptive_pool_to_reduce.hpp" @@ -146,7 +147,8 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr // Transformation call example, to check with the real model manager.register_pass(TypeVector{f8e4m3}, TypeVector{u4}); manager.register_pass( - TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f4e2m1, f8e4m3, f8e5m2, f8e8m0}); + TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); + manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); } if (!m_use_shapes) { manager.register_pass(); diff --git a/src/common/transformations/src/transformations/common_optimizations/shared_ops_optimization.cpp b/src/common/transformations/src/transformations/common_optimizations/shared_ops_optimization.cpp index 281a4dcf9d2973..585be3ceee12bc 100644 --- a/src/common/transformations/src/transformations/common_optimizations/shared_ops_optimization.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/shared_ops_optimization.cpp @@ -176,7 +176,7 @@ bool shared_node_optimization(const shared_ptr& model) { if (nodes_are_equal(root_op, child_op, node_attributes_cache)) { rewritten = - replace_output_update_name(child_op->output(0), root_op->output(0)) || rewritten; + replace_outputs_update_name(child_op->outputs(), root_op->outputs()) || rewritten; visited_nodes[j] = true; } } diff --git a/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp b/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp index 650babeabda543..634e42e85e42a5 100644 --- a/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp +++ b/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp @@ -30,7 +30,14 @@ using namespace ov; namespace { -const element::TypeVector kLowPrecisionTypes{element::u8, element::i8, element::u4, element::i4}; +const element::TypeVector kLowPrecisionTypes{element::u8, + element::i8, + element::u4, + element::i4, + element::f8e4m3, + element::f8e5m2, + element::f4e2m1, + element::f8e8m0}; } // namespace @@ -231,3 +238,129 @@ TEST_F(TransformationTestsF, LowPrecisionDequantize_NoopReshapeSkipped) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); } + +TEST_F(TransformationTestsF, LowPrecisionDequantize_f8e4m3) { + const Shape weights_shape{4, 16}; + + { + auto weights = op::v0::Constant::create(element::f8e4m3, weights_shape, {-2}); + auto scale = op::v0::Constant::create(element::f16, Shape{}, {0.2f}); + + auto out = decomposition::low_precision_dequantize(weights, scale); + model = std::make_shared(OutputVector{out}, ParameterVector{}); + } + + manager.register_pass(kLowPrecisionTypes); + manager.register_pass(); + + { + auto weights = op::v0::Constant::create(element::f8e4m3, weights_shape, {-2}); + auto convert = std::make_shared(weights, element::f16); + disable_constant_folding(convert); + + auto scale = op::v0::Constant::create(element::f16, Shape{}, {0.2f}); + auto multiply = std::make_shared(convert, scale); + mark_as_dequantization_node(multiply); + + model_ref = std::make_shared(OutputVector{multiply}, ParameterVector{}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); +} + +TEST_F(TransformationTestsF, LowPrecisionDequantize_f8e5m2) { + const Shape weights_shape{4, 16}; + + { + auto weights = op::v0::Constant::create(element::f8e5m2, weights_shape, {-2}); + auto scale = op::v0::Constant::create(element::f16, Shape{}, {0.2f}); + + auto out = decomposition::low_precision_dequantize(weights, scale); + model = std::make_shared(OutputVector{out}, ParameterVector{}); + } + + manager.register_pass(kLowPrecisionTypes); + manager.register_pass(); + + { + auto weights = op::v0::Constant::create(element::f8e5m2, weights_shape, {-2}); + auto convert = std::make_shared(weights, element::f16); + disable_constant_folding(convert); + + auto scale = op::v0::Constant::create(element::f16, Shape{}, {0.2f}); + auto multiply = std::make_shared(convert, scale); + mark_as_dequantization_node(multiply); + + model_ref = std::make_shared(OutputVector{multiply}, ParameterVector{}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); +} + +TEST_F(TransformationTestsF, LowPrecisionDequantize_mxf8e4m3) { + const Shape weights_shape{4, 32}; + + { + auto weights = op::v0::Constant::create(element::f8e4m3, weights_shape, {-2}); + auto scale = op::v0::Constant::create(element::f8e8m0, Shape{}, {0.2f}); + + auto out = decomposition::low_precision_dequantize(weights, scale, {}, {}, element::f16); + model = std::make_shared(OutputVector{out}, ParameterVector{}); + } + + manager.register_pass(kLowPrecisionTypes, false, false); + manager.register_pass(); + + { + auto weights = op::v0::Constant::create(element::f8e4m3, weights_shape, {-2}); + auto convert_weights = std::make_shared(weights, element::f16); + disable_constant_folding(convert_weights); + + auto scale = op::v0::Constant::create(element::f8e8m0, Shape{}, {0.2f}); + auto convert_scale = std::make_shared(scale, element::f16); + disable_constant_folding(convert_scale); + + auto multiply = std::make_shared(convert_weights, convert_scale); + mark_as_dequantization_node(multiply); + + model_ref = std::make_shared(OutputVector{multiply}, ParameterVector{}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); +} + +TEST_F(TransformationTestsF, LowPrecisionDequantize_mxf8e5m2) { + const Shape weights_shape{4, 32}; + + { + auto weights = op::v0::Constant::create(element::f8e5m2, weights_shape, {-2}); + auto scale = op::v0::Constant::create(element::f8e8m0, Shape{}, {0.2f}); + + auto out = decomposition::low_precision_dequantize(weights, scale, {}, {}, element::f16); + model = std::make_shared(OutputVector{out}, ParameterVector{}); + } + + manager.register_pass(kLowPrecisionTypes, false, false); + manager.register_pass(); + + { + auto weights = op::v0::Constant::create(element::f8e5m2, weights_shape, {-2}); + auto convert_weights = std::make_shared(weights, element::f16); + disable_constant_folding(convert_weights); + + auto scale = op::v0::Constant::create(element::f8e8m0, Shape{}, {0.2f}); + auto convert_scale = std::make_shared(scale, element::f16); + disable_constant_folding(convert_scale); + + auto multiply = std::make_shared(convert_weights, convert_scale); + mark_as_dequantization_node(multiply); + + model_ref = std::make_shared(OutputVector{multiply}, ParameterVector{}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); +} diff --git a/src/core/include/openvino/core/graph_util.hpp b/src/core/include/openvino/core/graph_util.hpp index f465c77fc57180..7c8500ad57a81f 100644 --- a/src/core/include/openvino/core/graph_util.hpp +++ b/src/core/include/openvino/core/graph_util.hpp @@ -292,6 +292,9 @@ bool compare_constants(const std::shared_ptr& n1, const std::shared_ptr node, const Output& node_input); +OPENVINO_API +bool replace_outputs_update_name(OutputVector nodes, const OutputVector& node_inputs); + OPENVINO_API bool replace_node_update_name(const std::shared_ptr& target, const std::shared_ptr& replacement); diff --git a/src/core/src/graph_util.cpp b/src/core/src/graph_util.cpp index ba031ae29c4d77..b64800d21f513d 100644 --- a/src/core/src/graph_util.cpp +++ b/src/core/src/graph_util.cpp @@ -65,6 +65,23 @@ void clone_ov_nodes(const std::vector>& nodes, } } +bool has_result_consumers(const ov::Output& port) { + const auto& consumers = port.get_target_inputs(); + return std::any_of(consumers.cbegin(), consumers.cend(), [](const ov::Input& consumer) { + return ov::is_type(consumer.get_node()); + }); +} + +bool can_output_be_replaced(const ov::Output& output, const ov::Output& replacement) { + if (has_result_consumers(output)) { + if (output.get_node()->get_output_size() != 1 || replacement.get_node()->get_output_size() != 1 || + ov::is_type(replacement.get_node()) || has_result_consumers(replacement)) { + return false; + } + } + return true; +} + } // namespace namespace ov { @@ -268,18 +285,11 @@ bool replace_output_update_name(Output output, const Output& replace // and has exactly one output port // In all other cases output name will be lost or changed, so we don't perform the replacement - auto has_result_consumers = [](const Output& port) { - const auto& consumers = port.get_target_inputs(); - return std::any_of(consumers.cbegin(), consumers.cend(), [](const Input& consumer) { - return ov::is_type(consumer.get_node()); - }); - }; + if (!can_output_be_replaced(output, replacement)) { + return false; + } if (has_result_consumers(output)) { - if (output.get_node()->get_output_size() != 1 || replacement.get_node()->get_output_size() != 1 || - is_type(replacement.get_node()) || has_result_consumers(replacement)) { - return false; - } replacement.get_node()->set_friendly_name(output.get_node()->get_friendly_name()); } @@ -290,6 +300,29 @@ bool replace_output_update_name(Output output, const Output& replace return true; } +bool replace_outputs_update_name(OutputVector outputs, const OutputVector& replacements) { + if (outputs.size() != replacements.size()) { + return false; + } + + for (size_t i = 0; i < outputs.size(); ++i) { + if (!can_output_be_replaced(outputs[i], replacements[i])) { + return false; + } + } + + for (size_t i = 0; i < outputs.size(); ++i) { + bool replaced = replace_output_update_name(outputs[i], replacements[i]); + OPENVINO_ASSERT(replaced, + "Failed to replace output ", + i, + " of node ", + outputs[i].get_node()->get_friendly_name()); + } + + return true; +} + bool replace_node_update_name(const std::shared_ptr& target, const std::shared_ptr& replacement) { for (auto& output : target->output(0).get_target_inputs()) { if (replacement->get_input_size() > 0 && diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index fb7ffeb096f9e3..b325f201fee5b5 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -8,6 +8,7 @@ #include "function_guard.hpp" #include "itt.hpp" #include "layout_utils.hpp" +#include "low_precision/low_precision.hpp" #include "openvino/core/model.hpp" #include "openvino/op/util/multi_subgraph_base.hpp" #include "openvino/pass/constant_folding.hpp" @@ -93,7 +94,7 @@ void transformation_pipeline(std::shared_ptr& model) { RTInfoCache rt_info_cache; rt_info_cache.store(model); - auto get_manager = []() { + auto get_manager = [](const ov::Model& model) { Manager manager("pre_post_processing"); manager.set_per_pass_validation(false); @@ -112,10 +113,12 @@ void transformation_pipeline(std::shared_ptr& model) { MarkGatherSubgraph, element::TypeVector{element::f8e4m3}, element::TypeVector{element::u4}); + REGISTER_PASS( manager, MarkDequantization, - TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f4e2m1, f8e4m3, f8e5m2, f8e8m0}); + TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); + REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); REGISTER_PASS(manager, DisableShapeOfConstantFolding, false); REGISTER_PASS(manager, DisableRandomUniformConstantFolding) // Mark quantized and f16/bf16 compressed constants to prevent CF for them, @@ -140,7 +143,7 @@ void transformation_pipeline(std::shared_ptr& model) { return manager; }; - static thread_local Manager manager = get_manager(); + static thread_local Manager manager = get_manager(*model); manager.run_passes(model); diff --git a/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py b/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py index 414c993f406440..83015585a0ebac 100644 --- a/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py +++ b/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 # Copyright (C) 2018-2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 @@ -8,6 +7,27 @@ import ntpath import re +def detect_guard_patterns(content): + # Detect include guard macros (#ifndef X paired with bodyless #define X for the + # same name) to avoid adding #undef for them. Without this, the guard is + # defeated when multiple kernels with the same inlined header are batched + # into a single compilation unit. + # The following patterns are supported: + # 1. #ifndef X + # #define X + # 2. #if !defined(X) + # #define X + # 3. #if !defined X + # #define X + # All patterns allowed to have arbitrary whitespace between the tokens + ifndef_pattern = re.compile(r'#\s*ifndef\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_pattern2 = re.compile(r'#\s*if\s+!\s*defined\s*\(\s*(\w+)\s*\)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_pattern3 = re.compile(r'#\s*if\s+!\s*defined\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_names = set(match.group(1) for match in ifndef_pattern.finditer(content) if match.group(1) == match.group(2)) + ifndef_names.update(match.group(1) for match in ifndef_pattern2.finditer(content) if match.group(1) == match.group(2)) + ifndef_names.update(match.group(1) for match in ifndef_pattern3.finditer(content) if match.group(1) == match.group(2)) + return ifndef_names + class Code2CHeaders(object): def __init__(self, kernels_folder, headers_folder, lang): self.kernels_folder = os.path.abspath(kernels_folder) @@ -77,8 +97,12 @@ def add_missing_undefs(self, content): defines = set(match.group(1) for match in define_pattern.finditer(content)) undef_directives = [] - content_lines = content.split("\n") + guard_patterns = detect_guard_patterns(content) + for define in defines: + if define in guard_patterns: + continue + # Regex to check if #undef for this define already exists undef_pattern = re.compile(r'#undef\s+' + re.escape(define) + r'\s*(?:\n|$)') diff --git a/src/plugins/intel_gpu/src/graph/debug_helper.cpp b/src/plugins/intel_gpu/src/graph/debug_helper.cpp index 5d7e93d889075e..86a0a377e46f09 100644 --- a/src/plugins/intel_gpu/src/graph/debug_helper.cpp +++ b/src/plugins/intel_gpu/src/graph/debug_helper.cpp @@ -280,6 +280,12 @@ void log_memory_to_file(memory::ptr mem, layout data_layout, stream& stream, std dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::u8) dump(actual_mem, stream, file_stream, dump_raw); + else if (mem_dt == cldnn::data_types::f8e5m2) + dump(actual_mem, stream, file_stream, dump_raw); + else if (mem_dt == cldnn::data_types::f8e4m3) + dump(actual_mem, stream, file_stream, dump_raw); + else if (mem_dt == cldnn::data_types::f8e8m0) + dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::boolean) dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::i4 || mem_dt == cldnn::data_types::u4) diff --git a/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp b/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp index 68d4d01c211c95..4d4442d4e89ba0 100644 --- a/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp +++ b/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp @@ -15,7 +15,11 @@ GPU_DEFINE_PRIMITIVE_TYPE_ID(dynamic_quantize); // We should skip dynamic_quantization execution for 2nd token of LLM because it does not show performance gain. // can_be_optimized flag will be turned on from primitive_inst::update_shape function -static bool should_skip_execution(dynamic_quantize_node const& node, const layout &act_layout) { +static bool should_skip_execution(const dynamic_quantize_node& node, const layout& act_layout, const dynamic_quantize::Attributes& attrs) { + if (cldnn::data_type_traits::is_floating_point(attrs.quantization_dt)) { + return false; // Execute unconditionally for floating point types. + } + if (!node.is_runtime_skippable() || !act_layout.is_static()) return false; @@ -45,7 +49,7 @@ static bool should_skip_execution(dynamic_quantize_node const& node, const layou layout dynamic_quantize_inst::calc_output_layout(dynamic_quantize_node const& /*node*/, kernel_impl_params const& impl_param) { auto desc = impl_param.typed_desc(); const auto& input_layout = impl_param.get_input_layout(); - auto output_type = data_types::i8; + auto output_type = desc->attrs.quantization_dt; auto output_format = input_layout.format; return layout(output_type, output_format, input_layout.get_tensor()); @@ -69,7 +73,7 @@ std::vector dynamic_quantize_inst::__calc_output_layouts(const dynamic_q std::vector output_layouts = { layout(output_shapes[0], attrs.quantization_dt, output_format), layout(output_shapes[1], attrs.scale_dt, output_format) }; - auto flag_skip_execution = should_skip_execution(node, act_layout); + auto flag_skip_execution = should_skip_execution(node, act_layout, attrs); GPU_DEBUG_TRACE_DETAIL << node.id() << " should_skip_execution " << flag_skip_execution << std::endl; diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/add_required_reorders.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/add_required_reorders.cpp index 3101630d9103f1..748d0a66f85bc0 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/add_required_reorders.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/add_required_reorders.cpp @@ -310,7 +310,7 @@ void add_required_reorders::run(program& p) { continue; max_in_dims = std::max(cldnn::format::dimension(node.first->get_output_layout().format), max_in_dims); } - // This list of preferred layouts has been selected arbitrary due to developers' experience + // This list of preferred layouts has been selected arbitrarily due to developers' experience preferred_layout_formats = { cldnn::format::get_default_format(max_in_dims) }; if (max_in_dims == 8) { preferred_layout_formats.push_back(cldnn::format::bfvuwzyx); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/dynamic_quantize.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/dynamic_quantize.cpp index ab39c731198bbc..875c6ee6383b7b 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/dynamic_quantize.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/dynamic_quantize.cpp @@ -75,7 +75,10 @@ attach_dynamic_quantize_impl::attach_dynamic_quantize_impl() { auto types = { data_types::f16, data_types::i8, - data_types::u8 + data_types::u8, + data_types::f8e4m3, + data_types::f8e5m2, + data_types::f8e8m0, }; auto formats = { diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp index 9b9e9011e133df..014506259fc35a 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp @@ -228,6 +228,12 @@ kernel_selector::data_type to_data_type(data_types dt) { return kernel_selector::data_type::F32; case cldnn::data_types::bf16: return kernel_selector::data_type::BF16; + case cldnn::data_types::f8e4m3: + return kernel_selector::data_type::F8E4M3; + case cldnn::data_types::f8e5m2: + return kernel_selector::data_type::F8E5M2; + case cldnn::data_types::f8e8m0: + return kernel_selector::data_type::F8E8M0; default: OPENVINO_THROW("[GPU] Unable to convert cldnn data type ", dt, " to kernel_selector data type"); } @@ -257,6 +263,12 @@ data_types from_data_type(kernel_selector::data_type dt) { return cldnn::data_types::f16; case kernel_selector::data_type::F32: return cldnn::data_types::f32; + case kernel_selector::data_type::F8E4M3: + return cldnn::data_types::f8e4m3; + case kernel_selector::data_type::F8E5M2: + return cldnn::data_types::f8e5m2; + case kernel_selector::data_type::F8E8M0: + return cldnn::data_types::f8e8m0; default: OPENVINO_THROW("[GPU] Unable to convert kernel_selector data type ", kernel_selector::toString(dt), " to cldnn data type"); } @@ -280,6 +292,12 @@ kernel_selector::weights_type to_weights_type(data_types dt) { return kernel_selector::weights_type::INT32; case cldnn::data_types::bf16: return kernel_selector::weights_type::BF16; + case cldnn::data_types::f8e4m3: + return kernel_selector::weights_type::F8E4M3; + case cldnn::data_types::f8e5m2: + return kernel_selector::weights_type::F8E5M2; + case cldnn::data_types::f8e8m0: + return kernel_selector::weights_type::F8E8M0; default: OPENVINO_THROW("[GPU] Unable to convert cldnn data type ", dt, " to kernel_selector weights type"); } @@ -301,6 +319,12 @@ data_types from_weights_type(kernel_selector::weights_type dt) { return data_types::f32; case kernel_selector::weights_type::INT32: return data_types::i32; + case kernel_selector::weights_type::F8E4M3: + return data_types::f8e4m3; + case kernel_selector::weights_type::F8E5M2: + return data_types::f8e5m2; + case kernel_selector::weights_type::F8E8M0: + return data_types::f8e8m0; default: OPENVINO_THROW("[GPU] Unable to convert kernel_selector weights type ", kernel_selector::toString(dt), " to cldnn data type"); } diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp index 56ed0f607de4df..f5f708dc21df78 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp @@ -320,6 +320,42 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: type_size = "4"; is_fp = true; break; + case ov::element::f8e4m3: + type = "fp8e4m3_t"; + max_val = "(fp8e4m3_t){as_char((char)0x7E)}"; // 448.0 + min_val = "(fp8e4m3_t){as_char((char)0xFE)}"; // -448.0 + val_one = "(fp8e4m3_t){as_char((char)0x38)}"; + val_zero = "(fp8e4m3_t){as_char((char)0x0)}"; + to_type = "_convert_fp8e4m3_t(v)"; + to_type_sat = "_convert_fp8e4m3_t_sat(v)"; + as_type = "as_fp8e4m3_t(v)"; + type_size = "1"; + is_fp = true; + break; + case ov::element::f8e5m2: + type = "fp8e5m2_t"; + max_val = "(fp8e5m2_t){as_uchar((uchar)0x7B)}"; // 57344.0 + min_val = "(fp8e5m2_t){as_uchar((uchar)0xFB)}"; // -57344.0 + val_one = "(fp8e5m2_t){as_uchar((uchar)0x3C)}"; + val_zero = "(fp8e5m2_t){as_uchar((uchar)0x0)}"; + to_type = "_convert_fp8e5m2_t(v)"; + to_type_sat = "_convert_fp8e5m2_t_sat(v)"; + as_type = "as_fp8e5m2_t(v)"; + type_size = "1"; + is_fp = true; + break; + case ov::element::f8e8m0: + type = "fp8e8m0_t"; + max_val = "(fp8e8m0_t){as_uchar((uchar)0xFE)}"; // 2^127 + min_val = "(fp8e8m0_t){as_uchar((uchar)0x00)}"; // 2^(-127) + val_one = "(fp8e8m0_t){as_uchar((uchar)0x7F)}"; + val_zero = ""; // There is no representation of zero in FP8E8M0 + to_type = "_convert_fp8e8m0_t(v)"; + to_type_sat = "_convert_fp8e8m0_t_sat(v)"; + as_type = "as_fp8e8m0_t(v)"; + type_size = "1"; + is_fp = true; + break; case ov::element::dynamic: type = "uchar"; max_val = "UCHAR_MAX"; diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp index 244f18aeda9e11..8fd4cd582fa7b1 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp @@ -69,25 +69,27 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { dnnl::memory::desc desc = onednn::layout_to_memory_desc_flatten(zp_mem->get_layout(), dnnl::memory::format_tag::a); args.insert({DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_WEIGHTS, zp_mem->get_onednn_memory(desc)}); } - bool is_dyn_quan_input = instance.get_input_layout(0).data_type == data_types::i8 || instance.get_input_layout(0).data_type == data_types::u8; + + const auto input_dt = instance.get_input_layout(0).data_type; + const bool is_dyn_quan_input = cldnn::one_of(input_dt, {data_types::i8, data_types::u8, data_types::f8e4m3, data_types::f8e5m2}); if (is_dyn_quan_input && prim->activation_scale.is_valid()) { - auto activation_scale_idx = idx++; - auto act_scale_mem = instance.dep_memory_ptr(activation_scale_idx); + const auto activation_scale_idx = idx++; + const auto act_scale_mem = instance.dep_memory_ptr(activation_scale_idx); dnnl::memory::desc desc = onednn::layout_to_memory_desc_flatten(act_scale_mem->get_layout(), dnnl::memory::format_tag::ab); args.insert({DNNL_ARG_ATTR_SCALES | DNNL_ARG_SRC_0, act_scale_mem->get_onednn_memory(desc)}); } if (is_dyn_quan_input && prim->activation_zero_point.is_valid()) { - auto activation_zp_idx = idx++; - auto act_zp_mem = instance.dep_memory_ptr(activation_zp_idx); + const auto activation_zp_idx = idx++; + const auto act_zp_mem = instance.dep_memory_ptr(activation_zp_idx); dnnl::memory::desc desc = onednn::layout_to_memory_desc_flatten(act_zp_mem->get_layout(), dnnl::memory::format_tag::ab); args.insert({DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_SRC_0, act_zp_mem->get_onednn_memory(desc)}); } if (is_dyn_quan_input && prim->activation_precomputed_reduction.is_valid()) { - auto activation_precomputed_reduction_idx = idx++; - auto act_precomputed_reduction_mem = instance.dep_memory_ptr(activation_precomputed_reduction_idx); + const auto activation_precomputed_reduction_idx = idx++; + const auto act_precomputed_reduction_mem = instance.dep_memory_ptr(activation_precomputed_reduction_idx); dnnl::memory::desc desc = onednn::layout_to_memory_desc_flatten(act_precomputed_reduction_mem->get_layout(), dnnl::memory::format_tag::ab); args.insert({DNNL_ARG_ATTR_PRECOMPUTED_REDUCTIONS | DNNL_ARG_SRC_0, act_precomputed_reduction_mem->get_onednn_memory(desc)}); } @@ -304,7 +306,8 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { } } - bool is_dyn_quan_input = impl_params->get_input_layout(0).data_type == data_types::i8 || impl_params->get_input_layout(0).data_type == data_types::u8; + const auto input_dt = impl_params->get_input_layout(0).data_type; + const bool is_dyn_quan_input = cldnn::one_of(input_dt, {data_types::i8, data_types::u8, data_types::f8e4m3, data_types::f8e5m2}); if (is_dyn_quan_input && dynamic_quantized_activation) { auto src_scale_idx = ++idx; auto partial_shape = impl_params->get_input_layout(0).get_partial_shape(); @@ -351,7 +354,8 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { int idx = !arg.bias_term() ? 1 : 2; if (prim->compressed_weights) { - bool is_dyn_quan_input = impl_params.get_input_layout(0).data_type == data_types::i8 || impl_params.get_input_layout(0).data_type == data_types::u8; + const auto input_dt = impl_params.get_input_layout(0).data_type; + const bool is_dyn_quan_input = cldnn::one_of(input_dt, {data_types::i8, data_types::u8, data_types::f8e4m3, data_types::f8e5m2}); if (is_dyn_quan_input) { OPENVINO_ASSERT(prim->input_size <= 3, "[GPU] Dynamic quantization for 4D matmul is not implemented"); } else { @@ -436,7 +440,6 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { } - auto prim_desc = get_matmul_primitive_descriptor(impl_params, impl_params.prog->get_engine(), prim->input_size, prim->weights_rank, prim->bias.is_valid(), *attr); diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp index fe555f05c4036e..8052aabb07f5a4 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp @@ -62,13 +62,15 @@ struct FullyConnectedImplementationManager : public ImplementationManager { one_of(in0_dt, {data_types::f16, data_types::bf16, data_types::f32, data_types::i8, data_types::u8}) && one_of(wei_dt, {data_types::u8, data_types::i8, data_types::u4, data_types::i4}) && one_of(out_dt, {data_types::f16, data_types::bf16, data_types::f32, data_types::u8, data_types::i8}); - if (!f16f16_case && !bf16bf16_case && !f32f32_case && !u8s8_case && !compressed_case) + const bool fp_compressed_case = fc_prim->compressed_weights && one_of(in0_dt, {data_types::f8e4m3, data_types::f8e5m2}) && + one_of(wei_dt, {data_types::f8e4m3, data_types::f8e5m2}) && one_of(out_dt, {data_types::f16, data_types::f32}); + if (!f16f16_case && !bf16bf16_case && !f32f32_case && !u8s8_case && !compressed_case && !fp_compressed_case) LOG_AND_RETURN_FALSE(node); if (fc_prim->compressed_weights) { if (fc_prim->decompression_zero_point.is_valid()) { - auto decompression_zp_idx = fc_prim->bias.is_valid() ? 4 : 3; - auto decompression_zp_dt = fc_node.get_input_layout(decompression_zp_idx).data_type; + const auto decompression_zp_idx = fc_prim->bias.is_valid() ? 4 : 3; + const auto decompression_zp_dt = fc_node.get_input_layout(decompression_zp_idx).data_type; if ((wei_dt != ov::element::Type_t::i4 && wei_dt != ov::element::Type_t::u4 && wei_dt != ov::element::Type_t::u8) || (decompression_zp_dt != ov::element::Type_t::i4 && decompression_zp_dt != ov::element::Type_t::u8 && decompression_zp_dt != ov::element::Type_t::i8)) { @@ -77,6 +79,18 @@ struct FullyConnectedImplementationManager : public ImplementationManager { } } + if (fp_compressed_case && fc_prim->decompression_scale.is_valid() && fc_prim->activation_scale.is_valid()) { + auto decompression_scale_idx = fc_prim->bias.is_valid() ? 3 : 2; + auto weight_scale_dt = fc_node.get_input_layout(decompression_scale_idx).data_type; + auto activation_scale_dt = fc_node.get_input_layout(decompression_scale_idx + 1).data_type; + if (activation_scale_dt != weight_scale_dt) { + LOG_AND_RETURN_FALSE(node); + } + if (!one_of(activation_scale_dt, {ov::element::Type_t::f8e8m0, ov::element::Type_t::f16})) { + LOG_AND_RETURN_FALSE(node); + } + } + return true; } diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp index 1b91790c1d5975..6d830c3c9e4a61 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp @@ -142,6 +142,9 @@ dnnl::memory::data_type convert_data_type(cldnn::data_types dt) { case cldnn::data_types::i32: return dnnl::memory::data_type::s32; case cldnn::data_types::i4: return dnnl::memory::data_type::s4; case cldnn::data_types::u4: return dnnl::memory::data_type::u4; + case cldnn::data_types::f8e4m3: return dnnl::memory::data_type::f8_e4m3; + case cldnn::data_types::f8e5m2: return dnnl::memory::data_type::f8_e5m2; + case cldnn::data_types::f8e8m0: return dnnl::memory::data_type::e8m0; case cldnn::data_types::bf16: return dnnl::memory::data_type::bf16; default: throw std::invalid_argument("[clDNN] Unsupported conversion from cldnn to onednn type"); } @@ -258,6 +261,9 @@ int64_t get_offset(const cldnn::layout& l, dnnl::memory::desc&& desc) { return offset / 2; case dnnl::memory::data_type::s8: case dnnl::memory::data_type::u8: + case dnnl::memory::data_type::f8_e4m3: + case dnnl::memory::data_type::f8_e5m2: + case dnnl::memory::data_type::e8m0: return offset; case dnnl::memory::data_type::f16: case dnnl::memory::data_type::bf16: diff --git a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp index e3e61a68c45e2b..95cb7470fda255 100644 --- a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp +++ b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp @@ -540,7 +540,7 @@ void primitive_inst::update_shape() { if (get_node().is_type() && get_flag(ExecutionFlags::SHAPE_CHANGED)) { auto &layout = _impl_params->get_output_layout(0); - OPENVINO_ASSERT(one_of(layout.data_type, {data_types::f16, data_types::i8, data_types::u8}), + OPENVINO_ASSERT(one_of(layout.data_type, {data_types::f16, data_types::i8, data_types::u8, data_types::f8e4m3, data_types::f8e5m2}), "[GPU] Unsupported data type of dynamic_quantize: ", layout.data_type); if (layout.data_type == data_types::f16) set_can_be_optimized(true); diff --git a/src/plugins/intel_gpu/src/graph/program.cpp b/src/plugins/intel_gpu/src/graph/program.cpp index 55a1a8e9fd38ce..c13b7bc87fc2ba 100644 --- a/src/plugins/intel_gpu/src/graph/program.cpp +++ b/src/plugins/intel_gpu/src/graph/program.cpp @@ -915,9 +915,15 @@ void program::add_intermediate(program_node& node, size_t prev_idx, bool connect_int_node_with_old_dep, bool move_usrs_of_prev_to_node) { - if (connect_int_node_with_old_dep && !node.dependencies.empty()) - throw std::invalid_argument( - "Node which is about to be added in between two other nodes should not have any existing dependencies"); + if (connect_int_node_with_old_dep && !node.dependencies.empty()) { + std::string deps; + for (auto& dep : node.dependencies) { + deps += dep.first->id() + " ( " + dep.first->get_primitive()->type_string() + " ), "; + } + OPENVINO_THROW("Node which is about to be added in between two other nodes should not have any existing dependencies. Node: " + node.id() + " ( " + + node.get_primitive()->type_string() + " )" + ". Next: " + next.id() + " ( " + next.get_primitive()->type_string() + + " ). Dependencies: " + deps); + } auto& prev = next.get_dependency(prev_idx); // firstly add connection, later replace dependency, so 'prev' won't become dangling and therefore removed diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index 09077d04e229a3..80bed3e059b05c 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -2,21 +2,41 @@ // SPDX-License-Identifier: Apache-2.0 // +#define IS_F8 (F8E5M2_OUTPUT || F8E4M3_OUTPUT) + #include "include/batch_headers/fetch_data.cl" +#if IS_F8 +#include "include/f8_utils.cl" +#endif #if OUTPUT_DIMS != 4 && OUTPUT_DIMS != 2 #error "dynamic_quantize_gpu_opt.cl: Unsupported output dimension" #endif +#if IS_F8 + #define SCALE_TYPE float + #define TO_SCALE_TYPE(x) _convert_float(x) + #define ACT_MIN_VAL 0.000000059604645h // min half dtype val + #define TO_TYPE_N_(type, n, x) convert_##type##n(x) + #define TO_TYPE_N_SAT_(type, n, x) _convert_##type##n##_sat(x) +#else + #define SCALE_TYPE half + #define TO_SCALE_TYPE(x) convert_half(x) + #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL + #define TO_TYPE_N_(type, n, x) convert_##type##n(x) + #define TO_TYPE_N_SAT_(type, n, x) convert_##type##n##_sat(x) +#endif + #define VLOAD_N CAT(vload, VEC_SIZE) #define VSTORE_N CAT(vstore, VEC_SIZE) #define CONVERT_UCHAR_N CAT(convert_uchar, VEC_SIZE) #define CONVERT_CHAR_N CAT(convert_char, VEC_SIZE) #define CONVERT_INT_N CAT(convert_int, VEC_SIZE) +#define TO_TYPE_N(type, n, x) TO_TYPE_N_(type, n, x) +#define TO_TYPE_N_SAT(type, n, x) TO_TYPE_N_SAT_(type, n, x) #define AS_TYPE_N_(type, n, x) as_##type##n(x) #define AS_TYPE_N(type, n, x) AS_TYPE_N_(type, n, x) #define AS_INPUT_TYPE_N(x) AS_TYPE_N(INPUT0_TYPE, VEC_SIZE, x) -#define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL #if GENERATE_PRECOMPUTED_REDUCTION #define FOR_PRECOMPUTED_REDUCTION(x) x @@ -60,7 +80,7 @@ KERNEL(dynamic_quantize_gpu_opt)( #endif const uint quantize_block = QUANTIZE_GROUP_SIZE / 4; half4 input_0[quantize_block]; - char4 quantized_value[quantize_block]; + MAKE_VECTOR_TYPE(OUTPUT_TYPE, 4) quantized_value[quantize_block]; half max[quantize_block]; unroll_for (uint i = 0 ; i < quantize_block; ++i) { @@ -73,13 +93,22 @@ KERNEL(dynamic_quantize_gpu_opt)( max_value = fmax(max_value, max[i]); } - half quan_scale = 127.0h / max_value; +#if IS_MXFP + SCALE_TYPE quan_scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / convert_float(max_value))))); +#else + SCALE_TYPE quan_scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_value; FOR_PRECOMPUTED_REDUCTION(int precomputed_reduction = 0); +#endif // MXFP unroll_for (uint i = 0 ; i < quantize_block; ++i) { +#if IS_F8 + quantized_value[i] = TO_TYPE_N_SAT(OUTPUT_TYPE, 4, convert_float4(input_0[i]) * (MAKE_VECTOR_TYPE(SCALE_TYPE, 4))quan_scale); + vstore4(quantized_value[i].data, 0, (char*)(&output[output_offset + i * 4])); +#else quantized_value[i] = convert_char4_rte(input_0[i] * (half4)quan_scale); FOR_PRECOMPUTED_REDUCTION(precomputed_reduction += quantized_value[i][0] + quantized_value[i][1] + quantized_value[i][2] + quantized_value[i][3]); vstore4(quantized_value[i], 0, &output[output_offset + i * 4]); +#endif // IS_F8 } #if OUTPUT_DIMS == 2 @@ -87,9 +116,11 @@ KERNEL(dynamic_quantize_gpu_opt)( #else const uint output_idx = OUTPUT1_GET_INDEX(b, f, y_grp, 0); #endif - output_scale[output_idx] = 1.0h / quan_scale; + output_scale[output_idx] = TO_OUTPUT1_TYPE(1.0h / quan_scale); +#if !(IS_MXFP) FOR_PRECOMPUTED_REDUCTION(output_precomputed_reduction[output_idx] = precomputed_reduction); +#endif } // *********************************************** @@ -199,14 +230,19 @@ KERNEL(dynamic_quantize_gpu_opt)( OUTPUT1_TYPE scale = (OUTPUT1_TYPE)((CHAR_MAX - CHAR_MIN) / (max_value - min_value)); OUTPUT2_TYPE zp = (OUTPUT2_TYPE)(-min_value * scale); #else - OUTPUT1_TYPE scale = 127.0h / max_value; + SCALE_TYPE scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_value; #endif +#if IS_F8 + val = TO_TYPE_N(INPUT0_TYPE, VEC_SIZE, TO_TYPE_N(SCALE_TYPE, VEC_SIZE, val) * (MAKE_VECTOR_TYPE(SCALE_TYPE, VEC_SIZE))scale); + MAKE_VECTOR_TYPE(OUTPUT_TYPE, VEC_SIZE) out = TO_TYPE_N_SAT(OUTPUT_TYPE, VEC_SIZE, val); + VSTORE_N(out.data, 0, (char*)(&output[output_offset + (blockid * block_size)])); +#elif ASYMMETRIC_QUANTIZATION val *= scale; -#if ASYMMETRIC_QUANTIZATION val += zp; VSTORE_N(CAT(CONVERT_UCHAR_N, _rte)(val), 0, output + output_offset + (blockid * block_size)); -#else +#else // i8 symmetric + val *= scale; VSTORE_N(CAT(CONVERT_CHAR_N, _rte)(val), 0, output + output_offset + (blockid * block_size)); #endif @@ -250,7 +286,9 @@ KERNEL(dynamic_quantize_gpu_opt)( #if ASYMMETRIC_QUANTIZATION output_zp[output_idx] = convert_uchar_rte(zp); #endif +#if !(IS_MXFP) FOR_PRECOMPUTED_REDUCTION(output_precomputed_reduction[output_idx] = precomputed_reduction); +#endif } } @@ -344,7 +382,7 @@ KERNEL(dynamic_quantize_gpu_opt)( OUTPUT1_TYPE scale = (OUTPUT1_TYPE)((CHAR_MAX - CHAR_MIN) / (max_value - min_value)); OUTPUT2_TYPE zp = (OUTPUT2_TYPE)(-min_value * scale); #else - OUTPUT1_TYPE scale = 127.0h / max_value; + SCALE_TYPE scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_value; #endif @@ -352,11 +390,14 @@ KERNEL(dynamic_quantize_gpu_opt)( if ((local_id * iteration + i) >= TOTAL_BLOCK_NUM) continue; - val[i] *= scale; -#if ASYMMETRIC_QUANTIZATION + val[i] = TO_TYPE_N(INPUT0_TYPE, VEC_SIZE, TO_TYPE_N(SCALE_TYPE, VEC_SIZE, val[i]) * (MAKE_VECTOR_TYPE(SCALE_TYPE, VEC_SIZE))scale); +#if IS_F8 + MAKE_VECTOR_TYPE(OUTPUT_TYPE, VEC_SIZE) out = TO_TYPE_N_SAT(OUTPUT_TYPE, VEC_SIZE, val[i]); + VSTORE_N(out.data, 0, (char*)(&output[offset + ((local_id * iteration + i) * block_size)])); +#elif ASYMMETRIC_QUANTIZATION val[i] += zp; VSTORE_N(CAT(CONVERT_UCHAR_N, _rte)(val[i]), 0, output + offset + ((local_id * iteration + i) * block_size)); -#else +#else // i8 symmetric VSTORE_N(CAT(CONVERT_CHAR_N, _rte)(val[i]), 0, output + offset + ((local_id * iteration + i) * block_size)); #endif } diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl index 56f906af15288b..dee2976bd61dbd 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl @@ -2,17 +2,39 @@ // SPDX-License-Identifier: Apache-2.0 // +#define IS_F8 (F8E5M2_OUTPUT || F8E4M3_OUTPUT) + #include "include/batch_headers/fetch_data.cl" +#if IS_F8 +#include "include/f8_utils.cl" +#endif #define UINT64_MAX 0xFFFFFFFFFFFFFFFF -#define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL -#if ASYMMETRIC_QUANTIZATION && UNSIGNED_OUTPUT - #define TO_OUTPUT_TYPE_RTE(val) convert_uchar_rte(val) - #define TO_OUTPUT_VEC_TYPE_RTE(val) convert_uchar8_rte(val) +#if IS_F8 + #define SCALE_TYPE float + #define TO_SCALE_TYPE(x) _convert_float(x) + #define TO_SCALE_TYPE_8(x) convert_float8(x) + #define ACT_MIN_VAL 0.000000059604645h // min half dtype val +#else + #define SCALE_TYPE half + #define TO_SCALE_TYPE(x) convert_half(x) + #define TO_SCALE_TYPE_8(x) convert_half8(x) + #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL +#endif + +#if F8E5M2_OUTPUT + #define TO_OUTPUT_TYPE_CUSTOM(val) _convert_fp8e5m2_t_sat(val) + #define TO_OUTPUT_VEC_TYPE_CUSTOM(val) _convert_fp8e5m2_t8_sat(val) +#elif F8E4M3_OUTPUT + #define TO_OUTPUT_TYPE_CUSTOM(val) _convert_fp8e4m3_t_sat(val) + #define TO_OUTPUT_VEC_TYPE_CUSTOM(val) _convert_fp8e4m3_t8_sat(val) +#elif (ASYMMETRIC_QUANTIZATION && UNSIGNED_OUTPUT) + #define TO_OUTPUT_TYPE_CUSTOM(val) convert_uchar_rte(val) + #define TO_OUTPUT_VEC_TYPE_CUSTOM(val) convert_uchar8_rte(val) #else - #define TO_OUTPUT_TYPE_RTE(val) convert_char_rte(val) - #define TO_OUTPUT_VEC_TYPE_RTE(val) convert_char8_rte(val) + #define TO_OUTPUT_TYPE_CUSTOM(val) convert_char_rte(val) + #define TO_OUTPUT_VEC_TYPE_CUSTOM(val) convert_char8_rte(val) #endif #if GENERATE_PRECOMPUTED_REDUCTION @@ -124,8 +146,12 @@ KERNEL(dynamic_quantize_gpu_ref)( OUTPUT1_TYPE scale = (OUTPUT1_TYPE)(scale_tmp); OUTPUT1_TYPE zp = (OUTPUT1_TYPE)(zp_tmp); #else // !ASYMMETRIC_QUANTIZATION - OUTPUT1_TYPE scale = 127.0h / max_val; -#endif +#if IS_MXFP + SCALE_TYPE scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / convert_float(max_val))))); +#else + SCALE_TYPE scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_val; +#endif // IS_FP8 +#endif // ASYMMETRIC_QUANTIZATION FOR_PRECOMPUTED_REDUCTION(OUTPUT2_TYPE precomputed_reduction = 0); @@ -141,7 +167,7 @@ KERNEL(dynamic_quantize_gpu_ref)( #if ASYMMETRIC_QUANTIZATION val += zp; #endif - OUTPUT_TYPE ival = TO_OUTPUT_TYPE_RTE(val); + OUTPUT_TYPE ival = TO_OUTPUT_TYPE_CUSTOM(val); output[out_offset] = ival; FOR_PRECOMPUTED_REDUCTION(precomputed_reduction += ival); #else // GROUP_SIZE_DIM3 != 1 @@ -150,13 +176,17 @@ KERNEL(dynamic_quantize_gpu_ref)( int x; for (x = 0; x < INPUT0_SIZE_X / 8; x++) { half8 val = as_half8(vload8(0, (ushort*)input + in_offset + x * 8)); - val *= scale; + val = convert_half8(TO_SCALE_TYPE_8(val) * (MAKE_VECTOR_TYPE(SCALE_TYPE, 8))scale); #if ASYMMETRIC_QUANTIZATION val += zp; #endif - MAKE_VECTOR_TYPE(OUTPUT_TYPE, 8) ival = TO_OUTPUT_VEC_TYPE_RTE(val); +#if IS_F8 + vstore8(TO_OUTPUT_VEC_TYPE_CUSTOM(val).data, 0, (char*)(&output[out_offset + x * 8])); +#else + MAKE_VECTOR_TYPE(OUTPUT_TYPE, 8) ival = TO_OUTPUT_VEC_TYPE_CUSTOM(val); vstore8(ival, 0, output + out_offset + x * 8); FOR_PRECOMPUTED_REDUCTION(precomputed_reduction += ((int)ival[0]) + ival[1] + ival[2] + ival[3] + ival[4] + ival[5] + ival[6] + ival[7]); +#endif } x *= 8; for (; x < INPUT0_SIZE_X; x++) { @@ -165,7 +195,7 @@ KERNEL(dynamic_quantize_gpu_ref)( #if ASYMMETRIC_QUANTIZATION val += zp; #endif - OUTPUT_TYPE ival = TO_OUTPUT_TYPE_RTE(val); + OUTPUT_TYPE ival = TO_OUTPUT_TYPE_CUSTOM(val); output[out_offset + x] = ival; FOR_PRECOMPUTED_REDUCTION(precomputed_reduction += ival); } @@ -174,7 +204,7 @@ KERNEL(dynamic_quantize_gpu_ref)( } } - output_scale[scale_idx] = 1.0h / scale; + output_scale[scale_idx] = TO_OUTPUT1_TYPE(1.0h / scale); FOR_PRECOMPUTED_REDUCTION(output_precomputed_reduction[scale_idx] = precomputed_reduction); #if ASYMMETRIC_QUANTIZATION && GROUP_SCALES_WITH_ZP output_scale[scale_idx + 1] = zp; diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index ac885a2be0ad6b..5bcd72ef1ff8c8 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -38,6 +38,21 @@ #define MAKE_VECTOR_TYPE_IMPL_16(elem_type) CAT(elem_type, 16) #define MAKE_VECTOR_TYPE(elem_type, size) CAT(MAKE_VECTOR_TYPE_IMPL_, size)(elem_type) +#define AS_TYPE_PREFIX_uchar as_ +#define AS_TYPE_PREFIX_char as_ +#define AS_TYPE_PREFIX_fp8e5m2_t _as_ +#define AS_TYPE_PREFIX_fp8e4m3_t _as_ +#define AS_TYPE_PREFIX_fp8e8m0_t _as_ +#define AS_TYPE_PREFIX_ushort as_ +#define AS_TYPE_PREFIX_short as_ +#define AS_TYPE_PREFIX_half as_ +#define AS_TYPE_PREFIX_int as_ +#define AS_TYPE_PREFIX_uint as_ +#define AS_TYPE_PREFIX_float as_ +#define AS_TYPE_PREFIX_ulong as_ +#define AS_TYPE_PREFIX_long as_ + +#define AS_TYPE_EXT(type, val, src_type) CAT(CAT(AS_TYPE_PREFIX_, src_type), type)(val) #define AS_TYPE(type, val) CAT(as_, type)(val) // ==================================================================================================================== @@ -46,6 +61,9 @@ // ==================================================================================================================== #define TYPE_SIZE_uchar 1 #define TYPE_SIZE_char 1 +#define TYPE_SIZE_fp8e5m2_t 1 +#define TYPE_SIZE_fp8e4m3_t 1 +#define TYPE_SIZE_fp8e8m0_t 1 #define TYPE_SIZE_ushort 2 #define TYPE_SIZE_short 2 #define TYPE_SIZE_half 2 diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl index 0dc7769a7fa284..f89ebe1eb02e2f 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl @@ -48,16 +48,16 @@ #define BLOCK_WRITEN_FUNC_size8(vector_size) BLOCK_WRITEN_FUNC_SIZE_DEF(8, vector_size) #define BLOCK_WRITEN_FUNC(type_size, vector_size) CAT(BLOCK_WRITEN_FUNC_size, type_size)(vector_size) -#define BLOCK_WRITEN_RAW(type_size, vector_size, addr_space, ptr, offset, val) \ +#define BLOCK_WRITEN_RAW(type_size, vector_size, addr_space, ptr, offset, val, src_type) \ BLOCK_WRITEN_FUNC(type_size, vector_size)( \ (addr_space BLOCK_WRITE_TYPE(type_size)*)(ptr) + (offset), \ - AS_TYPE(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val)) + AS_TYPE_EXT(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val, src_type)) #define BLOCK_WRITEN(type, vector_size, ptr, offset, val) \ - BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __global, ptr, offset, val) + BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __global, ptr, offset, val, type) #define BLOCK_WRITEN_SLM(type, vector_size, ptr, offset, val) \ - BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __local, ptr, offset, val) + BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __local, ptr, offset, val, type) #define DT_OUTPUT_BLOCK_WRITE(ptr, offset, val) BLOCK_WRITEN(OUTPUT_TYPE, 1, ptr, offset, val) #define DT_OUTPUT_BLOCK_WRITE2(ptr, offset, val) BLOCK_WRITEN(OUTPUT_TYPE, 2, ptr, offset, val) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/f8_utils.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/f8_utils.cl new file mode 100644 index 00000000000000..70ae7d37b64631 --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/f8_utils.cl @@ -0,0 +1,1285 @@ +// Copyright (C) 2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +// TODO: Replace `_intel_convert*` with bultins when ready, current implementations are copied from XeTLA: + +#ifndef OV_GPU_OCL_F8_UTILS_H +#define OV_GPU_OCL_F8_UTILS_H + +uchar _f16_to_bf8_universal(half val, bool is_saturation) { + half val_fp16 = val; + const ushort *p = (ushort *)&val_fp16; + const ushort mant = p[0] & 0x3FF; + const uchar exp_mask = 0x7C; + + uchar ret_tmp = p[0] >> 8; + const bool is_infnan = (ret_tmp & exp_mask) == exp_mask; + if (is_infnan) { + ret_tmp |= (mant != 0); // The bit signifying NaNness may have been cut off + } else { + ret_tmp += (mant & 0x80) && ((mant & 0x7F) || (mant & 0x100)); // RTE + if (is_saturation) { + bool is_overflow = (ret_tmp & exp_mask) == exp_mask; + ret_tmp -= is_overflow; + } + } + + return ret_tmp; +} + +uchar _intel_convert_f16_to_bf8(half val) { + return _f16_to_bf8_universal(val, false); +} + +uchar _intel_convert_f16_to_bf8_sat(half val) { + return _f16_to_bf8_universal(val, true); +} + +half _intel_convert_bf8_to_f16(uchar val) { + ushort temp = val; + temp = temp << 0x8; + half temp_fp16 = as_half(temp); + return temp_fp16; +} + +char _f16_to_hf8_universal(half val, bool is_saturation) { + half val_fp16 = val; + ushort *p = (ushort *)&val_fp16; + ushort src = p[0]; + const ushort src_exp_size = 5; + const ushort src_mant_size = 10; + const ushort src_exp_bias = (1 << (src_exp_size - 1)) - 1; + const ushort src_exp_mask = (1 << src_exp_size) - 1; + const ushort src_mant_mask = (1 << src_mant_size) - 1; + const short max_exp_unbiased = 8; + const short min_exp_unbiased = -6; + const short exp_bias = 7; + const ushort exp_size = 4; + const ushort mant_size = 3; + const uchar nan = 0x7f; + const uchar max_val = 0x7e; + + ushort src_sign = src >> (src_exp_size + src_mant_size); + ushort src_exp = (src >> src_mant_size) & src_exp_mask; + short src_exp_unbiased = src_exp - src_exp_bias; + ushort src_mant = src & src_mant_mask; + + bool is_src_inf_nan = src_exp == 0x1f; + bool is_overflow = (src_exp_unbiased > max_exp_unbiased) + || ((src_exp_unbiased == max_exp_unbiased) && (src_mant > 0x0340)); + bool is_zero = (src_exp_unbiased < (min_exp_unbiased - mant_size)); + bool is_denorm = (src_exp_unbiased < min_exp_unbiased) && (!is_zero); + + uchar dst_val; + if (is_src_inf_nan) { + dst_val = nan; + } else if (is_overflow) { + dst_val = is_saturation ? max_val : nan; + } else if (is_zero) { + dst_val = 0; + } else if (is_denorm) { + ushort src_m = src_mant | 0x0400; + short shift_out_bit = min_exp_unbiased - src_exp_unbiased; + bool sticky_flag = (src_m & ((1 << shift_out_bit) - 1)) != 0; + src_m = src_m >> shift_out_bit; + ushort tail_size = src_mant_size - mant_size; + sticky_flag + = sticky_flag || ((src_m & ((1 << (tail_size - 1)) - 1)) != 0); + bool lsb_bit = src_m & (1 << tail_size); + bool rnd_bit = src_m & (1 << (tail_size - 1)); + bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); + + dst_val = (src_m >> tail_size) + carry; + } else { + ushort tail_size = src_mant_size - mant_size; + bool sticky_flag = (src_mant & ((1 << (tail_size - 1)) - 1)) != 0; + bool lsb_bit = src_mant & (1 << tail_size); + bool rnd_bit = src_mant & (1 << (tail_size - 1)); + bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); + ushort src_m = (src_mant >> tail_size) + carry; + ushort src_e = src_exp_unbiased + exp_bias; + dst_val = (src_e << mant_size) + src_m; + } + return src_sign << (exp_size + mant_size) | dst_val; +} + +char _intel_convert_f16_to_hf8(half val) { + return _f16_to_hf8_universal(val, false); +} + +char _intel_convert_f16_to_hf8_sat(half val) { + return _f16_to_hf8_universal(val, true); +} + +half _intel_convert_hf8_to_f16(char val) { + const short exp_bias = 7; + char data = val; + ushort sign = data >> 7; + ushort exp = (data >> 3) & 0b1111; + ushort mant = data & 0x07; + ushort dst_val; + if ((exp == 0xf) && (mant == 0x7)) { + dst_val = 0x7fff; + } else if ((exp == 0) && (mant == 0)) { + dst_val = 0; + } else if ((exp == 0) && (mant != 0)) { + ushort lz_count = (mant > 3) ? 0 : ((mant > 1) ? 1 : 2); + ushort dst_exp = exp - exp_bias + 15 - lz_count; + ushort dst_mant = (mant << (lz_count + 1)) & 0x7; + dst_val = (dst_exp << 10) | (dst_mant << 7); + } else { + ushort dst_exp = exp - exp_bias + 15; + dst_val = (dst_exp << 10) | (mant << 7); + } + + ushort temp = (sign << 15) | dst_val; + half temp_fp16 = as_half(temp); + return temp_fp16; +} + +uchar _intel_convert_f32_fo_e8m0(float val) { + uint val_uint = as_uint(val); + return (uchar)((val_uint >> 23) & 0xFF); +} + +uchar _intel_convert_f32_fo_e8m0_sat(float val) { + return _intel_convert_f32_fo_e8m0(val); +} + +float _intel_convert_e8m0_to_f32(uchar val) { + if (val == 0xFF) { + return as_float(0xFFC00000); // NaN + } else if (val == 0) { + return as_float(0x00400000); // 2^(-127) + } + + uint temp = (uint)val; + temp = temp << 23; + float temp_fp32 = as_float(temp); + return temp_fp32; +} + +typedef struct fp8e5m2_t { uchar data; } fp8e5m2_t; // bf8 +typedef struct fp8e5m2_t1 { uchar data; } fp8e5m2_t1; +typedef struct fp8e5m2_t2 { uchar2 data; } fp8e5m2_t2; +typedef struct fp8e5m2_t3 { uchar3 data; } fp8e5m2_t3; +typedef struct fp8e5m2_t4 { uchar4 data; } fp8e5m2_t4; +typedef struct fp8e5m2_t8 { uchar8 data; } fp8e5m2_t8; +typedef struct fp8e5m2_t16 { uchar16 data; } fp8e5m2_t16; + +typedef struct fp8e4m3_t { char data; } fp8e4m3_t; // hf8 +typedef struct fp8e4m3_t1 { char data; } fp8e4m3_t1; +typedef struct fp8e4m3_t2 { char2 data; } fp8e4m3_t2; +typedef struct fp8e4m3_t3 { char3 data; } fp8e4m3_t3; +typedef struct fp8e4m3_t4 { char4 data; } fp8e4m3_t4; +typedef struct fp8e4m3_t8 { char8 data; } fp8e4m3_t8; +typedef struct fp8e4m3_t16 { char16 data; } fp8e4m3_t16; + +typedef struct fp8e8m0_t { uchar data; } fp8e8m0_t; // e8m0 +typedef struct fp8e8m0_t1 { uchar data; } fp8e8m0_t1; +typedef struct fp8e8m0_t2 { uchar2 data; } fp8e8m0_t2; +typedef struct fp8e8m0_t3 { uchar3 data; } fp8e8m0_t3; +typedef struct fp8e8m0_t4 { uchar4 data; } fp8e8m0_t4; +typedef struct fp8e8m0_t8 { uchar8 data; } fp8e8m0_t8; +typedef struct fp8e8m0_t16 { uchar16 data; } fp8e8m0_t16; + + +half __attribute__((overloadable)) _convert_half(fp8e5m2_t val) { + return _intel_convert_bf8_to_f16(val.data); +} +half __attribute__((overloadable)) _convert_half(fp8e5m2_t1 val) { + return _intel_convert_bf8_to_f16(val.data); +} +half2 __attribute__((overloadable)) _convert_half2(fp8e5m2_t2 val) { + return (half2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); +} +half3 __attribute__((overloadable)) _convert_half3(fp8e5m2_t3 val) { + return (half3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); +} +half4 __attribute__((overloadable)) _convert_half4(fp8e5m2_t4 val) { + return (half4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); +} +half8 __attribute__((overloadable)) _convert_half8(fp8e5m2_t8 val) { + return (half8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); +} +half16 __attribute__((overloadable)) _convert_half16(fp8e5m2_t16 val) { + return (half16)( + _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), + _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), + _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), + _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), + _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) + ); +} + +float __attribute__((overloadable)) _convert_float(fp8e5m2_t val) { + return (float)_intel_convert_bf8_to_f16(val.data); +} +float __attribute__((overloadable)) _convert_float(fp8e5m2_t1 val) { + return (float)_intel_convert_bf8_to_f16(val.data); +} +float2 __attribute__((overloadable)) _convert_float2(fp8e5m2_t2 val) { + return (float2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); +} +float3 __attribute__((overloadable)) _convert_float3(fp8e5m2_t3 val) { + return (float3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); +} +float4 __attribute__((overloadable)) _convert_float4(fp8e5m2_t4 val) { + return (float4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); +} +float8 __attribute__((overloadable)) _convert_float8(fp8e5m2_t8 val) { + return (float8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); +} +float16 __attribute__((overloadable)) _convert_float16(fp8e5m2_t16 val) { + return (float16)( + _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), + _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), + _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), + _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), + _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) + ); +} + +half __attribute__((overloadable)) _convert_half(fp8e4m3_t val) { + return _intel_convert_hf8_to_f16(val.data); +} +half __attribute__((overloadable)) _convert_half(fp8e4m3_t1 val) { + return _intel_convert_hf8_to_f16(val.data); +} +half2 __attribute__((overloadable)) _convert_half2(fp8e4m3_t2 val) { + return (half2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); +} +half3 __attribute__((overloadable)) _convert_half3(fp8e4m3_t3 val) { + return (half3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); +} +half4 __attribute__((overloadable)) _convert_half4(fp8e4m3_t4 val) { + return (half4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); +} +half8 __attribute__((overloadable)) _convert_half8(fp8e4m3_t8 val) { + return (half8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); +} +half16 __attribute__((overloadable)) _convert_half16(fp8e4m3_t16 val) { + return (half16)( + _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), + _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), + _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), + _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), + _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) + ); +} + +float __attribute__((overloadable)) _convert_float(fp8e4m3_t val) { + return (float)_intel_convert_hf8_to_f16(val.data); +} +float __attribute__((overloadable)) _convert_float(fp8e4m3_t1 val) { + return (float)_intel_convert_hf8_to_f16(val.data); +} +float2 __attribute__((overloadable)) _convert_float2(fp8e4m3_t2 val) { + return (float2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); +} +float3 __attribute__((overloadable)) _convert_float3(fp8e4m3_t3 val) { + return (float3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); +} +float4 __attribute__((overloadable)) _convert_float4(fp8e4m3_t4 val) { + return (float4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); +} +float8 __attribute__((overloadable)) _convert_float8(fp8e4m3_t8 val) { + return (float8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); +} +float16 __attribute__((overloadable)) _convert_float16(fp8e4m3_t16 val) { + return (float16)( + _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), + _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), + _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), + _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), + _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) + ); +} + +float __attribute__((overloadable)) _convert_float(fp8e8m0_t val) { + return (float)_intel_convert_e8m0_to_f32(val.data); +} +float __attribute__((overloadable)) _convert_float(fp8e8m0_t1 val) { + return (float)_intel_convert_e8m0_to_f32(val.data); +} +float2 __attribute__((overloadable)) _convert_float2(fp8e8m0_t2 val) { + return (float2)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1)); +} +float3 __attribute__((overloadable)) _convert_float3(fp8e8m0_t3 val) { + return (float3)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), _intel_convert_e8m0_to_f32(val.data.s2)); +} +float4 __attribute__((overloadable)) _convert_float4(fp8e8m0_t4 val) { + return (float4)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), + _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3)); +} +float8 __attribute__((overloadable)) _convert_float8(fp8e8m0_t8 val) { + return (float8)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), + _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), + _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), + _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7)); +} +float16 __attribute__((overloadable)) _convert_float16(fp8e8m0_t16 val) { + return (float16)( + _intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), + _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), + _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), + _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7), + _intel_convert_e8m0_to_f32(val.data.s8), _intel_convert_e8m0_to_f32(val.data.s9), + _intel_convert_e8m0_to_f32(val.data.sA), _intel_convert_e8m0_to_f32(val.data.sB), + _intel_convert_e8m0_to_f32(val.data.sC), _intel_convert_e8m0_to_f32(val.data.sD), + _intel_convert_e8m0_to_f32(val.data.sE), _intel_convert_e8m0_to_f32(val.data.sF) + ); +} + +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(float val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8((half)val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(float val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8((half)val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(float2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(float3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(float4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); + res.data.s3 = _intel_convert_f16_to_bf8((half)val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(float8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(float16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); + res.data.s8 = _intel_convert_f16_to_bf8((half)val.s8); + res.data.s9 = _intel_convert_f16_to_bf8((half)val.s9); + res.data.sA = _intel_convert_f16_to_bf8((half)val.sA); + res.data.sB = _intel_convert_f16_to_bf8((half)val.sB); + res.data.sC = _intel_convert_f16_to_bf8((half)val.sC); + res.data.sD = _intel_convert_f16_to_bf8((half)val.sD); + res.data.sE = _intel_convert_f16_to_bf8((half)val.sE); + res.data.sF = _intel_convert_f16_to_bf8((half)val.sF); + return res; +} + +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(half val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8(val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(half val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8(val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(half2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.x); + res.data.s1 = _intel_convert_f16_to_bf8(val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(half3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.x); + res.data.s1 = _intel_convert_f16_to_bf8(val.y); + res.data.s2 = _intel_convert_f16_to_bf8(val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(half4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.x); + res.data.s1 = _intel_convert_f16_to_bf8(val.y); + res.data.s2 = _intel_convert_f16_to_bf8(val.z); + res.data.s3 = _intel_convert_f16_to_bf8(val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(half8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8(val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(half16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8(val.s7); + res.data.s8 = _intel_convert_f16_to_bf8(val.s8); + res.data.s9 = _intel_convert_f16_to_bf8(val.s9); + res.data.sA = _intel_convert_f16_to_bf8(val.sA); + res.data.sB = _intel_convert_f16_to_bf8(val.sB); + res.data.sC = _intel_convert_f16_to_bf8(val.sC); + res.data.sD = _intel_convert_f16_to_bf8(val.sD); + res.data.sE = _intel_convert_f16_to_bf8(val.sE); + res.data.sF = _intel_convert_f16_to_bf8(val.sF); + return res; +} + +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(float val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8_sat((half)val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(float val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8_sat((half)val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(float2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(float3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(float4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); + res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(float8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(float16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); + res.data.s8 = _intel_convert_f16_to_bf8_sat((half)val.s8); + res.data.s9 = _intel_convert_f16_to_bf8_sat((half)val.s9); + res.data.sA = _intel_convert_f16_to_bf8_sat((half)val.sA); + res.data.sB = _intel_convert_f16_to_bf8_sat((half)val.sB); + res.data.sC = _intel_convert_f16_to_bf8_sat((half)val.sC); + res.data.sD = _intel_convert_f16_to_bf8_sat((half)val.sD); + res.data.sE = _intel_convert_f16_to_bf8_sat((half)val.sE); + res.data.sF = _intel_convert_f16_to_bf8_sat((half)val.sF); + return res; +} + +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(half val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8_sat(val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(half val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8_sat(val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(half2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(half3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(half4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); + res.data.s3 = _intel_convert_f16_to_bf8_sat(val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(half8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(half16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); + res.data.s8 = _intel_convert_f16_to_bf8_sat(val.s8); + res.data.s9 = _intel_convert_f16_to_bf8_sat(val.s9); + res.data.sA = _intel_convert_f16_to_bf8_sat(val.sA); + res.data.sB = _intel_convert_f16_to_bf8_sat(val.sB); + res.data.sC = _intel_convert_f16_to_bf8_sat(val.sC); + res.data.sD = _intel_convert_f16_to_bf8_sat(val.sD); + res.data.sE = _intel_convert_f16_to_bf8_sat(val.sE); + res.data.sF = _intel_convert_f16_to_bf8_sat(val.sF); + return res; +} + +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(float val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8((half)val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(float val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8((half)val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(float2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(float3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(float4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); + res.data.s3 = _intel_convert_f16_to_hf8((half)val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(float8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(float16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); + res.data.s8 = _intel_convert_f16_to_hf8((half)val.s8); + res.data.s9 = _intel_convert_f16_to_hf8((half)val.s9); + res.data.sA = _intel_convert_f16_to_hf8((half)val.sA); + res.data.sB = _intel_convert_f16_to_hf8((half)val.sB); + res.data.sC = _intel_convert_f16_to_hf8((half)val.sC); + res.data.sD = _intel_convert_f16_to_hf8((half)val.sD); + res.data.sE = _intel_convert_f16_to_hf8((half)val.sE); + res.data.sF = _intel_convert_f16_to_hf8((half)val.sF); + return res; +} + +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(half val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8(val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(half val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8(val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(half2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.x); + res.data.s1 = _intel_convert_f16_to_hf8(val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(half3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.x); + res.data.s1 = _intel_convert_f16_to_hf8(val.y); + res.data.s2 = _intel_convert_f16_to_hf8(val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(half4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.x); + res.data.s1 = _intel_convert_f16_to_hf8(val.y); + res.data.s2 = _intel_convert_f16_to_hf8(val.z); + res.data.s3 = _intel_convert_f16_to_hf8(val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(half8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8(val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(half16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8(val.s7); + res.data.s8 = _intel_convert_f16_to_hf8(val.s8); + res.data.s9 = _intel_convert_f16_to_hf8(val.s9); + res.data.sA = _intel_convert_f16_to_hf8(val.sA); + res.data.sB = _intel_convert_f16_to_hf8(val.sB); + res.data.sC = _intel_convert_f16_to_hf8(val.sC); + res.data.sD = _intel_convert_f16_to_hf8(val.sD); + res.data.sE = _intel_convert_f16_to_hf8(val.sE); + res.data.sF = _intel_convert_f16_to_hf8(val.sF); + return res; +} + +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(float val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8_sat((half)val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(float val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8_sat((half)val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(float2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(float3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(float4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); + res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(float8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(float16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); + res.data.s8 = _intel_convert_f16_to_hf8_sat((half)val.s8); + res.data.s9 = _intel_convert_f16_to_hf8_sat((half)val.s9); + res.data.sA = _intel_convert_f16_to_hf8_sat((half)val.sA); + res.data.sB = _intel_convert_f16_to_hf8_sat((half)val.sB); + res.data.sC = _intel_convert_f16_to_hf8_sat((half)val.sC); + res.data.sD = _intel_convert_f16_to_hf8_sat((half)val.sD); + res.data.sE = _intel_convert_f16_to_hf8_sat((half)val.sE); + res.data.sF = _intel_convert_f16_to_hf8_sat((half)val.sF); + return res; +} + +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(half val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8_sat(val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(half val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8_sat(val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(half2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(half3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(half4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); + res.data.s3 = _intel_convert_f16_to_hf8_sat(val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(half8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(half16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); + res.data.s8 = _intel_convert_f16_to_hf8_sat(val.s8); + res.data.s9 = _intel_convert_f16_to_hf8_sat(val.s9); + res.data.sA = _intel_convert_f16_to_hf8_sat(val.sA); + res.data.sB = _intel_convert_f16_to_hf8_sat(val.sB); + res.data.sC = _intel_convert_f16_to_hf8_sat(val.sC); + res.data.sD = _intel_convert_f16_to_hf8_sat(val.sD); + res.data.sE = _intel_convert_f16_to_hf8_sat(val.sE); + res.data.sF = _intel_convert_f16_to_hf8_sat(val.sF); + return res; +} + +fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(fp8e8m0_t val) { + return val; +} + +fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(float val) { + fp8e8m0_t res; + res.data = _intel_convert_f32_fo_e8m0(val); + return res; +} +fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1(float val[1]) { + fp8e8m0_t1 res; + res.data = _intel_convert_f32_fo_e8m0(val[0]); + return res; +} +fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2(float2 val) { + fp8e8m0_t2 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); + return res; +} +fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3(float3 val) { + fp8e8m0_t3 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); + return res; +} +fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4(float4 val) { + fp8e8m0_t4 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); + res.data.s3 = _intel_convert_f32_fo_e8m0(val.w); + return res; +} +fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8(float8 val) { + fp8e8m0_t8 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); + return res; +} +fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16(float16 val) { + fp8e8m0_t16 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); + res.data.s8 = _intel_convert_f32_fo_e8m0(val.s8); + res.data.s9 = _intel_convert_f32_fo_e8m0(val.s9); + res.data.sA = _intel_convert_f32_fo_e8m0(val.sA); + res.data.sB = _intel_convert_f32_fo_e8m0(val.sB); + res.data.sC = _intel_convert_f32_fo_e8m0(val.sC); + res.data.sD = _intel_convert_f32_fo_e8m0(val.sD); + res.data.sE = _intel_convert_f32_fo_e8m0(val.sE); + res.data.sF = _intel_convert_f32_fo_e8m0(val.sF); + return res; +} + +fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t_sat(float val) { + fp8e8m0_t res; + res.data = _intel_convert_f32_fo_e8m0_sat(val); + return res; +} +fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1_sat(float val[1]) { + fp8e8m0_t1 res; + res.data = _intel_convert_f32_fo_e8m0_sat(val[0]); + return res; +} +fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2_sat(float2 val) { + fp8e8m0_t2 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); + return res; +} +fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3_sat(float3 val) { + fp8e8m0_t3 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); + return res; +} +fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4_sat(float4 val) { + fp8e8m0_t4 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); + res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.w); + return res; +} +fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8_sat(float8 val) { + fp8e8m0_t8 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); + return res; +} +fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16_sat(float16 val) { + fp8e8m0_t16 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); + res.data.s8 = _intel_convert_f32_fo_e8m0_sat(val.s8); + res.data.s9 = _intel_convert_f32_fo_e8m0_sat(val.s9); + res.data.sA = _intel_convert_f32_fo_e8m0_sat(val.sA); + res.data.sB = _intel_convert_f32_fo_e8m0_sat(val.sB); + res.data.sC = _intel_convert_f32_fo_e8m0_sat(val.sC); + res.data.sD = _intel_convert_f32_fo_e8m0_sat(val.sD); + res.data.sE = _intel_convert_f32_fo_e8m0_sat(val.sE); + res.data.sF = _intel_convert_f32_fo_e8m0_sat(val.sF); + return res; +} + +fp8e5m2_t __attribute__((overloadable)) as_fp8e5m2_t(uchar val) { + fp8e5m2_t res; + res.data = val; + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) as_fp8e5m2_t1(uchar val[1]) { + fp8e5m2_t1 res; + res.data = val[0]; + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) as_fp8e5m2_t2(uchar2 val) { + fp8e5m2_t2 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) as_fp8e5m2_t3(uchar3 val) { + fp8e5m2_t3 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) as_fp8e5m2_t4(uchar4 val) { + fp8e5m2_t4 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + res.data.s3 = val.w; + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) as_fp8e5m2_t8(uchar8 val) { + fp8e5m2_t8 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) as_fp8e5m2_t16(uchar16 val) { + fp8e5m2_t16 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + res.data.s8 = val.s8; + res.data.s9 = val.s9; + res.data.sA = val.sA; + res.data.sB = val.sB; + res.data.sC = val.sC; + res.data.sD = val.sD; + res.data.sE = val.sE; + res.data.sF = val.sF; + return res; +} + +fp8e4m3_t __attribute__((overloadable)) as_fp8e4m3_t(uchar val) { + fp8e4m3_t res; + res.data = as_char(val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) as_fp8e4m3_t1(uchar val[1]) { + fp8e4m3_t1 res; + res.data = as_char(val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) as_fp8e4m3_t2(uchar2 val) { + fp8e4m3_t2 res; + res.data.s0 = as_char(val.x); + res.data.s1 = as_char(val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) as_fp8e4m3_t3(uchar3 val) { + fp8e4m3_t3 res; + res.data.s0 = as_char(val.x); + res.data.s1 = as_char(val.y); + res.data.s2 = as_char(val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) as_fp8e4m3_t4(uchar4 val) { + fp8e4m3_t4 res; + res.data.s0 = as_char(val.x); + res.data.s1 = as_char(val.y); + res.data.s2 = as_char(val.z); + res.data.s3 = as_char(val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) as_fp8e4m3_t8(uchar8 val) { + fp8e4m3_t8 res; + res.data.s0 = as_char(val.s0); + res.data.s1 = as_char(val.s1); + res.data.s2 = as_char(val.s2); + res.data.s3 = as_char(val.s3); + res.data.s4 = as_char(val.s4); + res.data.s5 = as_char(val.s5); + res.data.s6 = as_char(val.s6); + res.data.s7 = as_char(val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) as_fp8e4m3_t16(uchar16 val) { + fp8e4m3_t16 res; + res.data.s0 = as_char(val.s0); + res.data.s1 = as_char(val.s1); + res.data.s2 = as_char(val.s2); + res.data.s3 = as_char(val.s3); + res.data.s4 = as_char(val.s4); + res.data.s5 = as_char(val.s5); + res.data.s6 = as_char(val.s6); + res.data.s7 = as_char(val.s7); + res.data.s8 = as_char(val.s8); + res.data.s9 = as_char(val.s9); + res.data.sA = as_char(val.sA); + res.data.sB = as_char(val.sB); + res.data.sC = as_char(val.sC); + res.data.sD = as_char(val.sD); + res.data.sE = as_char(val.sE); + res.data.sF = as_char(val.sF); + return res; +} + +fp8e8m0_t __attribute__((overloadable)) as_fp8e8m0_t(uchar val) { + fp8e8m0_t res; + res.data = val; + return res; +} +fp8e8m0_t1 __attribute__((overloadable)) as_fp8e8m0_t1(uchar val[1]) { + fp8e8m0_t1 res; + res.data = val[0]; + return res; +} +fp8e8m0_t2 __attribute__((overloadable)) as_fp8e8m0_t2(uchar2 val) { + fp8e8m0_t2 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + return res; +} +fp8e8m0_t3 __attribute__((overloadable)) as_fp8e8m0_t3(uchar3 val) { + fp8e8m0_t3 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + return res; +} +fp8e8m0_t4 __attribute__((overloadable)) as_fp8e8m0_t4(uchar4 val) { + fp8e8m0_t4 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + res.data.s3 = val.w; + return res; +} +fp8e8m0_t8 __attribute__((overloadable)) as_fp8e8m0_t8(uchar8 val) { + fp8e8m0_t8 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + return res; +} +fp8e8m0_t16 __attribute__((overloadable)) as_fp8e8m0_t16(uchar16 val) { + fp8e8m0_t16 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + res.data.s8 = val.s8; + res.data.s9 = val.s9; + res.data.sA = val.sA; + res.data.sB = val.sB; + res.data.sC = val.sC; + res.data.sD = val.sD; + res.data.sE = val.sE; + res.data.sF = val.sF; + return res; +} + +uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t val) { + return val.data; +} +uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t1 val) { + return val.data; +} +uchar2 __attribute__((overloadable)) _as_uchar2(fp8e5m2_t2 val) { + return (uchar2)(val.data.s0, val.data.s1); +} +uchar3 __attribute__((overloadable)) _as_uchar3(fp8e5m2_t3 val) { + return (uchar3)(val.data.s0, val.data.s1, val.data.s2); +} +uchar4 __attribute__((overloadable)) _as_uchar4(fp8e5m2_t4 val) { + return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); +} +uchar8 __attribute__((overloadable)) _as_uchar8(fp8e5m2_t8 val) { + return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7); +} +uchar16 __attribute__((overloadable)) _as_uchar16(fp8e5m2_t16 val) { + return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7, + val.data.s8, val.data.s9, val.data.sA, val.data.sB, + val.data.sC, val.data.sD, val.data.sE, val.data.sF); +} + +uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t val) { + return as_uchar((char)val.data); +} +uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t1 val) { + return as_uchar((char)val.data); +} +uchar2 __attribute__((overloadable)) _as_uchar2(fp8e4m3_t2 val) { + return (uchar2)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1)); +} +uchar3 __attribute__((overloadable)) _as_uchar3(fp8e4m3_t3 val) { + return (uchar3)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), as_uchar((char)val.data.s2)); +} +uchar4 __attribute__((overloadable)) _as_uchar4(fp8e4m3_t4 val) { + return (uchar4)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), + as_uchar((char)val.data.s2), as_uchar((char)val.data.s3)); +} +uchar8 __attribute__((overloadable)) _as_uchar8(fp8e4m3_t8 val) { + return (uchar8)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), + as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), + as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), + as_uchar((char)val.data.s6), as_uchar((char)val.data.s7)); +} +uchar16 __attribute__((overloadable)) _as_uchar16(fp8e4m3_t16 val) { + return (uchar16)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), + as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), + as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), + as_uchar((char)val.data.s6), as_uchar((char)val.data.s7), + as_uchar((char)val.data.s8), as_uchar((char)val.data.s9), + as_uchar((char)val.data.sA), as_uchar((char)val.data.sB), + as_uchar((char)val.data.sC), as_uchar((char)val.data.sD), + as_uchar((char)val.data.sE), as_uchar((char)val.data.sF)); +} + +uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t val) { + return val.data; +} +uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t1 val) { + return val.data; +} +uchar2 __attribute__((overloadable)) _as_uchar2(fp8e8m0_t2 val) { + return (uchar2)(val.data.s0, val.data.s1); +} +uchar3 __attribute__((overloadable)) _as_uchar3(fp8e8m0_t3 val) { + return (uchar3)(val.data.s0, val.data.s1, val.data.s2); +} +uchar4 __attribute__((overloadable)) _as_uchar4(fp8e8m0_t4 val) { + return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); +} +uchar8 __attribute__((overloadable)) _as_uchar8(fp8e8m0_t8 val) { + return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7); +} +uchar16 __attribute__((overloadable)) _as_uchar16(fp8e8m0_t16 val) { + return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7, + val.data.s8, val.data.s9, val.data.sA, val.data.sB, + val.data.sC, val.data.sD, val.data.sE, val.data.sF); +} + +#endif + diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl index 6dd09d08dc629d..d3bc565d1977e3 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl @@ -2,6 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // +#define IS_F8 (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT || F8E5M2_OUTPUT || F8E4M3_OUTPUT || F8E8M0_OUTPUT) + +#if IS_F8 +#include "include/batch_headers/common.cl" +#include "include/f8_utils.cl" +#endif + #include "include/reshape_dims.cl" #include "include/fetch_utils.cl" @@ -153,6 +160,8 @@ KERNEL (reorder_data)( #elif defined UINT4_INPUT const uint uint4_idx = input_idx >> 1; OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(convert_as_uint4_float(input[uint4_idx], input_idx)); + #elif (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT) + OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(_convert_float(input[input_idx])); #else CALC_TYPE res = TO_CALC_TYPE(input[input_idx]); #endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/common_types.h b/src/plugins/intel_gpu/src/kernel_selector/common_types.h index 0f0c11712f33f6..37a34f83a80f11 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/common_types.h +++ b/src/plugins/intel_gpu/src/kernel_selector/common_types.h @@ -131,6 +131,9 @@ enum class Datatype { F16, F32, BF16, + F8E4M3, + F8E5M2, + F8E8M0, }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -145,7 +148,10 @@ enum class WeightsType { UINT4, INT4, INT32, - BF16 + BF16, + F8E4M3, + F8E5M2, + F8E8M0, }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp index d0112f65aecfdd..f639bffd5a0f30 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp @@ -1596,6 +1596,42 @@ JitConstants MakeTypeJitConstants(Datatype dataType, const std::string& macroNam type_size = "2"; is_fp = false; break; + case Datatype::F8E4M3: + type = "fp8e4m3_t"; + max_val = "(fp8e4m3_t){as_char((char)0x7E)}"; // 448.0 + min_val = "(fp8e4m3_t){as_char((char)0xFE)}"; // -448.0 + val_one = "(fp8e4m3_t){as_char((char)0x38)}"; + val_zero = "(fp8e4m3_t){as_char((char)0x0)}"; + to_type = "_convert_fp8e4m3_t(v)"; + to_type_sat = "_convert_fp8e4m3_t_sat(v)"; + as_type = "as_fp8e4m3_t(v)"; + type_size = "1"; + is_fp = true; + break; + case Datatype::F8E5M2: + type = "fp8e5m2_t"; + max_val = "(fp8e5m2_t){as_uchar((uchar)0x7B)}"; // 57344.0 + min_val = "(fp8e5m2_t){as_uchar((uchar)0xFB)}"; // -57344.0 + val_one = "(fp8e5m2_t){as_uchar((uchar)0x3C)}"; + val_zero = "(fp8e5m2_t){as_uchar((uchar)0x0)}"; + to_type = "_convert_fp8e5m2_t(v)"; + to_type_sat = "_convert_fp8e5m2_t_sat(v)"; + as_type = "as_fp8e5m2_t(v)"; + type_size = "1"; + is_fp = true; + break; + case Datatype::F8E8M0: + type = "fp8e8m0_t"; + max_val = "(fp8e8m0_t){as_uchar((uchar)0xFE)}"; // 2^127 + min_val = "(fp8e8m0_t){as_uchar((uchar)0x00)}"; // 2^(-127) + val_one = "(fp8e8m0_t){as_uchar((uchar)0x7F)}"; + val_zero = ""; // There is no representation of zero in FP8E8M0 + to_type = "_convert_fp8e8m0_t(v)"; + to_type_sat = "_convert_fp8e8m0_t_sat(v)"; + as_type = "as_fp8e8m0_t(v)"; + type_size = "1"; + is_fp = true; + break; default: type = "float"; max_val = "FLT_MAX"; @@ -1649,6 +1685,12 @@ JitConstants MakeTypeJitConstants(WeightsType weightsType, const std::string& ma return MakeTypeJitConstants(Datatype::INT32, macroName); case WeightsType::BF16: return MakeTypeJitConstants(Datatype::BF16, macroName); + case WeightsType::F8E4M3: + return MakeTypeJitConstants(Datatype::F8E4M3, macroName); + case WeightsType::F8E5M2: + return MakeTypeJitConstants(Datatype::F8E5M2, macroName); + case WeightsType::F8E8M0: + return MakeTypeJitConstants(Datatype::F8E8M0, macroName); } assert(false || "Unreachable!"); // FIXME: Is there some builtin_unreachable available? diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp index 4d5d9e8a000a4f..54fa4b16f446d6 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp @@ -157,6 +157,9 @@ std::string toString(Datatype dType) { case Datatype::INT64: return "INT64"; case Datatype::F16: return "F16"; case Datatype::F32: return "F32"; + case Datatype::F8E4M3: return "F8E4M3"; + case Datatype::F8E5M2: return "F8E5M2"; + case Datatype::F8E8M0: return "F8E8M0"; default: return ""; } } diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.cpp index ae255a0a9b83a9..af31dcd0539e3e 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.cpp @@ -83,6 +83,15 @@ void ParamsKey::EnableInputDataType(Datatype dt) { case Datatype::BF16: key.inputType.val.BF16 = 1; break; + case Datatype::F8E4M3: + key.inputType.val.F8E4M3 = 1; + break; + case Datatype::F8E5M2: + key.inputType.val.F8E5M2 = 1; + break; + case Datatype::F8E8M0: + key.inputType.val.F8E8M0 = 1; + break; default: break; } @@ -128,6 +137,15 @@ void ParamsKey::EnableOutputDataType(Datatype dt) { case Datatype::BF16: key.outputType.val.BF16 = 1; break; + case Datatype::F8E4M3: + key.outputType.val.F8E4M3 = 1; + break; + case Datatype::F8E5M2: + key.outputType.val.F8E5M2 = 1; + break; + case Datatype::F8E8M0: + key.outputType.val.F8E8M0 = 1; + break; default: break; } diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h index 98faaa15f14c8f..c14b55efa803c9 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h @@ -256,6 +256,9 @@ class ParamsKey { uint32_t F16 : 1; uint32_t F32 : 1; uint32_t BF16 : 1; + uint32_t F8E4M3 : 1; + uint32_t F8E5M2 : 1; + uint32_t F8E8M0 : 1; } val; uint32_t raw; } DataTypesKey; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp index e7b8a1af9070ae..987ae930c4af1b 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp @@ -63,6 +63,9 @@ ParamsKey DynamicQuantizeKernelOpt::GetSupportedKey() const { k.EnableInputDataType(Datatype::F16); k.EnableOutputDataType(Datatype::UINT8); k.EnableOutputDataType(Datatype::INT8); + k.EnableOutputDataType(Datatype::F8E4M3); + k.EnableOutputDataType(Datatype::F8E5M2); + k.EnableOutputDataType(Datatype::F8E8M0); k.EnableDifferentTypes(); k.EnableInputLayout(DataLayout::bf); k.EnableInputLayout(DataLayout::bfyx); @@ -92,6 +95,9 @@ JitConstants DynamicQuantizeKernelOpt::GetJitConstants(const dynamic_quantize_pa jit.AddConstant(MakeJitConstant("MODE_SMALL_GS", static_cast(DynQuanMode::SMALL_GS))); jit.AddConstant(MakeJitConstant("MODE_LARGE_GS", static_cast(DynQuanMode::LARGE_GS))); jit.AddConstant(MakeJitConstant("MODE_PER_TOKEN", static_cast(DynQuanMode::PER_TOKEN))); + jit.AddConstant(MakeJitConstant("F8E5M2_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E5M2 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E4M3_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E4M3 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("IS_MXFP", params.outputs[1].GetDType() == Datatype::F8E8M0 ? 1 : 0)); jit.Merge(GetTensorFriendlyWorkGroupsJit(params.outputs[0])); jit.AddConstant(MakeJitConstant("TOTAL_BLOCK_NUM", total_block_num)); size_t block_num = (total_block_num > 32) ? 32 : total_block_num; @@ -200,6 +206,10 @@ bool DynamicQuantizeKernelOpt::Validate(const Params& params) const { if (get_dynamic_quantize_mode(dq_params) == DynQuanMode::PER_TOKEN && dq_params.generate_precomputed_reduction) DO_NOT_USE_THIS_KERNEL(params.layerID); + if (dq_params.generate_precomputed_reduction && cldnn::one_of(dq_params.outputs[0].GetDType(), {Datatype::F8E4M3, Datatype::F8E5M2})) { + DO_NOT_USE_THIS_KERNEL(params.layerID); + } + auto bf = get_input_bf_size(dq_params); if (((bf.second) % (simd * 2)) != 0) DO_NOT_USE_THIS_KERNEL(params.layerID); diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp index 18567c3b55b878..4b36c2eded23ba 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp @@ -12,6 +12,9 @@ ParamsKey DynamicQuantizeKernelRef::GetSupportedKey() const { k.EnableInputDataType(Datatype::F16); k.EnableOutputDataType(Datatype::INT8); k.EnableOutputDataType(Datatype::UINT8); + k.EnableOutputDataType(Datatype::F8E4M3); + k.EnableOutputDataType(Datatype::F8E5M2); + k.EnableOutputDataType(Datatype::F8E8M0); k.EnableInputLayout(DataLayout::bfyx); k.EnableOutputLayout(DataLayout::bfyx); k.EnableTensorOffset(); @@ -56,6 +59,9 @@ JitConstants DynamicQuantizeKernelRef::GetJitConstants(const dynamic_quantize_pa jit.AddConstant(MakeJitConstant("GENERATE_PRECOMPUTED_REDUCTION", params.generate_precomputed_reduction)); jit.AddConstant(MakeJitConstant("GROUP_SCALES_WITH_ZP", params.combine_scales_and_zp)); jit.AddConstant(MakeJitConstant("UNSIGNED_OUTPUT", params.outputs[0].GetDType() == Datatype::UINT8 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E5M2_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E5M2 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E4M3_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E4M3 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("IS_MXFP", params.outputs[1].GetDType() == Datatype::F8E8M0 ? 1 : 0)); // Use FP32 accumulator type for scale/zp calculation jit.Merge(MakeTypeJitConstants(Datatype::F32, "ACCUMULATOR")); @@ -156,6 +162,11 @@ bool DynamicQuantizeKernelRef::Validate(const Params& params) const { if (!KernelBaseOpenCL::Validate(params)) DO_NOT_USE_THIS_KERNEL(params.layerID); + const auto& dq_params = static_cast(params); + if (dq_params.generate_precomputed_reduction && cldnn::one_of(dq_params.outputs[0].GetDType(), {Datatype::F8E4M3, Datatype::F8E5M2})) { + DO_NOT_USE_THIS_KERNEL(params.layerID); + } + return true; } } // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp index 50ef5198d87b95..e41c6824ce7b7a 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp @@ -20,6 +20,9 @@ ParamsKey ReorderKernelRef::GetSupportedKey() const { k.EnableInputDataType(Datatype::INT64); k.EnableInputDataType(Datatype::F16); k.EnableInputDataType(Datatype::F32); + k.EnableInputDataType(Datatype::F8E4M3); + k.EnableInputDataType(Datatype::F8E5M2); + k.EnableInputDataType(Datatype::F8E8M0); k.EnableOutputDataType(Datatype::F16); k.EnableOutputDataType(Datatype::F32); k.EnableOutputDataType(Datatype::INT8); @@ -32,6 +35,9 @@ ParamsKey ReorderKernelRef::GetSupportedKey() const { k.EnableOutputDataType(Datatype::UINT4); k.EnableOutputDataType(Datatype::INT4); k.EnableOutputDataType(Datatype::BF16); + k.EnableOutputDataType(Datatype::F8E4M3); + k.EnableOutputDataType(Datatype::F8E5M2); + k.EnableOutputDataType(Datatype::F8E8M0); k.EnableSurfaceInputSupport(); k.EnableDifferentTypes(); k.EnableAllInputLayout(); @@ -84,6 +90,13 @@ JitConstants ReorderKernelRef::GetJitConstants(const reorder_params& params) con jit.AddConstant(MakeJitConstant("INT4_OUTPUT", true)); } + jit.AddConstant(MakeJitConstant("F8E5M2_INPUT", params.inputs[0].GetDType() == Datatype::F8E5M2 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E4M3_INPUT", params.inputs[0].GetDType() == Datatype::F8E4M3 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E8M0_INPUT", params.inputs[0].GetDType() == Datatype::F8E8M0 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E5M2_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E5M2 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E4M3_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E4M3 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E8M0_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E8M0 ? 1 : 0)); + return jit; } diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel_base.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel_base.cpp index c76155f5ae8a84..136994090dc5ac 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel_base.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel_base.cpp @@ -119,9 +119,12 @@ JitConstants ReorderKernelBase::GetJitConstants(const reorder_params& params) co params.mode == MeanSubtractMode::NONE && params.activations.empty()); Datatype calc_type = useUshort ? Datatype::UINT16 : params.inputs[0].GetDType(); - if ( params.inputs[0].GetDType() == Datatype::BF16 ) { + if (params.inputs[0].GetDType() == Datatype::BF16 || params.inputs[0].GetDType() == Datatype::F8E8M0) { calc_type = Datatype::F32; } + if (params.inputs[0].GetDType() == Datatype::F8E4M3 || params.inputs[0].GetDType() == Datatype::F8E5M2) { + calc_type = Datatype::F16; + } Datatype output_reorder_type = useUshort ? Datatype::UINT16 : params.outputs[0].GetDType(); Datatype input_reorder_type = useUshort ? Datatype::UINT16 : params.inputs[0].GetDType(); diff --git a/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py b/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py index e7d2aa4e86f7d0..6310f71424a1de 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py +++ b/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py @@ -12,6 +12,10 @@ import glob import ntpath import re +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "graph", "common_utils")) +from kernels_db_gen import detect_guard_patterns class KernelLang(Enum): OCLC = 0 @@ -88,17 +92,22 @@ def append_undefs(self, filename): content = [] with open(filename) as f: content += f.readlines() + + guard_patterns = detect_guard_patterns("".join(content)) + for line in content: if '#define' in line: name = line.strip().split(" ")[1].split("(")[0] - undefs += "#ifdef " + name + "\n" - undefs += "#undef " + name + "\n" - undefs += "#endif\n" + if name not in guard_patterns: + undefs += "#ifdef " + name + "\n" + undefs += "#undef " + name + "\n" + undefs += "#endif\n" if '# define' in line: name = line.strip().split(" ")[2].split("(")[0] - undefs += "#ifdef " + name + "\n" - undefs += "#undef " + name + "\n" - undefs += "#endif\n" + if name not in guard_patterns: + undefs += "#ifdef " + name + "\n" + undefs += "#undef " + name + "\n" + undefs += "#endif\n" if filename in self.include_files: for include_file in self.include_files[filename]: include_file_undefs = self.append_undefs(include_file) diff --git a/src/plugins/intel_gpu/src/plugin/common_utils.cpp b/src/plugins/intel_gpu/src/plugin/common_utils.cpp index eefbbb6aa1b5ba..854c2fb08a5e84 100644 --- a/src/plugins/intel_gpu/src/plugin/common_utils.cpp +++ b/src/plugins/intel_gpu/src/plugin/common_utils.cpp @@ -201,8 +201,8 @@ bool is_supported(ov::element::Type_t et) { case ov::element::Type_t::u32: return true; // converted to i32 case ov::element::Type_t::u64: return true; // converted to i32 case ov::element::Type_t::nf4: return false; - case ov::element::Type_t::f8e4m3: return false; - case ov::element::Type_t::f8e5m2: return false; + case ov::element::Type_t::f8e4m3: return true; + case ov::element::Type_t::f8e5m2: return true; case ov::element::Type_t::string: return false; default: return false; } diff --git a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp index 0a1c1200ea8cbd..ed37b0b6460664 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp @@ -4,11 +4,14 @@ #pragma once +#include "openvino/pass/pattern/op/optional.hpp" + using namespace ov::pass::pattern; #define FC_COMPRESSED_WEIGHT_PATTERN\ auto compressed_constant = [](const ov::Output& output) {\ return (output.get_element_type() == ov::element::u8 || output.get_element_type() == ov::element::i8 ||\ - output.get_element_type() == ov::element::u4 || output.get_element_type() == ov::element::i4);\ + output.get_element_type() == ov::element::u4 || output.get_element_type() == ov::element::i4 ||\ + output.get_element_type() == ov::element::f8e4m3 || output.get_element_type() == ov::element::f8e5m2);\ };\ \ auto reshape_squeeze = [](const ov::Output& output) {\ @@ -36,7 +39,8 @@ using namespace ov::pass::pattern; \ auto mul_const_m = wrap_type();\ auto mul_with_sub_m = wrap_type({subtract_m, mul_const_m});\ - auto mul_no_sub_m = wrap_type({decompressed_weights_m, mul_const_m});\ + auto mul_const_convert_m = ov::pass::pattern::optional(mul_const_m);\ + auto mul_no_sub_m = wrap_type({decompressed_weights_m, mul_const_convert_m});\ auto mul_m = std::make_shared(OutputVector{mul_with_sub_m, mul_no_sub_m});\ \ auto reshape_const_m = wrap_type();\ diff --git a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp index 73a4e22b64b90f..e368ddc591611f 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp @@ -13,6 +13,7 @@ #include "openvino/pass/pattern/op/wrap_type.hpp" #include "transformations/utils/utils.hpp" #include "intel_gpu/runtime/debug_configuration.hpp" +#include "intel_gpu/runtime/utils.hpp" #include "openvino/core/graph_util.hpp" namespace ov::intel_gpu { @@ -47,7 +48,6 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size const bool has_wzp = m_fc->get_input_size() > 4; auto optional_w_zp = has_wzp ? m_fc->get_input_node_shared_ptr(4) : std::make_shared(); - auto rank = m_fc->get_input_partial_shape(0).size(); ov::op::internal::DynamicQuantize::Attributes config; const bool has_static_wzp = m_fc->get_input_size() > 4 && optional_w_zp->get_output_partial_shape(0).rank().is_static(); const bool is_wei_i8_u8 = cldnn::one_of(m_fc->get_input_element_type(1), {ov::element::i8, ov::element::u8}); @@ -85,6 +85,7 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size return false; } + const auto rank = m_fc->get_input_partial_shape(0).size(); if (adj_group_size < 32) { GPU_DEBUG_LOG << "Dynamic quantization: quantized activation by group size " << adj_group_size << " is not supported by onednn matmul if it is less than 32" << std::endl; @@ -94,9 +95,19 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size std::vector shape_group_size(rank, 1); shape_group_size.back() = adj_group_size; - config.quantization_dt = element::i8; + auto weight_dtype = m_fc->get_input_element_type(1); + auto scale_dtype = m_fc->get_input_element_type(3); + if (weight_dtype.is_integral()) { + config.quantization_dt = element::i8; + } else if (cldnn::one_of(weight_dtype, {element::f8e4m3, element::f8e5m2, element::f4e2m1})) { + config.quantization_dt = weight_dtype; + } else { + OPENVINO_THROW("Unexpected weight data type: ", weight_dtype); + } + const bool is_mxfp = scale_dtype == element::f8e8m0; + config.quantization_type = QuantizationType::Symmetric; - config.scale_dt = element::f16; + config.scale_dt = is_mxfp ? element::f8e8m0 : element::f16; config.group_sizes = shape_group_size; if (asymmetric && adj_group_size == UINT64_MAX) { @@ -105,7 +116,8 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size config.zp_dt = element::u8; // it supports u8 only now } - auto dyn_quan = std::make_shared(m_fc->input_value(0), config); + std::shared_ptr dyn_quan = std::make_shared(m_fc->input_value(0), config); + int dyn_quan_output_idx = 2; auto optional_a_zp = config.quantization_type == QuantizationType::Symmetric ? std::make_shared() : dyn_quan->output(dyn_quan_output_idx++); diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index cde95c93805ad9..9b7f30d9a8791c 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -297,9 +297,25 @@ static bool is_decompression_multiply(const std::shared_ptr node return true; }; - if (all_has_types(consumers, { ov::opset1::Reshape::get_type_info_static() })) { + if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()}) || all_has_types(consumers, {ov::op::v1::Transpose::get_type_info_static()})) { for (const auto& consumer : consumers) { - const auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + auto get_child_consumers = [&]() { + auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + + // Reshape + Transpose chain + if (all_has_types(child_consumers, { ov::op::v1::Transpose::get_type_info_static() })) { + std::set> next_child_consumers; + for (const auto& child_consumer : child_consumers) { + const auto grand_child_consumers = child_consumer.get_node()->get_output_target_inputs(0); + next_child_consumers.insert(grand_child_consumers.begin(), grand_child_consumers.end()); + } + return next_child_consumers; + } + return child_consumers; + }; + + auto child_consumers = get_child_consumers(); + for (const auto& child_consumer : child_consumers) { const auto& type_info = child_consumer.get_node()->get_type_info(); if (cldnn::one_of(type_info, target_consumers)) { @@ -470,7 +486,8 @@ void TransformationsPipeline::apply(std::shared_ptr func) { using const_node_ptr = const std::shared_ptr; const auto& defaultPrecisions = ov::pass::low_precision::precision_set::get_int8_support(); - const ov::element::TypeVector supported_woq_types = {ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}; + const ov::element::TypeVector supported_woq_types = + {ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}; bool enableInt8; bool unroll_loop = config.get_enable_loop_unrolling(); const bool disable_gated_mlp_fusion = GPU_DEBUG_VALUE_OR(config.get_disable_gated_mlp_fusion(), true); @@ -521,7 +538,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { // Transformation of SDPA to VLSDPA for QWen2.x-VL, // Note: this should be applied before TransposeFusion. manager.register_pass(); - // Disable SDPAToVLSDPA if XMX architectures is unavaiable or IGC incompatiable. + // Disable SDPAToVLSDPA if XMX architectures is unavaiable or IGC incompatible. pass_config->set_callback( [&](const_node_ptr &) -> bool { auto& engine = m_context->get_engine(); @@ -554,9 +571,15 @@ void TransformationsPipeline::apply(std::shared_ptr func) { } manager.register_pass(); - manager.register_pass( - std::vector{ ov::element::i8, ov::element::u8, ov::element::i4, ov::element::u4 }, - !device_info.supports_immad); + + manager.register_pass(std::vector{ov::element::i8, ov::element::u8, ov::element::i4, ov::element::u4}, + !device_info.supports_immad); + if (config.get_use_onednn() && m_context->get_engine().get_device_info().arch >= cldnn::gpu_arch::xe3p) { + manager.register_pass( + std::vector{ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1, ov::element::f8e8m0}, + true, + false); + } const bool is_pa = [&func]() { for (const auto& op : func->get_ops()) { @@ -1706,6 +1729,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { precomputed_reduction, use_gs128_for_int8_per_token, use_gs128_for_linear_attention); + manager.register_pass(); } } diff --git a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp index e31bddb35a94b9..13e155eac6ed34 100644 --- a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp +++ b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -247,6 +247,8 @@ const std::vector& disabled_test_patterns() { patterns.push_back(std::regex(R"(.*smoke_MatMulCompressedWeights_3D_weight.*)")); // MoE patterns are not supported on the platforms without immad (fusion transformation are disabled) patterns.push_back(std::regex(R"(.*smoke_MoE3GemmCompressedFusion.*)")); + // Dynamic quantization of fp8 and mxfp8 models supported only through oneDNN (disabled for non-systolic platforms) + patterns.push_back(std::regex(R"(.*smoke_MatMulCompressedWeights_dyn_quan_.*fp8.*)")); } return patterns; diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp index c799ead6124ad1..3d51f606e8332e 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp @@ -2,9 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // +#include + #include "common_test_utils/ov_tensor_utils.hpp" +#include "intel_gpu/runtime/utils.hpp" #include "shared_test_classes/base/ov_subgraph.hpp" #include "transformations/rt_info/decompression.hpp" +#include "intel_gpu/runtime/internal_properties.hpp" #include "openvino/op/parameter.hpp" #include "openvino/op/constant.hpp" @@ -54,6 +58,7 @@ struct ShapeParams { using MatmulWeightsDecompressionParams = std::tuple()[i] /= ov::float16(16.f); - else if (data_precision == ov::element::f32) + else if (scale_precision == ov::element::f32) scale_tensor.data()[i] /= 16.f; + else if (scale_precision == ov::element::f8e8m0) + scale_tensor.data()[i] /= ov::float8_e8m0(16.f); } + std::shared_ptr scale_const = std::make_shared(scale_tensor); + if (scale_precision != data_precision) { + auto scale_convert = std::make_shared(scale_const, data_precision); + scale_const = scale_convert; + } + if (reshape_on_decompression_constant) { auto scale_reshape_const = ov::op::v0::Constant::create(ov::element::i32, {scaleshift_target_shape.size()}, scaleshift_target_shape); auto scale_reshape = std::make_shared(scale_const, scale_reshape_const, false); @@ -287,6 +305,7 @@ class MatmulWeightsDecompression : public testing::WithParamInterface{32, 128, std::numeric_limits::max()}), + ::testing::Values(1.0f)), + MatmulWeightsDecompression::get_test_case_name); + INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_scalar_wzp, MatmulWeightsDecompressionScalarWeightZp, ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 1024}, {{1024, 1, 1024}}}, {1024, 1024}, 128}), ::testing::Values(ov::element::u4), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::Values(true), ::testing::Values(true), @@ -620,6 +687,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_redu {128, 128}, 16}), // shape ::testing::Values(ov::element::u8), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::ValuesIn(add_decompression_sub), ::testing::Values(true), @@ -636,6 +704,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_input_4d, {2048, 512}, 32}), // shape ::testing::Values(ov::element::u4), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::Values(false), ::testing::Values(false), diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp index 40e3d787bc5e28..f65dd9d15c6931 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp @@ -37,6 +37,7 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { const QuantizationType quantization_type = QuantizationType::Symmetric, uint64_t group_size = UINT64_MAX, data_types quant_dt = data_types::i8, + data_types scale_dt = data_types::f16, data_types zp_dt = data_types::dynamic, OutputStorageType storage_type = OutputStorageType::Planar, const std::string& impl_name = "", @@ -74,7 +75,7 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { dynamic_quantize::Attributes dq_config; dq_config.quantization_type = quantization_type; dq_config.quantization_dt = quant_dt; - dq_config.scale_dt = data_types::f16; + dq_config.scale_dt = scale_dt; dq_config.zp_dt = zp_dt; dq_config.group_sizes = group_sizes; dq_config.scales_zp_output_order = { 0, 1, 2}; @@ -184,9 +185,14 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { cldnn::mem_lock output_ptr(output_buffers[i], get_test_stream()); cldnn::mem_lock output_ptr_ref(ref_output_buffers[i], get_test_stream()); - for (size_t i = 0; i < output_ptr_ref.size(); ++i) { - const int abs_error_threshold = 2; - ASSERT_NEAR(output_ptr_ref[i], output_ptr[i], abs_error_threshold); + for (size_t j = 0; j < output_ptr_ref.size(); ++j) { + if (ov::element::Type(quant_dt).is_real()) { + auto abs_diff = std::abs(output_ptr_ref[j] - output_ptr[j]); + ASSERT_EQ(abs_diff, 0); + } else { // (u)int8 + const int abs_error_threshold = 2; + ASSERT_NEAR(output_ptr_ref[j], output_ptr[j], abs_error_threshold); + } } } @@ -234,6 +240,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_precompute_g QuantizationType::Symmetric, 32, data_types::i8, + data_types::f16, data_types::i8, OutputStorageType::Planar, "", @@ -247,6 +254,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_precompute_g QuantizationType::Symmetric, 128, data_types::i8, + data_types::f16, data_types::i8, OutputStorageType::Planar, "", @@ -260,6 +268,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_precompute_g QuantizationType::Symmetric, 128, data_types::i8, + data_types::f16, data_types::i8, OutputStorageType::Planar, "", @@ -274,6 +283,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_precompute_g QuantizationType::Symmetric, 128, data_types::i8, + data_types::f16, data_types::i8, OutputStorageType::Planar, "", @@ -284,7 +294,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_precompute_g TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_asym_act) { this->test_dynamic_quantization(false, {-1, 1, 1, 4096}, {1, 1, 1, 4096}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::u8, data_types::u8, OutputStorageType::Planar); + data_types::u8, data_types::f16, data_types::u8, OutputStorageType::Planar); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_grouped) { @@ -326,6 +336,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache) { QuantizationType::Symmetric, UINT64_MAX, data_types::i8, + data_types::f16, data_types::dynamic, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); @@ -338,6 +349,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched) { QuantizationType::Symmetric, UINT64_MAX, data_types::i8, + data_types::f16, data_types::dynamic, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); @@ -350,6 +362,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_reordered) { QuantizationType::Symmetric, UINT64_MAX, data_types::i8, + data_types::f16, data_types::dynamic, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); @@ -362,6 +375,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reorde QuantizationType::Symmetric, UINT64_MAX, data_types::i8, + data_types::f16, data_types::dynamic, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); @@ -369,91 +383,139 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reorde TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_asym_planar) { this->test_dynamic_quantization(false, {-1, 8, -1, 96}, {1, 8, 1, 96}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_asym_planar) { this->test_dynamic_quantization(false, {-1, 4, -1, 64}, {1, 4, 35, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_reordered_asym_planar) { this->test_dynamic_quantization(false, {-1, -1, 8, 96}, {1, 1, 8, 96}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reordered_asym_planar) { this->test_dynamic_quantization(false, {-1, -1, 4, 64}, {1, 35, 4, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_asym_interleaved) { this->test_dynamic_quantization(false, {-1, 8, -1, 96}, {1, 8, 1, 96}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_asym_interleaved) { this->test_dynamic_quantization(false, {-1, 4, -1, 64}, {1, 4, 35, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_reordered_asym_interleaved) { this->test_dynamic_quantization(false, {-1, -1, 8, 96}, {1, 1, 8, 96}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reordered_asym_interleaved) { this->test_dynamic_quantization(false, {-1, -1, 4, 64}, {1, 35, 4, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_asym_planar_i8_zp) { this->test_dynamic_quantization(false, {-1, 8, -1, 32}, {1, 8, 1, 32}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_asym_planar_i8_zp) { this->test_dynamic_quantization(false, {-1, 4, -1, 64}, {1, 4, 35, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_reordered_asym_planar_i8_zp) { this->test_dynamic_quantization(false, {-1, -1, 8, 96}, {1, 1, 8, 96}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reordered_asym_planar_i8_zp) { this->test_dynamic_quantization(false, {-1, -1, 4, 64}, {1, 35, 4, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_inner_most_dim_zero_values_asym) { this->test_dynamic_quantization(false, {-1, 8, -1, 128}, {1, 8, 52, 128}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache", SetInnerMostDimValuesZero::Yes); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache", SetInnerMostDimValuesZero::Yes); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e4m3) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + 32, + data_types::f8e4m3, + data_types::f8e8m0, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e5m2) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + 32, + data_types::f8e5m2, + data_types::f8e8m0, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e4m3) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + UINT64_MAX, + data_types::f8e4m3, + data_types::f16, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + UINT64_MAX, + data_types::f8e5m2, + data_types::f16, + data_types::dynamic, + OutputStorageType::Planar); } TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_opt_group_size_256) { this->test_dynamic_quantization(false, {1, 1, 8192}, {1, 1, 8192}, QuantizationType::Symmetric, 256, - data_types::i8, data_types::dynamic, OutputStorageType::Planar, + data_types::i8, data_types::f16, data_types::dynamic, OutputStorageType::Planar, "dynamic_quantize_gpu_opt"); } TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_opt_group_size_256_precompute_sum) { this->test_dynamic_quantization(false, {1, 1, 8192}, {1, 1, 8192}, QuantizationType::Symmetric, 256, - data_types::i8, data_types::i8, OutputStorageType::Planar, + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_opt", SetInnerMostDimValuesZero::No, PrecomputeSum::Enabled); } TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_group_size_8192_with_precompute_sum) { this->test_dynamic_quantization(false, {1, 1, 16384}, {1, 1, 16384}, QuantizationType::Symmetric, 8192, - data_types::i8, data_types::i8, OutputStorageType::Planar, + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "", SetInnerMostDimValuesZero::No, PrecomputeSum::Enabled); } TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_opt_gs128_K2560_sym_precompute_sum) { this->test_dynamic_quantization(false, {1, 1, 2560}, {1, 1, 2560}, QuantizationType::Symmetric, 128, - data_types::i8, data_types::i8, OutputStorageType::Planar, + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_opt", SetInnerMostDimValuesZero::No, PrecomputeSum::Enabled); } diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp index 24b185ac16c960..3c8f6105cb1f34 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp @@ -725,6 +725,104 @@ TEST_F(TransformationTestsF, ConvertFCToCompressed18_4dGroupedWith3dWeight) { model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input1}); } } + +TEST_F(TransformationTestsF, ConvertFCToCompressed19_mxfp8e4m3) { + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 32 }, { 1 }); + auto weights_convert = std::make_shared(weights_const, ov::element::f16); + auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); + auto scale_convert = std::make_shared(scale_const, ov::element::f16); + auto deq_mul = std::make_shared(weights_convert, scale_convert); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input1, deq_mul, no_bias); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input1}); + manager.register_pass(); + } + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 32 }, { 1 }); + auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(input1, weights_const, no_bias, scale_const); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input1}); + } +} + +TEST_F(TransformationTestsF, ConvertFCToCompressed20_mxfp8_e5m2) { + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 32 }, { 1 }); + auto weights_convert = std::make_shared(weights_const, ov::element::f16); + auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); + auto scale_convert = std::make_shared(scale_const, ov::element::f16); + auto deq_mul = std::make_shared(weights_convert, scale_convert); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input1, deq_mul, no_bias); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input1}); + manager.register_pass(); + } + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 32 }, { 1 }); + auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(input1, weights_const, no_bias, scale_const); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input1}); + } +} + +TEST_F(TransformationTestsF, ConvertFCToCompressed21_fp8e4m3) { + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 32 }, { 1 }); + auto weights_convert = std::make_shared(weights_const, ov::element::f16); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto deq_mul = std::make_shared(weights_convert, scale_const); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input1, deq_mul, no_bias); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input1}); + manager.register_pass(); + } + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 32 }, { 1 }); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(input1, weights_const, no_bias, scale_const); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input1}); + } +} + +TEST_F(TransformationTestsF, ConvertFCToCompressed22_fp8_e5m2) { + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 32 }, { 1 }); + auto weights_convert = std::make_shared(weights_const, ov::element::f16); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto deq_mul = std::make_shared(weights_convert, scale_const); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input1, deq_mul, no_bias); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input1}); + manager.register_pass(); + } + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 32 }, { 1 }); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(input1, weights_const, no_bias, scale_const); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input1}); + } +} } // namespace intel_gpu } // namespace test