From 395ff23e99d27ba20ab11a237d6a2c57c6550f05 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 13 Apr 2026 11:12:40 +0000 Subject: [PATCH 01/48] [GPU] Enable dynamic quantization for MXFP8 & regular FP8 dtypes --- .../include/low_precision/low_precision.hpp | 2 + .../src/low_precision.cpp | 28 + .../include/ov_ops/dynamic_quantize.hpp | 36 + .../moc_transformations.cpp | 11 +- src/core/src/preprocess/pre_post_process.cpp | 17 +- .../intel_gpu/src/graph/debug_helper.cpp | 6 + .../intel_gpu/src/graph/dynamic_quantize.cpp | 10 +- .../graph_optimizer/add_required_reorders.cpp | 2 +- .../src/graph/impls/ocl/dynamic_quantize.cpp | 5 +- .../impls/ocl/kernel_selector_helper.cpp | 24 + .../src/graph/impls/ocl_v2/utils/jitter.cpp | 44 +- .../impls/onednn/fully_connected_onednn.cpp | 23 +- .../impls/onednn/fully_connected_onednn.hpp | 20 +- .../src/graph/impls/onednn/utils.cpp | 6 + .../intel_gpu/src/graph/primitive_inst.cpp | 2 +- src/plugins/intel_gpu/src/graph/program.cpp | 12 +- .../cl_kernels/dynamic_quantize_gpu_opt.cl | 57 +- .../cl_kernels/dynamic_quantize_gpu_ref.cl | 57 +- .../include/batch_headers/common.cl | 86 ++ .../include/batch_headers/f8_utils.cl | 1275 +++++++++++++++++ .../batch_headers/sub_group_block_write.cl | 8 +- .../cl_kernels/reorder_data.cl | 7 + .../src/kernel_selector/common_types.h | 8 +- .../intel_gpu/src/kernel_selector/jitter.cpp | 50 +- .../kernel_selector_common.cpp | 3 + .../kernel_selector_params.cpp | 18 + .../kernel_selector/kernel_selector_params.h | 3 + .../dynamic_quantize_kernel_opt.cpp | 10 + .../dynamic_quantize_kernel_ref.cpp | 11 + .../kernels/reorder/reorder_kernel.cpp | 11 + .../kernels/reorder/reorder_kernel_base.cpp | 5 +- .../intel_gpu/src/plugin/common_utils.cpp | 4 +- .../compressed_weights_pattern.hpp | 8 +- .../dynamic_quantize_fully_connected.cpp | 42 +- .../dynamic_quantize_fully_connected.hpp | 1 + .../src/plugin/transformations_pipeline.cpp | 46 +- .../skip_tests_config.cpp | 2 + .../dynamic/matmul_weights_decompression.cpp | 161 ++- .../test_cases/dynamic_quantize_gpu_test.cpp | 98 +- .../convert_fc_to_compressed_test.cpp | 50 + 40 files changed, 2143 insertions(+), 126 deletions(-) create mode 100644 src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl diff --git a/src/common/low_precision_transformations/include/low_precision/low_precision.hpp b/src/common/low_precision_transformations/include/low_precision/low_precision.hpp index 0dacbd26ef8c..5604664ba03b 100644 --- a/src/common/low_precision_transformations/include/low_precision/low_precision.hpp +++ b/src/common/low_precision_transformations/include/low_precision/low_precision.hpp @@ -74,6 +74,8 @@ class LP_TRANSFORMATIONS_API ov::pass::low_precision::LowPrecision : public ov:: const std::set& supported_levels = all_levels, bool check_fake_convert = false); + static bool doesModelContainMXFPPatterns(const std::shared_ptr& model); + template std::shared_ptr add_main(Args&&... args) { const auto tr = std::make_shared(std::forward(args)...); diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index 96c6af78bd8b..d0bed1599341 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -31,6 +31,7 @@ #include "openvino/pass/constant_folding.hpp" #include "openvino/pass/manager.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" +#include "openvino/pass/pattern/op/pattern.hpp" #include "openvino/util/log.hpp" #include "ov_ops/type_relaxed.hpp" #include "transformations/common_optimizations/lin_op_sequence_fusion.hpp" @@ -332,3 +333,30 @@ bool LowPrecision::isFunctionQuantized(const std::shared_ptr& m } return false; } + +bool ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns( + const std::shared_ptr& model) { + using namespace ov::op; + using namespace ov::pass::pattern; + MATCHER_SCOPE(doesModelContainMXFPPatterns); + + auto weight_pattern = any_input( + type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && + shape_matches("[Dims..., 32]")); + auto weight_cvt_pattern = wrap_type({weight_pattern}); + + auto scale_pattern = any_input(type_matches(ov::element::Type_t::f8e8m0) && shape_matches("[Dims..., 1]")); + auto scale_cvt_pattern = wrap_type({scale_pattern}); + + auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); + + auto m = std::make_shared(mult_pattern, "doesModelContainMXFPPatterns"); + + for (const auto& n : model->get_ordered_ops()) { + if (m->match(n)) { + return true; + } + } + + return false; +} diff --git a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp index e760f23c83ff..24b89d318498 100644 --- a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp +++ b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp @@ -83,6 +83,42 @@ class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { static std::vector shape_infer(const DynamicQuantize* op, const std::vector& input_shapes); + bool is_config_equal(const Attributes& attr) const { + bool is_equal = + m_attrs.quantization_type == attr.quantization_type && + m_attrs.quantization_dt == attr.quantization_dt && + m_attrs.scale_dt == attr.scale_dt && + m_attrs.zp_dt == attr.zp_dt && + m_attrs.precomputed_reduction_dt == attr.precomputed_reduction_dt && + m_attrs.precomputed_reduction == attr.precomputed_reduction && + m_attrs.group_sizes == attr.group_sizes && + m_attrs.output_storage_type == attr.output_storage_type; + + if (!is_equal) { + return false; + } + + if (m_attrs.scales_zp_output_order == attr.scales_zp_output_order) { + return true; + } + + // Edge case: if zp_output_order is empty, DynamicQuantize constructor generates it with std::iota(). + // Check if this is the case here. + if (attr.scales_zp_output_order.empty()) { + for (size_t i = 0; i < m_attrs.scales_zp_output_order.size(); ++i) { + if (m_attrs.scales_zp_output_order[i] != i) { + return false; + } + } + } + + return true; + } + + bool operator==(const DynamicQuantize& rhs) const { + return is_config_equal(rhs.m_attrs); + } + protected: Attributes m_attrs; }; diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index c9329e7d3726..a27d83ac8180 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -7,6 +7,7 @@ #include #include "itt.hpp" +#include "low_precision/low_precision.hpp" #include "openvino/pass/constant_folding.hpp" #include "openvino/pass/manager.hpp" #include "transformations/common_optimizations/adaptive_pool_to_reduce.hpp" @@ -144,8 +145,16 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr if (m_low_precision_enabled) { // Transformation call example, to check with the real model manager.register_pass(TypeVector{f8e4m3}, TypeVector{u4}); + const bool does_model_contain_mxfp_patterns = ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(f); manager.register_pass( - TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f4e2m1, f8e4m3, f8e5m2, f8e8m0}); + TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, + false); + if (does_model_contain_mxfp_patterns) { + manager.register_pass( + TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, + false, + false); + } } if (!m_use_shapes) { manager.register_pass(); diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index fb7ffeb096f9..7da7b3ea2875 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -8,6 +8,7 @@ #include "function_guard.hpp" #include "itt.hpp" #include "layout_utils.hpp" +#include "low_precision/low_precision.hpp" #include "openvino/core/model.hpp" #include "openvino/op/util/multi_subgraph_base.hpp" #include "openvino/pass/constant_folding.hpp" @@ -93,7 +94,7 @@ void transformation_pipeline(std::shared_ptr& model) { RTInfoCache rt_info_cache; rt_info_cache.store(model); - auto get_manager = []() { + auto get_manager = [&]() { Manager manager("pre_post_processing"); manager.set_per_pass_validation(false); @@ -112,10 +113,16 @@ void transformation_pipeline(std::shared_ptr& model) { MarkGatherSubgraph, element::TypeVector{element::f8e4m3}, element::TypeVector{element::u4}); - REGISTER_PASS( - manager, - MarkDequantization, - TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f4e2m1, f8e4m3, f8e5m2, f8e8m0}); + + const bool does_model_contain_mxfp_patterns = + ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(model); + REGISTER_PASS(manager, + MarkDequantization, + TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, + false); + if (does_model_contain_mxfp_patterns) { + REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); + } REGISTER_PASS(manager, DisableShapeOfConstantFolding, false); REGISTER_PASS(manager, DisableRandomUniformConstantFolding) // Mark quantized and f16/bf16 compressed constants to prevent CF for them, diff --git a/src/plugins/intel_gpu/src/graph/debug_helper.cpp b/src/plugins/intel_gpu/src/graph/debug_helper.cpp index 56c0fb4a4b5a..07a80b621f6b 100644 --- a/src/plugins/intel_gpu/src/graph/debug_helper.cpp +++ b/src/plugins/intel_gpu/src/graph/debug_helper.cpp @@ -270,6 +270,12 @@ void log_memory_to_file(memory::ptr mem, layout data_layout, stream& stream, std dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::u8) dump(actual_mem, stream, file_stream, dump_raw); + else if (mem_dt == cldnn::data_types::f8e5m2) + dump(actual_mem, stream, file_stream, dump_raw); + else if (mem_dt == cldnn::data_types::f8e4m3) + dump(actual_mem, stream, file_stream, dump_raw); + else if (mem_dt == cldnn::data_types::f8e8m0) + dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::boolean) dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::i4 || mem_dt == cldnn::data_types::u4) diff --git a/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp b/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp index de6e2f485858..0a3e428291c1 100644 --- a/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp +++ b/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp @@ -15,7 +15,11 @@ GPU_DEFINE_PRIMITIVE_TYPE_ID(dynamic_quantize); // We should skip dynamic_quantization execution for 2nd token of LLM because it does not show performance gain. // can_be_optimized flag will be turned on from primitive_inst::update_shape function -static bool should_skip_execution(dynamic_quantize_node const& node, const layout &act_layout) { +static bool should_skip_execution(const dynamic_quantize_node& node, const layout& act_layout, const dynamic_quantize::Attributes& attrs) { + if (cldnn::data_type_traits::is_floating_point(attrs.quantization_dt)) { + return false; // Execute unconditionally for floating point types. + } + if (!node.is_runtime_skippable() || !act_layout.is_static()) return false; @@ -45,7 +49,7 @@ static bool should_skip_execution(dynamic_quantize_node const& node, const layou layout dynamic_quantize_inst::calc_output_layout(dynamic_quantize_node const& /*node*/, kernel_impl_params const& impl_param) { auto desc = impl_param.typed_desc(); const auto& input_layout = impl_param.get_input_layout(); - auto output_type = data_types::i8; + auto output_type = desc->attrs.quantization_dt; auto output_format = input_layout.format; return layout(output_type, output_format, input_layout.get_tensor()); @@ -69,7 +73,7 @@ std::vector dynamic_quantize_inst::__calc_output_layouts(const dynamic_q std::vector output_layouts = { layout(output_shapes[0], attrs.quantization_dt, output_format), layout(output_shapes[1], attrs.scale_dt, output_format) }; - auto flag_skip_execution = should_skip_execution(node, act_layout); + auto flag_skip_execution = should_skip_execution(node, act_layout, attrs); GPU_DEBUG_TRACE_DETAIL << node.id() << " should_skip_execution " << flag_skip_execution << std::endl; diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/add_required_reorders.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/add_required_reorders.cpp index 120c2011a619..45ec68f10730 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/add_required_reorders.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/add_required_reorders.cpp @@ -308,7 +308,7 @@ void add_required_reorders::run(program& p) { continue; max_in_dims = std::max(cldnn::format::dimension(node.first->get_output_layout().format), max_in_dims); } - // This list of preferred layouts has been selected arbitrary due to developers' experience + // This list of preferred layouts has been selected arbitrarily due to developers' experience preferred_layout_formats = { cldnn::format::get_default_format(max_in_dims) }; if (max_in_dims == 8) { preferred_layout_formats.push_back(cldnn::format::bfvuwzyx); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/dynamic_quantize.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/dynamic_quantize.cpp index ab39c731198b..875c6ee6383b 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/dynamic_quantize.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/dynamic_quantize.cpp @@ -75,7 +75,10 @@ attach_dynamic_quantize_impl::attach_dynamic_quantize_impl() { auto types = { data_types::f16, data_types::i8, - data_types::u8 + data_types::u8, + data_types::f8e4m3, + data_types::f8e5m2, + data_types::f8e8m0, }; auto formats = { diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp index 103169b46b58..612215b92c44 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp @@ -195,6 +195,12 @@ kernel_selector::data_type to_data_type(data_types dt) { return kernel_selector::data_type::F32; case cldnn::data_types::bf16: return kernel_selector::data_type::BF16; + case cldnn::data_types::f8e4m3: + return kernel_selector::data_type::F8E4M3; + case cldnn::data_types::f8e5m2: + return kernel_selector::data_type::F8E5M2; + case cldnn::data_types::f8e8m0: + return kernel_selector::data_type::F8E8M0; default: OPENVINO_THROW("[GPU] Unable to convert cldnn data type ", dt, " to kernel_selector data type"); } @@ -224,6 +230,12 @@ data_types from_data_type(kernel_selector::data_type dt) { return cldnn::data_types::f16; case kernel_selector::data_type::F32: return cldnn::data_types::f32; + case kernel_selector::data_type::F8E4M3: + return cldnn::data_types::f8e4m3; + case kernel_selector::data_type::F8E5M2: + return cldnn::data_types::f8e5m2; + case kernel_selector::data_type::F8E8M0: + return cldnn::data_types::f8e8m0; default: OPENVINO_THROW("[GPU] Unable to convert kernel_selector data type ", kernel_selector::toString(dt), " to cldnn data type"); } @@ -247,6 +259,12 @@ kernel_selector::weights_type to_weights_type(data_types dt) { return kernel_selector::weights_type::INT32; case cldnn::data_types::bf16: return kernel_selector::weights_type::BF16; + case cldnn::data_types::f8e4m3: + return kernel_selector::weights_type::F8E4M3; + case cldnn::data_types::f8e5m2: + return kernel_selector::weights_type::F8E5M2; + case cldnn::data_types::f8e8m0: + return kernel_selector::weights_type::F8E8M0; default: OPENVINO_THROW("[GPU] Unable to convert cldnn data type ", dt, " to kernel_selector weights type"); } @@ -268,6 +286,12 @@ data_types from_weights_type(kernel_selector::weights_type dt) { return data_types::f32; case kernel_selector::weights_type::INT32: return data_types::i32; + case kernel_selector::weights_type::F8E4M3: + return data_types::f8e4m3; + case kernel_selector::weights_type::F8E5M2: + return data_types::f8e5m2; + case kernel_selector::weights_type::F8E8M0: + return data_types::f8e8m0; default: OPENVINO_THROW("[GPU] Unable to convert kernel_selector weights type ", kernel_selector::toString(dt), " to cldnn data type"); } diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp index 0cb4dc777db6..f5ef87141034 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp @@ -277,8 +277,8 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: min_val = "-" + name + "_VAL_MAX"; val_one = "1.0h"; val_zero = "0.0h"; - to_type = "convert_half(v)"; - to_type_sat = "convert_half(v)"; + to_type = "_convert_half(v)"; + to_type_sat = "_convert_half(v)"; as_type = "as_half(v)"; max_func = "fmax"; min_func = "fmin"; @@ -311,8 +311,8 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: min_val = "-" + name + "_VAL_MAX"; val_one = "1.0f"; val_zero = "0.0f"; - to_type = "convert_float(v)"; - to_type_sat = "convert_float(v)"; + to_type = "_convert_float(v)"; + to_type_sat = "_convert_float(v)"; as_type = "as_float(v)"; max_func = "fmax"; min_func = "fmin"; @@ -320,6 +320,42 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: type_size = "4"; is_fp = true; break; + case ov::element::f8e4m3: + type = "fp8e4m3_t"; + max_val = "(fp8e4m3_t){as_char((char)0x7E)}"; // 448.0 + min_val = "(fp8e4m3_t){as_char((char)0xFE)}"; // -448.0 + val_one = "(fp8e4m3_t){as_char((char)0x38)}"; + val_zero = "(fp8e4m3_t){as_char((char)0x0)}"; + to_type = "_convert_fp8e4m3_t(v)"; + to_type_sat = "_convert_fp8e4m3_t_sat(v)"; + as_type = "as_fp8e4m3_t(v)"; + type_size = "1"; + is_fp = true; + break; + case ov::element::f8e5m2: + type = "fp8e5m2_t"; + max_val = "(fp8e5m2_t){as_uchar((uchar)0x7B)}"; // 57344.0 + min_val = "(fp8e5m2_t){as_uchar((uchar)0xFB)}"; // -57344.0 + val_one = "(fp8e5m2_t){as_uchar((uchar)0x3C)}"; + val_zero = "(fp8e5m2_t){as_uchar((uchar)0x0)}"; + to_type = "_convert_fp8e5m2_t(v)"; + to_type_sat = "_convert_fp8e5m2_t_sat(v)"; + as_type = "as_fp8e5m2_t(v)"; + type_size = "1"; + is_fp = true; + break; + case ov::element::f8e8m0: + type = "fp8e8m0_t"; + max_val = "(fp8e8m0_t){as_uchar((uchar)0xFE)}"; // 2^127 + min_val = "(fp8e8m0_t){as_uchar((uchar)0x00)}"; // 2^(-127) + val_one = "(fp8e8m0_t){as_uchar((uchar)0x7F)}"; + val_zero = ""; // There is no representation of zero in FP8E8M0 + to_type = "_convert_fp8e8m0_t(v)"; + to_type_sat = "_convert_fp8e8m0_t_sat(v)"; + as_type = "as_fp8e8m0_t(v)"; + type_size = "1"; + is_fp = true; + break; default: OPENVINO_THROW("[GPU] Jitter: unsupported data type: ", value); } diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp index 19601d38a865..930a6357016e 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp @@ -69,25 +69,27 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { dnnl::memory::desc desc = onednn::layout_to_memory_desc_flatten(zp_mem->get_layout(), dnnl::memory::format_tag::a); args.insert({DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_WEIGHTS, zp_mem->get_onednn_memory(desc)}); } - bool is_dyn_quan_input = instance.get_input_layout(0).data_type == data_types::i8 || instance.get_input_layout(0).data_type == data_types::u8; + + const auto input_dt = instance.get_input_layout(0).data_type; + const bool is_dyn_quan_input = cldnn::one_of(input_dt, {data_types::i8, data_types::u8, data_types::f8e4m3, data_types::f8e5m2}); if (is_dyn_quan_input && prim->activation_scale.is_valid()) { - auto activation_scale_idx = idx++; - auto act_scale_mem = instance.dep_memory_ptr(activation_scale_idx); + const auto activation_scale_idx = idx++; + const auto act_scale_mem = instance.dep_memory_ptr(activation_scale_idx); dnnl::memory::desc desc = onednn::layout_to_memory_desc_flatten(act_scale_mem->get_layout(), dnnl::memory::format_tag::ab); args.insert({DNNL_ARG_ATTR_SCALES | DNNL_ARG_SRC_0, act_scale_mem->get_onednn_memory(desc)}); } if (is_dyn_quan_input && prim->activation_zero_point.is_valid()) { - auto activation_zp_idx = idx++; - auto act_zp_mem = instance.dep_memory_ptr(activation_zp_idx); + const auto activation_zp_idx = idx++; + const auto act_zp_mem = instance.dep_memory_ptr(activation_zp_idx); dnnl::memory::desc desc = onednn::layout_to_memory_desc_flatten(act_zp_mem->get_layout(), dnnl::memory::format_tag::ab); args.insert({DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_SRC_0, act_zp_mem->get_onednn_memory(desc)}); } if (is_dyn_quan_input && prim->activation_precomputed_reduction.is_valid()) { - auto activation_precomputed_reduction_idx = idx++; - auto act_precomputed_reduction_mem = instance.dep_memory_ptr(activation_precomputed_reduction_idx); + const auto activation_precomputed_reduction_idx = idx++; + const auto act_precomputed_reduction_mem = instance.dep_memory_ptr(activation_precomputed_reduction_idx); dnnl::memory::desc desc = onednn::layout_to_memory_desc_flatten(act_precomputed_reduction_mem->get_layout(), dnnl::memory::format_tag::ab); args.insert({DNNL_ARG_ATTR_PRECOMPUTED_REDUCTIONS | DNNL_ARG_SRC_0, act_precomputed_reduction_mem->get_onednn_memory(desc)}); } @@ -304,7 +306,8 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { } } - bool is_dyn_quan_input = impl_params->get_input_layout(0).data_type == data_types::i8 || impl_params->get_input_layout(0).data_type == data_types::u8; + const auto input_dt = impl_params->get_input_layout(0).data_type; + const bool is_dyn_quan_input = cldnn::one_of(input_dt, {data_types::i8, data_types::u8, data_types::f8e4m3, data_types::f8e5m2}); if (is_dyn_quan_input && dynamic_quantized_activation) { auto src_scale_idx = ++idx; auto partial_shape = impl_params->get_input_layout(0).get_partial_shape(); @@ -351,7 +354,8 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { int idx = !arg.bias_term() ? 1 : 2; if (prim->compressed_weights) { - bool is_dyn_quan_input = impl_params.get_input_layout(0).data_type == data_types::i8 || impl_params.get_input_layout(0).data_type == data_types::u8; + const auto input_dt = impl_params.get_input_layout(0).data_type; + const bool is_dyn_quan_input = cldnn::one_of(input_dt, {data_types::i8, data_types::u8, data_types::f8e4m3, data_types::f8e5m2}); if (is_dyn_quan_input) { OPENVINO_ASSERT(prim->input_size <= 3, "[GPU] Dynamic quantization for 4D matmul is not implemented"); } else { @@ -436,7 +440,6 @@ struct fully_connected_onednn : typed_primitive_onednn_impl { } - auto prim_desc = get_matmul_primitive_descriptor(impl_params, impl_params.prog->get_engine(), prim->input_size, prim->weights_rank, prim->bias.is_valid(), *attr); diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp index e633dcc9749f..0100f3e27779 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp @@ -60,13 +60,15 @@ struct FullyConnectedImplementationManager : public ImplementationManager { one_of(in0_dt, {data_types::f16, data_types::f32, data_types::i8, data_types::u8}) && one_of(wei_dt, {data_types::u8, data_types::i8, data_types::u4, data_types::i4}) && one_of(out_dt, {data_types::f16, data_types::f32, data_types::u8, data_types::i8}); - if (!f16f16_case && !f32f32_case && !u8s8_case && !compressed_case) + const bool fp_compressed_case = fc_prim->compressed_weights && one_of(in0_dt, {data_types::f8e4m3, data_types::f8e5m2}) && + one_of(wei_dt, {data_types::f8e4m3, data_types::f8e5m2}) && one_of(out_dt, {data_types::f16, data_types::f32}); + if (!f16f16_case && !f32f32_case && !u8s8_case && !compressed_case && !fp_compressed_case) LOG_AND_RETURN_FALSE(node); if (fc_prim->compressed_weights) { if (fc_prim->decompression_zero_point.is_valid()) { - auto decompression_zp_idx = fc_prim->bias.is_valid() ? 4 : 3; - auto decompression_zp_dt = fc_node.get_input_layout(decompression_zp_idx).data_type; + const auto decompression_zp_idx = fc_prim->bias.is_valid() ? 4 : 3; + const auto decompression_zp_dt = fc_node.get_input_layout(decompression_zp_idx).data_type; if ((wei_dt != ov::element::Type_t::i4 && wei_dt != ov::element::Type_t::u4 && wei_dt != ov::element::Type_t::u8) || (decompression_zp_dt != ov::element::Type_t::i4 && decompression_zp_dt != ov::element::Type_t::u8 && decompression_zp_dt != ov::element::Type_t::i8)) { @@ -75,6 +77,18 @@ struct FullyConnectedImplementationManager : public ImplementationManager { } } + if (fp_compressed_case && fc_prim->decompression_scale.is_valid() && fc_prim->activation_scale.is_valid()) { + auto decompression_scale_idx = fc_prim->bias.is_valid() ? 3 : 2; + auto weight_scale_dt = fc_node.get_input_layout(decompression_scale_idx).data_type; + auto activation_scale_dt = fc_node.get_input_layout(decompression_scale_idx + 1).data_type; + if (activation_scale_dt != weight_scale_dt) { + LOG_AND_RETURN_FALSE(node); + } + if (!one_of(activation_scale_dt, {ov::element::Type_t::f8e8m0, ov::element::Type_t::f16})) { + LOG_AND_RETURN_FALSE(node); + } + } + return true; } diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp index ce1104785ae7..b0fe902872cc 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp @@ -142,6 +142,9 @@ dnnl::memory::data_type convert_data_type(cldnn::data_types dt) { case cldnn::data_types::i32: return dnnl::memory::data_type::s32; case cldnn::data_types::i4: return dnnl::memory::data_type::s4; case cldnn::data_types::u4: return dnnl::memory::data_type::u4; + case cldnn::data_types::f8e4m3: return dnnl::memory::data_type::f8_e4m3; + case cldnn::data_types::f8e5m2: return dnnl::memory::data_type::f8_e5m2; + case cldnn::data_types::f8e8m0: return dnnl::memory::data_type::e8m0; default: throw std::invalid_argument("[clDNN] Unsupported conversion from cldnn to onednn type"); } } @@ -257,6 +260,9 @@ int64_t get_offset(const cldnn::layout& l, dnnl::memory::desc&& desc) { return offset / 2; case dnnl::memory::data_type::s8: case dnnl::memory::data_type::u8: + case dnnl::memory::data_type::f8_e4m3: + case dnnl::memory::data_type::f8_e5m2: + case dnnl::memory::data_type::e8m0: return offset; case dnnl::memory::data_type::f16: case dnnl::memory::data_type::bf16: diff --git a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp index 9f68ae9603b7..f0973f99d640 100644 --- a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp +++ b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp @@ -538,7 +538,7 @@ void primitive_inst::update_shape() { if (get_node().is_type() && get_flag(ExecutionFlags::SHAPE_CHANGED)) { auto &layout = _impl_params->get_output_layout(0); - OPENVINO_ASSERT(one_of(layout.data_type, {data_types::f16, data_types::i8, data_types::u8}), + OPENVINO_ASSERT(one_of(layout.data_type, {data_types::f16, data_types::i8, data_types::u8, data_types::f8e4m3, data_types::f8e5m2}), "[GPU] Unsupported data type of dynamic_quantize: ", layout.data_type); if (layout.data_type == data_types::f16) set_can_be_optimized(true); diff --git a/src/plugins/intel_gpu/src/graph/program.cpp b/src/plugins/intel_gpu/src/graph/program.cpp index d783140eecc9..6de625a82ead 100644 --- a/src/plugins/intel_gpu/src/graph/program.cpp +++ b/src/plugins/intel_gpu/src/graph/program.cpp @@ -909,9 +909,15 @@ void program::add_intermediate(program_node& node, size_t prev_idx, bool connect_int_node_with_old_dep, bool move_usrs_of_prev_to_node) { - if (connect_int_node_with_old_dep && !node.dependencies.empty()) - throw std::invalid_argument( - "Node which is about to be added in between two other nodes should not have any existing dependencies"); + if (connect_int_node_with_old_dep && !node.dependencies.empty()) { + std::string deps; + for (auto& dep : node.dependencies) { + deps += dep.first->id() + " ( " + dep.first->get_primitive()->type_string() + " ), "; + } + OPENVINO_THROW("Node which is about to be added in between two other nodes should not have any existing dependencies. Node: " + node.id() + " ( " + + node.get_primitive()->type_string() + " )" + ". Next: " + next.id() + " ( " + next.get_primitive()->type_string() + + " ). Dependencies: " + deps); + } auto& prev = next.get_dependency(prev_idx); // firstly add connection, later replace dependency, so 'prev' won't become dangling and therefore removed diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index f7f184c4b813..c4e6cacac69b 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -2,7 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // +#define IS_F8 (F8E5M2_OUTPUT || F8E4M3_OUTPUT) + #include "include/batch_headers/fetch_data.cl" +#if IS_F8 +#include "include/batch_headers/common.cl" +#include "include/batch_headers/f8_utils.cl" +#endif #if OUTPUT_DIMS != 4 && OUTPUT_DIMS != 2 #error "dynamic_quantize_gpu_opt.cl: Unsupported output dimension" @@ -13,10 +19,22 @@ #define CONVERT_UCHAR_N CAT(convert_uchar, VEC_SIZE) #define CONVERT_CHAR_N CAT(convert_char, VEC_SIZE) #define CONVERT_INT_N CAT(convert_int, VEC_SIZE) +#define TO_TYPE_N_(type, n, x) _convert_##type##n(x) +#define TO_TYPE_N(type, n, x) TO_TYPE_N_(type, n, x) +#define TO_TYPE_N_SAT_(type, n, x) _convert_##type##n##_sat(x) +#define TO_TYPE_N_SAT(type, n, x) TO_TYPE_N_SAT_(type, n, x) #define AS_TYPE_N_(type, n, x) as_##type##n(x) #define AS_TYPE_N(type, n, x) AS_TYPE_N_(type, n, x) #define AS_INPUT_TYPE_N(x) AS_TYPE_N(INPUT0_TYPE, VEC_SIZE, x) -#define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL +#if IS_F8 + #define SCALE_TYPE float + #define TO_SCALE_TYPE(x) _convert_float(x) + #define ACT_MIN_VAL 0.000000059604645h // min half dtype val +#else + #define SCALE_TYPE half + #define TO_SCALE_TYPE(x) _convert_half(x) + #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL +#endif #if GENERATE_PRECOMPUTED_REDUCTION #define FOR_PRECOMPUTED_REDUCTION(x) x @@ -60,7 +78,7 @@ KERNEL(dynamic_quantize_gpu_opt)( #endif const uint quantize_block = QUANTIZE_GROUP_SIZE / 4; half4 input_0[quantize_block]; - char4 quantized_value[quantize_block]; + MAKE_VECTOR_TYPE(OUTPUT_TYPE, 4) quantized_value[quantize_block]; half max[quantize_block]; unroll_for (uint i = 0 ; i < quantize_block; ++i) { @@ -73,13 +91,22 @@ KERNEL(dynamic_quantize_gpu_opt)( max_value = fmax(max_value, max[i]); } - half quan_scale = 127.0h / max_value; +#if IS_MXFP + SCALE_TYPE quan_scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / _convert_float(max_value))))); +#else + SCALE_TYPE quan_scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_value; FOR_PRECOMPUTED_REDUCTION(int precomputed_reduction = 0); +#endif // MXFP unroll_for (uint i = 0 ; i < quantize_block; ++i) { +#if IS_F8 + quantized_value[i] = TO_TYPE_N_SAT(OUTPUT_TYPE, 4, convert_float4(input_0[i]) * (MAKE_VECTOR_TYPE(SCALE_TYPE, 4))quan_scale); + vstore4(quantized_value[i].data, 0, (char*)(&output[output_offset + i * 4])); +#else quantized_value[i] = convert_char4_rte(input_0[i] * (half4)quan_scale); FOR_PRECOMPUTED_REDUCTION(precomputed_reduction += quantized_value[i][0] + quantized_value[i][1] + quantized_value[i][2] + quantized_value[i][3]); vstore4(quantized_value[i], 0, &output[output_offset + i * 4]); +#endif // IS_F8 } #if OUTPUT_DIMS == 2 @@ -87,7 +114,7 @@ KERNEL(dynamic_quantize_gpu_opt)( #else const uint output_idx = OUTPUT1_GET_INDEX(b, f, y_grp, 0); #endif - output_scale[output_idx] = 1.0h / quan_scale; + output_scale[output_idx] = TO_OUTPUT1_TYPE(1.0h / quan_scale); FOR_PRECOMPUTED_REDUCTION(output_precomputed_reduction[output_idx] = precomputed_reduction); } @@ -189,14 +216,19 @@ KERNEL(dynamic_quantize_gpu_opt)( OUTPUT1_TYPE scale = (OUTPUT1_TYPE)((CHAR_MAX - CHAR_MIN) / (max_value - min_value)); OUTPUT2_TYPE zp = (OUTPUT2_TYPE)(-min_value * scale); #else - OUTPUT1_TYPE scale = 127.0h / max_value; + SCALE_TYPE scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_value; #endif +#if IS_F8 + val = TO_TYPE_N(INPUT0_TYPE, VEC_SIZE, TO_TYPE_N(SCALE_TYPE, VEC_SIZE, val) * (MAKE_VECTOR_TYPE(SCALE_TYPE, VEC_SIZE))scale); + MAKE_VECTOR_TYPE(OUTPUT_TYPE, VEC_SIZE) out = TO_TYPE_N_SAT(OUTPUT_TYPE, VEC_SIZE, val); + VSTORE_N(out.data, 0, (char*)(&output[output_offset + (blockid * block_size)])); +#elif ASYMMETRIC_QUANTIZATION val *= scale; -#if ASYMMETRIC_QUANTIZATION val += zp; VSTORE_N(CAT(CONVERT_UCHAR_N, _rte)(val), 0, output + output_offset + (blockid * block_size)); -#else +#else // i8 symmetric + val *= scale; VSTORE_N(CAT(CONVERT_CHAR_N, _rte)(val), 0, output + output_offset + (blockid * block_size)); #endif @@ -315,7 +347,7 @@ KERNEL(dynamic_quantize_gpu_opt)( OUTPUT1_TYPE scale = (OUTPUT1_TYPE)((CHAR_MAX - CHAR_MIN) / (max_value - min_value)); OUTPUT2_TYPE zp = (OUTPUT2_TYPE)(-min_value * scale); #else - OUTPUT1_TYPE scale = 127.0h / max_value; + SCALE_TYPE scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_value; #endif @@ -323,11 +355,14 @@ KERNEL(dynamic_quantize_gpu_opt)( if ((local_id * iteration + i) >= TOTAL_BLOCK_NUM) continue; - val[i] *= scale; -#if ASYMMETRIC_QUANTIZATION + val[i] = TO_TYPE_N(INPUT0_TYPE, VEC_SIZE, TO_TYPE_N(SCALE_TYPE, VEC_SIZE, val[i]) * (MAKE_VECTOR_TYPE(SCALE_TYPE, VEC_SIZE))scale); +#if IS_F8 + MAKE_VECTOR_TYPE(OUTPUT_TYPE, VEC_SIZE) out = TO_TYPE_N_SAT(OUTPUT_TYPE, VEC_SIZE, val[i]); + VSTORE_N(out.data, 0, (char*)(&output[offset + ((local_id * iteration + i) * block_size)])); +#elif ASYMMETRIC_QUANTIZATION val[i] += zp; VSTORE_N(CAT(CONVERT_UCHAR_N, _rte)(val[i]), 0, output + offset + ((local_id * iteration + i) * block_size)); -#else +#else // i8 symmetric VSTORE_N(CAT(CONVERT_CHAR_N, _rte)(val[i]), 0, output + offset + ((local_id * iteration + i) * block_size)); #endif } diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl index 56f906af1528..2b8859c552be 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl @@ -2,17 +2,40 @@ // SPDX-License-Identifier: Apache-2.0 // +#define IS_F8 (F8E5M2_OUTPUT || F8E4M3_OUTPUT) + #include "include/batch_headers/fetch_data.cl" +#if IS_F8 +#include "include/batch_headers/common.cl" +#include "include/batch_headers/f8_utils.cl" +#endif #define UINT64_MAX 0xFFFFFFFFFFFFFFFF -#define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL -#if ASYMMETRIC_QUANTIZATION && UNSIGNED_OUTPUT - #define TO_OUTPUT_TYPE_RTE(val) convert_uchar_rte(val) - #define TO_OUTPUT_VEC_TYPE_RTE(val) convert_uchar8_rte(val) +#if IS_F8 + #define SCALE_TYPE float + #define TO_SCALE_TYPE(x) _convert_float(x) + #define TO_SCALE_TYPE_8(x) _convert_float8(x) + #define ACT_MIN_VAL 0.000000059604645h // min half dtype val +#else + #define SCALE_TYPE half + #define TO_SCALE_TYPE(x) _convert_half(x) + #define TO_SCALE_TYPE_8(x) _convert_half8(x) + #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL +#endif + +#if F8E5M2_OUTPUT + #define TO_OUTPUT_TYPE_CUSTOM(val) _convert_fp8e5m2_t_sat(val) + #define TO_OUTPUT_VEC_TYPE_CUSTOM(val) _convert_fp8e5m2_t8_sat(val) +#elif F8E4M3_OUTPUT + #define TO_OUTPUT_TYPE_CUSTOM(val) _convert_fp8e4m3_t_sat(val) + #define TO_OUTPUT_VEC_TYPE_CUSTOM(val) _convert_fp8e4m3_t8_sat(val) +#elif (ASYMMETRIC_QUANTIZATION && UNSIGNED_OUTPUT) + #define TO_OUTPUT_TYPE_CUSTOM(val) convert_uchar_rte(val) + #define TO_OUTPUT_VEC_TYPE_CUSTOM(val) convert_uchar8_rte(val) #else - #define TO_OUTPUT_TYPE_RTE(val) convert_char_rte(val) - #define TO_OUTPUT_VEC_TYPE_RTE(val) convert_char8_rte(val) + #define TO_OUTPUT_TYPE_CUSTOM(val) convert_char_rte(val) + #define TO_OUTPUT_VEC_TYPE_CUSTOM(val) convert_char8_rte(val) #endif #if GENERATE_PRECOMPUTED_REDUCTION @@ -124,8 +147,12 @@ KERNEL(dynamic_quantize_gpu_ref)( OUTPUT1_TYPE scale = (OUTPUT1_TYPE)(scale_tmp); OUTPUT1_TYPE zp = (OUTPUT1_TYPE)(zp_tmp); #else // !ASYMMETRIC_QUANTIZATION - OUTPUT1_TYPE scale = 127.0h / max_val; -#endif +#if IS_MXFP + SCALE_TYPE scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / _convert_float(max_val))))); +#else + SCALE_TYPE scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_val; +#endif // IS_FP8 +#endif // ASYMMETRIC_QUANTIZATION FOR_PRECOMPUTED_REDUCTION(OUTPUT2_TYPE precomputed_reduction = 0); @@ -141,7 +168,7 @@ KERNEL(dynamic_quantize_gpu_ref)( #if ASYMMETRIC_QUANTIZATION val += zp; #endif - OUTPUT_TYPE ival = TO_OUTPUT_TYPE_RTE(val); + OUTPUT_TYPE ival = TO_OUTPUT_TYPE_CUSTOM(val); output[out_offset] = ival; FOR_PRECOMPUTED_REDUCTION(precomputed_reduction += ival); #else // GROUP_SIZE_DIM3 != 1 @@ -150,13 +177,17 @@ KERNEL(dynamic_quantize_gpu_ref)( int x; for (x = 0; x < INPUT0_SIZE_X / 8; x++) { half8 val = as_half8(vload8(0, (ushort*)input + in_offset + x * 8)); - val *= scale; + val = convert_half8(TO_SCALE_TYPE_8(val) * (MAKE_VECTOR_TYPE(SCALE_TYPE, 8))scale); #if ASYMMETRIC_QUANTIZATION val += zp; #endif - MAKE_VECTOR_TYPE(OUTPUT_TYPE, 8) ival = TO_OUTPUT_VEC_TYPE_RTE(val); +#if IS_F8 + vstore8(TO_OUTPUT_VEC_TYPE_CUSTOM(val).data, 0, (char*)(&output[out_offset + x * 8])); +#else + MAKE_VECTOR_TYPE(OUTPUT_TYPE, 8) ival = TO_OUTPUT_VEC_TYPE_CUSTOM(val); vstore8(ival, 0, output + out_offset + x * 8); FOR_PRECOMPUTED_REDUCTION(precomputed_reduction += ((int)ival[0]) + ival[1] + ival[2] + ival[3] + ival[4] + ival[5] + ival[6] + ival[7]); +#endif } x *= 8; for (; x < INPUT0_SIZE_X; x++) { @@ -165,7 +196,7 @@ KERNEL(dynamic_quantize_gpu_ref)( #if ASYMMETRIC_QUANTIZATION val += zp; #endif - OUTPUT_TYPE ival = TO_OUTPUT_TYPE_RTE(val); + OUTPUT_TYPE ival = TO_OUTPUT_TYPE_CUSTOM(val); output[out_offset + x] = ival; FOR_PRECOMPUTED_REDUCTION(precomputed_reduction += ival); } @@ -174,7 +205,7 @@ KERNEL(dynamic_quantize_gpu_ref)( } } - output_scale[scale_idx] = 1.0h / scale; + output_scale[scale_idx] = TO_OUTPUT1_TYPE(1.0h / scale); FOR_PRECOMPUTED_REDUCTION(output_precomputed_reduction[scale_idx] = precomputed_reduction); #if ASYMMETRIC_QUANTIZATION && GROUP_SCALES_WITH_ZP output_scale[scale_idx + 1] = zp; diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index ac885a2be0ad..ea7de660a57b 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -38,6 +38,21 @@ #define MAKE_VECTOR_TYPE_IMPL_16(elem_type) CAT(elem_type, 16) #define MAKE_VECTOR_TYPE(elem_type, size) CAT(MAKE_VECTOR_TYPE_IMPL_, size)(elem_type) +#define AS_TYPE_PREFIX_uchar as_ +#define AS_TYPE_PREFIX_char as_ +#define AS_TYPE_PREFIX_fp8e5m2_t _as_ +#define AS_TYPE_PREFIX_fp8e4m3_t _as_ +#define AS_TYPE_PREFIX_fp8e8m0_t _as_ +#define AS_TYPE_PREFIX_ushort as_ +#define AS_TYPE_PREFIX_short as_ +#define AS_TYPE_PREFIX_half as_ +#define AS_TYPE_PREFIX_int as_ +#define AS_TYPE_PREFIX_uint as_ +#define AS_TYPE_PREFIX_float as_ +#define AS_TYPE_PREFIX_ulong as_ +#define AS_TYPE_PREFIX_long as_ + +#define AS_TYPE_EXT(type, val, src_type) CAT(CAT(AS_TYPE_PREFIX_, src_type), type)(val) #define AS_TYPE(type, val) CAT(as_, type)(val) // ==================================================================================================================== @@ -46,6 +61,9 @@ // ==================================================================================================================== #define TYPE_SIZE_uchar 1 #define TYPE_SIZE_char 1 +#define TYPE_SIZE_fp8e5m2_t 1 +#define TYPE_SIZE_fp8e4m3_t 1 +#define TYPE_SIZE_fp8e8m0_t 1 #define TYPE_SIZE_ushort 2 #define TYPE_SIZE_short 2 #define TYPE_SIZE_half 2 @@ -61,3 +79,71 @@ #else #define REQD_SUB_GROUP_SIZE(sg_size) #endif + +#define DEFINE_VECTOR_CONVERT(target_type, source_type, vector_size) \ +MAKE_VECTOR_TYPE(target_type, vector_size) __attribute__((overloadable)) _convert_##target_type##vector_size(MAKE_VECTOR_TYPE(source_type, vector_size) val) { \ + return CAT(convert_, CAT(target_type, vector_size))(val); \ +} + +#define DEFINE_VECTOR_IDENTITY(type, vector_size) \ +MAKE_VECTOR_TYPE(type, vector_size) __attribute__((overloadable)) _convert_##type##vector_size(MAKE_VECTOR_TYPE(type, vector_size) val) { \ + return val; \ +} + +#define DEFINE_ALL_VECTOR_CONVERTS(target_type, source_type) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 2) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 3) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 4) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 8) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 16) + +#define DEFINE_ALL_VECTOR_IDENTITY(type) \ +DEFINE_VECTOR_IDENTITY(type, 2) \ +DEFINE_VECTOR_IDENTITY(type, 3) \ +DEFINE_VECTOR_IDENTITY(type, 4) \ +DEFINE_VECTOR_IDENTITY(type, 8) \ +DEFINE_VECTOR_IDENTITY(type, 16) + +float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(float val) { return val; } + +DEFINE_ALL_VECTOR_CONVERTS(float, char) +DEFINE_ALL_VECTOR_CONVERTS(float, uchar) +DEFINE_ALL_VECTOR_CONVERTS(float, short) +DEFINE_ALL_VECTOR_CONVERTS(float, ushort) +DEFINE_ALL_VECTOR_CONVERTS(float, int) +DEFINE_ALL_VECTOR_CONVERTS(float, uint) +DEFINE_ALL_VECTOR_CONVERTS(float, long) +DEFINE_ALL_VECTOR_CONVERTS(float, ulong) +DEFINE_ALL_VECTOR_CONVERTS(float, half) +DEFINE_ALL_VECTOR_IDENTITY(float) + +half __attribute__((overloadable)) _convert_half(char val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(uchar val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(short val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(ushort val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(int val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(uint val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(long val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(ulong val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(half val) { return val; } +half __attribute__((overloadable)) _convert_half(float val) { return convert_half(val); } + +DEFINE_ALL_VECTOR_CONVERTS(half, char) +DEFINE_ALL_VECTOR_CONVERTS(half, uchar) +DEFINE_ALL_VECTOR_CONVERTS(half, short) +DEFINE_ALL_VECTOR_CONVERTS(half, ushort) +DEFINE_ALL_VECTOR_CONVERTS(half, int) +DEFINE_ALL_VECTOR_CONVERTS(half, uint) +DEFINE_ALL_VECTOR_CONVERTS(half, long) +DEFINE_ALL_VECTOR_CONVERTS(half, ulong) +DEFINE_ALL_VECTOR_CONVERTS(half, float) +DEFINE_ALL_VECTOR_IDENTITY(half) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl new file mode 100644 index 000000000000..0e0a7b97a7d1 --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl @@ -0,0 +1,1275 @@ +// Copyright (C) 2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +// TODO: Replace `_intel_convert*` with bultins when ready, current implementations are copied from XeTLA: + +uchar _f16_to_bf8_universal(half val, bool is_saturation) { + half val_fp16 = val; + const ushort *p = (ushort *)&val_fp16; + const ushort mant = p[0] & 0x3FF; + const uchar exp_mask = 0x7C; + + uchar ret_tmp = p[0] >> 8; + const bool is_infnan = (ret_tmp & exp_mask) == exp_mask; + if (is_infnan) { + ret_tmp |= (mant != 0); // The bit signifying NaNness may have been cut off + } else { + ret_tmp += (mant & 0x80) && ((mant & 0x7F) || (mant & 0x100)); // RTE + if (is_saturation) { + bool is_overflow = (ret_tmp & exp_mask) == exp_mask; + ret_tmp -= is_overflow; + } + } + + return ret_tmp; +} + +uchar _intel_convert_f16_to_bf8(half val) { + return _f16_to_bf8_universal(val, false); +} + +uchar _intel_convert_f16_to_bf8_sat(half val) { + return _f16_to_bf8_universal(val, true); +} + +half _intel_convert_bf8_to_f16(uchar val) { + ushort temp = val; + temp = temp << 0x8; + half temp_fp16 = as_half(temp); + return temp_fp16; +} + +char _f16_to_hf8_universal(half val, bool is_saturation) { + half val_fp16 = val; + ushort *p = (ushort *)&val_fp16; + ushort src = p[0]; + const ushort src_exp_size = 5; + const ushort src_mant_size = 10; + const ushort src_exp_bias = (1 << (src_exp_size - 1)) - 1; + const ushort src_exp_mask = (1 << src_exp_size) - 1; + const ushort src_mant_mask = (1 << src_mant_size) - 1; + const short max_exp_unbiased = 8; + const short min_exp_unbiased = -6; + const short exp_bias = 7; + const ushort exp_size = 4; + const ushort mant_size = 3; + const uchar nan = 0x7f; + const uchar max_val = 0x7e; + + ushort src_sign = src >> (src_exp_size + src_mant_size); + ushort src_exp = (src >> src_mant_size) & src_exp_mask; + short src_exp_unbiased = src_exp - src_exp_bias; + ushort src_mant = src & src_mant_mask; + + bool is_src_inf_nan = src_exp == 0x1f; + bool is_overflow = (src_exp_unbiased > max_exp_unbiased) + || ((src_exp_unbiased == max_exp_unbiased) && (src_mant > 0x0340)); + bool is_zero = (src_exp_unbiased < (min_exp_unbiased - mant_size)); + bool is_denorm = (src_exp_unbiased < min_exp_unbiased) && (!is_zero); + + uchar dst_val; + if (is_src_inf_nan) { + dst_val = nan; + } else if (is_overflow) { + dst_val = is_saturation ? max_val : nan; + } else if (is_zero) { + dst_val = 0; + } else if (is_denorm) { + ushort src_m = src_mant | 0x0400; + short shift_out_bit = min_exp_unbiased - src_exp_unbiased; + bool sticky_flag = (src_m & ((1 << shift_out_bit) - 1)) != 0; + src_m = src_m >> shift_out_bit; + ushort tail_size = src_mant_size - mant_size; + sticky_flag + = sticky_flag || ((src_m & ((1 << (tail_size - 1)) - 1)) != 0); + bool lsb_bit = src_m & (1 << tail_size); + bool rnd_bit = src_m & (1 << (tail_size - 1)); + bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); + + dst_val = (src_m >> tail_size) + carry; + } else { + ushort tail_size = src_mant_size - mant_size; + bool sticky_flag = (src_mant & ((1 << (tail_size - 1)) - 1)) != 0; + bool lsb_bit = src_mant & (1 << tail_size); + bool rnd_bit = src_mant & (1 << (tail_size - 1)); + bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); + ushort src_m = (src_mant >> tail_size) + carry; + ushort src_e = src_exp_unbiased + exp_bias; + dst_val = (src_e << mant_size) + src_m; + } + return src_sign << (exp_size + mant_size) | dst_val; +} + +char _intel_convert_f16_to_hf8(half val) { + return _f16_to_hf8_universal(val, false); +} + +char _intel_convert_f16_to_hf8_sat(half val) { + return _f16_to_hf8_universal(val, true); +} + +half _intel_convert_hf8_to_f16(char val) { + const short exp_bias = 7; + char data = val; + ushort sign = data >> 7; + ushort exp = (data >> 3) & 0b1111; + ushort mant = data & 0x07; + ushort dst_val; + if ((exp == 0xf) && (mant == 0x7)) { + dst_val = 0x7fff; + } else if ((exp == 0) && (mant == 0)) { + dst_val = 0; + } else if ((exp == 0) && (mant != 0)) { + ushort lz_count = (mant > 3) ? 0 : ((mant > 1) ? 1 : 2); + ushort dst_exp = exp - exp_bias + 15 - lz_count; + ushort dst_mant = (mant << (lz_count + 1)) & 0x7; + dst_val = (dst_exp << 10) | (dst_mant << 7); + } else { + ushort dst_exp = exp - exp_bias + 15; + dst_val = (dst_exp << 10) | (mant << 7); + } + + ushort temp = (sign << 15) | dst_val; + half temp_fp16 = as_half(temp); + return temp_fp16; +} + +uchar _intel_convert_f32_fo_e8m0(float val) { + uint val_uint = as_uint(val); + return (uchar)((val_uint >> 23) & 0xFF); +} + +uchar _intel_convert_f32_fo_e8m0_sat(float val) { + return _intel_convert_f32_fo_e8m0(val); +} + +float _intel_convert_e8m0_to_f32(uchar val) { + if (val == 0xFF) { + return as_float(0xFFC00000); // NaN + } else if (val == 0) { + return as_float(0x00400000); // 2^(-127) + } + + uint temp = (uint)val; + temp = temp << 23; + float temp_fp32 = as_float(temp); + return temp_fp32; +} + +typedef struct fp8e5m2_t { uchar data; } fp8e5m2_t; // bf8 +typedef struct fp8e5m2_t1 { uchar data; } fp8e5m2_t1; +typedef struct fp8e5m2_t2 { uchar2 data; } fp8e5m2_t2; +typedef struct fp8e5m2_t3 { uchar3 data; } fp8e5m2_t3; +typedef struct fp8e5m2_t4 { uchar4 data; } fp8e5m2_t4; +typedef struct fp8e5m2_t8 { uchar8 data; } fp8e5m2_t8; +typedef struct fp8e5m2_t16 { uchar16 data; } fp8e5m2_t16; + +typedef struct fp8e4m3_t { char data; } fp8e4m3_t; // hf8 +typedef struct fp8e4m3_t1 { char data; } fp8e4m3_t1; +typedef struct fp8e4m3_t2 { char2 data; } fp8e4m3_t2; +typedef struct fp8e4m3_t3 { char3 data; } fp8e4m3_t3; +typedef struct fp8e4m3_t4 { char4 data; } fp8e4m3_t4; +typedef struct fp8e4m3_t8 { char8 data; } fp8e4m3_t8; +typedef struct fp8e4m3_t16 { char16 data; } fp8e4m3_t16; + +typedef struct fp8e8m0_t { uchar data; } fp8e8m0_t; // e8m0 +typedef struct fp8e8m0_t1 { uchar data; } fp8e8m0_t1; +typedef struct fp8e8m0_t2 { uchar2 data; } fp8e8m0_t2; +typedef struct fp8e8m0_t3 { uchar3 data; } fp8e8m0_t3; +typedef struct fp8e8m0_t4 { uchar4 data; } fp8e8m0_t4; +typedef struct fp8e8m0_t8 { uchar8 data; } fp8e8m0_t8; +typedef struct fp8e8m0_t16 { uchar16 data; } fp8e8m0_t16; + + +half __attribute__((overloadable)) _convert_half(fp8e5m2_t val) { + return _intel_convert_bf8_to_f16(val.data); +} +half __attribute__((overloadable)) _convert_half(fp8e5m2_t1 val) { + return _intel_convert_bf8_to_f16(val.data); +} +half2 __attribute__((overloadable)) _convert_half2(fp8e5m2_t2 val) { + return (half2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); +} +half3 __attribute__((overloadable)) _convert_half3(fp8e5m2_t3 val) { + return (half3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); +} +half4 __attribute__((overloadable)) _convert_half4(fp8e5m2_t4 val) { + return (half4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); +} +half8 __attribute__((overloadable)) _convert_half8(fp8e5m2_t8 val) { + return (half8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); +} +half16 __attribute__((overloadable)) _convert_half16(fp8e5m2_t16 val) { + return (half16)( + _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), + _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), + _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), + _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), + _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) + ); +} + +float __attribute__((overloadable)) _convert_float(fp8e5m2_t val) { + return (float)_intel_convert_bf8_to_f16(val.data); +} +float __attribute__((overloadable)) _convert_float(fp8e5m2_t1 val) { + return (float)_intel_convert_bf8_to_f16(val.data); +} +float2 __attribute__((overloadable)) _convert_float2(fp8e5m2_t2 val) { + return (float2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); +} +float3 __attribute__((overloadable)) _convert_float3(fp8e5m2_t3 val) { + return (float3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); +} +float4 __attribute__((overloadable)) _convert_float4(fp8e5m2_t4 val) { + return (float4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); +} +float8 __attribute__((overloadable)) _convert_float8(fp8e5m2_t8 val) { + return (float8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); +} +float16 __attribute__((overloadable)) _convert_float16(fp8e5m2_t16 val) { + return (float16)( + _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), + _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), + _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), + _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), + _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) + ); +} + +half __attribute__((overloadable)) _convert_half(fp8e4m3_t val) { + return _intel_convert_hf8_to_f16(val.data); +} +half __attribute__((overloadable)) _convert_half(fp8e4m3_t1 val) { + return _intel_convert_hf8_to_f16(val.data); +} +half2 __attribute__((overloadable)) _convert_half2(fp8e4m3_t2 val) { + return (half2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); +} +half3 __attribute__((overloadable)) _convert_half3(fp8e4m3_t3 val) { + return (half3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); +} +half4 __attribute__((overloadable)) _convert_half4(fp8e4m3_t4 val) { + return (half4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); +} +half8 __attribute__((overloadable)) _convert_half8(fp8e4m3_t8 val) { + return (half8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); +} +half16 __attribute__((overloadable)) _convert_half16(fp8e4m3_t16 val) { + return (half16)( + _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), + _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), + _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), + _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), + _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) + ); +} + +float __attribute__((overloadable)) _convert_float(fp8e4m3_t val) { + return (float)_intel_convert_hf8_to_f16(val.data); +} +float __attribute__((overloadable)) _convert_float(fp8e4m3_t1 val) { + return (float)_intel_convert_hf8_to_f16(val.data); +} +float2 __attribute__((overloadable)) _convert_float2(fp8e4m3_t2 val) { + return (float2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); +} +float3 __attribute__((overloadable)) _convert_float3(fp8e4m3_t3 val) { + return (float3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); +} +float4 __attribute__((overloadable)) _convert_float4(fp8e4m3_t4 val) { + return (float4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); +} +float8 __attribute__((overloadable)) _convert_float8(fp8e4m3_t8 val) { + return (float8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); +} +float16 __attribute__((overloadable)) _convert_float16(fp8e4m3_t16 val) { + return (float16)( + _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), + _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), + _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), + _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), + _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) + ); +} + +float __attribute__((overloadable)) _convert_float(fp8e8m0_t val) { + return (float)_intel_convert_e8m0_to_f32(val.data); +} +float __attribute__((overloadable)) _convert_float(fp8e8m0_t1 val) { + return (float)_intel_convert_e8m0_to_f32(val.data); +} +float2 __attribute__((overloadable)) _convert_float2(fp8e8m0_t2 val) { + return (float2)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1)); +} +float3 __attribute__((overloadable)) _convert_float3(fp8e8m0_t3 val) { + return (float3)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), _intel_convert_e8m0_to_f32(val.data.s2)); +} +float4 __attribute__((overloadable)) _convert_float4(fp8e8m0_t4 val) { + return (float4)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), + _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3)); +} +float8 __attribute__((overloadable)) _convert_float8(fp8e8m0_t8 val) { + return (float8)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), + _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), + _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), + _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7)); +} +float16 __attribute__((overloadable)) _convert_float16(fp8e8m0_t16 val) { + return (float16)( + _intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), + _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), + _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), + _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7), + _intel_convert_e8m0_to_f32(val.data.s8), _intel_convert_e8m0_to_f32(val.data.s9), + _intel_convert_e8m0_to_f32(val.data.sA), _intel_convert_e8m0_to_f32(val.data.sB), + _intel_convert_e8m0_to_f32(val.data.sC), _intel_convert_e8m0_to_f32(val.data.sD), + _intel_convert_e8m0_to_f32(val.data.sE), _intel_convert_e8m0_to_f32(val.data.sF) + ); +} + +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(float val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8((half)val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(float val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8((half)val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(float2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(float3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(float4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); + res.data.s3 = _intel_convert_f16_to_bf8((half)val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(float8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(float16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); + res.data.s8 = _intel_convert_f16_to_bf8((half)val.s8); + res.data.s9 = _intel_convert_f16_to_bf8((half)val.s9); + res.data.sA = _intel_convert_f16_to_bf8((half)val.sA); + res.data.sB = _intel_convert_f16_to_bf8((half)val.sB); + res.data.sC = _intel_convert_f16_to_bf8((half)val.sC); + res.data.sD = _intel_convert_f16_to_bf8((half)val.sD); + res.data.sE = _intel_convert_f16_to_bf8((half)val.sE); + res.data.sF = _intel_convert_f16_to_bf8((half)val.sF); + return res; +} + +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(half val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8(val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(half val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8(val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(half2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.x); + res.data.s1 = _intel_convert_f16_to_bf8(val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(half3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.x); + res.data.s1 = _intel_convert_f16_to_bf8(val.y); + res.data.s2 = _intel_convert_f16_to_bf8(val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(half4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.x); + res.data.s1 = _intel_convert_f16_to_bf8(val.y); + res.data.s2 = _intel_convert_f16_to_bf8(val.z); + res.data.s3 = _intel_convert_f16_to_bf8(val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(half8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8(val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(half16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8(val.s7); + res.data.s8 = _intel_convert_f16_to_bf8(val.s8); + res.data.s9 = _intel_convert_f16_to_bf8(val.s9); + res.data.sA = _intel_convert_f16_to_bf8(val.sA); + res.data.sB = _intel_convert_f16_to_bf8(val.sB); + res.data.sC = _intel_convert_f16_to_bf8(val.sC); + res.data.sD = _intel_convert_f16_to_bf8(val.sD); + res.data.sE = _intel_convert_f16_to_bf8(val.sE); + res.data.sF = _intel_convert_f16_to_bf8(val.sF); + return res; +} + +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(float val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8_sat((half)val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(float val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8_sat((half)val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(float2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(float3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(float4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); + res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(float8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(float16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); + res.data.s8 = _intel_convert_f16_to_bf8_sat((half)val.s8); + res.data.s9 = _intel_convert_f16_to_bf8_sat((half)val.s9); + res.data.sA = _intel_convert_f16_to_bf8_sat((half)val.sA); + res.data.sB = _intel_convert_f16_to_bf8_sat((half)val.sB); + res.data.sC = _intel_convert_f16_to_bf8_sat((half)val.sC); + res.data.sD = _intel_convert_f16_to_bf8_sat((half)val.sD); + res.data.sE = _intel_convert_f16_to_bf8_sat((half)val.sE); + res.data.sF = _intel_convert_f16_to_bf8_sat((half)val.sF); + return res; +} + +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(half val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8_sat(val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(half val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8_sat(val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(half2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(half3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(half4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); + res.data.s3 = _intel_convert_f16_to_bf8_sat(val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(half8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(half16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); + res.data.s8 = _intel_convert_f16_to_bf8_sat(val.s8); + res.data.s9 = _intel_convert_f16_to_bf8_sat(val.s9); + res.data.sA = _intel_convert_f16_to_bf8_sat(val.sA); + res.data.sB = _intel_convert_f16_to_bf8_sat(val.sB); + res.data.sC = _intel_convert_f16_to_bf8_sat(val.sC); + res.data.sD = _intel_convert_f16_to_bf8_sat(val.sD); + res.data.sE = _intel_convert_f16_to_bf8_sat(val.sE); + res.data.sF = _intel_convert_f16_to_bf8_sat(val.sF); + return res; +} + +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(float val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8((half)val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(float val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8((half)val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(float2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(float3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(float4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); + res.data.s3 = _intel_convert_f16_to_hf8((half)val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(float8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(float16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); + res.data.s8 = _intel_convert_f16_to_hf8((half)val.s8); + res.data.s9 = _intel_convert_f16_to_hf8((half)val.s9); + res.data.sA = _intel_convert_f16_to_hf8((half)val.sA); + res.data.sB = _intel_convert_f16_to_hf8((half)val.sB); + res.data.sC = _intel_convert_f16_to_hf8((half)val.sC); + res.data.sD = _intel_convert_f16_to_hf8((half)val.sD); + res.data.sE = _intel_convert_f16_to_hf8((half)val.sE); + res.data.sF = _intel_convert_f16_to_hf8((half)val.sF); + return res; +} + +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(half val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8(val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(half val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8(val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(half2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.x); + res.data.s1 = _intel_convert_f16_to_hf8(val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(half3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.x); + res.data.s1 = _intel_convert_f16_to_hf8(val.y); + res.data.s2 = _intel_convert_f16_to_hf8(val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(half4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.x); + res.data.s1 = _intel_convert_f16_to_hf8(val.y); + res.data.s2 = _intel_convert_f16_to_hf8(val.z); + res.data.s3 = _intel_convert_f16_to_hf8(val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(half8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8(val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(half16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8(val.s7); + res.data.s8 = _intel_convert_f16_to_hf8(val.s8); + res.data.s9 = _intel_convert_f16_to_hf8(val.s9); + res.data.sA = _intel_convert_f16_to_hf8(val.sA); + res.data.sB = _intel_convert_f16_to_hf8(val.sB); + res.data.sC = _intel_convert_f16_to_hf8(val.sC); + res.data.sD = _intel_convert_f16_to_hf8(val.sD); + res.data.sE = _intel_convert_f16_to_hf8(val.sE); + res.data.sF = _intel_convert_f16_to_hf8(val.sF); + return res; +} + +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(float val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8_sat((half)val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(float val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8_sat((half)val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(float2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(float3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(float4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); + res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(float8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(float16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); + res.data.s8 = _intel_convert_f16_to_hf8_sat((half)val.s8); + res.data.s9 = _intel_convert_f16_to_hf8_sat((half)val.s9); + res.data.sA = _intel_convert_f16_to_hf8_sat((half)val.sA); + res.data.sB = _intel_convert_f16_to_hf8_sat((half)val.sB); + res.data.sC = _intel_convert_f16_to_hf8_sat((half)val.sC); + res.data.sD = _intel_convert_f16_to_hf8_sat((half)val.sD); + res.data.sE = _intel_convert_f16_to_hf8_sat((half)val.sE); + res.data.sF = _intel_convert_f16_to_hf8_sat((half)val.sF); + return res; +} + +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(half val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8_sat(val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(half val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8_sat(val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(half2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(half3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(half4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); + res.data.s3 = _intel_convert_f16_to_hf8_sat(val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(half8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(half16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); + res.data.s8 = _intel_convert_f16_to_hf8_sat(val.s8); + res.data.s9 = _intel_convert_f16_to_hf8_sat(val.s9); + res.data.sA = _intel_convert_f16_to_hf8_sat(val.sA); + res.data.sB = _intel_convert_f16_to_hf8_sat(val.sB); + res.data.sC = _intel_convert_f16_to_hf8_sat(val.sC); + res.data.sD = _intel_convert_f16_to_hf8_sat(val.sD); + res.data.sE = _intel_convert_f16_to_hf8_sat(val.sE); + res.data.sF = _intel_convert_f16_to_hf8_sat(val.sF); + return res; +} + +fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(float val) { + fp8e8m0_t res; + res.data = _intel_convert_f32_fo_e8m0(val); + return res; +} +fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1(float val[1]) { + fp8e8m0_t1 res; + res.data = _intel_convert_f32_fo_e8m0(val[0]); + return res; +} +fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2(float2 val) { + fp8e8m0_t2 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); + return res; +} +fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3(float3 val) { + fp8e8m0_t3 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); + return res; +} +fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4(float4 val) { + fp8e8m0_t4 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); + res.data.s3 = _intel_convert_f32_fo_e8m0(val.w); + return res; +} +fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8(float8 val) { + fp8e8m0_t8 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); + return res; +} +fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16(float16 val) { + fp8e8m0_t16 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); + res.data.s8 = _intel_convert_f32_fo_e8m0(val.s8); + res.data.s9 = _intel_convert_f32_fo_e8m0(val.s9); + res.data.sA = _intel_convert_f32_fo_e8m0(val.sA); + res.data.sB = _intel_convert_f32_fo_e8m0(val.sB); + res.data.sC = _intel_convert_f32_fo_e8m0(val.sC); + res.data.sD = _intel_convert_f32_fo_e8m0(val.sD); + res.data.sE = _intel_convert_f32_fo_e8m0(val.sE); + res.data.sF = _intel_convert_f32_fo_e8m0(val.sF); + return res; +} + +fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t_sat(float val) { + fp8e8m0_t res; + res.data = _intel_convert_f32_fo_e8m0_sat(val); + return res; +} +fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1_sat(float val[1]) { + fp8e8m0_t1 res; + res.data = _intel_convert_f32_fo_e8m0_sat(val[0]); + return res; +} +fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2_sat(float2 val) { + fp8e8m0_t2 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); + return res; +} +fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3_sat(float3 val) { + fp8e8m0_t3 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); + return res; +} +fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4_sat(float4 val) { + fp8e8m0_t4 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); + res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.w); + return res; +} +fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8_sat(float8 val) { + fp8e8m0_t8 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); + return res; +} +fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16_sat(float16 val) { + fp8e8m0_t16 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); + res.data.s8 = _intel_convert_f32_fo_e8m0_sat(val.s8); + res.data.s9 = _intel_convert_f32_fo_e8m0_sat(val.s9); + res.data.sA = _intel_convert_f32_fo_e8m0_sat(val.sA); + res.data.sB = _intel_convert_f32_fo_e8m0_sat(val.sB); + res.data.sC = _intel_convert_f32_fo_e8m0_sat(val.sC); + res.data.sD = _intel_convert_f32_fo_e8m0_sat(val.sD); + res.data.sE = _intel_convert_f32_fo_e8m0_sat(val.sE); + res.data.sF = _intel_convert_f32_fo_e8m0_sat(val.sF); + return res; +} + +fp8e5m2_t __attribute__((overloadable)) as_fp8e5m2_t(uchar val) { + fp8e5m2_t res; + res.data = val; + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) as_fp8e5m2_t1(uchar val[1]) { + fp8e5m2_t1 res; + res.data = val[0]; + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) as_fp8e5m2_t2(uchar2 val) { + fp8e5m2_t2 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) as_fp8e5m2_t3(uchar3 val) { + fp8e5m2_t3 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) as_fp8e5m2_t4(uchar4 val) { + fp8e5m2_t4 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + res.data.s3 = val.w; + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) as_fp8e5m2_t8(uchar8 val) { + fp8e5m2_t8 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) as_fp8e5m2_t16(uchar16 val) { + fp8e5m2_t16 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + res.data.s8 = val.s8; + res.data.s9 = val.s9; + res.data.sA = val.sA; + res.data.sB = val.sB; + res.data.sC = val.sC; + res.data.sD = val.sD; + res.data.sE = val.sE; + res.data.sF = val.sF; + return res; +} + +fp8e4m3_t __attribute__((overloadable)) as_fp8e4m3_t(uchar val) { + fp8e4m3_t res; + res.data = as_char(val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) as_fp8e4m3_t1(uchar val[1]) { + fp8e4m3_t1 res; + res.data = as_char(val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) as_fp8e4m3_t2(uchar2 val) { + fp8e4m3_t2 res; + res.data.s0 = as_char(val.x); + res.data.s1 = as_char(val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) as_fp8e4m3_t3(uchar3 val) { + fp8e4m3_t3 res; + res.data.s0 = as_char(val.x); + res.data.s1 = as_char(val.y); + res.data.s2 = as_char(val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) as_fp8e4m3_t4(uchar4 val) { + fp8e4m3_t4 res; + res.data.s0 = as_char(val.x); + res.data.s1 = as_char(val.y); + res.data.s2 = as_char(val.z); + res.data.s3 = as_char(val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) as_fp8e4m3_t8(uchar8 val) { + fp8e4m3_t8 res; + res.data.s0 = as_char(val.s0); + res.data.s1 = as_char(val.s1); + res.data.s2 = as_char(val.s2); + res.data.s3 = as_char(val.s3); + res.data.s4 = as_char(val.s4); + res.data.s5 = as_char(val.s5); + res.data.s6 = as_char(val.s6); + res.data.s7 = as_char(val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) as_fp8e4m3_t16(uchar16 val) { + fp8e4m3_t16 res; + res.data.s0 = as_char(val.s0); + res.data.s1 = as_char(val.s1); + res.data.s2 = as_char(val.s2); + res.data.s3 = as_char(val.s3); + res.data.s4 = as_char(val.s4); + res.data.s5 = as_char(val.s5); + res.data.s6 = as_char(val.s6); + res.data.s7 = as_char(val.s7); + res.data.s8 = as_char(val.s8); + res.data.s9 = as_char(val.s9); + res.data.sA = as_char(val.sA); + res.data.sB = as_char(val.sB); + res.data.sC = as_char(val.sC); + res.data.sD = as_char(val.sD); + res.data.sE = as_char(val.sE); + res.data.sF = as_char(val.sF); + return res; +} + +fp8e8m0_t __attribute__((overloadable)) as_fp8e8m0_t(uchar val) { + fp8e8m0_t res; + res.data = val; + return res; +} +fp8e8m0_t1 __attribute__((overloadable)) as_fp8e8m0_t1(uchar val[1]) { + fp8e8m0_t1 res; + res.data = val[0]; + return res; +} +fp8e8m0_t2 __attribute__((overloadable)) as_fp8e8m0_t2(uchar2 val) { + fp8e8m0_t2 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + return res; +} +fp8e8m0_t3 __attribute__((overloadable)) as_fp8e8m0_t3(uchar3 val) { + fp8e8m0_t3 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + return res; +} +fp8e8m0_t4 __attribute__((overloadable)) as_fp8e8m0_t4(uchar4 val) { + fp8e8m0_t4 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + res.data.s3 = val.w; + return res; +} +fp8e8m0_t8 __attribute__((overloadable)) as_fp8e8m0_t8(uchar8 val) { + fp8e8m0_t8 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + return res; +} +fp8e8m0_t16 __attribute__((overloadable)) as_fp8e8m0_t16(uchar16 val) { + fp8e8m0_t16 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + res.data.s8 = val.s8; + res.data.s9 = val.s9; + res.data.sA = val.sA; + res.data.sB = val.sB; + res.data.sC = val.sC; + res.data.sD = val.sD; + res.data.sE = val.sE; + res.data.sF = val.sF; + return res; +} + +uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t val) { + return val.data; +} +uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t1 val) { + return val.data; +} +uchar2 __attribute__((overloadable)) _as_uchar2(fp8e5m2_t2 val) { + return (uchar2)(val.data.s0, val.data.s1); +} +uchar3 __attribute__((overloadable)) _as_uchar3(fp8e5m2_t3 val) { + return (uchar3)(val.data.s0, val.data.s1, val.data.s2); +} +uchar4 __attribute__((overloadable)) _as_uchar4(fp8e5m2_t4 val) { + return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); +} +uchar8 __attribute__((overloadable)) _as_uchar8(fp8e5m2_t8 val) { + return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7); +} +uchar16 __attribute__((overloadable)) _as_uchar16(fp8e5m2_t16 val) { + return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7, + val.data.s8, val.data.s9, val.data.sA, val.data.sB, + val.data.sC, val.data.sD, val.data.sE, val.data.sF); +} + +uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t val) { + return as_uchar((char)val.data); +} +uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t1 val) { + return as_uchar((char)val.data); +} +uchar2 __attribute__((overloadable)) _as_uchar2(fp8e4m3_t2 val) { + return (uchar2)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1)); +} +uchar3 __attribute__((overloadable)) _as_uchar3(fp8e4m3_t3 val) { + return (uchar3)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), as_uchar((char)val.data.s2)); +} +uchar4 __attribute__((overloadable)) _as_uchar4(fp8e4m3_t4 val) { + return (uchar4)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), + as_uchar((char)val.data.s2), as_uchar((char)val.data.s3)); +} +uchar8 __attribute__((overloadable)) _as_uchar8(fp8e4m3_t8 val) { + return (uchar8)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), + as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), + as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), + as_uchar((char)val.data.s6), as_uchar((char)val.data.s7)); +} +uchar16 __attribute__((overloadable)) _as_uchar16(fp8e4m3_t16 val) { + return (uchar16)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), + as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), + as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), + as_uchar((char)val.data.s6), as_uchar((char)val.data.s7), + as_uchar((char)val.data.s8), as_uchar((char)val.data.s9), + as_uchar((char)val.data.sA), as_uchar((char)val.data.sB), + as_uchar((char)val.data.sC), as_uchar((char)val.data.sD), + as_uchar((char)val.data.sE), as_uchar((char)val.data.sF)); +} + +uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t val) { + return val.data; +} +uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t1 val) { + return val.data; +} +uchar2 __attribute__((overloadable)) _as_uchar2(fp8e8m0_t2 val) { + return (uchar2)(val.data.s0, val.data.s1); +} +uchar3 __attribute__((overloadable)) _as_uchar3(fp8e8m0_t3 val) { + return (uchar3)(val.data.s0, val.data.s1, val.data.s2); +} +uchar4 __attribute__((overloadable)) _as_uchar4(fp8e8m0_t4 val) { + return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); +} +uchar8 __attribute__((overloadable)) _as_uchar8(fp8e8m0_t8 val) { + return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7); +} +uchar16 __attribute__((overloadable)) _as_uchar16(fp8e8m0_t16 val) { + return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7, + val.data.s8, val.data.s9, val.data.sA, val.data.sB, + val.data.sC, val.data.sD, val.data.sE, val.data.sF); +} diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl index 0dc7769a7fa2..f89ebe1eb02e 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl @@ -48,16 +48,16 @@ #define BLOCK_WRITEN_FUNC_size8(vector_size) BLOCK_WRITEN_FUNC_SIZE_DEF(8, vector_size) #define BLOCK_WRITEN_FUNC(type_size, vector_size) CAT(BLOCK_WRITEN_FUNC_size, type_size)(vector_size) -#define BLOCK_WRITEN_RAW(type_size, vector_size, addr_space, ptr, offset, val) \ +#define BLOCK_WRITEN_RAW(type_size, vector_size, addr_space, ptr, offset, val, src_type) \ BLOCK_WRITEN_FUNC(type_size, vector_size)( \ (addr_space BLOCK_WRITE_TYPE(type_size)*)(ptr) + (offset), \ - AS_TYPE(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val)) + AS_TYPE_EXT(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val, src_type)) #define BLOCK_WRITEN(type, vector_size, ptr, offset, val) \ - BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __global, ptr, offset, val) + BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __global, ptr, offset, val, type) #define BLOCK_WRITEN_SLM(type, vector_size, ptr, offset, val) \ - BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __local, ptr, offset, val) + BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __local, ptr, offset, val, type) #define DT_OUTPUT_BLOCK_WRITE(ptr, offset, val) BLOCK_WRITEN(OUTPUT_TYPE, 1, ptr, offset, val) #define DT_OUTPUT_BLOCK_WRITE2(ptr, offset, val) BLOCK_WRITEN(OUTPUT_TYPE, 2, ptr, offset, val) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl index 6dd09d08dc62..a2b6ee26419c 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl @@ -2,6 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // +#define IS_F8 (F8E5M2_INPUT || F8E4M3_INPUT || F8E5M2_OUTPUT || F8E4M3_OUTPUT) + +#if IS_F8 +#include "include/batch_headers/common.cl" +#include "include/batch_headers/f8_utils.cl" +#endif + #include "include/reshape_dims.cl" #include "include/fetch_utils.cl" diff --git a/src/plugins/intel_gpu/src/kernel_selector/common_types.h b/src/plugins/intel_gpu/src/kernel_selector/common_types.h index b7dbdf3b09cf..1bbd8e2fc25b 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/common_types.h +++ b/src/plugins/intel_gpu/src/kernel_selector/common_types.h @@ -131,6 +131,9 @@ enum class Datatype { F16, F32, BF16, + F8E4M3, + F8E5M2, + F8E8M0, }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -145,7 +148,10 @@ enum class WeightsType { UINT4, INT4, INT32, - BF16 + BF16, + F8E4M3, + F8E5M2, + F8E8M0, }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp index 9344d8057175..8cc42ec1ecb9 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp @@ -1505,8 +1505,8 @@ JitConstants MakeTypeJitConstants(Datatype dataType, const std::string& macroNam min_val = "-" + macroName + "_VAL_MAX"; val_one = "1.0h"; val_zero = "0.0h"; - to_type = "convert_half(v)"; - to_type_sat = "convert_half(v)"; + to_type = "_convert_half(v)"; + to_type_sat = "_convert_half(v)"; as_type = "as_half(v)"; max_func = "fmax"; min_func = "fmin"; @@ -1537,14 +1537,50 @@ JitConstants MakeTypeJitConstants(Datatype dataType, const std::string& macroNam type_size = "2"; is_fp = false; break; + case Datatype::F8E4M3: + type = "fp8e4m3_t"; + max_val = "(fp8e4m3_t){as_char((char)0x7E)}"; // 448.0 + min_val = "(fp8e4m3_t){as_char((char)0xFE)}"; // -448.0 + val_one = "(fp8e4m3_t){as_char((char)0x38)}"; + val_zero = "(fp8e4m3_t){as_char((char)0x0)}"; + to_type = "_convert_fp8e4m3_t(v)"; + to_type_sat = "_convert_fp8e4m3_t_sat(v)"; + as_type = "as_fp8e4m3_t(v)"; + type_size = "1"; + is_fp = true; + break; + case Datatype::F8E5M2: + type = "fp8e5m2_t"; + max_val = "(fp8e5m2_t){as_uchar((uchar)0x7B)}"; // 57344.0 + min_val = "(fp8e5m2_t){as_uchar((uchar)0xFB)}"; // -57344.0 + val_one = "(fp8e5m2_t){as_uchar((uchar)0x3C)}"; + val_zero = "(fp8e5m2_t){as_uchar((uchar)0x0)}"; + to_type = "_convert_fp8e5m2_t(v)"; + to_type_sat = "_convert_fp8e5m2_t_sat(v)"; + as_type = "as_fp8e5m2_t(v)"; + type_size = "1"; + is_fp = true; + break; + case Datatype::F8E8M0: + type = "fp8e8m0_t"; + max_val = "(fp8e8m0_t){as_uchar((uchar)0xFE)}"; // 2^127 + min_val = "(fp8e8m0_t){as_uchar((uchar)0x00)}"; // 2^(-127) + val_one = "(fp8e8m0_t){as_uchar((uchar)0x7F)}"; + val_zero = ""; // There is no representation of zero in FP8E8M0 + to_type = "_convert_fp8e8m0_t(v)"; + to_type_sat = "_convert_fp8e8m0_t_sat(v)"; + as_type = "as_fp8e8m0_t(v)"; + type_size = "1"; + is_fp = true; + break; default: type = "float"; max_val = "FLT_MAX"; min_val = "-" + macroName + "_VAL_MAX"; val_one = "1.0f"; val_zero = "0.0f"; - to_type = "convert_float(v)"; - to_type_sat = "convert_float(v)"; + to_type = "_convert_float(v)"; + to_type_sat = "_convert_float(v)"; as_type = "as_float(v)"; max_func = "fmax"; min_func = "fmin"; @@ -1590,6 +1626,12 @@ JitConstants MakeTypeJitConstants(WeightsType weightsType, const std::string& ma return MakeTypeJitConstants(Datatype::INT32, macroName); case WeightsType::BF16: return MakeTypeJitConstants(Datatype::BF16, macroName); + case WeightsType::F8E4M3: + return MakeTypeJitConstants(Datatype::F8E4M3, macroName); + case WeightsType::F8E5M2: + return MakeTypeJitConstants(Datatype::F8E5M2, macroName); + case WeightsType::F8E8M0: + return MakeTypeJitConstants(Datatype::F8E8M0, macroName); } assert(false || "Unreachable!"); // FIXME: Is there some builtin_unreachable available? diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp index d43b75e81c75..23fe16b5db08 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp @@ -156,6 +156,9 @@ std::string toString(Datatype dType) { case Datatype::INT64: return "INT64"; case Datatype::F16: return "F16"; case Datatype::F32: return "F32"; + case Datatype::F8E4M3: return "F8E4M3"; + case Datatype::F8E5M2: return "F8E5M2"; + case Datatype::F8E8M0: return "F8E8M0"; default: return ""; } } diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.cpp index ae255a0a9b83..af31dcd0539e 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.cpp @@ -83,6 +83,15 @@ void ParamsKey::EnableInputDataType(Datatype dt) { case Datatype::BF16: key.inputType.val.BF16 = 1; break; + case Datatype::F8E4M3: + key.inputType.val.F8E4M3 = 1; + break; + case Datatype::F8E5M2: + key.inputType.val.F8E5M2 = 1; + break; + case Datatype::F8E8M0: + key.inputType.val.F8E8M0 = 1; + break; default: break; } @@ -128,6 +137,15 @@ void ParamsKey::EnableOutputDataType(Datatype dt) { case Datatype::BF16: key.outputType.val.BF16 = 1; break; + case Datatype::F8E4M3: + key.outputType.val.F8E4M3 = 1; + break; + case Datatype::F8E5M2: + key.outputType.val.F8E5M2 = 1; + break; + case Datatype::F8E8M0: + key.outputType.val.F8E8M0 = 1; + break; default: break; } diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h index ef37c3759ed1..1d9839520f12 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h @@ -256,6 +256,9 @@ class ParamsKey { uint32_t F16 : 1; uint32_t F32 : 1; uint32_t BF16 : 1; + uint32_t F8E4M3 : 1; + uint32_t F8E5M2 : 1; + uint32_t F8E8M0 : 1; } val; uint32_t raw; } DataTypesKey; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp index 2460ea8db757..1ae72a64e623 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp @@ -63,6 +63,9 @@ ParamsKey DynamicQuantizeKernelOpt::GetSupportedKey() const { k.EnableInputDataType(Datatype::F16); k.EnableOutputDataType(Datatype::UINT8); k.EnableOutputDataType(Datatype::INT8); + k.EnableOutputDataType(Datatype::F8E4M3); + k.EnableOutputDataType(Datatype::F8E5M2); + k.EnableOutputDataType(Datatype::F8E8M0); k.EnableDifferentTypes(); k.EnableInputLayout(DataLayout::bf); k.EnableInputLayout(DataLayout::bfyx); @@ -92,6 +95,9 @@ JitConstants DynamicQuantizeKernelOpt::GetJitConstants(const dynamic_quantize_pa jit.AddConstant(MakeJitConstant("MODE_SMALL_GS", static_cast(DynQuanMode::SMALL_GS))); jit.AddConstant(MakeJitConstant("MODE_LARGE_GS", static_cast(DynQuanMode::LARGE_GS))); jit.AddConstant(MakeJitConstant("MODE_PER_TOKEN", static_cast(DynQuanMode::PER_TOKEN))); + jit.AddConstant(MakeJitConstant("F8E5M2_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E5M2 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E4M3_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E4M3 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("IS_MXFP", params.outputs[1].GetDType() == Datatype::F8E8M0 ? 1 : 0)); jit.Merge(GetTensorFriendlyWorkGroupsJit(params.outputs[0])); jit.AddConstant(MakeJitConstant("TOTAL_BLOCK_NUM", total_block_num)); size_t block_num = (total_block_num > 32) ? 32 : total_block_num; @@ -200,6 +206,10 @@ bool DynamicQuantizeKernelOpt::Validate(const Params& params) const { if (get_dynamic_quantize_mode(dq_params) == DynQuanMode::PER_TOKEN && dq_params.generate_precomputed_reduction) DO_NOT_USE_THIS_KERNEL(params.layerID); + if (dq_params.generate_precomputed_reduction && cldnn::one_of(dq_params.outputs[0].GetDType(), {Datatype::F8E4M3, Datatype::F8E5M2})) { + DO_NOT_USE_THIS_KERNEL(params.layerID); + } + auto bf = get_input_bf_size(dq_params); if (((bf.second) % (simd * 2)) != 0) DO_NOT_USE_THIS_KERNEL(params.layerID); diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp index 18567c3b55b8..4b36c2eded23 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp @@ -12,6 +12,9 @@ ParamsKey DynamicQuantizeKernelRef::GetSupportedKey() const { k.EnableInputDataType(Datatype::F16); k.EnableOutputDataType(Datatype::INT8); k.EnableOutputDataType(Datatype::UINT8); + k.EnableOutputDataType(Datatype::F8E4M3); + k.EnableOutputDataType(Datatype::F8E5M2); + k.EnableOutputDataType(Datatype::F8E8M0); k.EnableInputLayout(DataLayout::bfyx); k.EnableOutputLayout(DataLayout::bfyx); k.EnableTensorOffset(); @@ -56,6 +59,9 @@ JitConstants DynamicQuantizeKernelRef::GetJitConstants(const dynamic_quantize_pa jit.AddConstant(MakeJitConstant("GENERATE_PRECOMPUTED_REDUCTION", params.generate_precomputed_reduction)); jit.AddConstant(MakeJitConstant("GROUP_SCALES_WITH_ZP", params.combine_scales_and_zp)); jit.AddConstant(MakeJitConstant("UNSIGNED_OUTPUT", params.outputs[0].GetDType() == Datatype::UINT8 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E5M2_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E5M2 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E4M3_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E4M3 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("IS_MXFP", params.outputs[1].GetDType() == Datatype::F8E8M0 ? 1 : 0)); // Use FP32 accumulator type for scale/zp calculation jit.Merge(MakeTypeJitConstants(Datatype::F32, "ACCUMULATOR")); @@ -156,6 +162,11 @@ bool DynamicQuantizeKernelRef::Validate(const Params& params) const { if (!KernelBaseOpenCL::Validate(params)) DO_NOT_USE_THIS_KERNEL(params.layerID); + const auto& dq_params = static_cast(params); + if (dq_params.generate_precomputed_reduction && cldnn::one_of(dq_params.outputs[0].GetDType(), {Datatype::F8E4M3, Datatype::F8E5M2})) { + DO_NOT_USE_THIS_KERNEL(params.layerID); + } + return true; } } // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp index 50ef5198d87b..39476adbbd8d 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp @@ -20,6 +20,9 @@ ParamsKey ReorderKernelRef::GetSupportedKey() const { k.EnableInputDataType(Datatype::INT64); k.EnableInputDataType(Datatype::F16); k.EnableInputDataType(Datatype::F32); + k.EnableInputDataType(Datatype::F8E4M3); + k.EnableInputDataType(Datatype::F8E5M2); + k.EnableInputDataType(Datatype::F8E8M0); k.EnableOutputDataType(Datatype::F16); k.EnableOutputDataType(Datatype::F32); k.EnableOutputDataType(Datatype::INT8); @@ -32,6 +35,9 @@ ParamsKey ReorderKernelRef::GetSupportedKey() const { k.EnableOutputDataType(Datatype::UINT4); k.EnableOutputDataType(Datatype::INT4); k.EnableOutputDataType(Datatype::BF16); + k.EnableOutputDataType(Datatype::F8E4M3); + k.EnableOutputDataType(Datatype::F8E5M2); + k.EnableOutputDataType(Datatype::F8E8M0); k.EnableSurfaceInputSupport(); k.EnableDifferentTypes(); k.EnableAllInputLayout(); @@ -84,6 +90,11 @@ JitConstants ReorderKernelRef::GetJitConstants(const reorder_params& params) con jit.AddConstant(MakeJitConstant("INT4_OUTPUT", true)); } + jit.AddConstant(MakeJitConstant("F8E5M2_INPUT", params.inputs[0].GetDType() == Datatype::F8E5M2 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E4M3_INPUT", params.inputs[0].GetDType() == Datatype::F8E4M3 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E5M2_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E5M2 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E4M3_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E4M3 ? 1 : 0)); + return jit; } diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel_base.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel_base.cpp index c76155f5ae8a..136994090dc5 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel_base.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel_base.cpp @@ -119,9 +119,12 @@ JitConstants ReorderKernelBase::GetJitConstants(const reorder_params& params) co params.mode == MeanSubtractMode::NONE && params.activations.empty()); Datatype calc_type = useUshort ? Datatype::UINT16 : params.inputs[0].GetDType(); - if ( params.inputs[0].GetDType() == Datatype::BF16 ) { + if (params.inputs[0].GetDType() == Datatype::BF16 || params.inputs[0].GetDType() == Datatype::F8E8M0) { calc_type = Datatype::F32; } + if (params.inputs[0].GetDType() == Datatype::F8E4M3 || params.inputs[0].GetDType() == Datatype::F8E5M2) { + calc_type = Datatype::F16; + } Datatype output_reorder_type = useUshort ? Datatype::UINT16 : params.outputs[0].GetDType(); Datatype input_reorder_type = useUshort ? Datatype::UINT16 : params.inputs[0].GetDType(); diff --git a/src/plugins/intel_gpu/src/plugin/common_utils.cpp b/src/plugins/intel_gpu/src/plugin/common_utils.cpp index eefbbb6aa1b5..854c2fb08a5e 100644 --- a/src/plugins/intel_gpu/src/plugin/common_utils.cpp +++ b/src/plugins/intel_gpu/src/plugin/common_utils.cpp @@ -201,8 +201,8 @@ bool is_supported(ov::element::Type_t et) { case ov::element::Type_t::u32: return true; // converted to i32 case ov::element::Type_t::u64: return true; // converted to i32 case ov::element::Type_t::nf4: return false; - case ov::element::Type_t::f8e4m3: return false; - case ov::element::Type_t::f8e5m2: return false; + case ov::element::Type_t::f8e4m3: return true; + case ov::element::Type_t::f8e5m2: return true; case ov::element::Type_t::string: return false; default: return false; } diff --git a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp index 0a1c1200ea8c..ed37b0b64606 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/compressed_weights_pattern.hpp @@ -4,11 +4,14 @@ #pragma once +#include "openvino/pass/pattern/op/optional.hpp" + using namespace ov::pass::pattern; #define FC_COMPRESSED_WEIGHT_PATTERN\ auto compressed_constant = [](const ov::Output& output) {\ return (output.get_element_type() == ov::element::u8 || output.get_element_type() == ov::element::i8 ||\ - output.get_element_type() == ov::element::u4 || output.get_element_type() == ov::element::i4);\ + output.get_element_type() == ov::element::u4 || output.get_element_type() == ov::element::i4 ||\ + output.get_element_type() == ov::element::f8e4m3 || output.get_element_type() == ov::element::f8e5m2);\ };\ \ auto reshape_squeeze = [](const ov::Output& output) {\ @@ -36,7 +39,8 @@ using namespace ov::pass::pattern; \ auto mul_const_m = wrap_type();\ auto mul_with_sub_m = wrap_type({subtract_m, mul_const_m});\ - auto mul_no_sub_m = wrap_type({decompressed_weights_m, mul_const_m});\ + auto mul_const_convert_m = ov::pass::pattern::optional(mul_const_m);\ + auto mul_no_sub_m = wrap_type({decompressed_weights_m, mul_const_convert_m});\ auto mul_m = std::make_shared(OutputVector{mul_with_sub_m, mul_no_sub_m});\ \ auto reshape_const_m = wrap_type();\ diff --git a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp index c319e3a1bdc8..230783c8ccc7 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp @@ -13,6 +13,7 @@ #include "openvino/pass/pattern/op/wrap_type.hpp" #include "transformations/utils/utils.hpp" #include "intel_gpu/runtime/debug_configuration.hpp" +#include "intel_gpu/runtime/utils.hpp" #include "openvino/core/graph_util.hpp" namespace ov::intel_gpu { @@ -46,7 +47,6 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size const bool has_wzp = m_fc->get_input_size() > 4; auto optional_w_zp = has_wzp ? m_fc->get_input_node_shared_ptr(4) : std::make_shared(); - auto rank = m_fc->get_input_partial_shape(0).size(); ov::op::internal::DynamicQuantize::Attributes config; const bool has_static_wzp = m_fc->get_input_size() > 4 && optional_w_zp->get_output_partial_shape(0).rank().is_static(); const bool is_wei_i8_u8 = cldnn::one_of(m_fc->get_input_element_type(1), {ov::element::i8, ov::element::u8}); @@ -84,6 +84,7 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size return false; } + const auto rank = m_fc->get_input_partial_shape(0).size(); if (adj_group_size < 32) { GPU_DEBUG_LOG << "Dynamic quantization: quantized activation by group size " << adj_group_size << " is not supported by onednn matmul if it is less than 32" << std::endl; @@ -93,9 +94,23 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size std::vector shape_group_size(rank, 1); shape_group_size.back() = adj_group_size; - config.quantization_dt = element::i8; + auto weight_dtype = m_fc->get_input_element_type(1); + auto scale_dtype = m_fc->get_input_element_type(3); + if (weight_dtype.is_integral()) { + config.quantization_dt = element::i8; + } else if (weight_dtype == element::f8e4m3) { + config.quantization_dt = element::f8e4m3; + } else if (weight_dtype == element::f8e5m2) { + config.quantization_dt = element::f8e5m2; + } else if (weight_dtype == element::f4e2m1) { + config.quantization_dt = element::f4e2m1; + } else { + OPENVINO_THROW("Unexpected weight data type: " + weight_dtype.to_string()); + } + const bool is_mxfp = scale_dtype == element::f8e8m0; + config.quantization_type = QuantizationType::Symmetric; - config.scale_dt = element::f16; + config.scale_dt = is_mxfp ? element::f8e8m0 : element::f16; config.group_sizes = shape_group_size; if (asymmetric && adj_group_size == UINT64_MAX) { @@ -104,7 +119,26 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size config.zp_dt = element::u8; // it supports u8 only now } - auto dyn_quan = std::make_shared(m_fc->input_value(0), config); + std::shared_ptr dyn_quan = nullptr; + + // If the parent already has an identical dynamic quantize as a user, reuse it instead of creating a new one. + const auto siblings = m_fc->get_input_node_shared_ptr(0)->get_users(); + for (const auto& sibling : siblings) { + if (auto dyn_quan_sibling = as_type_ptr(sibling)) { + if (dyn_quan_sibling->is_config_equal(config)) { + dyn_quan = dyn_quan_sibling; + } + } + + if (dyn_quan) { + break; + } + } + + if (!dyn_quan) { + dyn_quan = std::make_shared(m_fc->input_value(0), config); + } + int dyn_quan_output_idx = 2; auto optional_a_zp = config.quantization_type == QuantizationType::Symmetric ? std::make_shared() : dyn_quan->output(dyn_quan_output_idx++); diff --git a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.hpp b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.hpp index a364c664548b..f51769ab7f61 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.hpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.hpp @@ -5,6 +5,7 @@ #pragma once #include "openvino/pass/graph_rewrite.hpp" +#include "openvino/runtime/properties.hpp" namespace ov::intel_gpu { diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 58eec00a30f1..bdde9866ae9c 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -294,9 +294,25 @@ static bool is_decompression_multiply(const std::shared_ptr node return true; }; - if (all_has_types(consumers, { ov::opset1::Reshape::get_type_info_static() })) { + if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()}) || all_has_types(consumers, {ov::op::v1::Transpose::get_type_info_static()})) { for (const auto& consumer : consumers) { - const auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + auto get_child_consumers = [&]() { + auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + + // Reshape + Transpose chain + if (all_has_types(child_consumers, { ov::op::v1::Transpose::get_type_info_static() })) { + std::set> next_child_consumers; + for (const auto& child_consumer : child_consumers) { + const auto grand_child_consumers = child_consumer.get_node()->get_output_target_inputs(0); + next_child_consumers.insert(grand_child_consumers.begin(), grand_child_consumers.end()); + } + return next_child_consumers; + } + return child_consumers; + }; + + auto child_consumers = get_child_consumers(); + for (const auto& child_consumer : child_consumers) { const auto& type_info = child_consumer.get_node()->get_type_info(); if (cldnn::one_of(type_info, target_consumers)) { @@ -391,7 +407,8 @@ void TransformationsPipeline::apply(std::shared_ptr func) { using const_node_ptr = const std::shared_ptr; const auto& defaultPrecisions = ov::pass::low_precision::precision_set::get_int8_support(); - const ov::element::TypeVector supported_woq_types = {ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}; + const ov::element::TypeVector supported_woq_types = + {ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4, ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f8e8m0}; bool enableInt8; bool unroll_loop = config.get_enable_loop_unrolling(); const bool disable_gated_mlp_fusion = GPU_DEBUG_VALUE_OR(config.get_disable_gated_mlp_fusion(), true); @@ -441,7 +458,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { // Transformation of SDPA to VLSDPA for QWen2.x-VL, // Note: this should be applied before TransposeFusion. manager.register_pass(); - // Disable SDPAToVLSDPA if XMX architectures is unavaiable or IGC incompatiable. + // Disable SDPAToVLSDPA if XMX architectures is unavaiable or IGC incompatible. pass_config->set_callback( [&](const_node_ptr &) -> bool { auto& engine = m_context->get_engine(); @@ -474,9 +491,24 @@ void TransformationsPipeline::apply(std::shared_ptr func) { } manager.register_pass(); - manager.register_pass( - std::vector{ ov::element::i8, ov::element::u8, ov::element::i4, ov::element::u4 }, - !device_info.supports_immad); + const bool does_model_contain_mxfp_patterns = ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(func); + + manager.register_pass(std::vector{ov::element::i8, + ov::element::u8, + ov::element::i4, + ov::element::u4, + ov::element::f8e4m3, + ov::element::f8e5m2, + ov::element::f4e2m1, + ov::element::f8e8m0}, + !device_info.supports_immad); + if (does_model_contain_mxfp_patterns) { + manager.register_pass( + std::vector{ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1, ov::element::f8e8m0}, + !device_info.supports_immad, + !device_info.supports_immad); + } + const bool disable_moe_opt = GPU_DEBUG_VALUE_OR(config.get_disable_moe_opt(), false); if (!disable_moe_opt) { manager.register_pass(); diff --git a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp index e31bddb35a94..13e155eac6ed 100644 --- a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp +++ b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -247,6 +247,8 @@ const std::vector& disabled_test_patterns() { patterns.push_back(std::regex(R"(.*smoke_MatMulCompressedWeights_3D_weight.*)")); // MoE patterns are not supported on the platforms without immad (fusion transformation are disabled) patterns.push_back(std::regex(R"(.*smoke_MoE3GemmCompressedFusion.*)")); + // Dynamic quantization of fp8 and mxfp8 models supported only through oneDNN (disabled for non-systolic platforms) + patterns.push_back(std::regex(R"(.*smoke_MatMulCompressedWeights_dyn_quan_.*fp8.*)")); } return patterns; diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp index 948fb8a5595b..cfebec1796de 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp @@ -2,9 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // +#include + #include "common_test_utils/ov_tensor_utils.hpp" +#include "intel_gpu/runtime/utils.hpp" #include "shared_test_classes/base/ov_subgraph.hpp" #include "transformations/rt_info/decompression.hpp" +#include "intel_gpu/runtime/internal_properties.hpp" #include "openvino/op/parameter.hpp" #include "openvino/op/constant.hpp" @@ -61,7 +65,8 @@ using MatmulWeightsDecompressionParams = std::tuple; class MatmulWeightsDecompression : public testing::WithParamInterface, @@ -78,7 +83,8 @@ class MatmulWeightsDecompression : public testing::WithParamInterface(weights_precision, transformed_weights_shape); params.push_back(ov::as_type_ptr(weights)); } else { - auto weights_tensor = ov::test::utils::create_and_fill_tensor(weights_precision, transformed_weights_shape); + ov::test::utils::InputGenerateData wei_data; + wei_data.start_from = -0.05; + wei_data.range = 0.1; + wei_data.resolution = 30000; + auto weights_tensor = ov::test::utils::create_and_fill_tensor(weights_precision, transformed_weights_shape, wei_data); weights = std::make_shared(weights_tensor); } weights->set_friendly_name("Compressed_weights"); @@ -239,18 +253,26 @@ class MatmulWeightsDecompression : public testing::WithParamInterface(weights_convert, shift_convert); } + const ov::element::Type scale_data_precision = is_mxfp ? ov::element::f8e8m0 : data_precision; + ov::test::utils::InputGenerateData in_data; - in_data.start_from = -0.5; + in_data.start_from = is_mxfp ? 0 : -0.5; in_data.range = 1; in_data.resolution = 30000; - auto scale_tensor = ov::test::utils::create_and_fill_tensor(data_precision, scaleshift_const_shape, in_data); + auto scale_tensor = ov::test::utils::create_and_fill_tensor(scale_data_precision, scaleshift_const_shape, in_data); for (size_t i = 0; i < scale_tensor.get_size(); i++) { - if (data_precision == ov::element::f16) + if (scale_data_precision == ov::element::f16) scale_tensor.data()[i] /= ov::float16(16.f); - else if (data_precision == ov::element::f32) + else if (scale_data_precision == ov::element::f32) scale_tensor.data()[i] /= 16.f; } + std::shared_ptr scale_const = std::make_shared(scale_tensor); + if (scale_data_precision != data_precision) { + auto scale_convert = std::make_shared(scale_const, data_precision); + scale_const = scale_convert; + } + if (reshape_on_decompression_constant) { auto scale_reshape_const = ov::op::v0::Constant::create(ov::element::i32, {scaleshift_target_shape.size()}, scaleshift_target_shape); auto scale_reshape = std::make_shared(scale_const, scale_reshape_const, false); @@ -294,7 +316,8 @@ class MatmulWeightsDecompression : public testing::WithParamInterface input_shapes_corner_cases_basic = { @@ -470,7 +498,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_corner_cases_basic, ::testing::ValuesIn(per_tensor_zp), ::testing::ValuesIn(param_weights), ::testing::Values(0), - ::testing::Values(1.0f)), + ::testing::Values(1.0f), + ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(MatMulCompressedWeights_corner_cases_big, @@ -485,7 +514,8 @@ INSTANTIATE_TEST_SUITE_P(MatMulCompressedWeights_corner_cases_big, ::testing::ValuesIn(per_tensor_zp), ::testing::ValuesIn(param_weights), ::testing::Values(0), - ::testing::Values(1.0f)), + ::testing::Values(1.0f), + ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); @@ -506,7 +536,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan, ::testing::Values(true), // per_tensor_zp ::testing::Values(false), ::testing::ValuesIn(group_size), - ::testing::Values(2.0f)), // Note: this is because of potential cldnn accuracy issue + ::testing::Values(2.0f), // Note: this is because of potential cldnn accuracy issue + ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_reduction, @@ -522,7 +553,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_redu ::testing::Values(false), // per_tensor_zp ::testing::Values(false), ::testing::Values(128), - ::testing::Values(2.0f)), // Note: this is because of potential cldnn accuracy issue + ::testing::Values(2.0f), // Note: this is because of potential cldnn accuracy issue + ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_unaligned, // dyn_quan is turned off because of innermost-shape @@ -539,7 +571,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_unaligned, / ::testing::Values(true), // per_tensor_zp ::testing::Values(false), ::testing::Values(std::numeric_limits::max()), - ::testing::Values(2.0f)), // Note: this is because of potential cldnn accuracy issue + ::testing::Values(2.0f), // Note: this is because of potential cldnn accuracy issue + ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_no_slm, @@ -555,7 +588,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_no_slm, ::testing::Values(true), // per_tensor_zp ::testing::Values(false), ::testing::ValuesIn(group_size), - ::testing::Values(2.0f)), // Note: this is because of potential cldnn accuracy issue + ::testing::Values(2.0f), // Note: this is because of potential cldnn accuracy issue + ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_3D_weight, @@ -571,9 +605,79 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_3D_weight, ::testing::Values(true), ::testing::Values(false), ::testing::Values(0), - ::testing::Values(2.0f)), + ::testing::Values(2.0f), + ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); +INSTANTIATE_TEST_SUITE_P( + smoke_MatMulCompressedWeights_dyn_quan_mxfp8_e4m3, + MatmulWeightsDecompression, + ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 4096}, {{1, 1, 4096}, {8, 1, 4096}}}, {4096, 1024}, 32}), // shape + ::testing::Values(ov::element::f8e4m3), + ::testing::Values(ov::element::f16), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(32), + ::testing::Values(2.0f), + ::testing::Values(true)), + MatmulWeightsDecompression::get_test_case_name); + +INSTANTIATE_TEST_SUITE_P( + smoke_MatMulCompressedWeights_dyn_quan_mxfp8_e5m2, + MatmulWeightsDecompression, + ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 4096}, {{1, 1, 4096}, {8, 1, 4096}}}, {4096, 1024}, 32}), // shape + ::testing::Values(ov::element::f8e5m2), + ::testing::Values(ov::element::f16), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(32), + ::testing::Values(2.0f), + ::testing::Values(true)), + MatmulWeightsDecompression::get_test_case_name); + +INSTANTIATE_TEST_SUITE_P( + smoke_MatMulCompressedWeights_dyn_quan_fp8e4m3_fp8e4m3, + MatmulWeightsDecompression, + ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape + ::testing::Values(ov::element::f8e4m3), + ::testing::Values(ov::element::f16), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(std::numeric_limits::max()), + ::testing::Values(0.2f), + ::testing::Values(false)), + MatmulWeightsDecompression::get_test_case_name); + +INSTANTIATE_TEST_SUITE_P( + smoke_MatMulCompressedWeights_dyn_quan_fp8e5m2_fp8e5m2, + MatmulWeightsDecompression, + ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape + ::testing::Values(ov::element::f8e5m2), + ::testing::Values(ov::element::f16), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(std::numeric_limits::max()), + ::testing::Values(0.2f), + ::testing::Values(false)), + MatmulWeightsDecompression::get_test_case_name); + + INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_scalar_wzp, MatmulWeightsDecompressionScalarWeightZp, ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 1024}, {{1024, 1, 1024}}}, @@ -587,7 +691,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_scalar_wzp, ::testing::Values(true), ::testing::Values(false), ::testing::Values(128), - ::testing::Values(2.0f)), + ::testing::Values(2.0f), + ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_reduction_with_gs16, @@ -603,7 +708,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_redu ::testing::Values(false), ::testing::Values(false), ::testing::Values(128), - ::testing::Values(2.0f)), + ::testing::Values(2.0f), + ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_input_4d, @@ -619,6 +725,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_input_4d, ::testing::Values(true), ::testing::Values(false), ::testing::Values(0), - ::testing::Values(1.0f)), + ::testing::Values(1.0f), + ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); } // namespace diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp index 0d01d99f2c5f..7e26fc14b9b1 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp @@ -37,6 +37,7 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { const QuantizationType quantization_type = QuantizationType::Symmetric, uint64_t group_size = UINT64_MAX, data_types quant_dt = data_types::i8, + data_types scale_dt = data_types::f16, data_types zp_dt = data_types::dynamic, OutputStorageType storage_type = OutputStorageType::Planar, const std::string& impl_name = "", @@ -74,7 +75,7 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { dynamic_quantize::Attributes dq_config; dq_config.quantization_type = quantization_type; dq_config.quantization_dt = quant_dt; - dq_config.scale_dt = data_types::f16; + dq_config.scale_dt = scale_dt; dq_config.zp_dt = zp_dt; dq_config.group_sizes = group_sizes; dq_config.scales_zp_output_order = { 0, 1, 2}; @@ -184,9 +185,14 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { cldnn::mem_lock output_ptr(output_buffers[i], get_test_stream()); cldnn::mem_lock output_ptr_ref(ref_output_buffers[i], get_test_stream()); - for (size_t i = 0; i < output_ptr_ref.size(); ++i) { - const int abs_error_threshold = 2; - ASSERT_NEAR(output_ptr_ref[i], output_ptr[i], abs_error_threshold); + for (size_t j = 0; j < output_ptr_ref.size(); ++j) { + if (ov::element::Type(quant_dt).is_real()) { + auto abs_diff = std::abs(output_ptr_ref[j] - output_ptr[j]); + ASSERT_EQ(abs_diff, 0); + } else { // (u)int8 + const int abs_error_threshold = 2; + ASSERT_NEAR(output_ptr_ref[j], output_ptr[j], abs_error_threshold); + } } } @@ -234,6 +240,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_precompute_g QuantizationType::Symmetric, 32, data_types::i8, + data_types::f16, data_types::i8, OutputStorageType::Planar, "", @@ -247,6 +254,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_precompute_g QuantizationType::Symmetric, 128, data_types::i8, + data_types::f16, data_types::i8, OutputStorageType::Planar, "", @@ -260,6 +268,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_precompute_g QuantizationType::Symmetric, 128, data_types::i8, + data_types::f16, data_types::i8, OutputStorageType::Planar, "", @@ -274,6 +283,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_precompute_g QuantizationType::Symmetric, 128, data_types::i8, + data_types::f16, data_types::i8, OutputStorageType::Planar, "", @@ -284,7 +294,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_precompute_g TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_asym_act) { this->test_dynamic_quantization(false, {-1, 1, 1, 4096}, {1, 1, 1, 4096}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::u8, data_types::u8, OutputStorageType::Planar); + data_types::u8, data_types::f16, data_types::u8, OutputStorageType::Planar); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size_grouped) { @@ -326,6 +336,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache) { QuantizationType::Symmetric, UINT64_MAX, data_types::i8, + data_types::f16, data_types::dynamic, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); @@ -338,6 +349,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched) { QuantizationType::Symmetric, UINT64_MAX, data_types::i8, + data_types::f16, data_types::dynamic, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); @@ -350,6 +362,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_reordered) { QuantizationType::Symmetric, UINT64_MAX, data_types::i8, + data_types::f16, data_types::dynamic, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); @@ -362,6 +375,7 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reorde QuantizationType::Symmetric, UINT64_MAX, data_types::i8, + data_types::f16, data_types::dynamic, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); @@ -369,66 +383,114 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reorde TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_asym_planar) { this->test_dynamic_quantization(false, {-1, 8, -1, 96}, {1, 8, 1, 96}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_asym_planar) { this->test_dynamic_quantization(false, {-1, 4, -1, 64}, {1, 4, 35, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_reordered_asym_planar) { this->test_dynamic_quantization(false, {-1, -1, 8, 96}, {1, 1, 8, 96}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reordered_asym_planar) { this->test_dynamic_quantization(false, {-1, -1, 4, 64}, {1, 35, 4, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_asym_interleaved) { this->test_dynamic_quantization(false, {-1, 8, -1, 96}, {1, 8, 1, 96}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_asym_interleaved) { this->test_dynamic_quantization(false, {-1, 4, -1, 64}, {1, 4, 35, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_reordered_asym_interleaved) { this->test_dynamic_quantization(false, {-1, -1, 8, 96}, {1, 1, 8, 96}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reordered_asym_interleaved) { this->test_dynamic_quantization(false, {-1, -1, 4, 64}, {1, 35, 4, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_asym_planar_i8_zp) { this->test_dynamic_quantization(false, {-1, 8, -1, 32}, {1, 8, 1, 32}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_asym_planar_i8_zp) { this->test_dynamic_quantization(false, {-1, 4, -1, 64}, {1, 4, 35, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_reordered_asym_planar_i8_zp) { this->test_dynamic_quantization(false, {-1, -1, 8, 96}, {1, 1, 8, 96}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reordered_asym_planar_i8_zp) { this->test_dynamic_quantization(false, {-1, -1, 4, 64}, {1, 35, 4, 64}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_kv_cache"); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_inner_most_dim_zero_values_asym) { this->test_dynamic_quantization(false, {-1, 8, -1, 128}, {1, 8, 52, 128}, QuantizationType::Asymmetric, UINT64_MAX, - data_types::i8, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache", SetInnerMostDimValuesZero::Yes); + data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache", SetInnerMostDimValuesZero::Yes); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e4m3) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + 32, + data_types::f8e4m3, + data_types::f8e8m0, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e5m2) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + 32, + data_types::f8e5m2, + data_types::f8e8m0, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e4m3) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + UINT64_MAX, + data_types::f8e4m3, + data_types::f16, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + UINT64_MAX, + data_types::f8e5m2, + data_types::f16, + data_types::dynamic, + OutputStorageType::Planar); } diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp index 26f96ed84b7d..671bc16934d9 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp @@ -662,6 +662,56 @@ TEST_F(TransformationTestsF, ConvertFCToCompressed16_i8ZpNotConvertedToU8) { } } +TEST_F(TransformationTestsF, ConvertFCToCompressed17) { + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 16 }, { 1 }); + auto weights_convert = std::make_shared(weights_const, ov::element::f16); + auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); + auto scale_convert = std::make_shared(scale_const, ov::element::f16); + auto scale = std::make_shared(weights_convert, scale_convert); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input1, scale, no_bias); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input1}); + manager.register_pass(); + } + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 16 }, { 1 }); + auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(input1, weights_const, no_bias, scale_const); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input1}); + } +} + +TEST_F(TransformationTestsF, ConvertFCToCompressed18) { + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 16 }, { 1 }); + auto weights_convert = std::make_shared(weights_const, ov::element::f16); + auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); + auto scale_convert = std::make_shared(scale_const, ov::element::f16); + auto scale = std::make_shared(weights_convert, scale_convert); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input1, scale, no_bias); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input1}); + manager.register_pass(); + } + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 16 }, { 1 }); + auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(input1, weights_const, no_bias, scale_const); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input1}); + } +} + } // namespace intel_gpu } // namespace test } // namespace ov From fb6ff109ce99dd2af905e6360afcc2dc4e8eb0c8 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 13 Apr 2026 11:33:21 +0000 Subject: [PATCH 02/48] Code formatting --- .../include/ov_ops/dynamic_quantize.hpp | 9 +++------ .../common_optimizations/moc_transformations.cpp | 10 +++++----- src/core/src/preprocess/pre_post_process.cpp | 9 +++++---- .../src/graph/impls/ocl_v2/utils/jitter.cpp | 14 +++++++------- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp index 24b89d318498..11f694f1cfce 100644 --- a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp +++ b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp @@ -85,13 +85,10 @@ class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { bool is_config_equal(const Attributes& attr) const { bool is_equal = - m_attrs.quantization_type == attr.quantization_type && - m_attrs.quantization_dt == attr.quantization_dt && - m_attrs.scale_dt == attr.scale_dt && - m_attrs.zp_dt == attr.zp_dt && + m_attrs.quantization_type == attr.quantization_type && m_attrs.quantization_dt == attr.quantization_dt && + m_attrs.scale_dt == attr.scale_dt && m_attrs.zp_dt == attr.zp_dt && m_attrs.precomputed_reduction_dt == attr.precomputed_reduction_dt && - m_attrs.precomputed_reduction == attr.precomputed_reduction && - m_attrs.group_sizes == attr.group_sizes && + m_attrs.precomputed_reduction == attr.precomputed_reduction && m_attrs.group_sizes == attr.group_sizes && m_attrs.output_storage_type == attr.output_storage_type; if (!is_equal) { diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index a27d83ac8180..3f063a63e68f 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -145,15 +145,15 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr if (m_low_precision_enabled) { // Transformation call example, to check with the real model manager.register_pass(TypeVector{f8e4m3}, TypeVector{u4}); - const bool does_model_contain_mxfp_patterns = ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(f); + const bool does_model_contain_mxfp_patterns = + ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(f); manager.register_pass( TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false); if (does_model_contain_mxfp_patterns) { - manager.register_pass( - TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, - false, - false); + manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, + false, + false); } } if (!m_use_shapes) { diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index 7da7b3ea2875..83922424e1ac 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -116,10 +116,11 @@ void transformation_pipeline(std::shared_ptr& model) { const bool does_model_contain_mxfp_patterns = ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(model); - REGISTER_PASS(manager, - MarkDequantization, - TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, - false); + REGISTER_PASS( + manager, + MarkDequantization, + TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, + false); if (does_model_contain_mxfp_patterns) { REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); } diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp index f5ef87141034..9a15e44d6be8 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp @@ -322,8 +322,8 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: break; case ov::element::f8e4m3: type = "fp8e4m3_t"; - max_val = "(fp8e4m3_t){as_char((char)0x7E)}"; // 448.0 - min_val = "(fp8e4m3_t){as_char((char)0xFE)}"; // -448.0 + max_val = "(fp8e4m3_t){as_char((char)0x7E)}"; // 448.0 + min_val = "(fp8e4m3_t){as_char((char)0xFE)}"; // -448.0 val_one = "(fp8e4m3_t){as_char((char)0x38)}"; val_zero = "(fp8e4m3_t){as_char((char)0x0)}"; to_type = "_convert_fp8e4m3_t(v)"; @@ -334,8 +334,8 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: break; case ov::element::f8e5m2: type = "fp8e5m2_t"; - max_val = "(fp8e5m2_t){as_uchar((uchar)0x7B)}"; // 57344.0 - min_val = "(fp8e5m2_t){as_uchar((uchar)0xFB)}"; // -57344.0 + max_val = "(fp8e5m2_t){as_uchar((uchar)0x7B)}"; // 57344.0 + min_val = "(fp8e5m2_t){as_uchar((uchar)0xFB)}"; // -57344.0 val_one = "(fp8e5m2_t){as_uchar((uchar)0x3C)}"; val_zero = "(fp8e5m2_t){as_uchar((uchar)0x0)}"; to_type = "_convert_fp8e5m2_t(v)"; @@ -346,10 +346,10 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: break; case ov::element::f8e8m0: type = "fp8e8m0_t"; - max_val = "(fp8e8m0_t){as_uchar((uchar)0xFE)}"; // 2^127 - min_val = "(fp8e8m0_t){as_uchar((uchar)0x00)}"; // 2^(-127) + max_val = "(fp8e8m0_t){as_uchar((uchar)0xFE)}"; // 2^127 + min_val = "(fp8e8m0_t){as_uchar((uchar)0x00)}"; // 2^(-127) val_one = "(fp8e8m0_t){as_uchar((uchar)0x7F)}"; - val_zero = ""; // There is no representation of zero in FP8E8M0 + val_zero = ""; // There is no representation of zero in FP8E8M0 to_type = "_convert_fp8e8m0_t(v)"; to_type_sat = "_convert_fp8e8m0_t_sat(v)"; as_type = "as_fp8e8m0_t(v)"; From 2556a2b87022d1bbae15ea3c1f5c4d1707d7b23e Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 13 Apr 2026 13:43:11 +0000 Subject: [PATCH 03/48] Remove unnecessary matcher macro --- src/common/low_precision_transformations/src/low_precision.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index d0bed1599341..b5307f8368b2 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -338,7 +338,6 @@ bool ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns( const std::shared_ptr& model) { using namespace ov::op; using namespace ov::pass::pattern; - MATCHER_SCOPE(doesModelContainMXFPPatterns); auto weight_pattern = any_input( type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && From 15807e89af810b7556b1c873557c2911479052b7 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 14 Apr 2026 08:55:19 +0000 Subject: [PATCH 04/48] Apply review suggestions --- .../common_optimizations/moc_transformations.cpp | 4 +--- src/core/src/preprocess/pre_post_process.cpp | 4 +--- .../kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl | 5 ++++- .../kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl | 1 - .../intel_gpu/src/plugin/transformations_pipeline.cpp | 4 +--- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index 3f063a63e68f..c0b7a659fe5e 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -145,12 +145,10 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr if (m_low_precision_enabled) { // Transformation call example, to check with the real model manager.register_pass(TypeVector{f8e4m3}, TypeVector{u4}); - const bool does_model_contain_mxfp_patterns = - ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(f); manager.register_pass( TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false); - if (does_model_contain_mxfp_patterns) { + if (ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(f)) { manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index 83922424e1ac..75bfa0405613 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -114,14 +114,12 @@ void transformation_pipeline(std::shared_ptr& model) { element::TypeVector{element::f8e4m3}, element::TypeVector{element::u4}); - const bool does_model_contain_mxfp_patterns = - ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(model); REGISTER_PASS( manager, MarkDequantization, TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false); - if (does_model_contain_mxfp_patterns) { + if (ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(model)) { REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); } REGISTER_PASS(manager, DisableShapeOfConstantFolding, false); diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index c4e6cacac69b..9102ea573aec 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -6,7 +6,6 @@ #include "include/batch_headers/fetch_data.cl" #if IS_F8 -#include "include/batch_headers/common.cl" #include "include/batch_headers/f8_utils.cl" #endif @@ -116,7 +115,9 @@ KERNEL(dynamic_quantize_gpu_opt)( #endif output_scale[output_idx] = TO_OUTPUT1_TYPE(1.0h / quan_scale); +#if !(IS_MXFP) FOR_PRECOMPUTED_REDUCTION(output_precomputed_reduction[output_idx] = precomputed_reduction); +#endif } // *********************************************** @@ -253,7 +254,9 @@ KERNEL(dynamic_quantize_gpu_opt)( #if ASYMMETRIC_QUANTIZATION output_zp[output_idx] = convert_uchar_rte(zp); #endif +#if !(IS_MXFP) FOR_PRECOMPUTED_REDUCTION(output_precomputed_reduction[output_idx] = precomputed_reduction); +#endif } } diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl index 2b8859c552be..62d96aa3fba1 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl @@ -6,7 +6,6 @@ #include "include/batch_headers/fetch_data.cl" #if IS_F8 -#include "include/batch_headers/common.cl" #include "include/batch_headers/f8_utils.cl" #endif diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index bdde9866ae9c..5fc415e918f3 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -491,8 +491,6 @@ void TransformationsPipeline::apply(std::shared_ptr func) { } manager.register_pass(); - const bool does_model_contain_mxfp_patterns = ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(func); - manager.register_pass(std::vector{ov::element::i8, ov::element::u8, ov::element::i4, @@ -502,7 +500,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { ov::element::f4e2m1, ov::element::f8e8m0}, !device_info.supports_immad); - if (does_model_contain_mxfp_patterns) { + if (ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(func)) { manager.register_pass( std::vector{ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1, ov::element::f8e8m0}, !device_info.supports_immad, From 6e5833b39da67bf7d0834962ff12767fbae80f18 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Wed, 15 Apr 2026 06:01:56 +0000 Subject: [PATCH 05/48] Fix potential memory issue --- src/common/low_precision_transformations/src/low_precision.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index b5307f8368b2..fe50559fbd21 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -351,7 +351,8 @@ bool ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns( auto m = std::make_shared(mult_pattern, "doesModelContainMXFPPatterns"); - for (const auto& n : model->get_ordered_ops()) { + const auto ops = model->get_ops(); + for (const auto& n : ops) { if (m->match(n)) { return true; } From 2d59ee8f7e45404a50dbed4dfe3b2394c9d3485b Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 21 Apr 2026 08:22:40 +0000 Subject: [PATCH 06/48] CI out of resources issue test --- .../src/low_precision.cpp | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index fe50559fbd21..e68dea2bd067 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -336,27 +336,28 @@ bool LowPrecision::isFunctionQuantized(const std::shared_ptr& m bool ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns( const std::shared_ptr& model) { - using namespace ov::op; - using namespace ov::pass::pattern; + return false; + // using namespace ov::op; + // using namespace ov::pass::pattern; - auto weight_pattern = any_input( - type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && - shape_matches("[Dims..., 32]")); - auto weight_cvt_pattern = wrap_type({weight_pattern}); + // auto weight_pattern = any_input( + // type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && + // shape_matches("[Dims..., 32]")); + // auto weight_cvt_pattern = wrap_type({weight_pattern}); - auto scale_pattern = any_input(type_matches(ov::element::Type_t::f8e8m0) && shape_matches("[Dims..., 1]")); - auto scale_cvt_pattern = wrap_type({scale_pattern}); + // auto scale_pattern = any_input(type_matches(ov::element::Type_t::f8e8m0) && shape_matches("[Dims..., 1]")); + // auto scale_cvt_pattern = wrap_type({scale_pattern}); - auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); + // auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); - auto m = std::make_shared(mult_pattern, "doesModelContainMXFPPatterns"); + // auto m = std::make_shared(mult_pattern, "doesModelContainMXFPPatterns"); - const auto ops = model->get_ops(); - for (const auto& n : ops) { - if (m->match(n)) { - return true; - } - } + // const auto ops = model->get_ops(); + // for (const auto& n : ops) { + // if (m->match(n)) { + // return true; + // } + // } - return false; + // return false; } From 60db5d346d1af4665abac52f15fb4fbcd7079d26 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Wed, 22 Apr 2026 06:22:37 +0000 Subject: [PATCH 07/48] Revert "CI out of resources issue test" This reverts commit 2d59ee8f7e45404a50dbed4dfe3b2394c9d3485b. --- .../src/low_precision.cpp | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index e68dea2bd067..fe50559fbd21 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -336,28 +336,27 @@ bool LowPrecision::isFunctionQuantized(const std::shared_ptr& m bool ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns( const std::shared_ptr& model) { - return false; - // using namespace ov::op; - // using namespace ov::pass::pattern; + using namespace ov::op; + using namespace ov::pass::pattern; - // auto weight_pattern = any_input( - // type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && - // shape_matches("[Dims..., 32]")); - // auto weight_cvt_pattern = wrap_type({weight_pattern}); + auto weight_pattern = any_input( + type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && + shape_matches("[Dims..., 32]")); + auto weight_cvt_pattern = wrap_type({weight_pattern}); - // auto scale_pattern = any_input(type_matches(ov::element::Type_t::f8e8m0) && shape_matches("[Dims..., 1]")); - // auto scale_cvt_pattern = wrap_type({scale_pattern}); + auto scale_pattern = any_input(type_matches(ov::element::Type_t::f8e8m0) && shape_matches("[Dims..., 1]")); + auto scale_cvt_pattern = wrap_type({scale_pattern}); - // auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); + auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); - // auto m = std::make_shared(mult_pattern, "doesModelContainMXFPPatterns"); + auto m = std::make_shared(mult_pattern, "doesModelContainMXFPPatterns"); - // const auto ops = model->get_ops(); - // for (const auto& n : ops) { - // if (m->match(n)) { - // return true; - // } - // } + const auto ops = model->get_ops(); + for (const auto& n : ops) { + if (m->match(n)) { + return true; + } + } - // return false; + return false; } From 3befb0edf4361dea45bc9f4eaa8e9bb08ffc289d Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Wed, 22 Apr 2026 07:19:14 +0000 Subject: [PATCH 08/48] CI out of resources issue test - nested node search --- .../include/ov_ops/dynamic_quantize.hpp | 4 +- .../src/plugin/transformations_pipeline.cpp | 38 ++++++++++--------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp index 11f694f1cfce..306a65d8d297 100644 --- a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp +++ b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp @@ -95,8 +95,8 @@ class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { return false; } - if (m_attrs.scales_zp_output_order == attr.scales_zp_output_order) { - return true; + if (m_attrs.scales_zp_output_order != attr.scales_zp_output_order) { + return false; } // Edge case: if zp_output_order is empty, DynamicQuantize constructor generates it with std::iota(). diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 11f584f1e25e..bdd9c42b0dfa 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -294,24 +294,28 @@ static bool is_decompression_multiply(const std::shared_ptr node return true; }; - if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()}) || all_has_types(consumers, {ov::op::v1::Transpose::get_type_info_static()})) { + // if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()}) || all_has_types(consumers, {ov::op::v1::Transpose::get_type_info_static()})) { + // for (const auto& consumer : consumers) { + // auto get_child_consumers = [&]() { + // auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + + // // Reshape + Transpose chain + // if (all_has_types(child_consumers, { ov::op::v1::Transpose::get_type_info_static() })) { + // std::set> next_child_consumers; + // for (const auto& child_consumer : child_consumers) { + // const auto grand_child_consumers = child_consumer.get_node()->get_output_target_inputs(0); + // next_child_consumers.insert(grand_child_consumers.begin(), grand_child_consumers.end()); + // } + // return next_child_consumers; + // } + // return child_consumers; + // }; + + // auto child_consumers = get_child_consumers(); + + if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()})) { for (const auto& consumer : consumers) { - auto get_child_consumers = [&]() { - auto child_consumers = consumer.get_node()->get_output_target_inputs(0); - - // Reshape + Transpose chain - if (all_has_types(child_consumers, { ov::op::v1::Transpose::get_type_info_static() })) { - std::set> next_child_consumers; - for (const auto& child_consumer : child_consumers) { - const auto grand_child_consumers = child_consumer.get_node()->get_output_target_inputs(0); - next_child_consumers.insert(grand_child_consumers.begin(), grand_child_consumers.end()); - } - return next_child_consumers; - } - return child_consumers; - }; - - auto child_consumers = get_child_consumers(); + auto child_consumers = consumer.get_node()->get_output_target_inputs(0); for (const auto& child_consumer : child_consumers) { const auto& type_info = child_consumer.get_node()->get_type_info(); From 355b0fa1f818063a7ed76b9c56367f2cdb4052f9 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Fri, 24 Apr 2026 07:48:50 +0000 Subject: [PATCH 09/48] Revert "CI out of resources issue test - nested node search" This reverts commit 3befb0edf4361dea45bc9f4eaa8e9bb08ffc289d. --- .../include/ov_ops/dynamic_quantize.hpp | 4 +- .../src/plugin/transformations_pipeline.cpp | 38 +++++++++---------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp index 306a65d8d297..11f694f1cfce 100644 --- a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp +++ b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp @@ -95,8 +95,8 @@ class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { return false; } - if (m_attrs.scales_zp_output_order != attr.scales_zp_output_order) { - return false; + if (m_attrs.scales_zp_output_order == attr.scales_zp_output_order) { + return true; } // Edge case: if zp_output_order is empty, DynamicQuantize constructor generates it with std::iota(). diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index bdd9c42b0dfa..11f584f1e25e 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -294,28 +294,24 @@ static bool is_decompression_multiply(const std::shared_ptr node return true; }; - // if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()}) || all_has_types(consumers, {ov::op::v1::Transpose::get_type_info_static()})) { - // for (const auto& consumer : consumers) { - // auto get_child_consumers = [&]() { - // auto child_consumers = consumer.get_node()->get_output_target_inputs(0); - - // // Reshape + Transpose chain - // if (all_has_types(child_consumers, { ov::op::v1::Transpose::get_type_info_static() })) { - // std::set> next_child_consumers; - // for (const auto& child_consumer : child_consumers) { - // const auto grand_child_consumers = child_consumer.get_node()->get_output_target_inputs(0); - // next_child_consumers.insert(grand_child_consumers.begin(), grand_child_consumers.end()); - // } - // return next_child_consumers; - // } - // return child_consumers; - // }; - - // auto child_consumers = get_child_consumers(); - - if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()})) { + if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()}) || all_has_types(consumers, {ov::op::v1::Transpose::get_type_info_static()})) { for (const auto& consumer : consumers) { - auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + auto get_child_consumers = [&]() { + auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + + // Reshape + Transpose chain + if (all_has_types(child_consumers, { ov::op::v1::Transpose::get_type_info_static() })) { + std::set> next_child_consumers; + for (const auto& child_consumer : child_consumers) { + const auto grand_child_consumers = child_consumer.get_node()->get_output_target_inputs(0); + next_child_consumers.insert(grand_child_consumers.begin(), grand_child_consumers.end()); + } + return next_child_consumers; + } + return child_consumers; + }; + + auto child_consumers = get_child_consumers(); for (const auto& child_consumer : child_consumers) { const auto& type_info = child_consumer.get_node()->get_type_info(); From c953fce18e5f3ef972ff7e787988b3d95913d7ea Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Fri, 24 Apr 2026 12:47:28 +0000 Subject: [PATCH 10/48] Bring back block write macro --- .../intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp | 8 ++++---- .../cl_kernels/dynamic_quantize_gpu_opt.cl | 6 +++--- .../cl_kernels/dynamic_quantize_gpu_ref.cl | 4 ++-- src/plugins/intel_gpu/src/kernel_selector/jitter.cpp | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp index 9a15e44d6be8..bd1a5a6be789 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp @@ -277,8 +277,8 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: min_val = "-" + name + "_VAL_MAX"; val_one = "1.0h"; val_zero = "0.0h"; - to_type = "_convert_half(v)"; - to_type_sat = "_convert_half(v)"; + to_type = "convert_half(v)"; + to_type_sat = "convert_half(v)"; as_type = "as_half(v)"; max_func = "fmax"; min_func = "fmin"; @@ -311,8 +311,8 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: min_val = "-" + name + "_VAL_MAX"; val_one = "1.0f"; val_zero = "0.0f"; - to_type = "_convert_float(v)"; - to_type_sat = "_convert_float(v)"; + to_type = "convert_float(v)"; + to_type_sat = "convert_float(v)"; as_type = "as_float(v)"; max_func = "fmax"; min_func = "fmin"; diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index 9102ea573aec..c8b003e32ad1 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -18,9 +18,9 @@ #define CONVERT_UCHAR_N CAT(convert_uchar, VEC_SIZE) #define CONVERT_CHAR_N CAT(convert_char, VEC_SIZE) #define CONVERT_INT_N CAT(convert_int, VEC_SIZE) -#define TO_TYPE_N_(type, n, x) _convert_##type##n(x) +#define TO_TYPE_N_(type, n, x) convert_##type##n(x) #define TO_TYPE_N(type, n, x) TO_TYPE_N_(type, n, x) -#define TO_TYPE_N_SAT_(type, n, x) _convert_##type##n##_sat(x) +#define TO_TYPE_N_SAT_(type, n, x) convert_##type##n##_sat(x) #define TO_TYPE_N_SAT(type, n, x) TO_TYPE_N_SAT_(type, n, x) #define AS_TYPE_N_(type, n, x) as_##type##n(x) #define AS_TYPE_N(type, n, x) AS_TYPE_N_(type, n, x) @@ -31,7 +31,7 @@ #define ACT_MIN_VAL 0.000000059604645h // min half dtype val #else #define SCALE_TYPE half - #define TO_SCALE_TYPE(x) _convert_half(x) + #define TO_SCALE_TYPE(x) convert_half(x) #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL #endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl index 62d96aa3fba1..37e1d48f7230 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl @@ -18,8 +18,8 @@ #define ACT_MIN_VAL 0.000000059604645h // min half dtype val #else #define SCALE_TYPE half - #define TO_SCALE_TYPE(x) _convert_half(x) - #define TO_SCALE_TYPE_8(x) _convert_half8(x) + #define TO_SCALE_TYPE(x) convert_half(x) + #define TO_SCALE_TYPE_8(x) convert_half8(x) #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL #endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp index 8cc42ec1ecb9..8a8b20a1b28f 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp @@ -1505,8 +1505,8 @@ JitConstants MakeTypeJitConstants(Datatype dataType, const std::string& macroNam min_val = "-" + macroName + "_VAL_MAX"; val_one = "1.0h"; val_zero = "0.0h"; - to_type = "_convert_half(v)"; - to_type_sat = "_convert_half(v)"; + to_type = "convert_half(v)"; + to_type_sat = "convert_half(v)"; as_type = "as_half(v)"; max_func = "fmax"; min_func = "fmin"; @@ -1579,8 +1579,8 @@ JitConstants MakeTypeJitConstants(Datatype dataType, const std::string& macroNam min_val = "-" + macroName + "_VAL_MAX"; val_one = "1.0f"; val_zero = "0.0f"; - to_type = "_convert_float(v)"; - to_type_sat = "_convert_float(v)"; + to_type = "convert_float(v)"; + to_type_sat = "convert_float(v)"; as_type = "as_float(v)"; max_func = "fmax"; min_func = "fmin"; From e22b817b66438f811f460d5c4dc17ca7dc35fe4b Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 27 Apr 2026 08:43:59 +0000 Subject: [PATCH 11/48] Revert "Bring back block write macro" This reverts commit c953fce18e5f3ef972ff7e787988b3d95913d7ea. --- .../intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp | 8 ++++---- .../cl_kernels/dynamic_quantize_gpu_opt.cl | 6 +++--- .../cl_kernels/dynamic_quantize_gpu_ref.cl | 4 ++-- src/plugins/intel_gpu/src/kernel_selector/jitter.cpp | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp index bd1a5a6be789..9a15e44d6be8 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp @@ -277,8 +277,8 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: min_val = "-" + name + "_VAL_MAX"; val_one = "1.0h"; val_zero = "0.0h"; - to_type = "convert_half(v)"; - to_type_sat = "convert_half(v)"; + to_type = "_convert_half(v)"; + to_type_sat = "_convert_half(v)"; as_type = "as_half(v)"; max_func = "fmax"; min_func = "fmin"; @@ -311,8 +311,8 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: min_val = "-" + name + "_VAL_MAX"; val_one = "1.0f"; val_zero = "0.0f"; - to_type = "convert_float(v)"; - to_type_sat = "convert_float(v)"; + to_type = "_convert_float(v)"; + to_type_sat = "_convert_float(v)"; as_type = "as_float(v)"; max_func = "fmax"; min_func = "fmin"; diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index c8b003e32ad1..9102ea573aec 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -18,9 +18,9 @@ #define CONVERT_UCHAR_N CAT(convert_uchar, VEC_SIZE) #define CONVERT_CHAR_N CAT(convert_char, VEC_SIZE) #define CONVERT_INT_N CAT(convert_int, VEC_SIZE) -#define TO_TYPE_N_(type, n, x) convert_##type##n(x) +#define TO_TYPE_N_(type, n, x) _convert_##type##n(x) #define TO_TYPE_N(type, n, x) TO_TYPE_N_(type, n, x) -#define TO_TYPE_N_SAT_(type, n, x) convert_##type##n##_sat(x) +#define TO_TYPE_N_SAT_(type, n, x) _convert_##type##n##_sat(x) #define TO_TYPE_N_SAT(type, n, x) TO_TYPE_N_SAT_(type, n, x) #define AS_TYPE_N_(type, n, x) as_##type##n(x) #define AS_TYPE_N(type, n, x) AS_TYPE_N_(type, n, x) @@ -31,7 +31,7 @@ #define ACT_MIN_VAL 0.000000059604645h // min half dtype val #else #define SCALE_TYPE half - #define TO_SCALE_TYPE(x) convert_half(x) + #define TO_SCALE_TYPE(x) _convert_half(x) #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL #endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl index 37e1d48f7230..62d96aa3fba1 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl @@ -18,8 +18,8 @@ #define ACT_MIN_VAL 0.000000059604645h // min half dtype val #else #define SCALE_TYPE half - #define TO_SCALE_TYPE(x) convert_half(x) - #define TO_SCALE_TYPE_8(x) convert_half8(x) + #define TO_SCALE_TYPE(x) _convert_half(x) + #define TO_SCALE_TYPE_8(x) _convert_half8(x) #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL #endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp index 8a8b20a1b28f..8cc42ec1ecb9 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp @@ -1505,8 +1505,8 @@ JitConstants MakeTypeJitConstants(Datatype dataType, const std::string& macroNam min_val = "-" + macroName + "_VAL_MAX"; val_one = "1.0h"; val_zero = "0.0h"; - to_type = "convert_half(v)"; - to_type_sat = "convert_half(v)"; + to_type = "_convert_half(v)"; + to_type_sat = "_convert_half(v)"; as_type = "as_half(v)"; max_func = "fmax"; min_func = "fmin"; @@ -1579,8 +1579,8 @@ JitConstants MakeTypeJitConstants(Datatype dataType, const std::string& macroNam min_val = "-" + macroName + "_VAL_MAX"; val_one = "1.0f"; val_zero = "0.0f"; - to_type = "convert_float(v)"; - to_type_sat = "convert_float(v)"; + to_type = "_convert_float(v)"; + to_type_sat = "_convert_float(v)"; as_type = "as_float(v)"; max_func = "fmax"; min_func = "fmin"; From a4a70e2ddb04494633d84a7d6c8ecf8c46f0decf Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 27 Apr 2026 13:22:33 +0000 Subject: [PATCH 12/48] Inline expensive wrapper functions --- .../cl_kernels/dynamic_quantize_gpu_opt.cl | 25 +++++++++++-------- .../cl_kernels/dynamic_quantize_gpu_ref.cl | 4 +-- .../include/batch_headers/common.cl | 20 +++++++-------- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index 9102ea573aec..45f56b53cb21 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -13,27 +13,30 @@ #error "dynamic_quantize_gpu_opt.cl: Unsupported output dimension" #endif +#if IS_F8 + #define SCALE_TYPE float + #define TO_SCALE_TYPE(x) _convert_float(x) + #define ACT_MIN_VAL 0.000000059604645h // min half dtype val + #define TO_TYPE_N_(type, n, x) _convert_##type##n(x) + #define TO_TYPE_N_SAT_(type, n, x) _convert_##type##n##_sat(x) +#else + #define SCALE_TYPE half + #define TO_SCALE_TYPE(x) convert_half(x) + #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL + #define TO_TYPE_N_(type, n, x) convert_##type##n(x) + #define TO_TYPE_N_SAT_(type, n, x) convert_##type##n##_sat(x) +#endif + #define VLOAD_N CAT(vload, VEC_SIZE) #define VSTORE_N CAT(vstore, VEC_SIZE) #define CONVERT_UCHAR_N CAT(convert_uchar, VEC_SIZE) #define CONVERT_CHAR_N CAT(convert_char, VEC_SIZE) #define CONVERT_INT_N CAT(convert_int, VEC_SIZE) -#define TO_TYPE_N_(type, n, x) _convert_##type##n(x) #define TO_TYPE_N(type, n, x) TO_TYPE_N_(type, n, x) -#define TO_TYPE_N_SAT_(type, n, x) _convert_##type##n##_sat(x) #define TO_TYPE_N_SAT(type, n, x) TO_TYPE_N_SAT_(type, n, x) #define AS_TYPE_N_(type, n, x) as_##type##n(x) #define AS_TYPE_N(type, n, x) AS_TYPE_N_(type, n, x) #define AS_INPUT_TYPE_N(x) AS_TYPE_N(INPUT0_TYPE, VEC_SIZE, x) -#if IS_F8 - #define SCALE_TYPE float - #define TO_SCALE_TYPE(x) _convert_float(x) - #define ACT_MIN_VAL 0.000000059604645h // min half dtype val -#else - #define SCALE_TYPE half - #define TO_SCALE_TYPE(x) _convert_half(x) - #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL -#endif #if GENERATE_PRECOMPUTED_REDUCTION #define FOR_PRECOMPUTED_REDUCTION(x) x diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl index 62d96aa3fba1..37e1d48f7230 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl @@ -18,8 +18,8 @@ #define ACT_MIN_VAL 0.000000059604645h // min half dtype val #else #define SCALE_TYPE half - #define TO_SCALE_TYPE(x) _convert_half(x) - #define TO_SCALE_TYPE_8(x) _convert_half8(x) + #define TO_SCALE_TYPE(x) convert_half(x) + #define TO_SCALE_TYPE_8(x) convert_half8(x) #define ACT_MIN_VAL 0.003h // Too small value may generate inf during 127/ACT_MIN_VAL #endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index ea7de660a57b..f23736bf5e33 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -104,16 +104,16 @@ DEFINE_VECTOR_IDENTITY(type, 4) \ DEFINE_VECTOR_IDENTITY(type, 8) \ DEFINE_VECTOR_IDENTITY(type, 16) -float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(float val) { return val; } +inline float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } +inline float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } +inline float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } +inline float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } +inline float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } +inline float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } +inline float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } +inline float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } +inline float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } +inline float __attribute__((overloadable)) _convert_float(float val) { return val; } DEFINE_ALL_VECTOR_CONVERTS(float, char) DEFINE_ALL_VECTOR_CONVERTS(float, uchar) From 71777fddecb7f253d9f9302e973c031564b79fa6 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 28 Apr 2026 11:17:24 +0000 Subject: [PATCH 13/48] Get rid of expensive convert wrappers for non-fp dtypes --- .../src/graph/impls/ocl_v2/utils/jitter.cpp | 8 ++++---- .../include/batch_headers/common.cl | 20 +++++++++---------- .../include/batch_headers/f8_utils.cl | 4 ++++ .../cl_kernels/reorder_data.cl | 4 +++- .../intel_gpu/src/kernel_selector/jitter.cpp | 8 ++++---- .../kernels/reorder/reorder_kernel.cpp | 2 ++ 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp index 9a15e44d6be8..bd1a5a6be789 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl_v2/utils/jitter.cpp @@ -277,8 +277,8 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: min_val = "-" + name + "_VAL_MAX"; val_one = "1.0h"; val_zero = "0.0h"; - to_type = "_convert_half(v)"; - to_type_sat = "_convert_half(v)"; + to_type = "convert_half(v)"; + to_type_sat = "convert_half(v)"; as_type = "as_half(v)"; max_func = "fmax"; min_func = "fmin"; @@ -311,8 +311,8 @@ JitConstants make_type_jit_constants(const std::string& name, const ov::element: min_val = "-" + name + "_VAL_MAX"; val_one = "1.0f"; val_zero = "0.0f"; - to_type = "_convert_float(v)"; - to_type_sat = "_convert_float(v)"; + to_type = "convert_float(v)"; + to_type_sat = "convert_float(v)"; as_type = "as_float(v)"; max_func = "fmax"; min_func = "fmin"; diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index f23736bf5e33..ea7de660a57b 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -104,16 +104,16 @@ DEFINE_VECTOR_IDENTITY(type, 4) \ DEFINE_VECTOR_IDENTITY(type, 8) \ DEFINE_VECTOR_IDENTITY(type, 16) -inline float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } -inline float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } -inline float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } -inline float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } -inline float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } -inline float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } -inline float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } -inline float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } -inline float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } -inline float __attribute__((overloadable)) _convert_float(float val) { return val; } +float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(float val) { return val; } DEFINE_ALL_VECTOR_CONVERTS(float, char) DEFINE_ALL_VECTOR_CONVERTS(float, uchar) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl index 0e0a7b97a7d1..051fcf380d21 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl @@ -869,6 +869,10 @@ fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(half16 val) { return res; } +fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(fp8e8m0_t val) { + return val; +} + fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(float val) { fp8e8m0_t res; res.data = _intel_convert_f32_fo_e8m0(val); diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl index a2b6ee26419c..b7252b033712 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#define IS_F8 (F8E5M2_INPUT || F8E4M3_INPUT || F8E5M2_OUTPUT || F8E4M3_OUTPUT) +#define IS_F8 (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT || F8E5M2_OUTPUT || F8E4M3_OUTPUT || F8E8M0_OUTPUT) #if IS_F8 #include "include/batch_headers/common.cl" @@ -160,6 +160,8 @@ KERNEL (reorder_data)( #elif defined UINT4_INPUT const uint uint4_idx = input_idx >> 1; OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(convert_as_uint4_float(input[uint4_idx], input_idx)); + #elif (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT) + OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(_convert_float(input[input_idx])); #else CALC_TYPE res = TO_CALC_TYPE(input[input_idx]); #endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp index 8cc42ec1ecb9..8a8b20a1b28f 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp @@ -1505,8 +1505,8 @@ JitConstants MakeTypeJitConstants(Datatype dataType, const std::string& macroNam min_val = "-" + macroName + "_VAL_MAX"; val_one = "1.0h"; val_zero = "0.0h"; - to_type = "_convert_half(v)"; - to_type_sat = "_convert_half(v)"; + to_type = "convert_half(v)"; + to_type_sat = "convert_half(v)"; as_type = "as_half(v)"; max_func = "fmax"; min_func = "fmin"; @@ -1579,8 +1579,8 @@ JitConstants MakeTypeJitConstants(Datatype dataType, const std::string& macroNam min_val = "-" + macroName + "_VAL_MAX"; val_one = "1.0f"; val_zero = "0.0f"; - to_type = "_convert_float(v)"; - to_type_sat = "_convert_float(v)"; + to_type = "convert_float(v)"; + to_type_sat = "convert_float(v)"; as_type = "as_float(v)"; max_func = "fmax"; min_func = "fmin"; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp index 39476adbbd8d..e41c6824ce7b 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_kernel.cpp @@ -92,8 +92,10 @@ JitConstants ReorderKernelRef::GetJitConstants(const reorder_params& params) con jit.AddConstant(MakeJitConstant("F8E5M2_INPUT", params.inputs[0].GetDType() == Datatype::F8E5M2 ? 1 : 0)); jit.AddConstant(MakeJitConstant("F8E4M3_INPUT", params.inputs[0].GetDType() == Datatype::F8E4M3 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E8M0_INPUT", params.inputs[0].GetDType() == Datatype::F8E8M0 ? 1 : 0)); jit.AddConstant(MakeJitConstant("F8E5M2_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E5M2 ? 1 : 0)); jit.AddConstant(MakeJitConstant("F8E4M3_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E4M3 ? 1 : 0)); + jit.AddConstant(MakeJitConstant("F8E8M0_OUTPUT", params.outputs[0].GetDType() == Datatype::F8E8M0 ? 1 : 0)); return jit; } From 531257721a4dbc4a63bf8c31d4081ab7f919c3de Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Wed, 29 Apr 2026 09:20:37 +0000 Subject: [PATCH 14/48] Remove unnecessary preprocessor defines --- .../cl_kernels/dynamic_quantize_gpu_opt.cl | 4 +- .../include/batch_headers/common.cl | 68 ------------------- 2 files changed, 2 insertions(+), 70 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index 45f56b53cb21..99a75652f78c 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -94,7 +94,7 @@ KERNEL(dynamic_quantize_gpu_opt)( } #if IS_MXFP - SCALE_TYPE quan_scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / _convert_float(max_value))))); + SCALE_TYPE quan_scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / convert_float(max_value))))); #else SCALE_TYPE quan_scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_value; FOR_PRECOMPUTED_REDUCTION(int precomputed_reduction = 0); @@ -361,7 +361,7 @@ KERNEL(dynamic_quantize_gpu_opt)( if ((local_id * iteration + i) >= TOTAL_BLOCK_NUM) continue; - val[i] = TO_TYPE_N(INPUT0_TYPE, VEC_SIZE, TO_TYPE_N(SCALE_TYPE, VEC_SIZE, val[i]) * (MAKE_VECTOR_TYPE(SCALE_TYPE, VEC_SIZE))scale); + val[i] *= TO_INPUT0_TYPE(scale); #if IS_F8 MAKE_VECTOR_TYPE(OUTPUT_TYPE, VEC_SIZE) out = TO_TYPE_N_SAT(OUTPUT_TYPE, VEC_SIZE, val[i]); VSTORE_N(out.data, 0, (char*)(&output[offset + ((local_id * iteration + i) * block_size)])); diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index ea7de660a57b..5bcd72ef1ff8 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -79,71 +79,3 @@ #else #define REQD_SUB_GROUP_SIZE(sg_size) #endif - -#define DEFINE_VECTOR_CONVERT(target_type, source_type, vector_size) \ -MAKE_VECTOR_TYPE(target_type, vector_size) __attribute__((overloadable)) _convert_##target_type##vector_size(MAKE_VECTOR_TYPE(source_type, vector_size) val) { \ - return CAT(convert_, CAT(target_type, vector_size))(val); \ -} - -#define DEFINE_VECTOR_IDENTITY(type, vector_size) \ -MAKE_VECTOR_TYPE(type, vector_size) __attribute__((overloadable)) _convert_##type##vector_size(MAKE_VECTOR_TYPE(type, vector_size) val) { \ - return val; \ -} - -#define DEFINE_ALL_VECTOR_CONVERTS(target_type, source_type) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 2) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 3) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 4) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 8) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 16) - -#define DEFINE_ALL_VECTOR_IDENTITY(type) \ -DEFINE_VECTOR_IDENTITY(type, 2) \ -DEFINE_VECTOR_IDENTITY(type, 3) \ -DEFINE_VECTOR_IDENTITY(type, 4) \ -DEFINE_VECTOR_IDENTITY(type, 8) \ -DEFINE_VECTOR_IDENTITY(type, 16) - -float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(float val) { return val; } - -DEFINE_ALL_VECTOR_CONVERTS(float, char) -DEFINE_ALL_VECTOR_CONVERTS(float, uchar) -DEFINE_ALL_VECTOR_CONVERTS(float, short) -DEFINE_ALL_VECTOR_CONVERTS(float, ushort) -DEFINE_ALL_VECTOR_CONVERTS(float, int) -DEFINE_ALL_VECTOR_CONVERTS(float, uint) -DEFINE_ALL_VECTOR_CONVERTS(float, long) -DEFINE_ALL_VECTOR_CONVERTS(float, ulong) -DEFINE_ALL_VECTOR_CONVERTS(float, half) -DEFINE_ALL_VECTOR_IDENTITY(float) - -half __attribute__((overloadable)) _convert_half(char val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(uchar val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(short val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(ushort val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(int val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(uint val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(long val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(ulong val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(half val) { return val; } -half __attribute__((overloadable)) _convert_half(float val) { return convert_half(val); } - -DEFINE_ALL_VECTOR_CONVERTS(half, char) -DEFINE_ALL_VECTOR_CONVERTS(half, uchar) -DEFINE_ALL_VECTOR_CONVERTS(half, short) -DEFINE_ALL_VECTOR_CONVERTS(half, ushort) -DEFINE_ALL_VECTOR_CONVERTS(half, int) -DEFINE_ALL_VECTOR_CONVERTS(half, uint) -DEFINE_ALL_VECTOR_CONVERTS(half, long) -DEFINE_ALL_VECTOR_CONVERTS(half, ulong) -DEFINE_ALL_VECTOR_CONVERTS(half, float) -DEFINE_ALL_VECTOR_IDENTITY(half) From 5295b31da398210ab8c3be850d54b3d85bbe75f1 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Wed, 29 Apr 2026 14:06:08 +0000 Subject: [PATCH 15/48] Revert "Remove unnecessary preprocessor defines" This reverts commit 531257721a4dbc4a63bf8c31d4081ab7f919c3de. --- .../cl_kernels/dynamic_quantize_gpu_opt.cl | 4 +- .../include/batch_headers/common.cl | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index 99a75652f78c..45f56b53cb21 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -94,7 +94,7 @@ KERNEL(dynamic_quantize_gpu_opt)( } #if IS_MXFP - SCALE_TYPE quan_scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / convert_float(max_value))))); + SCALE_TYPE quan_scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / _convert_float(max_value))))); #else SCALE_TYPE quan_scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_value; FOR_PRECOMPUTED_REDUCTION(int precomputed_reduction = 0); @@ -361,7 +361,7 @@ KERNEL(dynamic_quantize_gpu_opt)( if ((local_id * iteration + i) >= TOTAL_BLOCK_NUM) continue; - val[i] *= TO_INPUT0_TYPE(scale); + val[i] = TO_TYPE_N(INPUT0_TYPE, VEC_SIZE, TO_TYPE_N(SCALE_TYPE, VEC_SIZE, val[i]) * (MAKE_VECTOR_TYPE(SCALE_TYPE, VEC_SIZE))scale); #if IS_F8 MAKE_VECTOR_TYPE(OUTPUT_TYPE, VEC_SIZE) out = TO_TYPE_N_SAT(OUTPUT_TYPE, VEC_SIZE, val[i]); VSTORE_N(out.data, 0, (char*)(&output[offset + ((local_id * iteration + i) * block_size)])); diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index 5bcd72ef1ff8..ea7de660a57b 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -79,3 +79,71 @@ #else #define REQD_SUB_GROUP_SIZE(sg_size) #endif + +#define DEFINE_VECTOR_CONVERT(target_type, source_type, vector_size) \ +MAKE_VECTOR_TYPE(target_type, vector_size) __attribute__((overloadable)) _convert_##target_type##vector_size(MAKE_VECTOR_TYPE(source_type, vector_size) val) { \ + return CAT(convert_, CAT(target_type, vector_size))(val); \ +} + +#define DEFINE_VECTOR_IDENTITY(type, vector_size) \ +MAKE_VECTOR_TYPE(type, vector_size) __attribute__((overloadable)) _convert_##type##vector_size(MAKE_VECTOR_TYPE(type, vector_size) val) { \ + return val; \ +} + +#define DEFINE_ALL_VECTOR_CONVERTS(target_type, source_type) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 2) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 3) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 4) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 8) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 16) + +#define DEFINE_ALL_VECTOR_IDENTITY(type) \ +DEFINE_VECTOR_IDENTITY(type, 2) \ +DEFINE_VECTOR_IDENTITY(type, 3) \ +DEFINE_VECTOR_IDENTITY(type, 4) \ +DEFINE_VECTOR_IDENTITY(type, 8) \ +DEFINE_VECTOR_IDENTITY(type, 16) + +float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(float val) { return val; } + +DEFINE_ALL_VECTOR_CONVERTS(float, char) +DEFINE_ALL_VECTOR_CONVERTS(float, uchar) +DEFINE_ALL_VECTOR_CONVERTS(float, short) +DEFINE_ALL_VECTOR_CONVERTS(float, ushort) +DEFINE_ALL_VECTOR_CONVERTS(float, int) +DEFINE_ALL_VECTOR_CONVERTS(float, uint) +DEFINE_ALL_VECTOR_CONVERTS(float, long) +DEFINE_ALL_VECTOR_CONVERTS(float, ulong) +DEFINE_ALL_VECTOR_CONVERTS(float, half) +DEFINE_ALL_VECTOR_IDENTITY(float) + +half __attribute__((overloadable)) _convert_half(char val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(uchar val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(short val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(ushort val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(int val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(uint val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(long val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(ulong val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(half val) { return val; } +half __attribute__((overloadable)) _convert_half(float val) { return convert_half(val); } + +DEFINE_ALL_VECTOR_CONVERTS(half, char) +DEFINE_ALL_VECTOR_CONVERTS(half, uchar) +DEFINE_ALL_VECTOR_CONVERTS(half, short) +DEFINE_ALL_VECTOR_CONVERTS(half, ushort) +DEFINE_ALL_VECTOR_CONVERTS(half, int) +DEFINE_ALL_VECTOR_CONVERTS(half, uint) +DEFINE_ALL_VECTOR_CONVERTS(half, long) +DEFINE_ALL_VECTOR_CONVERTS(half, ulong) +DEFINE_ALL_VECTOR_CONVERTS(half, float) +DEFINE_ALL_VECTOR_IDENTITY(half) From d776d23573d4be008f12176712a339040d7f6e3c Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 4 May 2026 09:09:34 +0000 Subject: [PATCH 16/48] Test block write macro impact --- .../cl_kernels/include/batch_headers/sub_group_block_write.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl index f89ebe1eb02e..0257e101bc2d 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl @@ -51,7 +51,7 @@ #define BLOCK_WRITEN_RAW(type_size, vector_size, addr_space, ptr, offset, val, src_type) \ BLOCK_WRITEN_FUNC(type_size, vector_size)( \ (addr_space BLOCK_WRITE_TYPE(type_size)*)(ptr) + (offset), \ - AS_TYPE_EXT(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val, src_type)) + AS_TYPE(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val)) #define BLOCK_WRITEN(type, vector_size, ptr, offset, val) \ BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __global, ptr, offset, val, type) From 35ceecd954ce04289c2b1d3650c9e65e743fa899 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 4 May 2026 09:12:16 +0000 Subject: [PATCH 17/48] Correct logical mistake --- .../transformations/include/ov_ops/dynamic_quantize.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp index 11f694f1cfce..306a65d8d297 100644 --- a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp +++ b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp @@ -95,8 +95,8 @@ class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { return false; } - if (m_attrs.scales_zp_output_order == attr.scales_zp_output_order) { - return true; + if (m_attrs.scales_zp_output_order != attr.scales_zp_output_order) { + return false; } // Edge case: if zp_output_order is empty, DynamicQuantize constructor generates it with std::iota(). From db90ea6331f8343bb8b3e8dc704f6c50aea28953 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 4 May 2026 13:00:20 +0000 Subject: [PATCH 18/48] Revert "Test block write macro impact" This reverts commit d776d23573d4be008f12176712a339040d7f6e3c. --- .../cl_kernels/include/batch_headers/sub_group_block_write.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl index 0257e101bc2d..f89ebe1eb02e 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl @@ -51,7 +51,7 @@ #define BLOCK_WRITEN_RAW(type_size, vector_size, addr_space, ptr, offset, val, src_type) \ BLOCK_WRITEN_FUNC(type_size, vector_size)( \ (addr_space BLOCK_WRITE_TYPE(type_size)*)(ptr) + (offset), \ - AS_TYPE(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val)) + AS_TYPE_EXT(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val, src_type)) #define BLOCK_WRITEN(type, vector_size, ptr, offset, val) \ BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __global, ptr, offset, val, type) From 3dea2a793a208080ebac3db91c194c812713957f Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 4 May 2026 13:02:02 +0000 Subject: [PATCH 19/48] Check utils file include --- .../src/kernel_selector/cl_kernels/reorder_data.cl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl index b7252b033712..64245163c98c 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl @@ -4,10 +4,10 @@ #define IS_F8 (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT || F8E5M2_OUTPUT || F8E4M3_OUTPUT || F8E8M0_OUTPUT) -#if IS_F8 -#include "include/batch_headers/common.cl" -#include "include/batch_headers/f8_utils.cl" -#endif +//#if IS_F8 +//#include "include/batch_headers/common.cl" +//#include "include/batch_headers/f8_utils.cl" +//#endif #include "include/reshape_dims.cl" #include "include/fetch_utils.cl" From 2f049ca82673c70c00961cbc954b6be97c63ce1e Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 4 May 2026 16:48:53 +0000 Subject: [PATCH 20/48] Retry removing extra defines --- .../cl_kernels/dynamic_quantize_gpu_opt.cl | 2 +- .../include/batch_headers/common.cl | 67 ------------------- 2 files changed, 1 insertion(+), 68 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index 45f56b53cb21..2077bc4bf356 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -361,7 +361,7 @@ KERNEL(dynamic_quantize_gpu_opt)( if ((local_id * iteration + i) >= TOTAL_BLOCK_NUM) continue; - val[i] = TO_TYPE_N(INPUT0_TYPE, VEC_SIZE, TO_TYPE_N(SCALE_TYPE, VEC_SIZE, val[i]) * (MAKE_VECTOR_TYPE(SCALE_TYPE, VEC_SIZE))scale); + val[i] *= TO_INPUT0_TYPE(scale); #if IS_F8 MAKE_VECTOR_TYPE(OUTPUT_TYPE, VEC_SIZE) out = TO_TYPE_N_SAT(OUTPUT_TYPE, VEC_SIZE, val[i]); VSTORE_N(out.data, 0, (char*)(&output[offset + ((local_id * iteration + i) * block_size)])); diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index ea7de660a57b..b81208fa976f 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -80,70 +80,3 @@ #define REQD_SUB_GROUP_SIZE(sg_size) #endif -#define DEFINE_VECTOR_CONVERT(target_type, source_type, vector_size) \ -MAKE_VECTOR_TYPE(target_type, vector_size) __attribute__((overloadable)) _convert_##target_type##vector_size(MAKE_VECTOR_TYPE(source_type, vector_size) val) { \ - return CAT(convert_, CAT(target_type, vector_size))(val); \ -} - -#define DEFINE_VECTOR_IDENTITY(type, vector_size) \ -MAKE_VECTOR_TYPE(type, vector_size) __attribute__((overloadable)) _convert_##type##vector_size(MAKE_VECTOR_TYPE(type, vector_size) val) { \ - return val; \ -} - -#define DEFINE_ALL_VECTOR_CONVERTS(target_type, source_type) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 2) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 3) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 4) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 8) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 16) - -#define DEFINE_ALL_VECTOR_IDENTITY(type) \ -DEFINE_VECTOR_IDENTITY(type, 2) \ -DEFINE_VECTOR_IDENTITY(type, 3) \ -DEFINE_VECTOR_IDENTITY(type, 4) \ -DEFINE_VECTOR_IDENTITY(type, 8) \ -DEFINE_VECTOR_IDENTITY(type, 16) - -float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(float val) { return val; } - -DEFINE_ALL_VECTOR_CONVERTS(float, char) -DEFINE_ALL_VECTOR_CONVERTS(float, uchar) -DEFINE_ALL_VECTOR_CONVERTS(float, short) -DEFINE_ALL_VECTOR_CONVERTS(float, ushort) -DEFINE_ALL_VECTOR_CONVERTS(float, int) -DEFINE_ALL_VECTOR_CONVERTS(float, uint) -DEFINE_ALL_VECTOR_CONVERTS(float, long) -DEFINE_ALL_VECTOR_CONVERTS(float, ulong) -DEFINE_ALL_VECTOR_CONVERTS(float, half) -DEFINE_ALL_VECTOR_IDENTITY(float) - -half __attribute__((overloadable)) _convert_half(char val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(uchar val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(short val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(ushort val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(int val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(uint val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(long val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(ulong val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(half val) { return val; } -half __attribute__((overloadable)) _convert_half(float val) { return convert_half(val); } - -DEFINE_ALL_VECTOR_CONVERTS(half, char) -DEFINE_ALL_VECTOR_CONVERTS(half, uchar) -DEFINE_ALL_VECTOR_CONVERTS(half, short) -DEFINE_ALL_VECTOR_CONVERTS(half, ushort) -DEFINE_ALL_VECTOR_CONVERTS(half, int) -DEFINE_ALL_VECTOR_CONVERTS(half, uint) -DEFINE_ALL_VECTOR_CONVERTS(half, long) -DEFINE_ALL_VECTOR_CONVERTS(half, ulong) -DEFINE_ALL_VECTOR_CONVERTS(half, float) -DEFINE_ALL_VECTOR_IDENTITY(half) From 90660f00e86cb3164889c4674c7777b514058e1f Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 5 May 2026 08:41:07 +0000 Subject: [PATCH 21/48] Revert "Check utils file include" This reverts commit 3dea2a793a208080ebac3db91c194c812713957f. --- .../src/kernel_selector/cl_kernels/reorder_data.cl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl index 64245163c98c..b7252b033712 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl @@ -4,10 +4,10 @@ #define IS_F8 (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT || F8E5M2_OUTPUT || F8E4M3_OUTPUT || F8E8M0_OUTPUT) -//#if IS_F8 -//#include "include/batch_headers/common.cl" -//#include "include/batch_headers/f8_utils.cl" -//#endif +#if IS_F8 +#include "include/batch_headers/common.cl" +#include "include/batch_headers/f8_utils.cl" +#endif #include "include/reshape_dims.cl" #include "include/fetch_utils.cl" From 2f62b6202ae268075d75a6bcf3edd44153c1937a Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 5 May 2026 08:41:16 +0000 Subject: [PATCH 22/48] Revert "Retry removing extra defines" This reverts commit 2f049ca82673c70c00961cbc954b6be97c63ce1e. --- .../cl_kernels/dynamic_quantize_gpu_opt.cl | 2 +- .../include/batch_headers/common.cl | 67 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index 2077bc4bf356..45f56b53cb21 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -361,7 +361,7 @@ KERNEL(dynamic_quantize_gpu_opt)( if ((local_id * iteration + i) >= TOTAL_BLOCK_NUM) continue; - val[i] *= TO_INPUT0_TYPE(scale); + val[i] = TO_TYPE_N(INPUT0_TYPE, VEC_SIZE, TO_TYPE_N(SCALE_TYPE, VEC_SIZE, val[i]) * (MAKE_VECTOR_TYPE(SCALE_TYPE, VEC_SIZE))scale); #if IS_F8 MAKE_VECTOR_TYPE(OUTPUT_TYPE, VEC_SIZE) out = TO_TYPE_N_SAT(OUTPUT_TYPE, VEC_SIZE, val[i]); VSTORE_N(out.data, 0, (char*)(&output[offset + ((local_id * iteration + i) * block_size)])); diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index b81208fa976f..ea7de660a57b 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -80,3 +80,70 @@ #define REQD_SUB_GROUP_SIZE(sg_size) #endif +#define DEFINE_VECTOR_CONVERT(target_type, source_type, vector_size) \ +MAKE_VECTOR_TYPE(target_type, vector_size) __attribute__((overloadable)) _convert_##target_type##vector_size(MAKE_VECTOR_TYPE(source_type, vector_size) val) { \ + return CAT(convert_, CAT(target_type, vector_size))(val); \ +} + +#define DEFINE_VECTOR_IDENTITY(type, vector_size) \ +MAKE_VECTOR_TYPE(type, vector_size) __attribute__((overloadable)) _convert_##type##vector_size(MAKE_VECTOR_TYPE(type, vector_size) val) { \ + return val; \ +} + +#define DEFINE_ALL_VECTOR_CONVERTS(target_type, source_type) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 2) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 3) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 4) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 8) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 16) + +#define DEFINE_ALL_VECTOR_IDENTITY(type) \ +DEFINE_VECTOR_IDENTITY(type, 2) \ +DEFINE_VECTOR_IDENTITY(type, 3) \ +DEFINE_VECTOR_IDENTITY(type, 4) \ +DEFINE_VECTOR_IDENTITY(type, 8) \ +DEFINE_VECTOR_IDENTITY(type, 16) + +float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(float val) { return val; } + +DEFINE_ALL_VECTOR_CONVERTS(float, char) +DEFINE_ALL_VECTOR_CONVERTS(float, uchar) +DEFINE_ALL_VECTOR_CONVERTS(float, short) +DEFINE_ALL_VECTOR_CONVERTS(float, ushort) +DEFINE_ALL_VECTOR_CONVERTS(float, int) +DEFINE_ALL_VECTOR_CONVERTS(float, uint) +DEFINE_ALL_VECTOR_CONVERTS(float, long) +DEFINE_ALL_VECTOR_CONVERTS(float, ulong) +DEFINE_ALL_VECTOR_CONVERTS(float, half) +DEFINE_ALL_VECTOR_IDENTITY(float) + +half __attribute__((overloadable)) _convert_half(char val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(uchar val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(short val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(ushort val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(int val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(uint val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(long val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(ulong val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(half val) { return val; } +half __attribute__((overloadable)) _convert_half(float val) { return convert_half(val); } + +DEFINE_ALL_VECTOR_CONVERTS(half, char) +DEFINE_ALL_VECTOR_CONVERTS(half, uchar) +DEFINE_ALL_VECTOR_CONVERTS(half, short) +DEFINE_ALL_VECTOR_CONVERTS(half, ushort) +DEFINE_ALL_VECTOR_CONVERTS(half, int) +DEFINE_ALL_VECTOR_CONVERTS(half, uint) +DEFINE_ALL_VECTOR_CONVERTS(half, long) +DEFINE_ALL_VECTOR_CONVERTS(half, ulong) +DEFINE_ALL_VECTOR_CONVERTS(half, float) +DEFINE_ALL_VECTOR_IDENTITY(half) From c2213f58b6b450c5f689561f40e280c301c2db5f Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Mon, 18 May 2026 09:53:41 +0000 Subject: [PATCH 23/48] Revert pre_post_process.cpp changes --- src/core/src/preprocess/pre_post_process.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index 75bfa0405613..32ef35c65052 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -117,11 +117,7 @@ void transformation_pipeline(std::shared_ptr& model) { REGISTER_PASS( manager, MarkDequantization, - TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, - false); - if (ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(model)) { - REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); - } + TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); REGISTER_PASS(manager, DisableShapeOfConstantFolding, false); REGISTER_PASS(manager, DisableRandomUniformConstantFolding) // Mark quantized and f16/bf16 compressed constants to prevent CF for them, From c20a01c6c8d551c11fb9a39f434b92d1aa306167 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 19 May 2026 10:44:13 +0000 Subject: [PATCH 24/48] Check if execution of new tests is connected to the issue --- .../subgraph_tests/dynamic/matmul_weights_decompression.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp index cfebec1796de..08200647db96 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp @@ -319,6 +319,10 @@ class MatmulWeightsDecompression : public testing::WithParamInterface Date: Wed, 20 May 2026 07:46:38 +0000 Subject: [PATCH 25/48] For real now --- .../dynamic/matmul_weights_decompression.cpp | 138 +++++++++--------- 1 file changed, 67 insertions(+), 71 deletions(-) diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp index 08200647db96..761992d49ea2 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp @@ -319,10 +319,6 @@ class MatmulWeightsDecompression : public testing::WithParamInterface::max()), - ::testing::Values(0.2f), - ::testing::Values(false)), - MatmulWeightsDecompression::get_test_case_name); - -INSTANTIATE_TEST_SUITE_P( - smoke_MatMulCompressedWeights_dyn_quan_fp8e5m2_fp8e5m2, - MatmulWeightsDecompression, - ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape - ::testing::Values(ov::element::f8e5m2), - ::testing::Values(ov::element::f16), - ::testing::Values(true), - ::testing::Values(false), - ::testing::Values(false), - ::testing::Values(true), - ::testing::Values(false), - ::testing::Values(false), - ::testing::Values(std::numeric_limits::max()), - ::testing::Values(0.2f), - ::testing::Values(false)), - MatmulWeightsDecompression::get_test_case_name); +//INSTANTIATE_TEST_SUITE_P( +// smoke_MatMulCompressedWeights_dyn_quan_mxfp8_e4m3, +// MatmulWeightsDecompression, +// ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 4096}, {{1, 1, 4096}, {8, 1, 4096}}}, {4096, 1024}, 32}), // shape +// ::testing::Values(ov::element::f8e4m3), +// ::testing::Values(ov::element::f16), +// ::testing::Values(true), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(32), +// ::testing::Values(2.0f), +// ::testing::Values(true)), +// MatmulWeightsDecompression::get_test_case_name); +// +//INSTANTIATE_TEST_SUITE_P( +// smoke_MatMulCompressedWeights_dyn_quan_mxfp8_e5m2, +// MatmulWeightsDecompression, +// ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 4096}, {{1, 1, 4096}, {8, 1, 4096}}}, {4096, 1024}, 32}), // shape +// ::testing::Values(ov::element::f8e5m2), +// ::testing::Values(ov::element::f16), +// ::testing::Values(true), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(32), +// ::testing::Values(2.0f), +// ::testing::Values(true)), +// MatmulWeightsDecompression::get_test_case_name); +// +//INSTANTIATE_TEST_SUITE_P( +// smoke_MatMulCompressedWeights_dyn_quan_fp8e4m3_fp8e4m3, +// MatmulWeightsDecompression, +// ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape +// ::testing::Values(ov::element::f8e4m3), +// ::testing::Values(ov::element::f16), +// ::testing::Values(true), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(true), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(std::numeric_limits::max()), +// ::testing::Values(0.2f), +// ::testing::Values(false)), +// MatmulWeightsDecompression::get_test_case_name); +// +//INSTANTIATE_TEST_SUITE_P( +// smoke_MatMulCompressedWeights_dyn_quan_fp8e5m2_fp8e5m2, +// MatmulWeightsDecompression, +// ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape +// ::testing::Values(ov::element::f8e5m2), +// ::testing::Values(ov::element::f16), +// ::testing::Values(true), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(true), +// ::testing::Values(false), +// ::testing::Values(false), +// ::testing::Values(std::numeric_limits::max()), +// ::testing::Values(0.2f), +// ::testing::Values(false)), +// MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_scalar_wzp, From e57ea3868a06bab40594c3604553889b12e920d6 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Wed, 20 May 2026 11:50:17 +0000 Subject: [PATCH 26/48] Frfr --- .../test_cases/dynamic_quantize_gpu_test.cpp | 94 +++++++++---------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp index 7e26fc14b9b1..0d09bd23ae7b 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp @@ -447,50 +447,50 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_inner_most_dim data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache", SetInnerMostDimValuesZero::Yes); } -TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e4m3) { - this->test_dynamic_quantization(false, - {1, 1, 4096}, - {1, 1, 4096}, - QuantizationType::Symmetric, - 32, - data_types::f8e4m3, - data_types::f8e8m0, - data_types::dynamic, - OutputStorageType::Planar); -} - -TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e5m2) { - this->test_dynamic_quantization(false, - {1, 1, 4096}, - {1, 1, 4096}, - QuantizationType::Symmetric, - 32, - data_types::f8e5m2, - data_types::f8e8m0, - data_types::dynamic, - OutputStorageType::Planar); -} - -TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e4m3) { - this->test_dynamic_quantization(false, - {1, 1, 4096}, - {1, 1, 4096}, - QuantizationType::Symmetric, - UINT64_MAX, - data_types::f8e4m3, - data_types::f16, - data_types::dynamic, - OutputStorageType::Planar); -} - -TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { - this->test_dynamic_quantization(false, - {1, 1, 4096}, - {1, 1, 4096}, - QuantizationType::Symmetric, - UINT64_MAX, - data_types::f8e5m2, - data_types::f16, - data_types::dynamic, - OutputStorageType::Planar); -} +//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e4m3) { +// this->test_dynamic_quantization(false, +// {1, 1, 4096}, +// {1, 1, 4096}, +// QuantizationType::Symmetric, +// 32, +// data_types::f8e4m3, +// data_types::f8e8m0, +// data_types::dynamic, +// OutputStorageType::Planar); +//} +// +//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e5m2) { +// this->test_dynamic_quantization(false, +// {1, 1, 4096}, +// {1, 1, 4096}, +// QuantizationType::Symmetric, +// 32, +// data_types::f8e5m2, +// data_types::f8e8m0, +// data_types::dynamic, +// OutputStorageType::Planar); +//} +// +//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e4m3) { +// this->test_dynamic_quantization(false, +// {1, 1, 4096}, +// {1, 1, 4096}, +// QuantizationType::Symmetric, +// UINT64_MAX, +// data_types::f8e4m3, +// data_types::f16, +// data_types::dynamic, +// OutputStorageType::Planar); +//} +// +//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { +// this->test_dynamic_quantization(false, +// {1, 1, 4096}, +// {1, 1, 4096}, +// QuantizationType::Symmetric, +// UINT64_MAX, +// data_types::f8e5m2, +// data_types::f16, +// data_types::dynamic, +// OutputStorageType::Planar); +//} From e50d868667b1c2f49ef626748f5edac5480c1574 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Thu, 21 May 2026 11:53:15 +0000 Subject: [PATCH 27/48] Revert "Revert pre_post_process.cpp changes" This reverts commit c2213f58b6b450c5f689561f40e280c301c2db5f. --- src/core/src/preprocess/pre_post_process.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index 32ef35c65052..75bfa0405613 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -117,7 +117,11 @@ void transformation_pipeline(std::shared_ptr& model) { REGISTER_PASS( manager, MarkDequantization, - TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); + TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, + false); + if (ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(model)) { + REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); + } REGISTER_PASS(manager, DisableShapeOfConstantFolding, false); REGISTER_PASS(manager, DisableRandomUniformConstantFolding) // Mark quantized and f16/bf16 compressed constants to prevent CF for them, From 3a9f2feb2156afbd4b7bab3ccc4cbcb7d839e7c7 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Fri, 22 May 2026 06:01:02 +0000 Subject: [PATCH 28/48] Bring back tests --- .../dynamic/matmul_weights_decompression.cpp | 134 +++++++++--------- .../test_cases/dynamic_quantize_gpu_test.cpp | 94 ++++++------ 2 files changed, 114 insertions(+), 114 deletions(-) diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp index 761992d49ea2..43e52b538f0b 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp @@ -609,73 +609,73 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_3D_weight, ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); -//INSTANTIATE_TEST_SUITE_P( -// smoke_MatMulCompressedWeights_dyn_quan_mxfp8_e4m3, -// MatmulWeightsDecompression, -// ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 4096}, {{1, 1, 4096}, {8, 1, 4096}}}, {4096, 1024}, 32}), // shape -// ::testing::Values(ov::element::f8e4m3), -// ::testing::Values(ov::element::f16), -// ::testing::Values(true), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(32), -// ::testing::Values(2.0f), -// ::testing::Values(true)), -// MatmulWeightsDecompression::get_test_case_name); -// -//INSTANTIATE_TEST_SUITE_P( -// smoke_MatMulCompressedWeights_dyn_quan_mxfp8_e5m2, -// MatmulWeightsDecompression, -// ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 4096}, {{1, 1, 4096}, {8, 1, 4096}}}, {4096, 1024}, 32}), // shape -// ::testing::Values(ov::element::f8e5m2), -// ::testing::Values(ov::element::f16), -// ::testing::Values(true), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(32), -// ::testing::Values(2.0f), -// ::testing::Values(true)), -// MatmulWeightsDecompression::get_test_case_name); -// -//INSTANTIATE_TEST_SUITE_P( -// smoke_MatMulCompressedWeights_dyn_quan_fp8e4m3_fp8e4m3, -// MatmulWeightsDecompression, -// ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape -// ::testing::Values(ov::element::f8e4m3), -// ::testing::Values(ov::element::f16), -// ::testing::Values(true), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(true), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(std::numeric_limits::max()), -// ::testing::Values(0.2f), -// ::testing::Values(false)), -// MatmulWeightsDecompression::get_test_case_name); -// -//INSTANTIATE_TEST_SUITE_P( -// smoke_MatMulCompressedWeights_dyn_quan_fp8e5m2_fp8e5m2, -// MatmulWeightsDecompression, -// ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape -// ::testing::Values(ov::element::f8e5m2), -// ::testing::Values(ov::element::f16), -// ::testing::Values(true), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(true), -// ::testing::Values(false), -// ::testing::Values(false), -// ::testing::Values(std::numeric_limits::max()), -// ::testing::Values(0.2f), -// ::testing::Values(false)), -// MatmulWeightsDecompression::get_test_case_name); +INSTANTIATE_TEST_SUITE_P( + smoke_MatMulCompressedWeights_dyn_quan_mxfp8_e4m3, + MatmulWeightsDecompression, + ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 4096}, {{1, 1, 4096}, {8, 1, 4096}}}, {4096, 1024}, 32}), // shape + ::testing::Values(ov::element::f8e4m3), + ::testing::Values(ov::element::f16), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(32), + ::testing::Values(2.0f), + ::testing::Values(true)), + MatmulWeightsDecompression::get_test_case_name); + +INSTANTIATE_TEST_SUITE_P( + smoke_MatMulCompressedWeights_dyn_quan_mxfp8_e5m2, + MatmulWeightsDecompression, + ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 4096}, {{1, 1, 4096}, {8, 1, 4096}}}, {4096, 1024}, 32}), // shape + ::testing::Values(ov::element::f8e5m2), + ::testing::Values(ov::element::f16), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(32), + ::testing::Values(2.0f), + ::testing::Values(true)), + MatmulWeightsDecompression::get_test_case_name); + +INSTANTIATE_TEST_SUITE_P( + smoke_MatMulCompressedWeights_dyn_quan_fp8e4m3_fp8e4m3, + MatmulWeightsDecompression, + ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape + ::testing::Values(ov::element::f8e4m3), + ::testing::Values(ov::element::f16), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(std::numeric_limits::max()), + ::testing::Values(0.2f), + ::testing::Values(false)), + MatmulWeightsDecompression::get_test_case_name); + +INSTANTIATE_TEST_SUITE_P( + smoke_MatMulCompressedWeights_dyn_quan_fp8e5m2_fp8e5m2, + MatmulWeightsDecompression, + ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape + ::testing::Values(ov::element::f8e5m2), + ::testing::Values(ov::element::f16), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(true), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(std::numeric_limits::max()), + ::testing::Values(0.2f), + ::testing::Values(false)), + MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_scalar_wzp, diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp index 0d09bd23ae7b..7e26fc14b9b1 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp @@ -447,50 +447,50 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_inner_most_dim data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache", SetInnerMostDimValuesZero::Yes); } -//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e4m3) { -// this->test_dynamic_quantization(false, -// {1, 1, 4096}, -// {1, 1, 4096}, -// QuantizationType::Symmetric, -// 32, -// data_types::f8e4m3, -// data_types::f8e8m0, -// data_types::dynamic, -// OutputStorageType::Planar); -//} -// -//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e5m2) { -// this->test_dynamic_quantization(false, -// {1, 1, 4096}, -// {1, 1, 4096}, -// QuantizationType::Symmetric, -// 32, -// data_types::f8e5m2, -// data_types::f8e8m0, -// data_types::dynamic, -// OutputStorageType::Planar); -//} -// -//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e4m3) { -// this->test_dynamic_quantization(false, -// {1, 1, 4096}, -// {1, 1, 4096}, -// QuantizationType::Symmetric, -// UINT64_MAX, -// data_types::f8e4m3, -// data_types::f16, -// data_types::dynamic, -// OutputStorageType::Planar); -//} -// -//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { -// this->test_dynamic_quantization(false, -// {1, 1, 4096}, -// {1, 1, 4096}, -// QuantizationType::Symmetric, -// UINT64_MAX, -// data_types::f8e5m2, -// data_types::f16, -// data_types::dynamic, -// OutputStorageType::Planar); -//} +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e4m3) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + 32, + data_types::f8e4m3, + data_types::f8e8m0, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e5m2) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + 32, + data_types::f8e5m2, + data_types::f8e8m0, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e4m3) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + UINT64_MAX, + data_types::f8e4m3, + data_types::f16, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + UINT64_MAX, + data_types::f8e5m2, + data_types::f16, + data_types::dynamic, + OutputStorageType::Planar); +} From 3c8e92b8675e0549586660eec26b78dac3f909aa Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 9 Jun 2026 09:26:01 +0000 Subject: [PATCH 29/48] Apply review suggestions --- .../include/low_precision/low_precision.hpp | 2 +- .../low_precision_transformations/src/low_precision.cpp | 4 ++-- .../common_optimizations/moc_transformations.cpp | 2 +- src/core/src/preprocess/pre_post_process.cpp | 4 ++-- src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/common/low_precision_transformations/include/low_precision/low_precision.hpp b/src/common/low_precision_transformations/include/low_precision/low_precision.hpp index 0c7155e435f3..e38540632ae1 100644 --- a/src/common/low_precision_transformations/include/low_precision/low_precision.hpp +++ b/src/common/low_precision_transformations/include/low_precision/low_precision.hpp @@ -76,7 +76,7 @@ class LP_TRANSFORMATIONS_API ov::pass::low_precision::LowPrecision : public ov:: const std::set& supported_levels = all_levels, bool check_fake_convert = false); - static bool doesModelContainMXFPPatterns(const std::shared_ptr& model); + static bool does_model_contain_mxfp_patterns(const std::shared_ptr& model); template std::shared_ptr add_main(Args&&... args) { diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index c5bd32b61046..e7ce046bad8c 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -345,7 +345,7 @@ bool LowPrecision::isFunctionQuantized(const std::shared_ptr& m return false; } -bool ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns( +bool ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns( const std::shared_ptr& model) { using namespace ov::op; using namespace ov::pass::pattern; @@ -360,7 +360,7 @@ bool ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns( auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); - auto m = std::make_shared(mult_pattern, "doesModelContainMXFPPatterns"); + auto m = std::make_shared(mult_pattern, "does_model_contain_mxfp_patterns"); const auto ops = model->get_ops(); for (const auto& n : ops) { diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index 9cee1146038d..262010bdee29 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -149,7 +149,7 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr manager.register_pass( TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false); - if (ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(f)) { + if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(f)) { manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index 75bfa0405613..0987b77a52df 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -94,7 +94,7 @@ void transformation_pipeline(std::shared_ptr& model) { RTInfoCache rt_info_cache; rt_info_cache.store(model); - auto get_manager = [&]() { + auto get_manager = [&model]() { Manager manager("pre_post_processing"); manager.set_per_pass_validation(false); @@ -119,7 +119,7 @@ void transformation_pipeline(std::shared_ptr& model) { MarkDequantization, TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false); - if (ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(model)) { + if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(model)) { REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); } REGISTER_PASS(manager, DisableShapeOfConstantFolding, false); diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 66fa22ae693c..53aeb57d18d6 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -576,7 +576,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { ov::element::f4e2m1, ov::element::f8e8m0}, !device_info.supports_immad); - if (ov::pass::low_precision::LowPrecision::doesModelContainMXFPPatterns(func)) { + if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(func)) { manager.register_pass( std::vector{ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1, ov::element::f8e8m0}, !device_info.supports_immad, From 360cc3e9be9eac32e92c37d436ade616f0e9094c Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 9 Jun 2026 12:27:57 +0000 Subject: [PATCH 30/48] Add missing test modifications --- .../tests/unit/test_cases/dynamic_quantize_gpu_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp index e35d6fc1b0c4..d13bac29402c 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp @@ -497,18 +497,18 @@ TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_opt_group_size_256) { this->test_dynamic_quantization(false, {1, 1, 8192}, {1, 1, 8192}, QuantizationType::Symmetric, 256, - data_types::i8, data_types::dynamic, OutputStorageType::Planar, + data_types::i8, data_types::f16, data_types::dynamic, OutputStorageType::Planar, "dynamic_quantize_gpu_opt"); } TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_opt_group_size_256_precompute_sum) { this->test_dynamic_quantization(false, {1, 1, 8192}, {1, 1, 8192}, QuantizationType::Symmetric, 256, - data_types::i8, data_types::i8, OutputStorageType::Planar, + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_opt", SetInnerMostDimValuesZero::No, PrecomputeSum::Enabled); } TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_group_size_8192_with_precompute_sum) { this->test_dynamic_quantization(false, {1, 1, 16384}, {1, 1, 16384}, QuantizationType::Symmetric, 8192, - data_types::i8, data_types::i8, OutputStorageType::Planar, + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "", SetInnerMostDimValuesZero::No, PrecomputeSum::Enabled); } From 89b7d28ba71bce3d205fce58484696d9b465c8e1 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Thu, 11 Jun 2026 14:10:51 +0000 Subject: [PATCH 31/48] Double check that it's not unit tests' fault --- .../test_cases/dynamic_quantize_gpu_test.cpp | 94 +++++++++---------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp index d13bac29402c..c671b20a3d78 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp @@ -447,53 +447,53 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_inner_most_dim data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache", SetInnerMostDimValuesZero::Yes); } -TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e4m3) { - this->test_dynamic_quantization(false, - {1, 1, 4096}, - {1, 1, 4096}, - QuantizationType::Symmetric, - 32, - data_types::f8e4m3, - data_types::f8e8m0, - data_types::dynamic, - OutputStorageType::Planar); -} - -TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e5m2) { - this->test_dynamic_quantization(false, - {1, 1, 4096}, - {1, 1, 4096}, - QuantizationType::Symmetric, - 32, - data_types::f8e5m2, - data_types::f8e8m0, - data_types::dynamic, - OutputStorageType::Planar); -} - -TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e4m3) { - this->test_dynamic_quantization(false, - {1, 1, 4096}, - {1, 1, 4096}, - QuantizationType::Symmetric, - UINT64_MAX, - data_types::f8e4m3, - data_types::f16, - data_types::dynamic, - OutputStorageType::Planar); -} - -TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { - this->test_dynamic_quantization(false, - {1, 1, 4096}, - {1, 1, 4096}, - QuantizationType::Symmetric, - UINT64_MAX, - data_types::f8e5m2, - data_types::f16, - data_types::dynamic, - OutputStorageType::Planar); -} +//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e4m3) { +// this->test_dynamic_quantization(false, +// {1, 1, 4096}, +// {1, 1, 4096}, +// QuantizationType::Symmetric, +// 32, +// data_types::f8e4m3, +// data_types::f8e8m0, +// data_types::dynamic, +// OutputStorageType::Planar); +//} +// +//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e5m2) { +// this->test_dynamic_quantization(false, +// {1, 1, 4096}, +// {1, 1, 4096}, +// QuantizationType::Symmetric, +// 32, +// data_types::f8e5m2, +// data_types::f8e8m0, +// data_types::dynamic, +// OutputStorageType::Planar); +//} +// +//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e4m3) { +// this->test_dynamic_quantization(false, +// {1, 1, 4096}, +// {1, 1, 4096}, +// QuantizationType::Symmetric, +// UINT64_MAX, +// data_types::f8e4m3, +// data_types::f16, +// data_types::dynamic, +// OutputStorageType::Planar); +//} +// +//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { +// this->test_dynamic_quantization(false, +// {1, 1, 4096}, +// {1, 1, 4096}, +// QuantizationType::Symmetric, +// UINT64_MAX, +// data_types::f8e5m2, +// data_types::f16, +// data_types::dynamic, +// OutputStorageType::Planar); +//} TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_opt_group_size_256) { this->test_dynamic_quantization(false, {1, 1, 8192}, {1, 1, 8192}, QuantizationType::Symmetric, 256, From 3d2c6a9492b9a1b688659c88f71834dc00cc5343 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Fri, 12 Jun 2026 07:41:46 +0000 Subject: [PATCH 32/48] Comment out troubleshooting part 1 --- .../include/low_precision/low_precision.hpp | 2 +- .../src/low_precision.cpp | 42 +- .../moc_transformations.cpp | 15 +- src/core/src/preprocess/pre_post_process.cpp | 13 +- .../cl_kernels/dynamic_quantize_gpu_opt.cl | 6 +- .../include/batch_headers/common.cl | 170 +- .../include/batch_headers/f8_utils.cl | 2466 ++++++++--------- .../batch_headers/sub_group_block_write.cl | 8 +- .../cl_kernels/reorder_data.cl | 9 - .../src/plugin/transformations_pipeline.cpp | 49 +- 10 files changed, 1386 insertions(+), 1394 deletions(-) diff --git a/src/common/low_precision_transformations/include/low_precision/low_precision.hpp b/src/common/low_precision_transformations/include/low_precision/low_precision.hpp index e38540632ae1..9c3f61a9806b 100644 --- a/src/common/low_precision_transformations/include/low_precision/low_precision.hpp +++ b/src/common/low_precision_transformations/include/low_precision/low_precision.hpp @@ -76,7 +76,7 @@ class LP_TRANSFORMATIONS_API ov::pass::low_precision::LowPrecision : public ov:: const std::set& supported_levels = all_levels, bool check_fake_convert = false); - static bool does_model_contain_mxfp_patterns(const std::shared_ptr& model); + //static bool does_model_contain_mxfp_patterns(const std::shared_ptr& model); template std::shared_ptr add_main(Args&&... args) { diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index e7ce046bad8c..2cc69cc9b9a0 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -31,7 +31,7 @@ #include "openvino/pass/constant_folding.hpp" #include "openvino/pass/manager.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" -#include "openvino/pass/pattern/op/pattern.hpp" +// #include "openvino/pass/pattern/op/pattern.hpp" #include "openvino/util/log.hpp" #include "ov_ops/type_relaxed.hpp" #include "transformations/common_optimizations/lin_op_sequence_fusion.hpp" @@ -345,29 +345,29 @@ bool LowPrecision::isFunctionQuantized(const std::shared_ptr& m return false; } -bool ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns( - const std::shared_ptr& model) { - using namespace ov::op; - using namespace ov::pass::pattern; +// bool ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns( +// const std::shared_ptr& model) { +// using namespace ov::op; +// using namespace ov::pass::pattern; - auto weight_pattern = any_input( - type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && - shape_matches("[Dims..., 32]")); - auto weight_cvt_pattern = wrap_type({weight_pattern}); +// auto weight_pattern = any_input( +// type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && +// shape_matches("[Dims..., 32]")); +// auto weight_cvt_pattern = wrap_type({weight_pattern}); - auto scale_pattern = any_input(type_matches(ov::element::Type_t::f8e8m0) && shape_matches("[Dims..., 1]")); - auto scale_cvt_pattern = wrap_type({scale_pattern}); +// auto scale_pattern = any_input(type_matches(ov::element::Type_t::f8e8m0) && shape_matches("[Dims..., 1]")); +// auto scale_cvt_pattern = wrap_type({scale_pattern}); - auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); +// auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); - auto m = std::make_shared(mult_pattern, "does_model_contain_mxfp_patterns"); +// auto m = std::make_shared(mult_pattern, "does_model_contain_mxfp_patterns"); - const auto ops = model->get_ops(); - for (const auto& n : ops) { - if (m->match(n)) { - return true; - } - } +// const auto ops = model->get_ops(); +// for (const auto& n : ops) { +// if (m->match(n)) { +// return true; +// } +// } - return false; -} +// return false; +// } diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index 262010bdee29..371ee4b7af55 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -7,7 +7,7 @@ #include #include "itt.hpp" -#include "low_precision/low_precision.hpp" +// #include "low_precision/low_precision.hpp" #include "openvino/pass/constant_folding.hpp" #include "openvino/pass/manager.hpp" #include "transformations/common_optimizations/adaptive_pool_to_reduce.hpp" @@ -147,13 +147,12 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr // Transformation call example, to check with the real model manager.register_pass(TypeVector{f8e4m3}, TypeVector{u4}); manager.register_pass( - TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, - false); - if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(f)) { - manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, - false, - false); - } + TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); + // if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(f)) { + // manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, + // false, + // false); + // } } if (!m_use_shapes) { manager.register_pass(); diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index 0987b77a52df..1620ee644fd6 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -8,7 +8,7 @@ #include "function_guard.hpp" #include "itt.hpp" #include "layout_utils.hpp" -#include "low_precision/low_precision.hpp" +// #include "low_precision/low_precision.hpp" #include "openvino/core/model.hpp" #include "openvino/op/util/multi_subgraph_base.hpp" #include "openvino/pass/constant_folding.hpp" @@ -94,7 +94,7 @@ void transformation_pipeline(std::shared_ptr& model) { RTInfoCache rt_info_cache; rt_info_cache.store(model); - auto get_manager = [&model]() { + auto get_manager = []() { Manager manager("pre_post_processing"); manager.set_per_pass_validation(false); @@ -117,11 +117,10 @@ void transformation_pipeline(std::shared_ptr& model) { REGISTER_PASS( manager, MarkDequantization, - TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}, - false); - if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(model)) { - REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); - } + TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); + // if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(model)) { + // REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); + // } REGISTER_PASS(manager, DisableShapeOfConstantFolding, false); REGISTER_PASS(manager, DisableRandomUniformConstantFolding) // Mark quantized and f16/bf16 compressed constants to prevent CF for them, diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index c91ca1d9ba51..335295d2d355 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -5,9 +5,9 @@ #define IS_F8 (F8E5M2_OUTPUT || F8E4M3_OUTPUT) #include "include/batch_headers/fetch_data.cl" -#if IS_F8 -#include "include/batch_headers/f8_utils.cl" -#endif +// #if IS_F8 +// #include "include/batch_headers/f8_utils.cl" +// #endif #if OUTPUT_DIMS != 4 && OUTPUT_DIMS != 2 #error "dynamic_quantize_gpu_opt.cl: Unsupported output dimension" diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index ea7de660a57b..5d8309112861 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -38,21 +38,21 @@ #define MAKE_VECTOR_TYPE_IMPL_16(elem_type) CAT(elem_type, 16) #define MAKE_VECTOR_TYPE(elem_type, size) CAT(MAKE_VECTOR_TYPE_IMPL_, size)(elem_type) -#define AS_TYPE_PREFIX_uchar as_ -#define AS_TYPE_PREFIX_char as_ -#define AS_TYPE_PREFIX_fp8e5m2_t _as_ -#define AS_TYPE_PREFIX_fp8e4m3_t _as_ -#define AS_TYPE_PREFIX_fp8e8m0_t _as_ -#define AS_TYPE_PREFIX_ushort as_ -#define AS_TYPE_PREFIX_short as_ -#define AS_TYPE_PREFIX_half as_ -#define AS_TYPE_PREFIX_int as_ -#define AS_TYPE_PREFIX_uint as_ -#define AS_TYPE_PREFIX_float as_ -#define AS_TYPE_PREFIX_ulong as_ -#define AS_TYPE_PREFIX_long as_ - -#define AS_TYPE_EXT(type, val, src_type) CAT(CAT(AS_TYPE_PREFIX_, src_type), type)(val) +//#define AS_TYPE_PREFIX_uchar as_ +//#define AS_TYPE_PREFIX_char as_ +//#define AS_TYPE_PREFIX_fp8e5m2_t _as_ +//#define AS_TYPE_PREFIX_fp8e4m3_t _as_ +//#define AS_TYPE_PREFIX_fp8e8m0_t _as_ +//#define AS_TYPE_PREFIX_ushort as_ +//#define AS_TYPE_PREFIX_short as_ +//#define AS_TYPE_PREFIX_half as_ +//#define AS_TYPE_PREFIX_int as_ +//#define AS_TYPE_PREFIX_uint as_ +//#define AS_TYPE_PREFIX_float as_ +//#define AS_TYPE_PREFIX_ulong as_ +//#define AS_TYPE_PREFIX_long as_ + +// #define AS_TYPE_EXT(type, val, src_type) CAT(CAT(AS_TYPE_PREFIX_, src_type), type)(val) #define AS_TYPE(type, val) CAT(as_, type)(val) // ==================================================================================================================== @@ -61,9 +61,9 @@ // ==================================================================================================================== #define TYPE_SIZE_uchar 1 #define TYPE_SIZE_char 1 -#define TYPE_SIZE_fp8e5m2_t 1 -#define TYPE_SIZE_fp8e4m3_t 1 -#define TYPE_SIZE_fp8e8m0_t 1 +// #define TYPE_SIZE_fp8e5m2_t 1 +// #define TYPE_SIZE_fp8e4m3_t 1 +// #define TYPE_SIZE_fp8e8m0_t 1 #define TYPE_SIZE_ushort 2 #define TYPE_SIZE_short 2 #define TYPE_SIZE_half 2 @@ -80,70 +80,70 @@ #define REQD_SUB_GROUP_SIZE(sg_size) #endif -#define DEFINE_VECTOR_CONVERT(target_type, source_type, vector_size) \ -MAKE_VECTOR_TYPE(target_type, vector_size) __attribute__((overloadable)) _convert_##target_type##vector_size(MAKE_VECTOR_TYPE(source_type, vector_size) val) { \ - return CAT(convert_, CAT(target_type, vector_size))(val); \ -} - -#define DEFINE_VECTOR_IDENTITY(type, vector_size) \ -MAKE_VECTOR_TYPE(type, vector_size) __attribute__((overloadable)) _convert_##type##vector_size(MAKE_VECTOR_TYPE(type, vector_size) val) { \ - return val; \ -} - -#define DEFINE_ALL_VECTOR_CONVERTS(target_type, source_type) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 2) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 3) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 4) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 8) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 16) - -#define DEFINE_ALL_VECTOR_IDENTITY(type) \ -DEFINE_VECTOR_IDENTITY(type, 2) \ -DEFINE_VECTOR_IDENTITY(type, 3) \ -DEFINE_VECTOR_IDENTITY(type, 4) \ -DEFINE_VECTOR_IDENTITY(type, 8) \ -DEFINE_VECTOR_IDENTITY(type, 16) - -float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(float val) { return val; } - -DEFINE_ALL_VECTOR_CONVERTS(float, char) -DEFINE_ALL_VECTOR_CONVERTS(float, uchar) -DEFINE_ALL_VECTOR_CONVERTS(float, short) -DEFINE_ALL_VECTOR_CONVERTS(float, ushort) -DEFINE_ALL_VECTOR_CONVERTS(float, int) -DEFINE_ALL_VECTOR_CONVERTS(float, uint) -DEFINE_ALL_VECTOR_CONVERTS(float, long) -DEFINE_ALL_VECTOR_CONVERTS(float, ulong) -DEFINE_ALL_VECTOR_CONVERTS(float, half) -DEFINE_ALL_VECTOR_IDENTITY(float) - -half __attribute__((overloadable)) _convert_half(char val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(uchar val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(short val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(ushort val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(int val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(uint val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(long val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(ulong val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(half val) { return val; } -half __attribute__((overloadable)) _convert_half(float val) { return convert_half(val); } - -DEFINE_ALL_VECTOR_CONVERTS(half, char) -DEFINE_ALL_VECTOR_CONVERTS(half, uchar) -DEFINE_ALL_VECTOR_CONVERTS(half, short) -DEFINE_ALL_VECTOR_CONVERTS(half, ushort) -DEFINE_ALL_VECTOR_CONVERTS(half, int) -DEFINE_ALL_VECTOR_CONVERTS(half, uint) -DEFINE_ALL_VECTOR_CONVERTS(half, long) -DEFINE_ALL_VECTOR_CONVERTS(half, ulong) -DEFINE_ALL_VECTOR_CONVERTS(half, float) -DEFINE_ALL_VECTOR_IDENTITY(half) +// #define DEFINE_VECTOR_CONVERT(target_type, source_type, vector_size) \ +// MAKE_VECTOR_TYPE(target_type, vector_size) __attribute__((overloadable)) _convert_##target_type##vector_size(MAKE_VECTOR_TYPE(source_type, vector_size) val) { \ +// return CAT(convert_, CAT(target_type, vector_size))(val); \ +// } + +// #define DEFINE_VECTOR_IDENTITY(type, vector_size) \ +// MAKE_VECTOR_TYPE(type, vector_size) __attribute__((overloadable)) _convert_##type##vector_size(MAKE_VECTOR_TYPE(type, vector_size) val) { \ +// return val; \ +// } + +// #define DEFINE_ALL_VECTOR_CONVERTS(target_type, source_type) \ +// DEFINE_VECTOR_CONVERT(target_type, source_type, 2) \ +// DEFINE_VECTOR_CONVERT(target_type, source_type, 3) \ +// DEFINE_VECTOR_CONVERT(target_type, source_type, 4) \ +// DEFINE_VECTOR_CONVERT(target_type, source_type, 8) \ +// DEFINE_VECTOR_CONVERT(target_type, source_type, 16) + +// #define DEFINE_ALL_VECTOR_IDENTITY(type) \ +// DEFINE_VECTOR_IDENTITY(type, 2) \ +// DEFINE_VECTOR_IDENTITY(type, 3) \ +// DEFINE_VECTOR_IDENTITY(type, 4) \ +// DEFINE_VECTOR_IDENTITY(type, 8) \ +// DEFINE_VECTOR_IDENTITY(type, 16) + +// float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } +// float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } +// float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } +// float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } +// float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } +// float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } +// float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } +// float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } +// float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } +// float __attribute__((overloadable)) _convert_float(float val) { return val; } + +// DEFINE_ALL_VECTOR_CONVERTS(float, char) +// DEFINE_ALL_VECTOR_CONVERTS(float, uchar) +// DEFINE_ALL_VECTOR_CONVERTS(float, short) +// DEFINE_ALL_VECTOR_CONVERTS(float, ushort) +// DEFINE_ALL_VECTOR_CONVERTS(float, int) +// DEFINE_ALL_VECTOR_CONVERTS(float, uint) +// DEFINE_ALL_VECTOR_CONVERTS(float, long) +// DEFINE_ALL_VECTOR_CONVERTS(float, ulong) +// DEFINE_ALL_VECTOR_CONVERTS(float, half) +// DEFINE_ALL_VECTOR_IDENTITY(float) + +// half __attribute__((overloadable)) _convert_half(char val) { return convert_half(val); } +// half __attribute__((overloadable)) _convert_half(uchar val) { return convert_half(val); } +// half __attribute__((overloadable)) _convert_half(short val) { return convert_half(val); } +// half __attribute__((overloadable)) _convert_half(ushort val) { return convert_half(val); } +// half __attribute__((overloadable)) _convert_half(int val) { return convert_half(val); } +// half __attribute__((overloadable)) _convert_half(uint val) { return convert_half(val); } +// half __attribute__((overloadable)) _convert_half(long val) { return convert_half(val); } +// half __attribute__((overloadable)) _convert_half(ulong val) { return convert_half(val); } +// half __attribute__((overloadable)) _convert_half(half val) { return val; } +// half __attribute__((overloadable)) _convert_half(float val) { return convert_half(val); } + +// DEFINE_ALL_VECTOR_CONVERTS(half, char) +// DEFINE_ALL_VECTOR_CONVERTS(half, uchar) +// DEFINE_ALL_VECTOR_CONVERTS(half, short) +// DEFINE_ALL_VECTOR_CONVERTS(half, ushort) +// DEFINE_ALL_VECTOR_CONVERTS(half, int) +// DEFINE_ALL_VECTOR_CONVERTS(half, uint) +// DEFINE_ALL_VECTOR_CONVERTS(half, long) +// DEFINE_ALL_VECTOR_CONVERTS(half, ulong) +// DEFINE_ALL_VECTOR_CONVERTS(half, float) +// DEFINE_ALL_VECTOR_IDENTITY(half) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl index 051fcf380d21..230cae04aaff 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl @@ -1,1279 +1,1279 @@ -// Copyright (C) 2026 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// +// // Copyright (C) 2026 Intel Corporation +// // SPDX-License-Identifier: Apache-2.0 +// // -// TODO: Replace `_intel_convert*` with bultins when ready, current implementations are copied from XeTLA: +// // TODO: Replace `_intel_convert*` with bultins when ready, current implementations are copied from XeTLA: -uchar _f16_to_bf8_universal(half val, bool is_saturation) { - half val_fp16 = val; - const ushort *p = (ushort *)&val_fp16; - const ushort mant = p[0] & 0x3FF; - const uchar exp_mask = 0x7C; +// uchar _f16_to_bf8_universal(half val, bool is_saturation) { +// half val_fp16 = val; +// const ushort *p = (ushort *)&val_fp16; +// const ushort mant = p[0] & 0x3FF; +// const uchar exp_mask = 0x7C; - uchar ret_tmp = p[0] >> 8; - const bool is_infnan = (ret_tmp & exp_mask) == exp_mask; - if (is_infnan) { - ret_tmp |= (mant != 0); // The bit signifying NaNness may have been cut off - } else { - ret_tmp += (mant & 0x80) && ((mant & 0x7F) || (mant & 0x100)); // RTE - if (is_saturation) { - bool is_overflow = (ret_tmp & exp_mask) == exp_mask; - ret_tmp -= is_overflow; - } - } +// uchar ret_tmp = p[0] >> 8; +// const bool is_infnan = (ret_tmp & exp_mask) == exp_mask; +// if (is_infnan) { +// ret_tmp |= (mant != 0); // The bit signifying NaNness may have been cut off +// } else { +// ret_tmp += (mant & 0x80) && ((mant & 0x7F) || (mant & 0x100)); // RTE +// if (is_saturation) { +// bool is_overflow = (ret_tmp & exp_mask) == exp_mask; +// ret_tmp -= is_overflow; +// } +// } - return ret_tmp; -} +// return ret_tmp; +// } -uchar _intel_convert_f16_to_bf8(half val) { - return _f16_to_bf8_universal(val, false); -} +// uchar _intel_convert_f16_to_bf8(half val) { +// return _f16_to_bf8_universal(val, false); +// } -uchar _intel_convert_f16_to_bf8_sat(half val) { - return _f16_to_bf8_universal(val, true); -} +// uchar _intel_convert_f16_to_bf8_sat(half val) { +// return _f16_to_bf8_universal(val, true); +// } -half _intel_convert_bf8_to_f16(uchar val) { - ushort temp = val; - temp = temp << 0x8; - half temp_fp16 = as_half(temp); - return temp_fp16; -} +// half _intel_convert_bf8_to_f16(uchar val) { +// ushort temp = val; +// temp = temp << 0x8; +// half temp_fp16 = as_half(temp); +// return temp_fp16; +// } -char _f16_to_hf8_universal(half val, bool is_saturation) { - half val_fp16 = val; - ushort *p = (ushort *)&val_fp16; - ushort src = p[0]; - const ushort src_exp_size = 5; - const ushort src_mant_size = 10; - const ushort src_exp_bias = (1 << (src_exp_size - 1)) - 1; - const ushort src_exp_mask = (1 << src_exp_size) - 1; - const ushort src_mant_mask = (1 << src_mant_size) - 1; - const short max_exp_unbiased = 8; - const short min_exp_unbiased = -6; - const short exp_bias = 7; - const ushort exp_size = 4; - const ushort mant_size = 3; - const uchar nan = 0x7f; - const uchar max_val = 0x7e; +// char _f16_to_hf8_universal(half val, bool is_saturation) { +// half val_fp16 = val; +// ushort *p = (ushort *)&val_fp16; +// ushort src = p[0]; +// const ushort src_exp_size = 5; +// const ushort src_mant_size = 10; +// const ushort src_exp_bias = (1 << (src_exp_size - 1)) - 1; +// const ushort src_exp_mask = (1 << src_exp_size) - 1; +// const ushort src_mant_mask = (1 << src_mant_size) - 1; +// const short max_exp_unbiased = 8; +// const short min_exp_unbiased = -6; +// const short exp_bias = 7; +// const ushort exp_size = 4; +// const ushort mant_size = 3; +// const uchar nan = 0x7f; +// const uchar max_val = 0x7e; - ushort src_sign = src >> (src_exp_size + src_mant_size); - ushort src_exp = (src >> src_mant_size) & src_exp_mask; - short src_exp_unbiased = src_exp - src_exp_bias; - ushort src_mant = src & src_mant_mask; +// ushort src_sign = src >> (src_exp_size + src_mant_size); +// ushort src_exp = (src >> src_mant_size) & src_exp_mask; +// short src_exp_unbiased = src_exp - src_exp_bias; +// ushort src_mant = src & src_mant_mask; - bool is_src_inf_nan = src_exp == 0x1f; - bool is_overflow = (src_exp_unbiased > max_exp_unbiased) - || ((src_exp_unbiased == max_exp_unbiased) && (src_mant > 0x0340)); - bool is_zero = (src_exp_unbiased < (min_exp_unbiased - mant_size)); - bool is_denorm = (src_exp_unbiased < min_exp_unbiased) && (!is_zero); +// bool is_src_inf_nan = src_exp == 0x1f; +// bool is_overflow = (src_exp_unbiased > max_exp_unbiased) +// || ((src_exp_unbiased == max_exp_unbiased) && (src_mant > 0x0340)); +// bool is_zero = (src_exp_unbiased < (min_exp_unbiased - mant_size)); +// bool is_denorm = (src_exp_unbiased < min_exp_unbiased) && (!is_zero); - uchar dst_val; - if (is_src_inf_nan) { - dst_val = nan; - } else if (is_overflow) { - dst_val = is_saturation ? max_val : nan; - } else if (is_zero) { - dst_val = 0; - } else if (is_denorm) { - ushort src_m = src_mant | 0x0400; - short shift_out_bit = min_exp_unbiased - src_exp_unbiased; - bool sticky_flag = (src_m & ((1 << shift_out_bit) - 1)) != 0; - src_m = src_m >> shift_out_bit; - ushort tail_size = src_mant_size - mant_size; - sticky_flag - = sticky_flag || ((src_m & ((1 << (tail_size - 1)) - 1)) != 0); - bool lsb_bit = src_m & (1 << tail_size); - bool rnd_bit = src_m & (1 << (tail_size - 1)); - bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); +// uchar dst_val; +// if (is_src_inf_nan) { +// dst_val = nan; +// } else if (is_overflow) { +// dst_val = is_saturation ? max_val : nan; +// } else if (is_zero) { +// dst_val = 0; +// } else if (is_denorm) { +// ushort src_m = src_mant | 0x0400; +// short shift_out_bit = min_exp_unbiased - src_exp_unbiased; +// bool sticky_flag = (src_m & ((1 << shift_out_bit) - 1)) != 0; +// src_m = src_m >> shift_out_bit; +// ushort tail_size = src_mant_size - mant_size; +// sticky_flag +// = sticky_flag || ((src_m & ((1 << (tail_size - 1)) - 1)) != 0); +// bool lsb_bit = src_m & (1 << tail_size); +// bool rnd_bit = src_m & (1 << (tail_size - 1)); +// bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); - dst_val = (src_m >> tail_size) + carry; - } else { - ushort tail_size = src_mant_size - mant_size; - bool sticky_flag = (src_mant & ((1 << (tail_size - 1)) - 1)) != 0; - bool lsb_bit = src_mant & (1 << tail_size); - bool rnd_bit = src_mant & (1 << (tail_size - 1)); - bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); - ushort src_m = (src_mant >> tail_size) + carry; - ushort src_e = src_exp_unbiased + exp_bias; - dst_val = (src_e << mant_size) + src_m; - } - return src_sign << (exp_size + mant_size) | dst_val; -} +// dst_val = (src_m >> tail_size) + carry; +// } else { +// ushort tail_size = src_mant_size - mant_size; +// bool sticky_flag = (src_mant & ((1 << (tail_size - 1)) - 1)) != 0; +// bool lsb_bit = src_mant & (1 << tail_size); +// bool rnd_bit = src_mant & (1 << (tail_size - 1)); +// bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); +// ushort src_m = (src_mant >> tail_size) + carry; +// ushort src_e = src_exp_unbiased + exp_bias; +// dst_val = (src_e << mant_size) + src_m; +// } +// return src_sign << (exp_size + mant_size) | dst_val; +// } -char _intel_convert_f16_to_hf8(half val) { - return _f16_to_hf8_universal(val, false); -} +// char _intel_convert_f16_to_hf8(half val) { +// return _f16_to_hf8_universal(val, false); +// } -char _intel_convert_f16_to_hf8_sat(half val) { - return _f16_to_hf8_universal(val, true); -} +// char _intel_convert_f16_to_hf8_sat(half val) { +// return _f16_to_hf8_universal(val, true); +// } -half _intel_convert_hf8_to_f16(char val) { - const short exp_bias = 7; - char data = val; - ushort sign = data >> 7; - ushort exp = (data >> 3) & 0b1111; - ushort mant = data & 0x07; - ushort dst_val; - if ((exp == 0xf) && (mant == 0x7)) { - dst_val = 0x7fff; - } else if ((exp == 0) && (mant == 0)) { - dst_val = 0; - } else if ((exp == 0) && (mant != 0)) { - ushort lz_count = (mant > 3) ? 0 : ((mant > 1) ? 1 : 2); - ushort dst_exp = exp - exp_bias + 15 - lz_count; - ushort dst_mant = (mant << (lz_count + 1)) & 0x7; - dst_val = (dst_exp << 10) | (dst_mant << 7); - } else { - ushort dst_exp = exp - exp_bias + 15; - dst_val = (dst_exp << 10) | (mant << 7); - } +// half _intel_convert_hf8_to_f16(char val) { +// const short exp_bias = 7; +// char data = val; +// ushort sign = data >> 7; +// ushort exp = (data >> 3) & 0b1111; +// ushort mant = data & 0x07; +// ushort dst_val; +// if ((exp == 0xf) && (mant == 0x7)) { +// dst_val = 0x7fff; +// } else if ((exp == 0) && (mant == 0)) { +// dst_val = 0; +// } else if ((exp == 0) && (mant != 0)) { +// ushort lz_count = (mant > 3) ? 0 : ((mant > 1) ? 1 : 2); +// ushort dst_exp = exp - exp_bias + 15 - lz_count; +// ushort dst_mant = (mant << (lz_count + 1)) & 0x7; +// dst_val = (dst_exp << 10) | (dst_mant << 7); +// } else { +// ushort dst_exp = exp - exp_bias + 15; +// dst_val = (dst_exp << 10) | (mant << 7); +// } - ushort temp = (sign << 15) | dst_val; - half temp_fp16 = as_half(temp); - return temp_fp16; -} +// ushort temp = (sign << 15) | dst_val; +// half temp_fp16 = as_half(temp); +// return temp_fp16; +// } -uchar _intel_convert_f32_fo_e8m0(float val) { - uint val_uint = as_uint(val); - return (uchar)((val_uint >> 23) & 0xFF); -} +// uchar _intel_convert_f32_fo_e8m0(float val) { +// uint val_uint = as_uint(val); +// return (uchar)((val_uint >> 23) & 0xFF); +// } -uchar _intel_convert_f32_fo_e8m0_sat(float val) { - return _intel_convert_f32_fo_e8m0(val); -} +// uchar _intel_convert_f32_fo_e8m0_sat(float val) { +// return _intel_convert_f32_fo_e8m0(val); +// } -float _intel_convert_e8m0_to_f32(uchar val) { - if (val == 0xFF) { - return as_float(0xFFC00000); // NaN - } else if (val == 0) { - return as_float(0x00400000); // 2^(-127) - } +// float _intel_convert_e8m0_to_f32(uchar val) { +// if (val == 0xFF) { +// return as_float(0xFFC00000); // NaN +// } else if (val == 0) { +// return as_float(0x00400000); // 2^(-127) +// } - uint temp = (uint)val; - temp = temp << 23; - float temp_fp32 = as_float(temp); - return temp_fp32; -} +// uint temp = (uint)val; +// temp = temp << 23; +// float temp_fp32 = as_float(temp); +// return temp_fp32; +// } -typedef struct fp8e5m2_t { uchar data; } fp8e5m2_t; // bf8 -typedef struct fp8e5m2_t1 { uchar data; } fp8e5m2_t1; -typedef struct fp8e5m2_t2 { uchar2 data; } fp8e5m2_t2; -typedef struct fp8e5m2_t3 { uchar3 data; } fp8e5m2_t3; -typedef struct fp8e5m2_t4 { uchar4 data; } fp8e5m2_t4; -typedef struct fp8e5m2_t8 { uchar8 data; } fp8e5m2_t8; -typedef struct fp8e5m2_t16 { uchar16 data; } fp8e5m2_t16; +// typedef struct fp8e5m2_t { uchar data; } fp8e5m2_t; // bf8 +// typedef struct fp8e5m2_t1 { uchar data; } fp8e5m2_t1; +// typedef struct fp8e5m2_t2 { uchar2 data; } fp8e5m2_t2; +// typedef struct fp8e5m2_t3 { uchar3 data; } fp8e5m2_t3; +// typedef struct fp8e5m2_t4 { uchar4 data; } fp8e5m2_t4; +// typedef struct fp8e5m2_t8 { uchar8 data; } fp8e5m2_t8; +// typedef struct fp8e5m2_t16 { uchar16 data; } fp8e5m2_t16; -typedef struct fp8e4m3_t { char data; } fp8e4m3_t; // hf8 -typedef struct fp8e4m3_t1 { char data; } fp8e4m3_t1; -typedef struct fp8e4m3_t2 { char2 data; } fp8e4m3_t2; -typedef struct fp8e4m3_t3 { char3 data; } fp8e4m3_t3; -typedef struct fp8e4m3_t4 { char4 data; } fp8e4m3_t4; -typedef struct fp8e4m3_t8 { char8 data; } fp8e4m3_t8; -typedef struct fp8e4m3_t16 { char16 data; } fp8e4m3_t16; +// typedef struct fp8e4m3_t { char data; } fp8e4m3_t; // hf8 +// typedef struct fp8e4m3_t1 { char data; } fp8e4m3_t1; +// typedef struct fp8e4m3_t2 { char2 data; } fp8e4m3_t2; +// typedef struct fp8e4m3_t3 { char3 data; } fp8e4m3_t3; +// typedef struct fp8e4m3_t4 { char4 data; } fp8e4m3_t4; +// typedef struct fp8e4m3_t8 { char8 data; } fp8e4m3_t8; +// typedef struct fp8e4m3_t16 { char16 data; } fp8e4m3_t16; -typedef struct fp8e8m0_t { uchar data; } fp8e8m0_t; // e8m0 -typedef struct fp8e8m0_t1 { uchar data; } fp8e8m0_t1; -typedef struct fp8e8m0_t2 { uchar2 data; } fp8e8m0_t2; -typedef struct fp8e8m0_t3 { uchar3 data; } fp8e8m0_t3; -typedef struct fp8e8m0_t4 { uchar4 data; } fp8e8m0_t4; -typedef struct fp8e8m0_t8 { uchar8 data; } fp8e8m0_t8; -typedef struct fp8e8m0_t16 { uchar16 data; } fp8e8m0_t16; +// typedef struct fp8e8m0_t { uchar data; } fp8e8m0_t; // e8m0 +// typedef struct fp8e8m0_t1 { uchar data; } fp8e8m0_t1; +// typedef struct fp8e8m0_t2 { uchar2 data; } fp8e8m0_t2; +// typedef struct fp8e8m0_t3 { uchar3 data; } fp8e8m0_t3; +// typedef struct fp8e8m0_t4 { uchar4 data; } fp8e8m0_t4; +// typedef struct fp8e8m0_t8 { uchar8 data; } fp8e8m0_t8; +// typedef struct fp8e8m0_t16 { uchar16 data; } fp8e8m0_t16; -half __attribute__((overloadable)) _convert_half(fp8e5m2_t val) { - return _intel_convert_bf8_to_f16(val.data); -} -half __attribute__((overloadable)) _convert_half(fp8e5m2_t1 val) { - return _intel_convert_bf8_to_f16(val.data); -} -half2 __attribute__((overloadable)) _convert_half2(fp8e5m2_t2 val) { - return (half2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); -} -half3 __attribute__((overloadable)) _convert_half3(fp8e5m2_t3 val) { - return (half3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); -} -half4 __attribute__((overloadable)) _convert_half4(fp8e5m2_t4 val) { - return (half4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), - _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); -} -half8 __attribute__((overloadable)) _convert_half8(fp8e5m2_t8 val) { - return (half8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), - _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), - _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), - _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); -} -half16 __attribute__((overloadable)) _convert_half16(fp8e5m2_t16 val) { - return (half16)( - _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), - _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), - _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), - _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), - _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), - _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), - _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), - _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) - ); -} +// half __attribute__((overloadable)) _convert_half(fp8e5m2_t val) { +// return _intel_convert_bf8_to_f16(val.data); +// } +// half __attribute__((overloadable)) _convert_half(fp8e5m2_t1 val) { +// return _intel_convert_bf8_to_f16(val.data); +// } +// half2 __attribute__((overloadable)) _convert_half2(fp8e5m2_t2 val) { +// return (half2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); +// } +// half3 __attribute__((overloadable)) _convert_half3(fp8e5m2_t3 val) { +// return (half3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); +// } +// half4 __attribute__((overloadable)) _convert_half4(fp8e5m2_t4 val) { +// return (half4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), +// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); +// } +// half8 __attribute__((overloadable)) _convert_half8(fp8e5m2_t8 val) { +// return (half8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), +// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), +// _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), +// _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); +// } +// half16 __attribute__((overloadable)) _convert_half16(fp8e5m2_t16 val) { +// return (half16)( +// _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), +// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), +// _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), +// _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), +// _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), +// _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), +// _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), +// _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) +// ); +// } -float __attribute__((overloadable)) _convert_float(fp8e5m2_t val) { - return (float)_intel_convert_bf8_to_f16(val.data); -} -float __attribute__((overloadable)) _convert_float(fp8e5m2_t1 val) { - return (float)_intel_convert_bf8_to_f16(val.data); -} -float2 __attribute__((overloadable)) _convert_float2(fp8e5m2_t2 val) { - return (float2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); -} -float3 __attribute__((overloadable)) _convert_float3(fp8e5m2_t3 val) { - return (float3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); -} -float4 __attribute__((overloadable)) _convert_float4(fp8e5m2_t4 val) { - return (float4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), - _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); -} -float8 __attribute__((overloadable)) _convert_float8(fp8e5m2_t8 val) { - return (float8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), - _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), - _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), - _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); -} -float16 __attribute__((overloadable)) _convert_float16(fp8e5m2_t16 val) { - return (float16)( - _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), - _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), - _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), - _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), - _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), - _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), - _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), - _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) - ); -} +// float __attribute__((overloadable)) _convert_float(fp8e5m2_t val) { +// return (float)_intel_convert_bf8_to_f16(val.data); +// } +// float __attribute__((overloadable)) _convert_float(fp8e5m2_t1 val) { +// return (float)_intel_convert_bf8_to_f16(val.data); +// } +// float2 __attribute__((overloadable)) _convert_float2(fp8e5m2_t2 val) { +// return (float2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); +// } +// float3 __attribute__((overloadable)) _convert_float3(fp8e5m2_t3 val) { +// return (float3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); +// } +// float4 __attribute__((overloadable)) _convert_float4(fp8e5m2_t4 val) { +// return (float4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), +// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); +// } +// float8 __attribute__((overloadable)) _convert_float8(fp8e5m2_t8 val) { +// return (float8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), +// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), +// _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), +// _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); +// } +// float16 __attribute__((overloadable)) _convert_float16(fp8e5m2_t16 val) { +// return (float16)( +// _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), +// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), +// _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), +// _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), +// _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), +// _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), +// _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), +// _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) +// ); +// } -half __attribute__((overloadable)) _convert_half(fp8e4m3_t val) { - return _intel_convert_hf8_to_f16(val.data); -} -half __attribute__((overloadable)) _convert_half(fp8e4m3_t1 val) { - return _intel_convert_hf8_to_f16(val.data); -} -half2 __attribute__((overloadable)) _convert_half2(fp8e4m3_t2 val) { - return (half2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); -} -half3 __attribute__((overloadable)) _convert_half3(fp8e4m3_t3 val) { - return (half3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); -} -half4 __attribute__((overloadable)) _convert_half4(fp8e4m3_t4 val) { - return (half4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), - _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); -} -half8 __attribute__((overloadable)) _convert_half8(fp8e4m3_t8 val) { - return (half8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), - _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), - _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), - _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); -} -half16 __attribute__((overloadable)) _convert_half16(fp8e4m3_t16 val) { - return (half16)( - _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), - _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), - _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), - _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), - _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), - _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), - _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), - _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) - ); -} +// half __attribute__((overloadable)) _convert_half(fp8e4m3_t val) { +// return _intel_convert_hf8_to_f16(val.data); +// } +// half __attribute__((overloadable)) _convert_half(fp8e4m3_t1 val) { +// return _intel_convert_hf8_to_f16(val.data); +// } +// half2 __attribute__((overloadable)) _convert_half2(fp8e4m3_t2 val) { +// return (half2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); +// } +// half3 __attribute__((overloadable)) _convert_half3(fp8e4m3_t3 val) { +// return (half3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); +// } +// half4 __attribute__((overloadable)) _convert_half4(fp8e4m3_t4 val) { +// return (half4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), +// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); +// } +// half8 __attribute__((overloadable)) _convert_half8(fp8e4m3_t8 val) { +// return (half8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), +// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), +// _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), +// _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); +// } +// half16 __attribute__((overloadable)) _convert_half16(fp8e4m3_t16 val) { +// return (half16)( +// _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), +// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), +// _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), +// _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), +// _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), +// _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), +// _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), +// _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) +// ); +// } -float __attribute__((overloadable)) _convert_float(fp8e4m3_t val) { - return (float)_intel_convert_hf8_to_f16(val.data); -} -float __attribute__((overloadable)) _convert_float(fp8e4m3_t1 val) { - return (float)_intel_convert_hf8_to_f16(val.data); -} -float2 __attribute__((overloadable)) _convert_float2(fp8e4m3_t2 val) { - return (float2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); -} -float3 __attribute__((overloadable)) _convert_float3(fp8e4m3_t3 val) { - return (float3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); -} -float4 __attribute__((overloadable)) _convert_float4(fp8e4m3_t4 val) { - return (float4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), - _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); -} -float8 __attribute__((overloadable)) _convert_float8(fp8e4m3_t8 val) { - return (float8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), - _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), - _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), - _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); -} -float16 __attribute__((overloadable)) _convert_float16(fp8e4m3_t16 val) { - return (float16)( - _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), - _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), - _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), - _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), - _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), - _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), - _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), - _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) - ); -} +// float __attribute__((overloadable)) _convert_float(fp8e4m3_t val) { +// return (float)_intel_convert_hf8_to_f16(val.data); +// } +// float __attribute__((overloadable)) _convert_float(fp8e4m3_t1 val) { +// return (float)_intel_convert_hf8_to_f16(val.data); +// } +// float2 __attribute__((overloadable)) _convert_float2(fp8e4m3_t2 val) { +// return (float2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); +// } +// float3 __attribute__((overloadable)) _convert_float3(fp8e4m3_t3 val) { +// return (float3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); +// } +// float4 __attribute__((overloadable)) _convert_float4(fp8e4m3_t4 val) { +// return (float4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), +// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); +// } +// float8 __attribute__((overloadable)) _convert_float8(fp8e4m3_t8 val) { +// return (float8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), +// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), +// _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), +// _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); +// } +// float16 __attribute__((overloadable)) _convert_float16(fp8e4m3_t16 val) { +// return (float16)( +// _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), +// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), +// _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), +// _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), +// _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), +// _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), +// _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), +// _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) +// ); +// } -float __attribute__((overloadable)) _convert_float(fp8e8m0_t val) { - return (float)_intel_convert_e8m0_to_f32(val.data); -} -float __attribute__((overloadable)) _convert_float(fp8e8m0_t1 val) { - return (float)_intel_convert_e8m0_to_f32(val.data); -} -float2 __attribute__((overloadable)) _convert_float2(fp8e8m0_t2 val) { - return (float2)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1)); -} -float3 __attribute__((overloadable)) _convert_float3(fp8e8m0_t3 val) { - return (float3)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), _intel_convert_e8m0_to_f32(val.data.s2)); -} -float4 __attribute__((overloadable)) _convert_float4(fp8e8m0_t4 val) { - return (float4)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), - _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3)); -} -float8 __attribute__((overloadable)) _convert_float8(fp8e8m0_t8 val) { - return (float8)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), - _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), - _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), - _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7)); -} -float16 __attribute__((overloadable)) _convert_float16(fp8e8m0_t16 val) { - return (float16)( - _intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), - _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), - _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), - _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7), - _intel_convert_e8m0_to_f32(val.data.s8), _intel_convert_e8m0_to_f32(val.data.s9), - _intel_convert_e8m0_to_f32(val.data.sA), _intel_convert_e8m0_to_f32(val.data.sB), - _intel_convert_e8m0_to_f32(val.data.sC), _intel_convert_e8m0_to_f32(val.data.sD), - _intel_convert_e8m0_to_f32(val.data.sE), _intel_convert_e8m0_to_f32(val.data.sF) - ); -} +// float __attribute__((overloadable)) _convert_float(fp8e8m0_t val) { +// return (float)_intel_convert_e8m0_to_f32(val.data); +// } +// float __attribute__((overloadable)) _convert_float(fp8e8m0_t1 val) { +// return (float)_intel_convert_e8m0_to_f32(val.data); +// } +// float2 __attribute__((overloadable)) _convert_float2(fp8e8m0_t2 val) { +// return (float2)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1)); +// } +// float3 __attribute__((overloadable)) _convert_float3(fp8e8m0_t3 val) { +// return (float3)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), _intel_convert_e8m0_to_f32(val.data.s2)); +// } +// float4 __attribute__((overloadable)) _convert_float4(fp8e8m0_t4 val) { +// return (float4)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), +// _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3)); +// } +// float8 __attribute__((overloadable)) _convert_float8(fp8e8m0_t8 val) { +// return (float8)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), +// _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), +// _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), +// _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7)); +// } +// float16 __attribute__((overloadable)) _convert_float16(fp8e8m0_t16 val) { +// return (float16)( +// _intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), +// _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), +// _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), +// _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7), +// _intel_convert_e8m0_to_f32(val.data.s8), _intel_convert_e8m0_to_f32(val.data.s9), +// _intel_convert_e8m0_to_f32(val.data.sA), _intel_convert_e8m0_to_f32(val.data.sB), +// _intel_convert_e8m0_to_f32(val.data.sC), _intel_convert_e8m0_to_f32(val.data.sD), +// _intel_convert_e8m0_to_f32(val.data.sE), _intel_convert_e8m0_to_f32(val.data.sF) +// ); +// } -fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(float val) { - fp8e5m2_t res; - res.data = _intel_convert_f16_to_bf8((half)val); - return res; -} -fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(float val[1]) { - fp8e5m2_t1 res; - res.data = _intel_convert_f16_to_bf8((half)val[0]); - return res; -} -fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(float2 val) { - fp8e5m2_t2 res; - res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); - res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); - return res; -} -fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(float3 val) { - fp8e5m2_t3 res; - res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); - res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); - res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); - return res; -} -fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(float4 val) { - fp8e5m2_t4 res; - res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); - res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); - res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); - res.data.s3 = _intel_convert_f16_to_bf8((half)val.w); - return res; -} -fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(float8 val) { - fp8e5m2_t8 res; - res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); - res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); - res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); - res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); - res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); - res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); - res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); - res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); - return res; -} -fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(float16 val) { - fp8e5m2_t16 res; - res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); - res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); - res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); - res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); - res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); - res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); - res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); - res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); - res.data.s8 = _intel_convert_f16_to_bf8((half)val.s8); - res.data.s9 = _intel_convert_f16_to_bf8((half)val.s9); - res.data.sA = _intel_convert_f16_to_bf8((half)val.sA); - res.data.sB = _intel_convert_f16_to_bf8((half)val.sB); - res.data.sC = _intel_convert_f16_to_bf8((half)val.sC); - res.data.sD = _intel_convert_f16_to_bf8((half)val.sD); - res.data.sE = _intel_convert_f16_to_bf8((half)val.sE); - res.data.sF = _intel_convert_f16_to_bf8((half)val.sF); - return res; -} +// fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(float val) { +// fp8e5m2_t res; +// res.data = _intel_convert_f16_to_bf8((half)val); +// return res; +// } +// fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(float val[1]) { +// fp8e5m2_t1 res; +// res.data = _intel_convert_f16_to_bf8((half)val[0]); +// return res; +// } +// fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(float2 val) { +// fp8e5m2_t2 res; +// res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); +// res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); +// return res; +// } +// fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(float3 val) { +// fp8e5m2_t3 res; +// res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); +// res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); +// res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); +// return res; +// } +// fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(float4 val) { +// fp8e5m2_t4 res; +// res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); +// res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); +// res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); +// res.data.s3 = _intel_convert_f16_to_bf8((half)val.w); +// return res; +// } +// fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(float8 val) { +// fp8e5m2_t8 res; +// res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); +// res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); +// res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); +// res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); +// res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); +// res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); +// res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); +// res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); +// return res; +// } +// fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(float16 val) { +// fp8e5m2_t16 res; +// res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); +// res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); +// res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); +// res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); +// res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); +// res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); +// res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); +// res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); +// res.data.s8 = _intel_convert_f16_to_bf8((half)val.s8); +// res.data.s9 = _intel_convert_f16_to_bf8((half)val.s9); +// res.data.sA = _intel_convert_f16_to_bf8((half)val.sA); +// res.data.sB = _intel_convert_f16_to_bf8((half)val.sB); +// res.data.sC = _intel_convert_f16_to_bf8((half)val.sC); +// res.data.sD = _intel_convert_f16_to_bf8((half)val.sD); +// res.data.sE = _intel_convert_f16_to_bf8((half)val.sE); +// res.data.sF = _intel_convert_f16_to_bf8((half)val.sF); +// return res; +// } -fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(half val) { - fp8e5m2_t res; - res.data = _intel_convert_f16_to_bf8(val); - return res; -} -fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(half val[1]) { - fp8e5m2_t1 res; - res.data = _intel_convert_f16_to_bf8(val[0]); - return res; -} -fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(half2 val) { - fp8e5m2_t2 res; - res.data.s0 = _intel_convert_f16_to_bf8(val.x); - res.data.s1 = _intel_convert_f16_to_bf8(val.y); - return res; -} -fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(half3 val) { - fp8e5m2_t3 res; - res.data.s0 = _intel_convert_f16_to_bf8(val.x); - res.data.s1 = _intel_convert_f16_to_bf8(val.y); - res.data.s2 = _intel_convert_f16_to_bf8(val.z); - return res; -} -fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(half4 val) { - fp8e5m2_t4 res; - res.data.s0 = _intel_convert_f16_to_bf8(val.x); - res.data.s1 = _intel_convert_f16_to_bf8(val.y); - res.data.s2 = _intel_convert_f16_to_bf8(val.z); - res.data.s3 = _intel_convert_f16_to_bf8(val.w); - return res; -} -fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(half8 val) { - fp8e5m2_t8 res; - res.data.s0 = _intel_convert_f16_to_bf8(val.s0); - res.data.s1 = _intel_convert_f16_to_bf8(val.s1); - res.data.s2 = _intel_convert_f16_to_bf8(val.s2); - res.data.s3 = _intel_convert_f16_to_bf8(val.s3); - res.data.s4 = _intel_convert_f16_to_bf8(val.s4); - res.data.s5 = _intel_convert_f16_to_bf8(val.s5); - res.data.s6 = _intel_convert_f16_to_bf8(val.s6); - res.data.s7 = _intel_convert_f16_to_bf8(val.s7); - return res; -} -fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(half16 val) { - fp8e5m2_t16 res; - res.data.s0 = _intel_convert_f16_to_bf8(val.s0); - res.data.s1 = _intel_convert_f16_to_bf8(val.s1); - res.data.s2 = _intel_convert_f16_to_bf8(val.s2); - res.data.s3 = _intel_convert_f16_to_bf8(val.s3); - res.data.s4 = _intel_convert_f16_to_bf8(val.s4); - res.data.s5 = _intel_convert_f16_to_bf8(val.s5); - res.data.s6 = _intel_convert_f16_to_bf8(val.s6); - res.data.s7 = _intel_convert_f16_to_bf8(val.s7); - res.data.s8 = _intel_convert_f16_to_bf8(val.s8); - res.data.s9 = _intel_convert_f16_to_bf8(val.s9); - res.data.sA = _intel_convert_f16_to_bf8(val.sA); - res.data.sB = _intel_convert_f16_to_bf8(val.sB); - res.data.sC = _intel_convert_f16_to_bf8(val.sC); - res.data.sD = _intel_convert_f16_to_bf8(val.sD); - res.data.sE = _intel_convert_f16_to_bf8(val.sE); - res.data.sF = _intel_convert_f16_to_bf8(val.sF); - return res; -} +// fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(half val) { +// fp8e5m2_t res; +// res.data = _intel_convert_f16_to_bf8(val); +// return res; +// } +// fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(half val[1]) { +// fp8e5m2_t1 res; +// res.data = _intel_convert_f16_to_bf8(val[0]); +// return res; +// } +// fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(half2 val) { +// fp8e5m2_t2 res; +// res.data.s0 = _intel_convert_f16_to_bf8(val.x); +// res.data.s1 = _intel_convert_f16_to_bf8(val.y); +// return res; +// } +// fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(half3 val) { +// fp8e5m2_t3 res; +// res.data.s0 = _intel_convert_f16_to_bf8(val.x); +// res.data.s1 = _intel_convert_f16_to_bf8(val.y); +// res.data.s2 = _intel_convert_f16_to_bf8(val.z); +// return res; +// } +// fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(half4 val) { +// fp8e5m2_t4 res; +// res.data.s0 = _intel_convert_f16_to_bf8(val.x); +// res.data.s1 = _intel_convert_f16_to_bf8(val.y); +// res.data.s2 = _intel_convert_f16_to_bf8(val.z); +// res.data.s3 = _intel_convert_f16_to_bf8(val.w); +// return res; +// } +// fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(half8 val) { +// fp8e5m2_t8 res; +// res.data.s0 = _intel_convert_f16_to_bf8(val.s0); +// res.data.s1 = _intel_convert_f16_to_bf8(val.s1); +// res.data.s2 = _intel_convert_f16_to_bf8(val.s2); +// res.data.s3 = _intel_convert_f16_to_bf8(val.s3); +// res.data.s4 = _intel_convert_f16_to_bf8(val.s4); +// res.data.s5 = _intel_convert_f16_to_bf8(val.s5); +// res.data.s6 = _intel_convert_f16_to_bf8(val.s6); +// res.data.s7 = _intel_convert_f16_to_bf8(val.s7); +// return res; +// } +// fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(half16 val) { +// fp8e5m2_t16 res; +// res.data.s0 = _intel_convert_f16_to_bf8(val.s0); +// res.data.s1 = _intel_convert_f16_to_bf8(val.s1); +// res.data.s2 = _intel_convert_f16_to_bf8(val.s2); +// res.data.s3 = _intel_convert_f16_to_bf8(val.s3); +// res.data.s4 = _intel_convert_f16_to_bf8(val.s4); +// res.data.s5 = _intel_convert_f16_to_bf8(val.s5); +// res.data.s6 = _intel_convert_f16_to_bf8(val.s6); +// res.data.s7 = _intel_convert_f16_to_bf8(val.s7); +// res.data.s8 = _intel_convert_f16_to_bf8(val.s8); +// res.data.s9 = _intel_convert_f16_to_bf8(val.s9); +// res.data.sA = _intel_convert_f16_to_bf8(val.sA); +// res.data.sB = _intel_convert_f16_to_bf8(val.sB); +// res.data.sC = _intel_convert_f16_to_bf8(val.sC); +// res.data.sD = _intel_convert_f16_to_bf8(val.sD); +// res.data.sE = _intel_convert_f16_to_bf8(val.sE); +// res.data.sF = _intel_convert_f16_to_bf8(val.sF); +// return res; +// } -fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(float val) { - fp8e5m2_t res; - res.data = _intel_convert_f16_to_bf8_sat((half)val); - return res; -} -fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(float val[1]) { - fp8e5m2_t1 res; - res.data = _intel_convert_f16_to_bf8_sat((half)val[0]); - return res; -} -fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(float2 val) { - fp8e5m2_t2 res; - res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); - res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); - return res; -} -fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(float3 val) { - fp8e5m2_t3 res; - res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); - res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); - res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); - return res; -} -fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(float4 val) { - fp8e5m2_t4 res; - res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); - res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); - res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); - res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.w); - return res; -} -fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(float8 val) { - fp8e5m2_t8 res; - res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); - res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); - res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); - res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); - res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); - res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); - res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); - res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); - return res; -} -fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(float16 val) { - fp8e5m2_t16 res; - res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); - res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); - res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); - res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); - res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); - res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); - res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); - res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); - res.data.s8 = _intel_convert_f16_to_bf8_sat((half)val.s8); - res.data.s9 = _intel_convert_f16_to_bf8_sat((half)val.s9); - res.data.sA = _intel_convert_f16_to_bf8_sat((half)val.sA); - res.data.sB = _intel_convert_f16_to_bf8_sat((half)val.sB); - res.data.sC = _intel_convert_f16_to_bf8_sat((half)val.sC); - res.data.sD = _intel_convert_f16_to_bf8_sat((half)val.sD); - res.data.sE = _intel_convert_f16_to_bf8_sat((half)val.sE); - res.data.sF = _intel_convert_f16_to_bf8_sat((half)val.sF); - return res; -} +// fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(float val) { +// fp8e5m2_t res; +// res.data = _intel_convert_f16_to_bf8_sat((half)val); +// return res; +// } +// fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(float val[1]) { +// fp8e5m2_t1 res; +// res.data = _intel_convert_f16_to_bf8_sat((half)val[0]); +// return res; +// } +// fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(float2 val) { +// fp8e5m2_t2 res; +// res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); +// res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); +// return res; +// } +// fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(float3 val) { +// fp8e5m2_t3 res; +// res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); +// res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); +// res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); +// return res; +// } +// fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(float4 val) { +// fp8e5m2_t4 res; +// res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); +// res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); +// res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); +// res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.w); +// return res; +// } +// fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(float8 val) { +// fp8e5m2_t8 res; +// res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); +// res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); +// res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); +// res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); +// res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); +// res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); +// res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); +// res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); +// return res; +// } +// fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(float16 val) { +// fp8e5m2_t16 res; +// res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); +// res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); +// res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); +// res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); +// res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); +// res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); +// res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); +// res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); +// res.data.s8 = _intel_convert_f16_to_bf8_sat((half)val.s8); +// res.data.s9 = _intel_convert_f16_to_bf8_sat((half)val.s9); +// res.data.sA = _intel_convert_f16_to_bf8_sat((half)val.sA); +// res.data.sB = _intel_convert_f16_to_bf8_sat((half)val.sB); +// res.data.sC = _intel_convert_f16_to_bf8_sat((half)val.sC); +// res.data.sD = _intel_convert_f16_to_bf8_sat((half)val.sD); +// res.data.sE = _intel_convert_f16_to_bf8_sat((half)val.sE); +// res.data.sF = _intel_convert_f16_to_bf8_sat((half)val.sF); +// return res; +// } -fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(half val) { - fp8e5m2_t res; - res.data = _intel_convert_f16_to_bf8_sat(val); - return res; -} -fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(half val[1]) { - fp8e5m2_t1 res; - res.data = _intel_convert_f16_to_bf8_sat(val[0]); - return res; -} -fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(half2 val) { - fp8e5m2_t2 res; - res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); - res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); - return res; -} -fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(half3 val) { - fp8e5m2_t3 res; - res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); - res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); - res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); - return res; -} -fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(half4 val) { - fp8e5m2_t4 res; - res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); - res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); - res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); - res.data.s3 = _intel_convert_f16_to_bf8_sat(val.w); - return res; -} -fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(half8 val) { - fp8e5m2_t8 res; - res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); - res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); - res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); - res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); - res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); - res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); - res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); - res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); - return res; -} -fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(half16 val) { - fp8e5m2_t16 res; - res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); - res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); - res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); - res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); - res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); - res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); - res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); - res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); - res.data.s8 = _intel_convert_f16_to_bf8_sat(val.s8); - res.data.s9 = _intel_convert_f16_to_bf8_sat(val.s9); - res.data.sA = _intel_convert_f16_to_bf8_sat(val.sA); - res.data.sB = _intel_convert_f16_to_bf8_sat(val.sB); - res.data.sC = _intel_convert_f16_to_bf8_sat(val.sC); - res.data.sD = _intel_convert_f16_to_bf8_sat(val.sD); - res.data.sE = _intel_convert_f16_to_bf8_sat(val.sE); - res.data.sF = _intel_convert_f16_to_bf8_sat(val.sF); - return res; -} +// fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(half val) { +// fp8e5m2_t res; +// res.data = _intel_convert_f16_to_bf8_sat(val); +// return res; +// } +// fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(half val[1]) { +// fp8e5m2_t1 res; +// res.data = _intel_convert_f16_to_bf8_sat(val[0]); +// return res; +// } +// fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(half2 val) { +// fp8e5m2_t2 res; +// res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); +// res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); +// return res; +// } +// fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(half3 val) { +// fp8e5m2_t3 res; +// res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); +// res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); +// res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); +// return res; +// } +// fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(half4 val) { +// fp8e5m2_t4 res; +// res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); +// res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); +// res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); +// res.data.s3 = _intel_convert_f16_to_bf8_sat(val.w); +// return res; +// } +// fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(half8 val) { +// fp8e5m2_t8 res; +// res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); +// res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); +// res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); +// res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); +// res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); +// res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); +// res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); +// res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); +// return res; +// } +// fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(half16 val) { +// fp8e5m2_t16 res; +// res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); +// res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); +// res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); +// res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); +// res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); +// res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); +// res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); +// res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); +// res.data.s8 = _intel_convert_f16_to_bf8_sat(val.s8); +// res.data.s9 = _intel_convert_f16_to_bf8_sat(val.s9); +// res.data.sA = _intel_convert_f16_to_bf8_sat(val.sA); +// res.data.sB = _intel_convert_f16_to_bf8_sat(val.sB); +// res.data.sC = _intel_convert_f16_to_bf8_sat(val.sC); +// res.data.sD = _intel_convert_f16_to_bf8_sat(val.sD); +// res.data.sE = _intel_convert_f16_to_bf8_sat(val.sE); +// res.data.sF = _intel_convert_f16_to_bf8_sat(val.sF); +// return res; +// } -fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(float val) { - fp8e4m3_t res; - res.data = _intel_convert_f16_to_hf8((half)val); - return res; -} -fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(float val[1]) { - fp8e4m3_t1 res; - res.data = _intel_convert_f16_to_hf8((half)val[0]); - return res; -} -fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(float2 val) { - fp8e4m3_t2 res; - res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); - res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); - return res; -} -fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(float3 val) { - fp8e4m3_t3 res; - res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); - res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); - res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); - return res; -} -fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(float4 val) { - fp8e4m3_t4 res; - res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); - res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); - res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); - res.data.s3 = _intel_convert_f16_to_hf8((half)val.w); - return res; -} -fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(float8 val) { - fp8e4m3_t8 res; - res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); - res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); - res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); - res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); - res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); - res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); - res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); - res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); - return res; -} -fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(float16 val) { - fp8e4m3_t16 res; - res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); - res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); - res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); - res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); - res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); - res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); - res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); - res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); - res.data.s8 = _intel_convert_f16_to_hf8((half)val.s8); - res.data.s9 = _intel_convert_f16_to_hf8((half)val.s9); - res.data.sA = _intel_convert_f16_to_hf8((half)val.sA); - res.data.sB = _intel_convert_f16_to_hf8((half)val.sB); - res.data.sC = _intel_convert_f16_to_hf8((half)val.sC); - res.data.sD = _intel_convert_f16_to_hf8((half)val.sD); - res.data.sE = _intel_convert_f16_to_hf8((half)val.sE); - res.data.sF = _intel_convert_f16_to_hf8((half)val.sF); - return res; -} +// fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(float val) { +// fp8e4m3_t res; +// res.data = _intel_convert_f16_to_hf8((half)val); +// return res; +// } +// fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(float val[1]) { +// fp8e4m3_t1 res; +// res.data = _intel_convert_f16_to_hf8((half)val[0]); +// return res; +// } +// fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(float2 val) { +// fp8e4m3_t2 res; +// res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); +// res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); +// return res; +// } +// fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(float3 val) { +// fp8e4m3_t3 res; +// res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); +// res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); +// res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); +// return res; +// } +// fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(float4 val) { +// fp8e4m3_t4 res; +// res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); +// res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); +// res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); +// res.data.s3 = _intel_convert_f16_to_hf8((half)val.w); +// return res; +// } +// fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(float8 val) { +// fp8e4m3_t8 res; +// res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); +// res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); +// res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); +// res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); +// res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); +// res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); +// res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); +// res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); +// return res; +// } +// fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(float16 val) { +// fp8e4m3_t16 res; +// res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); +// res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); +// res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); +// res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); +// res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); +// res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); +// res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); +// res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); +// res.data.s8 = _intel_convert_f16_to_hf8((half)val.s8); +// res.data.s9 = _intel_convert_f16_to_hf8((half)val.s9); +// res.data.sA = _intel_convert_f16_to_hf8((half)val.sA); +// res.data.sB = _intel_convert_f16_to_hf8((half)val.sB); +// res.data.sC = _intel_convert_f16_to_hf8((half)val.sC); +// res.data.sD = _intel_convert_f16_to_hf8((half)val.sD); +// res.data.sE = _intel_convert_f16_to_hf8((half)val.sE); +// res.data.sF = _intel_convert_f16_to_hf8((half)val.sF); +// return res; +// } -fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(half val) { - fp8e4m3_t res; - res.data = _intel_convert_f16_to_hf8(val); - return res; -} -fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(half val[1]) { - fp8e4m3_t1 res; - res.data = _intel_convert_f16_to_hf8(val[0]); - return res; -} -fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(half2 val) { - fp8e4m3_t2 res; - res.data.s0 = _intel_convert_f16_to_hf8(val.x); - res.data.s1 = _intel_convert_f16_to_hf8(val.y); - return res; -} -fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(half3 val) { - fp8e4m3_t3 res; - res.data.s0 = _intel_convert_f16_to_hf8(val.x); - res.data.s1 = _intel_convert_f16_to_hf8(val.y); - res.data.s2 = _intel_convert_f16_to_hf8(val.z); - return res; -} -fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(half4 val) { - fp8e4m3_t4 res; - res.data.s0 = _intel_convert_f16_to_hf8(val.x); - res.data.s1 = _intel_convert_f16_to_hf8(val.y); - res.data.s2 = _intel_convert_f16_to_hf8(val.z); - res.data.s3 = _intel_convert_f16_to_hf8(val.w); - return res; -} -fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(half8 val) { - fp8e4m3_t8 res; - res.data.s0 = _intel_convert_f16_to_hf8(val.s0); - res.data.s1 = _intel_convert_f16_to_hf8(val.s1); - res.data.s2 = _intel_convert_f16_to_hf8(val.s2); - res.data.s3 = _intel_convert_f16_to_hf8(val.s3); - res.data.s4 = _intel_convert_f16_to_hf8(val.s4); - res.data.s5 = _intel_convert_f16_to_hf8(val.s5); - res.data.s6 = _intel_convert_f16_to_hf8(val.s6); - res.data.s7 = _intel_convert_f16_to_hf8(val.s7); - return res; -} -fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(half16 val) { - fp8e4m3_t16 res; - res.data.s0 = _intel_convert_f16_to_hf8(val.s0); - res.data.s1 = _intel_convert_f16_to_hf8(val.s1); - res.data.s2 = _intel_convert_f16_to_hf8(val.s2); - res.data.s3 = _intel_convert_f16_to_hf8(val.s3); - res.data.s4 = _intel_convert_f16_to_hf8(val.s4); - res.data.s5 = _intel_convert_f16_to_hf8(val.s5); - res.data.s6 = _intel_convert_f16_to_hf8(val.s6); - res.data.s7 = _intel_convert_f16_to_hf8(val.s7); - res.data.s8 = _intel_convert_f16_to_hf8(val.s8); - res.data.s9 = _intel_convert_f16_to_hf8(val.s9); - res.data.sA = _intel_convert_f16_to_hf8(val.sA); - res.data.sB = _intel_convert_f16_to_hf8(val.sB); - res.data.sC = _intel_convert_f16_to_hf8(val.sC); - res.data.sD = _intel_convert_f16_to_hf8(val.sD); - res.data.sE = _intel_convert_f16_to_hf8(val.sE); - res.data.sF = _intel_convert_f16_to_hf8(val.sF); - return res; -} +// fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(half val) { +// fp8e4m3_t res; +// res.data = _intel_convert_f16_to_hf8(val); +// return res; +// } +// fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(half val[1]) { +// fp8e4m3_t1 res; +// res.data = _intel_convert_f16_to_hf8(val[0]); +// return res; +// } +// fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(half2 val) { +// fp8e4m3_t2 res; +// res.data.s0 = _intel_convert_f16_to_hf8(val.x); +// res.data.s1 = _intel_convert_f16_to_hf8(val.y); +// return res; +// } +// fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(half3 val) { +// fp8e4m3_t3 res; +// res.data.s0 = _intel_convert_f16_to_hf8(val.x); +// res.data.s1 = _intel_convert_f16_to_hf8(val.y); +// res.data.s2 = _intel_convert_f16_to_hf8(val.z); +// return res; +// } +// fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(half4 val) { +// fp8e4m3_t4 res; +// res.data.s0 = _intel_convert_f16_to_hf8(val.x); +// res.data.s1 = _intel_convert_f16_to_hf8(val.y); +// res.data.s2 = _intel_convert_f16_to_hf8(val.z); +// res.data.s3 = _intel_convert_f16_to_hf8(val.w); +// return res; +// } +// fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(half8 val) { +// fp8e4m3_t8 res; +// res.data.s0 = _intel_convert_f16_to_hf8(val.s0); +// res.data.s1 = _intel_convert_f16_to_hf8(val.s1); +// res.data.s2 = _intel_convert_f16_to_hf8(val.s2); +// res.data.s3 = _intel_convert_f16_to_hf8(val.s3); +// res.data.s4 = _intel_convert_f16_to_hf8(val.s4); +// res.data.s5 = _intel_convert_f16_to_hf8(val.s5); +// res.data.s6 = _intel_convert_f16_to_hf8(val.s6); +// res.data.s7 = _intel_convert_f16_to_hf8(val.s7); +// return res; +// } +// fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(half16 val) { +// fp8e4m3_t16 res; +// res.data.s0 = _intel_convert_f16_to_hf8(val.s0); +// res.data.s1 = _intel_convert_f16_to_hf8(val.s1); +// res.data.s2 = _intel_convert_f16_to_hf8(val.s2); +// res.data.s3 = _intel_convert_f16_to_hf8(val.s3); +// res.data.s4 = _intel_convert_f16_to_hf8(val.s4); +// res.data.s5 = _intel_convert_f16_to_hf8(val.s5); +// res.data.s6 = _intel_convert_f16_to_hf8(val.s6); +// res.data.s7 = _intel_convert_f16_to_hf8(val.s7); +// res.data.s8 = _intel_convert_f16_to_hf8(val.s8); +// res.data.s9 = _intel_convert_f16_to_hf8(val.s9); +// res.data.sA = _intel_convert_f16_to_hf8(val.sA); +// res.data.sB = _intel_convert_f16_to_hf8(val.sB); +// res.data.sC = _intel_convert_f16_to_hf8(val.sC); +// res.data.sD = _intel_convert_f16_to_hf8(val.sD); +// res.data.sE = _intel_convert_f16_to_hf8(val.sE); +// res.data.sF = _intel_convert_f16_to_hf8(val.sF); +// return res; +// } -fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(float val) { - fp8e4m3_t res; - res.data = _intel_convert_f16_to_hf8_sat((half)val); - return res; -} -fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(float val[1]) { - fp8e4m3_t1 res; - res.data = _intel_convert_f16_to_hf8_sat((half)val[0]); - return res; -} -fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(float2 val) { - fp8e4m3_t2 res; - res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); - res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); - return res; -} -fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(float3 val) { - fp8e4m3_t3 res; - res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); - res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); - res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); - return res; -} -fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(float4 val) { - fp8e4m3_t4 res; - res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); - res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); - res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); - res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.w); - return res; -} -fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(float8 val) { - fp8e4m3_t8 res; - res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); - res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); - res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); - res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); - res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); - res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); - res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); - res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); - return res; -} -fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(float16 val) { - fp8e4m3_t16 res; - res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); - res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); - res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); - res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); - res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); - res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); - res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); - res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); - res.data.s8 = _intel_convert_f16_to_hf8_sat((half)val.s8); - res.data.s9 = _intel_convert_f16_to_hf8_sat((half)val.s9); - res.data.sA = _intel_convert_f16_to_hf8_sat((half)val.sA); - res.data.sB = _intel_convert_f16_to_hf8_sat((half)val.sB); - res.data.sC = _intel_convert_f16_to_hf8_sat((half)val.sC); - res.data.sD = _intel_convert_f16_to_hf8_sat((half)val.sD); - res.data.sE = _intel_convert_f16_to_hf8_sat((half)val.sE); - res.data.sF = _intel_convert_f16_to_hf8_sat((half)val.sF); - return res; -} +// fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(float val) { +// fp8e4m3_t res; +// res.data = _intel_convert_f16_to_hf8_sat((half)val); +// return res; +// } +// fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(float val[1]) { +// fp8e4m3_t1 res; +// res.data = _intel_convert_f16_to_hf8_sat((half)val[0]); +// return res; +// } +// fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(float2 val) { +// fp8e4m3_t2 res; +// res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); +// res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); +// return res; +// } +// fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(float3 val) { +// fp8e4m3_t3 res; +// res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); +// res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); +// res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); +// return res; +// } +// fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(float4 val) { +// fp8e4m3_t4 res; +// res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); +// res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); +// res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); +// res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.w); +// return res; +// } +// fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(float8 val) { +// fp8e4m3_t8 res; +// res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); +// res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); +// res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); +// res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); +// res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); +// res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); +// res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); +// res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); +// return res; +// } +// fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(float16 val) { +// fp8e4m3_t16 res; +// res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); +// res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); +// res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); +// res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); +// res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); +// res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); +// res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); +// res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); +// res.data.s8 = _intel_convert_f16_to_hf8_sat((half)val.s8); +// res.data.s9 = _intel_convert_f16_to_hf8_sat((half)val.s9); +// res.data.sA = _intel_convert_f16_to_hf8_sat((half)val.sA); +// res.data.sB = _intel_convert_f16_to_hf8_sat((half)val.sB); +// res.data.sC = _intel_convert_f16_to_hf8_sat((half)val.sC); +// res.data.sD = _intel_convert_f16_to_hf8_sat((half)val.sD); +// res.data.sE = _intel_convert_f16_to_hf8_sat((half)val.sE); +// res.data.sF = _intel_convert_f16_to_hf8_sat((half)val.sF); +// return res; +// } -fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(half val) { - fp8e4m3_t res; - res.data = _intel_convert_f16_to_hf8_sat(val); - return res; -} -fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(half val[1]) { - fp8e4m3_t1 res; - res.data = _intel_convert_f16_to_hf8_sat(val[0]); - return res; -} -fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(half2 val) { - fp8e4m3_t2 res; - res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); - res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); - return res; -} -fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(half3 val) { - fp8e4m3_t3 res; - res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); - res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); - res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); - return res; -} -fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(half4 val) { - fp8e4m3_t4 res; - res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); - res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); - res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); - res.data.s3 = _intel_convert_f16_to_hf8_sat(val.w); - return res; -} -fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(half8 val) { - fp8e4m3_t8 res; - res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); - res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); - res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); - res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); - res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); - res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); - res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); - res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); - return res; -} -fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(half16 val) { - fp8e4m3_t16 res; - res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); - res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); - res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); - res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); - res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); - res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); - res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); - res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); - res.data.s8 = _intel_convert_f16_to_hf8_sat(val.s8); - res.data.s9 = _intel_convert_f16_to_hf8_sat(val.s9); - res.data.sA = _intel_convert_f16_to_hf8_sat(val.sA); - res.data.sB = _intel_convert_f16_to_hf8_sat(val.sB); - res.data.sC = _intel_convert_f16_to_hf8_sat(val.sC); - res.data.sD = _intel_convert_f16_to_hf8_sat(val.sD); - res.data.sE = _intel_convert_f16_to_hf8_sat(val.sE); - res.data.sF = _intel_convert_f16_to_hf8_sat(val.sF); - return res; -} +// fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(half val) { +// fp8e4m3_t res; +// res.data = _intel_convert_f16_to_hf8_sat(val); +// return res; +// } +// fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(half val[1]) { +// fp8e4m3_t1 res; +// res.data = _intel_convert_f16_to_hf8_sat(val[0]); +// return res; +// } +// fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(half2 val) { +// fp8e4m3_t2 res; +// res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); +// res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); +// return res; +// } +// fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(half3 val) { +// fp8e4m3_t3 res; +// res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); +// res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); +// res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); +// return res; +// } +// fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(half4 val) { +// fp8e4m3_t4 res; +// res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); +// res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); +// res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); +// res.data.s3 = _intel_convert_f16_to_hf8_sat(val.w); +// return res; +// } +// fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(half8 val) { +// fp8e4m3_t8 res; +// res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); +// res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); +// res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); +// res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); +// res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); +// res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); +// res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); +// res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); +// return res; +// } +// fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(half16 val) { +// fp8e4m3_t16 res; +// res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); +// res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); +// res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); +// res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); +// res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); +// res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); +// res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); +// res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); +// res.data.s8 = _intel_convert_f16_to_hf8_sat(val.s8); +// res.data.s9 = _intel_convert_f16_to_hf8_sat(val.s9); +// res.data.sA = _intel_convert_f16_to_hf8_sat(val.sA); +// res.data.sB = _intel_convert_f16_to_hf8_sat(val.sB); +// res.data.sC = _intel_convert_f16_to_hf8_sat(val.sC); +// res.data.sD = _intel_convert_f16_to_hf8_sat(val.sD); +// res.data.sE = _intel_convert_f16_to_hf8_sat(val.sE); +// res.data.sF = _intel_convert_f16_to_hf8_sat(val.sF); +// return res; +// } -fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(fp8e8m0_t val) { - return val; -} +// fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(fp8e8m0_t val) { +// return val; +// } -fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(float val) { - fp8e8m0_t res; - res.data = _intel_convert_f32_fo_e8m0(val); - return res; -} -fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1(float val[1]) { - fp8e8m0_t1 res; - res.data = _intel_convert_f32_fo_e8m0(val[0]); - return res; -} -fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2(float2 val) { - fp8e8m0_t2 res; - res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); - res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); - return res; -} -fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3(float3 val) { - fp8e8m0_t3 res; - res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); - res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); - res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); - return res; -} -fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4(float4 val) { - fp8e8m0_t4 res; - res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); - res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); - res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); - res.data.s3 = _intel_convert_f32_fo_e8m0(val.w); - return res; -} -fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8(float8 val) { - fp8e8m0_t8 res; - res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); - res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); - res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); - res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); - res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); - res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); - res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); - res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); - return res; -} -fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16(float16 val) { - fp8e8m0_t16 res; - res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); - res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); - res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); - res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); - res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); - res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); - res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); - res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); - res.data.s8 = _intel_convert_f32_fo_e8m0(val.s8); - res.data.s9 = _intel_convert_f32_fo_e8m0(val.s9); - res.data.sA = _intel_convert_f32_fo_e8m0(val.sA); - res.data.sB = _intel_convert_f32_fo_e8m0(val.sB); - res.data.sC = _intel_convert_f32_fo_e8m0(val.sC); - res.data.sD = _intel_convert_f32_fo_e8m0(val.sD); - res.data.sE = _intel_convert_f32_fo_e8m0(val.sE); - res.data.sF = _intel_convert_f32_fo_e8m0(val.sF); - return res; -} +// fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(float val) { +// fp8e8m0_t res; +// res.data = _intel_convert_f32_fo_e8m0(val); +// return res; +// } +// fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1(float val[1]) { +// fp8e8m0_t1 res; +// res.data = _intel_convert_f32_fo_e8m0(val[0]); +// return res; +// } +// fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2(float2 val) { +// fp8e8m0_t2 res; +// res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); +// res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); +// return res; +// } +// fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3(float3 val) { +// fp8e8m0_t3 res; +// res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); +// res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); +// res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); +// return res; +// } +// fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4(float4 val) { +// fp8e8m0_t4 res; +// res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); +// res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); +// res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); +// res.data.s3 = _intel_convert_f32_fo_e8m0(val.w); +// return res; +// } +// fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8(float8 val) { +// fp8e8m0_t8 res; +// res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); +// res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); +// res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); +// res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); +// res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); +// res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); +// res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); +// res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); +// return res; +// } +// fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16(float16 val) { +// fp8e8m0_t16 res; +// res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); +// res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); +// res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); +// res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); +// res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); +// res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); +// res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); +// res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); +// res.data.s8 = _intel_convert_f32_fo_e8m0(val.s8); +// res.data.s9 = _intel_convert_f32_fo_e8m0(val.s9); +// res.data.sA = _intel_convert_f32_fo_e8m0(val.sA); +// res.data.sB = _intel_convert_f32_fo_e8m0(val.sB); +// res.data.sC = _intel_convert_f32_fo_e8m0(val.sC); +// res.data.sD = _intel_convert_f32_fo_e8m0(val.sD); +// res.data.sE = _intel_convert_f32_fo_e8m0(val.sE); +// res.data.sF = _intel_convert_f32_fo_e8m0(val.sF); +// return res; +// } -fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t_sat(float val) { - fp8e8m0_t res; - res.data = _intel_convert_f32_fo_e8m0_sat(val); - return res; -} -fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1_sat(float val[1]) { - fp8e8m0_t1 res; - res.data = _intel_convert_f32_fo_e8m0_sat(val[0]); - return res; -} -fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2_sat(float2 val) { - fp8e8m0_t2 res; - res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); - res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); - return res; -} -fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3_sat(float3 val) { - fp8e8m0_t3 res; - res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); - res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); - res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); - return res; -} -fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4_sat(float4 val) { - fp8e8m0_t4 res; - res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); - res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); - res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); - res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.w); - return res; -} -fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8_sat(float8 val) { - fp8e8m0_t8 res; - res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); - res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); - res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); - res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); - res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); - res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); - res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); - res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); - return res; -} -fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16_sat(float16 val) { - fp8e8m0_t16 res; - res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); - res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); - res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); - res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); - res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); - res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); - res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); - res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); - res.data.s8 = _intel_convert_f32_fo_e8m0_sat(val.s8); - res.data.s9 = _intel_convert_f32_fo_e8m0_sat(val.s9); - res.data.sA = _intel_convert_f32_fo_e8m0_sat(val.sA); - res.data.sB = _intel_convert_f32_fo_e8m0_sat(val.sB); - res.data.sC = _intel_convert_f32_fo_e8m0_sat(val.sC); - res.data.sD = _intel_convert_f32_fo_e8m0_sat(val.sD); - res.data.sE = _intel_convert_f32_fo_e8m0_sat(val.sE); - res.data.sF = _intel_convert_f32_fo_e8m0_sat(val.sF); - return res; -} +// fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t_sat(float val) { +// fp8e8m0_t res; +// res.data = _intel_convert_f32_fo_e8m0_sat(val); +// return res; +// } +// fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1_sat(float val[1]) { +// fp8e8m0_t1 res; +// res.data = _intel_convert_f32_fo_e8m0_sat(val[0]); +// return res; +// } +// fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2_sat(float2 val) { +// fp8e8m0_t2 res; +// res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); +// res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); +// return res; +// } +// fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3_sat(float3 val) { +// fp8e8m0_t3 res; +// res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); +// res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); +// res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); +// return res; +// } +// fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4_sat(float4 val) { +// fp8e8m0_t4 res; +// res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); +// res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); +// res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); +// res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.w); +// return res; +// } +// fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8_sat(float8 val) { +// fp8e8m0_t8 res; +// res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); +// res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); +// res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); +// res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); +// res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); +// res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); +// res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); +// res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); +// return res; +// } +// fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16_sat(float16 val) { +// fp8e8m0_t16 res; +// res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); +// res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); +// res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); +// res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); +// res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); +// res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); +// res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); +// res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); +// res.data.s8 = _intel_convert_f32_fo_e8m0_sat(val.s8); +// res.data.s9 = _intel_convert_f32_fo_e8m0_sat(val.s9); +// res.data.sA = _intel_convert_f32_fo_e8m0_sat(val.sA); +// res.data.sB = _intel_convert_f32_fo_e8m0_sat(val.sB); +// res.data.sC = _intel_convert_f32_fo_e8m0_sat(val.sC); +// res.data.sD = _intel_convert_f32_fo_e8m0_sat(val.sD); +// res.data.sE = _intel_convert_f32_fo_e8m0_sat(val.sE); +// res.data.sF = _intel_convert_f32_fo_e8m0_sat(val.sF); +// return res; +// } -fp8e5m2_t __attribute__((overloadable)) as_fp8e5m2_t(uchar val) { - fp8e5m2_t res; - res.data = val; - return res; -} -fp8e5m2_t1 __attribute__((overloadable)) as_fp8e5m2_t1(uchar val[1]) { - fp8e5m2_t1 res; - res.data = val[0]; - return res; -} -fp8e5m2_t2 __attribute__((overloadable)) as_fp8e5m2_t2(uchar2 val) { - fp8e5m2_t2 res; - res.data.s0 = val.x; - res.data.s1 = val.y; - return res; -} -fp8e5m2_t3 __attribute__((overloadable)) as_fp8e5m2_t3(uchar3 val) { - fp8e5m2_t3 res; - res.data.s0 = val.x; - res.data.s1 = val.y; - res.data.s2 = val.z; - return res; -} -fp8e5m2_t4 __attribute__((overloadable)) as_fp8e5m2_t4(uchar4 val) { - fp8e5m2_t4 res; - res.data.s0 = val.x; - res.data.s1 = val.y; - res.data.s2 = val.z; - res.data.s3 = val.w; - return res; -} -fp8e5m2_t8 __attribute__((overloadable)) as_fp8e5m2_t8(uchar8 val) { - fp8e5m2_t8 res; - res.data.s0 = val.s0; - res.data.s1 = val.s1; - res.data.s2 = val.s2; - res.data.s3 = val.s3; - res.data.s4 = val.s4; - res.data.s5 = val.s5; - res.data.s6 = val.s6; - res.data.s7 = val.s7; - return res; -} -fp8e5m2_t16 __attribute__((overloadable)) as_fp8e5m2_t16(uchar16 val) { - fp8e5m2_t16 res; - res.data.s0 = val.s0; - res.data.s1 = val.s1; - res.data.s2 = val.s2; - res.data.s3 = val.s3; - res.data.s4 = val.s4; - res.data.s5 = val.s5; - res.data.s6 = val.s6; - res.data.s7 = val.s7; - res.data.s8 = val.s8; - res.data.s9 = val.s9; - res.data.sA = val.sA; - res.data.sB = val.sB; - res.data.sC = val.sC; - res.data.sD = val.sD; - res.data.sE = val.sE; - res.data.sF = val.sF; - return res; -} +// fp8e5m2_t __attribute__((overloadable)) as_fp8e5m2_t(uchar val) { +// fp8e5m2_t res; +// res.data = val; +// return res; +// } +// fp8e5m2_t1 __attribute__((overloadable)) as_fp8e5m2_t1(uchar val[1]) { +// fp8e5m2_t1 res; +// res.data = val[0]; +// return res; +// } +// fp8e5m2_t2 __attribute__((overloadable)) as_fp8e5m2_t2(uchar2 val) { +// fp8e5m2_t2 res; +// res.data.s0 = val.x; +// res.data.s1 = val.y; +// return res; +// } +// fp8e5m2_t3 __attribute__((overloadable)) as_fp8e5m2_t3(uchar3 val) { +// fp8e5m2_t3 res; +// res.data.s0 = val.x; +// res.data.s1 = val.y; +// res.data.s2 = val.z; +// return res; +// } +// fp8e5m2_t4 __attribute__((overloadable)) as_fp8e5m2_t4(uchar4 val) { +// fp8e5m2_t4 res; +// res.data.s0 = val.x; +// res.data.s1 = val.y; +// res.data.s2 = val.z; +// res.data.s3 = val.w; +// return res; +// } +// fp8e5m2_t8 __attribute__((overloadable)) as_fp8e5m2_t8(uchar8 val) { +// fp8e5m2_t8 res; +// res.data.s0 = val.s0; +// res.data.s1 = val.s1; +// res.data.s2 = val.s2; +// res.data.s3 = val.s3; +// res.data.s4 = val.s4; +// res.data.s5 = val.s5; +// res.data.s6 = val.s6; +// res.data.s7 = val.s7; +// return res; +// } +// fp8e5m2_t16 __attribute__((overloadable)) as_fp8e5m2_t16(uchar16 val) { +// fp8e5m2_t16 res; +// res.data.s0 = val.s0; +// res.data.s1 = val.s1; +// res.data.s2 = val.s2; +// res.data.s3 = val.s3; +// res.data.s4 = val.s4; +// res.data.s5 = val.s5; +// res.data.s6 = val.s6; +// res.data.s7 = val.s7; +// res.data.s8 = val.s8; +// res.data.s9 = val.s9; +// res.data.sA = val.sA; +// res.data.sB = val.sB; +// res.data.sC = val.sC; +// res.data.sD = val.sD; +// res.data.sE = val.sE; +// res.data.sF = val.sF; +// return res; +// } -fp8e4m3_t __attribute__((overloadable)) as_fp8e4m3_t(uchar val) { - fp8e4m3_t res; - res.data = as_char(val); - return res; -} -fp8e4m3_t1 __attribute__((overloadable)) as_fp8e4m3_t1(uchar val[1]) { - fp8e4m3_t1 res; - res.data = as_char(val[0]); - return res; -} -fp8e4m3_t2 __attribute__((overloadable)) as_fp8e4m3_t2(uchar2 val) { - fp8e4m3_t2 res; - res.data.s0 = as_char(val.x); - res.data.s1 = as_char(val.y); - return res; -} -fp8e4m3_t3 __attribute__((overloadable)) as_fp8e4m3_t3(uchar3 val) { - fp8e4m3_t3 res; - res.data.s0 = as_char(val.x); - res.data.s1 = as_char(val.y); - res.data.s2 = as_char(val.z); - return res; -} -fp8e4m3_t4 __attribute__((overloadable)) as_fp8e4m3_t4(uchar4 val) { - fp8e4m3_t4 res; - res.data.s0 = as_char(val.x); - res.data.s1 = as_char(val.y); - res.data.s2 = as_char(val.z); - res.data.s3 = as_char(val.w); - return res; -} -fp8e4m3_t8 __attribute__((overloadable)) as_fp8e4m3_t8(uchar8 val) { - fp8e4m3_t8 res; - res.data.s0 = as_char(val.s0); - res.data.s1 = as_char(val.s1); - res.data.s2 = as_char(val.s2); - res.data.s3 = as_char(val.s3); - res.data.s4 = as_char(val.s4); - res.data.s5 = as_char(val.s5); - res.data.s6 = as_char(val.s6); - res.data.s7 = as_char(val.s7); - return res; -} -fp8e4m3_t16 __attribute__((overloadable)) as_fp8e4m3_t16(uchar16 val) { - fp8e4m3_t16 res; - res.data.s0 = as_char(val.s0); - res.data.s1 = as_char(val.s1); - res.data.s2 = as_char(val.s2); - res.data.s3 = as_char(val.s3); - res.data.s4 = as_char(val.s4); - res.data.s5 = as_char(val.s5); - res.data.s6 = as_char(val.s6); - res.data.s7 = as_char(val.s7); - res.data.s8 = as_char(val.s8); - res.data.s9 = as_char(val.s9); - res.data.sA = as_char(val.sA); - res.data.sB = as_char(val.sB); - res.data.sC = as_char(val.sC); - res.data.sD = as_char(val.sD); - res.data.sE = as_char(val.sE); - res.data.sF = as_char(val.sF); - return res; -} +// fp8e4m3_t __attribute__((overloadable)) as_fp8e4m3_t(uchar val) { +// fp8e4m3_t res; +// res.data = as_char(val); +// return res; +// } +// fp8e4m3_t1 __attribute__((overloadable)) as_fp8e4m3_t1(uchar val[1]) { +// fp8e4m3_t1 res; +// res.data = as_char(val[0]); +// return res; +// } +// fp8e4m3_t2 __attribute__((overloadable)) as_fp8e4m3_t2(uchar2 val) { +// fp8e4m3_t2 res; +// res.data.s0 = as_char(val.x); +// res.data.s1 = as_char(val.y); +// return res; +// } +// fp8e4m3_t3 __attribute__((overloadable)) as_fp8e4m3_t3(uchar3 val) { +// fp8e4m3_t3 res; +// res.data.s0 = as_char(val.x); +// res.data.s1 = as_char(val.y); +// res.data.s2 = as_char(val.z); +// return res; +// } +// fp8e4m3_t4 __attribute__((overloadable)) as_fp8e4m3_t4(uchar4 val) { +// fp8e4m3_t4 res; +// res.data.s0 = as_char(val.x); +// res.data.s1 = as_char(val.y); +// res.data.s2 = as_char(val.z); +// res.data.s3 = as_char(val.w); +// return res; +// } +// fp8e4m3_t8 __attribute__((overloadable)) as_fp8e4m3_t8(uchar8 val) { +// fp8e4m3_t8 res; +// res.data.s0 = as_char(val.s0); +// res.data.s1 = as_char(val.s1); +// res.data.s2 = as_char(val.s2); +// res.data.s3 = as_char(val.s3); +// res.data.s4 = as_char(val.s4); +// res.data.s5 = as_char(val.s5); +// res.data.s6 = as_char(val.s6); +// res.data.s7 = as_char(val.s7); +// return res; +// } +// fp8e4m3_t16 __attribute__((overloadable)) as_fp8e4m3_t16(uchar16 val) { +// fp8e4m3_t16 res; +// res.data.s0 = as_char(val.s0); +// res.data.s1 = as_char(val.s1); +// res.data.s2 = as_char(val.s2); +// res.data.s3 = as_char(val.s3); +// res.data.s4 = as_char(val.s4); +// res.data.s5 = as_char(val.s5); +// res.data.s6 = as_char(val.s6); +// res.data.s7 = as_char(val.s7); +// res.data.s8 = as_char(val.s8); +// res.data.s9 = as_char(val.s9); +// res.data.sA = as_char(val.sA); +// res.data.sB = as_char(val.sB); +// res.data.sC = as_char(val.sC); +// res.data.sD = as_char(val.sD); +// res.data.sE = as_char(val.sE); +// res.data.sF = as_char(val.sF); +// return res; +// } -fp8e8m0_t __attribute__((overloadable)) as_fp8e8m0_t(uchar val) { - fp8e8m0_t res; - res.data = val; - return res; -} -fp8e8m0_t1 __attribute__((overloadable)) as_fp8e8m0_t1(uchar val[1]) { - fp8e8m0_t1 res; - res.data = val[0]; - return res; -} -fp8e8m0_t2 __attribute__((overloadable)) as_fp8e8m0_t2(uchar2 val) { - fp8e8m0_t2 res; - res.data.s0 = val.x; - res.data.s1 = val.y; - return res; -} -fp8e8m0_t3 __attribute__((overloadable)) as_fp8e8m0_t3(uchar3 val) { - fp8e8m0_t3 res; - res.data.s0 = val.x; - res.data.s1 = val.y; - res.data.s2 = val.z; - return res; -} -fp8e8m0_t4 __attribute__((overloadable)) as_fp8e8m0_t4(uchar4 val) { - fp8e8m0_t4 res; - res.data.s0 = val.x; - res.data.s1 = val.y; - res.data.s2 = val.z; - res.data.s3 = val.w; - return res; -} -fp8e8m0_t8 __attribute__((overloadable)) as_fp8e8m0_t8(uchar8 val) { - fp8e8m0_t8 res; - res.data.s0 = val.s0; - res.data.s1 = val.s1; - res.data.s2 = val.s2; - res.data.s3 = val.s3; - res.data.s4 = val.s4; - res.data.s5 = val.s5; - res.data.s6 = val.s6; - res.data.s7 = val.s7; - return res; -} -fp8e8m0_t16 __attribute__((overloadable)) as_fp8e8m0_t16(uchar16 val) { - fp8e8m0_t16 res; - res.data.s0 = val.s0; - res.data.s1 = val.s1; - res.data.s2 = val.s2; - res.data.s3 = val.s3; - res.data.s4 = val.s4; - res.data.s5 = val.s5; - res.data.s6 = val.s6; - res.data.s7 = val.s7; - res.data.s8 = val.s8; - res.data.s9 = val.s9; - res.data.sA = val.sA; - res.data.sB = val.sB; - res.data.sC = val.sC; - res.data.sD = val.sD; - res.data.sE = val.sE; - res.data.sF = val.sF; - return res; -} +// fp8e8m0_t __attribute__((overloadable)) as_fp8e8m0_t(uchar val) { +// fp8e8m0_t res; +// res.data = val; +// return res; +// } +// fp8e8m0_t1 __attribute__((overloadable)) as_fp8e8m0_t1(uchar val[1]) { +// fp8e8m0_t1 res; +// res.data = val[0]; +// return res; +// } +// fp8e8m0_t2 __attribute__((overloadable)) as_fp8e8m0_t2(uchar2 val) { +// fp8e8m0_t2 res; +// res.data.s0 = val.x; +// res.data.s1 = val.y; +// return res; +// } +// fp8e8m0_t3 __attribute__((overloadable)) as_fp8e8m0_t3(uchar3 val) { +// fp8e8m0_t3 res; +// res.data.s0 = val.x; +// res.data.s1 = val.y; +// res.data.s2 = val.z; +// return res; +// } +// fp8e8m0_t4 __attribute__((overloadable)) as_fp8e8m0_t4(uchar4 val) { +// fp8e8m0_t4 res; +// res.data.s0 = val.x; +// res.data.s1 = val.y; +// res.data.s2 = val.z; +// res.data.s3 = val.w; +// return res; +// } +// fp8e8m0_t8 __attribute__((overloadable)) as_fp8e8m0_t8(uchar8 val) { +// fp8e8m0_t8 res; +// res.data.s0 = val.s0; +// res.data.s1 = val.s1; +// res.data.s2 = val.s2; +// res.data.s3 = val.s3; +// res.data.s4 = val.s4; +// res.data.s5 = val.s5; +// res.data.s6 = val.s6; +// res.data.s7 = val.s7; +// return res; +// } +// fp8e8m0_t16 __attribute__((overloadable)) as_fp8e8m0_t16(uchar16 val) { +// fp8e8m0_t16 res; +// res.data.s0 = val.s0; +// res.data.s1 = val.s1; +// res.data.s2 = val.s2; +// res.data.s3 = val.s3; +// res.data.s4 = val.s4; +// res.data.s5 = val.s5; +// res.data.s6 = val.s6; +// res.data.s7 = val.s7; +// res.data.s8 = val.s8; +// res.data.s9 = val.s9; +// res.data.sA = val.sA; +// res.data.sB = val.sB; +// res.data.sC = val.sC; +// res.data.sD = val.sD; +// res.data.sE = val.sE; +// res.data.sF = val.sF; +// return res; +// } -uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t val) { - return val.data; -} -uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t1 val) { - return val.data; -} -uchar2 __attribute__((overloadable)) _as_uchar2(fp8e5m2_t2 val) { - return (uchar2)(val.data.s0, val.data.s1); -} -uchar3 __attribute__((overloadable)) _as_uchar3(fp8e5m2_t3 val) { - return (uchar3)(val.data.s0, val.data.s1, val.data.s2); -} -uchar4 __attribute__((overloadable)) _as_uchar4(fp8e5m2_t4 val) { - return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); -} -uchar8 __attribute__((overloadable)) _as_uchar8(fp8e5m2_t8 val) { - return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, - val.data.s4, val.data.s5, val.data.s6, val.data.s7); -} -uchar16 __attribute__((overloadable)) _as_uchar16(fp8e5m2_t16 val) { - return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, - val.data.s4, val.data.s5, val.data.s6, val.data.s7, - val.data.s8, val.data.s9, val.data.sA, val.data.sB, - val.data.sC, val.data.sD, val.data.sE, val.data.sF); -} +// uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t val) { +// return val.data; +// } +// uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t1 val) { +// return val.data; +// } +// uchar2 __attribute__((overloadable)) _as_uchar2(fp8e5m2_t2 val) { +// return (uchar2)(val.data.s0, val.data.s1); +// } +// uchar3 __attribute__((overloadable)) _as_uchar3(fp8e5m2_t3 val) { +// return (uchar3)(val.data.s0, val.data.s1, val.data.s2); +// } +// uchar4 __attribute__((overloadable)) _as_uchar4(fp8e5m2_t4 val) { +// return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); +// } +// uchar8 __attribute__((overloadable)) _as_uchar8(fp8e5m2_t8 val) { +// return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, +// val.data.s4, val.data.s5, val.data.s6, val.data.s7); +// } +// uchar16 __attribute__((overloadable)) _as_uchar16(fp8e5m2_t16 val) { +// return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, +// val.data.s4, val.data.s5, val.data.s6, val.data.s7, +// val.data.s8, val.data.s9, val.data.sA, val.data.sB, +// val.data.sC, val.data.sD, val.data.sE, val.data.sF); +// } -uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t val) { - return as_uchar((char)val.data); -} -uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t1 val) { - return as_uchar((char)val.data); -} -uchar2 __attribute__((overloadable)) _as_uchar2(fp8e4m3_t2 val) { - return (uchar2)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1)); -} -uchar3 __attribute__((overloadable)) _as_uchar3(fp8e4m3_t3 val) { - return (uchar3)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), as_uchar((char)val.data.s2)); -} -uchar4 __attribute__((overloadable)) _as_uchar4(fp8e4m3_t4 val) { - return (uchar4)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), - as_uchar((char)val.data.s2), as_uchar((char)val.data.s3)); -} -uchar8 __attribute__((overloadable)) _as_uchar8(fp8e4m3_t8 val) { - return (uchar8)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), - as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), - as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), - as_uchar((char)val.data.s6), as_uchar((char)val.data.s7)); -} -uchar16 __attribute__((overloadable)) _as_uchar16(fp8e4m3_t16 val) { - return (uchar16)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), - as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), - as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), - as_uchar((char)val.data.s6), as_uchar((char)val.data.s7), - as_uchar((char)val.data.s8), as_uchar((char)val.data.s9), - as_uchar((char)val.data.sA), as_uchar((char)val.data.sB), - as_uchar((char)val.data.sC), as_uchar((char)val.data.sD), - as_uchar((char)val.data.sE), as_uchar((char)val.data.sF)); -} +// uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t val) { +// return as_uchar((char)val.data); +// } +// uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t1 val) { +// return as_uchar((char)val.data); +// } +// uchar2 __attribute__((overloadable)) _as_uchar2(fp8e4m3_t2 val) { +// return (uchar2)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1)); +// } +// uchar3 __attribute__((overloadable)) _as_uchar3(fp8e4m3_t3 val) { +// return (uchar3)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), as_uchar((char)val.data.s2)); +// } +// uchar4 __attribute__((overloadable)) _as_uchar4(fp8e4m3_t4 val) { +// return (uchar4)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), +// as_uchar((char)val.data.s2), as_uchar((char)val.data.s3)); +// } +// uchar8 __attribute__((overloadable)) _as_uchar8(fp8e4m3_t8 val) { +// return (uchar8)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), +// as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), +// as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), +// as_uchar((char)val.data.s6), as_uchar((char)val.data.s7)); +// } +// uchar16 __attribute__((overloadable)) _as_uchar16(fp8e4m3_t16 val) { +// return (uchar16)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), +// as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), +// as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), +// as_uchar((char)val.data.s6), as_uchar((char)val.data.s7), +// as_uchar((char)val.data.s8), as_uchar((char)val.data.s9), +// as_uchar((char)val.data.sA), as_uchar((char)val.data.sB), +// as_uchar((char)val.data.sC), as_uchar((char)val.data.sD), +// as_uchar((char)val.data.sE), as_uchar((char)val.data.sF)); +// } -uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t val) { - return val.data; -} -uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t1 val) { - return val.data; -} -uchar2 __attribute__((overloadable)) _as_uchar2(fp8e8m0_t2 val) { - return (uchar2)(val.data.s0, val.data.s1); -} -uchar3 __attribute__((overloadable)) _as_uchar3(fp8e8m0_t3 val) { - return (uchar3)(val.data.s0, val.data.s1, val.data.s2); -} -uchar4 __attribute__((overloadable)) _as_uchar4(fp8e8m0_t4 val) { - return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); -} -uchar8 __attribute__((overloadable)) _as_uchar8(fp8e8m0_t8 val) { - return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, - val.data.s4, val.data.s5, val.data.s6, val.data.s7); -} -uchar16 __attribute__((overloadable)) _as_uchar16(fp8e8m0_t16 val) { - return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, - val.data.s4, val.data.s5, val.data.s6, val.data.s7, - val.data.s8, val.data.s9, val.data.sA, val.data.sB, - val.data.sC, val.data.sD, val.data.sE, val.data.sF); -} +// uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t val) { +// return val.data; +// } +// uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t1 val) { +// return val.data; +// } +// uchar2 __attribute__((overloadable)) _as_uchar2(fp8e8m0_t2 val) { +// return (uchar2)(val.data.s0, val.data.s1); +// } +// uchar3 __attribute__((overloadable)) _as_uchar3(fp8e8m0_t3 val) { +// return (uchar3)(val.data.s0, val.data.s1, val.data.s2); +// } +// uchar4 __attribute__((overloadable)) _as_uchar4(fp8e8m0_t4 val) { +// return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); +// } +// uchar8 __attribute__((overloadable)) _as_uchar8(fp8e8m0_t8 val) { +// return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, +// val.data.s4, val.data.s5, val.data.s6, val.data.s7); +// } +// uchar16 __attribute__((overloadable)) _as_uchar16(fp8e8m0_t16 val) { +// return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, +// val.data.s4, val.data.s5, val.data.s6, val.data.s7, +// val.data.s8, val.data.s9, val.data.sA, val.data.sB, +// val.data.sC, val.data.sD, val.data.sE, val.data.sF); +// } diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl index f89ebe1eb02e..822952765a04 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl @@ -48,16 +48,16 @@ #define BLOCK_WRITEN_FUNC_size8(vector_size) BLOCK_WRITEN_FUNC_SIZE_DEF(8, vector_size) #define BLOCK_WRITEN_FUNC(type_size, vector_size) CAT(BLOCK_WRITEN_FUNC_size, type_size)(vector_size) -#define BLOCK_WRITEN_RAW(type_size, vector_size, addr_space, ptr, offset, val, src_type) \ +#define BLOCK_WRITEN_RAW(type_size, vector_size, addr_space, ptr, offset, val) \ BLOCK_WRITEN_FUNC(type_size, vector_size)( \ (addr_space BLOCK_WRITE_TYPE(type_size)*)(ptr) + (offset), \ - AS_TYPE_EXT(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val, src_type)) + AS_TYPE(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val)) #define BLOCK_WRITEN(type, vector_size, ptr, offset, val) \ - BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __global, ptr, offset, val, type) + BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __global, ptr, offset, val) #define BLOCK_WRITEN_SLM(type, vector_size, ptr, offset, val) \ - BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __local, ptr, offset, val, type) + BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __local, ptr, offset, val) #define DT_OUTPUT_BLOCK_WRITE(ptr, offset, val) BLOCK_WRITEN(OUTPUT_TYPE, 1, ptr, offset, val) #define DT_OUTPUT_BLOCK_WRITE2(ptr, offset, val) BLOCK_WRITEN(OUTPUT_TYPE, 2, ptr, offset, val) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl index b7252b033712..6dd09d08dc62 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl @@ -2,13 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // -#define IS_F8 (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT || F8E5M2_OUTPUT || F8E4M3_OUTPUT || F8E8M0_OUTPUT) - -#if IS_F8 -#include "include/batch_headers/common.cl" -#include "include/batch_headers/f8_utils.cl" -#endif - #include "include/reshape_dims.cl" #include "include/fetch_utils.cl" @@ -160,8 +153,6 @@ KERNEL (reorder_data)( #elif defined UINT4_INPUT const uint uint4_idx = input_idx >> 1; OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(convert_as_uint4_float(input[uint4_idx], input_idx)); - #elif (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT) - OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(_convert_float(input[input_idx])); #else CALC_TYPE res = TO_CALC_TYPE(input[input_idx]); #endif diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 85e33f300bdf..30e4b5694abd 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -296,24 +296,27 @@ static bool is_decompression_multiply(const std::shared_ptr node return true; }; - if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()}) || all_has_types(consumers, {ov::op::v1::Transpose::get_type_info_static()})) { + // if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()}) || all_has_types(consumers, {ov::op::v1::Transpose::get_type_info_static()})) { + // for (const auto& consumer : consumers) { + // auto get_child_consumers = [&]() { + // auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + + // // Reshape + Transpose chain + // if (all_has_types(child_consumers, { ov::op::v1::Transpose::get_type_info_static() })) { + // std::set> next_child_consumers; + // for (const auto& child_consumer : child_consumers) { + // const auto grand_child_consumers = child_consumer.get_node()->get_output_target_inputs(0); + // next_child_consumers.insert(grand_child_consumers.begin(), grand_child_consumers.end()); + // } + // return next_child_consumers; + // } + // return child_consumers; + // }; + + // auto child_consumers = get_child_consumers(); + if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()})) { for (const auto& consumer : consumers) { - auto get_child_consumers = [&]() { - auto child_consumers = consumer.get_node()->get_output_target_inputs(0); - - // Reshape + Transpose chain - if (all_has_types(child_consumers, { ov::op::v1::Transpose::get_type_info_static() })) { - std::set> next_child_consumers; - for (const auto& child_consumer : child_consumers) { - const auto grand_child_consumers = child_consumer.get_node()->get_output_target_inputs(0); - next_child_consumers.insert(grand_child_consumers.begin(), grand_child_consumers.end()); - } - return next_child_consumers; - } - return child_consumers; - }; - - auto child_consumers = get_child_consumers(); + const auto child_consumers = consumer.get_node()->get_output_target_inputs(0); for (const auto& child_consumer : child_consumers) { const auto& type_info = child_consumer.get_node()->get_type_info(); @@ -578,12 +581,12 @@ void TransformationsPipeline::apply(std::shared_ptr func) { ov::element::f4e2m1, ov::element::f8e8m0}, !device_info.supports_immad); - if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(func)) { - manager.register_pass( - std::vector{ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1, ov::element::f8e8m0}, - !device_info.supports_immad, - !device_info.supports_immad); - } + // if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(func)) { + // manager.register_pass( + // std::vector{ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1, ov::element::f8e8m0}, + // !device_info.supports_immad, + // !device_info.supports_immad); + // } const bool is_pa = [&func]() { for (const auto& op : func->get_ops()) { From 619c0d682b50511f37c7aa3d8ff67747bb66b30f Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 16 Jun 2026 12:21:18 +0000 Subject: [PATCH 33/48] Bring back cpp part --- .../include/low_precision/low_precision.hpp | 2 +- .../src/low_precision.cpp | 42 ++++++++-------- .../moc_transformations.cpp | 12 ++--- src/core/src/preprocess/pre_post_process.cpp | 12 ++--- .../src/plugin/transformations_pipeline.cpp | 49 +++++++++---------- 5 files changed, 57 insertions(+), 60 deletions(-) diff --git a/src/common/low_precision_transformations/include/low_precision/low_precision.hpp b/src/common/low_precision_transformations/include/low_precision/low_precision.hpp index 9c3f61a9806b..5dddc0c68572 100644 --- a/src/common/low_precision_transformations/include/low_precision/low_precision.hpp +++ b/src/common/low_precision_transformations/include/low_precision/low_precision.hpp @@ -76,7 +76,7 @@ class LP_TRANSFORMATIONS_API ov::pass::low_precision::LowPrecision : public ov:: const std::set& supported_levels = all_levels, bool check_fake_convert = false); - //static bool does_model_contain_mxfp_patterns(const std::shared_ptr& model); + static bool does_model_contain_mxfp_patterns(const ov::Model& model); template std::shared_ptr add_main(Args&&... args) { diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index 2cc69cc9b9a0..ae76c8d6592c 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -31,7 +31,7 @@ #include "openvino/pass/constant_folding.hpp" #include "openvino/pass/manager.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" -// #include "openvino/pass/pattern/op/pattern.hpp" +#include "openvino/pass/pattern/op/pattern.hpp" #include "openvino/util/log.hpp" #include "ov_ops/type_relaxed.hpp" #include "transformations/common_optimizations/lin_op_sequence_fusion.hpp" @@ -345,29 +345,29 @@ bool LowPrecision::isFunctionQuantized(const std::shared_ptr& m return false; } -// bool ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns( -// const std::shared_ptr& model) { -// using namespace ov::op; -// using namespace ov::pass::pattern; +bool ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns( + const ov::Model& model) { + using namespace ov::op; + using namespace ov::pass::pattern; -// auto weight_pattern = any_input( -// type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && -// shape_matches("[Dims..., 32]")); -// auto weight_cvt_pattern = wrap_type({weight_pattern}); + auto weight_pattern = any_input( + type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && + shape_matches("[Dims..., 32]")); + auto weight_cvt_pattern = wrap_type({weight_pattern}); -// auto scale_pattern = any_input(type_matches(ov::element::Type_t::f8e8m0) && shape_matches("[Dims..., 1]")); -// auto scale_cvt_pattern = wrap_type({scale_pattern}); + auto scale_pattern = any_input(type_matches(ov::element::Type_t::f8e8m0) && shape_matches("[Dims..., 1]")); + auto scale_cvt_pattern = wrap_type({scale_pattern}); -// auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); + auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); -// auto m = std::make_shared(mult_pattern, "does_model_contain_mxfp_patterns"); + auto m = std::make_shared(mult_pattern, "does_model_contain_mxfp_patterns"); -// const auto ops = model->get_ops(); -// for (const auto& n : ops) { -// if (m->match(n)) { -// return true; -// } -// } + const auto ops = model.get_ops(); + for (const auto& n : ops) { + if (m->match(n)) { + return true; + } + } -// return false; -// } + return false; +} diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index 371ee4b7af55..f07156dd58c3 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -7,7 +7,7 @@ #include #include "itt.hpp" -// #include "low_precision/low_precision.hpp" +#include "low_precision/low_precision.hpp" #include "openvino/pass/constant_folding.hpp" #include "openvino/pass/manager.hpp" #include "transformations/common_optimizations/adaptive_pool_to_reduce.hpp" @@ -148,11 +148,11 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr manager.register_pass(TypeVector{f8e4m3}, TypeVector{u4}); manager.register_pass( TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); - // if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(f)) { - // manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, - // false, - // false); - // } + if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(*f)) { + manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, + false, + false); + } } if (!m_use_shapes) { manager.register_pass(); diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index 1620ee644fd6..92149cf8ba9f 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -8,7 +8,7 @@ #include "function_guard.hpp" #include "itt.hpp" #include "layout_utils.hpp" -// #include "low_precision/low_precision.hpp" +#include "low_precision/low_precision.hpp" #include "openvino/core/model.hpp" #include "openvino/op/util/multi_subgraph_base.hpp" #include "openvino/pass/constant_folding.hpp" @@ -94,7 +94,7 @@ void transformation_pipeline(std::shared_ptr& model) { RTInfoCache rt_info_cache; rt_info_cache.store(model); - auto get_manager = []() { + auto get_manager = [](const ov::Model& model) { Manager manager("pre_post_processing"); manager.set_per_pass_validation(false); @@ -118,9 +118,9 @@ void transformation_pipeline(std::shared_ptr& model) { manager, MarkDequantization, TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); - // if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(model)) { - // REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); - // } + if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(model)) { + REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); + } REGISTER_PASS(manager, DisableShapeOfConstantFolding, false); REGISTER_PASS(manager, DisableRandomUniformConstantFolding) // Mark quantized and f16/bf16 compressed constants to prevent CF for them, @@ -145,7 +145,7 @@ void transformation_pipeline(std::shared_ptr& model) { return manager; }; - static thread_local Manager manager = get_manager(); + static thread_local Manager manager = get_manager(*model); manager.run_passes(model); diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 0644cfaaac90..28ccae34847f 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -297,27 +297,24 @@ static bool is_decompression_multiply(const std::shared_ptr node return true; }; - // if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()}) || all_has_types(consumers, {ov::op::v1::Transpose::get_type_info_static()})) { - // for (const auto& consumer : consumers) { - // auto get_child_consumers = [&]() { - // auto child_consumers = consumer.get_node()->get_output_target_inputs(0); - - // // Reshape + Transpose chain - // if (all_has_types(child_consumers, { ov::op::v1::Transpose::get_type_info_static() })) { - // std::set> next_child_consumers; - // for (const auto& child_consumer : child_consumers) { - // const auto grand_child_consumers = child_consumer.get_node()->get_output_target_inputs(0); - // next_child_consumers.insert(grand_child_consumers.begin(), grand_child_consumers.end()); - // } - // return next_child_consumers; - // } - // return child_consumers; - // }; - - // auto child_consumers = get_child_consumers(); - if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()})) { + if (all_has_types(consumers, {ov::opset1::Reshape::get_type_info_static()}) || all_has_types(consumers, {ov::op::v1::Transpose::get_type_info_static()})) { for (const auto& consumer : consumers) { - const auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + auto get_child_consumers = [&]() { + auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + + // Reshape + Transpose chain + if (all_has_types(child_consumers, { ov::op::v1::Transpose::get_type_info_static() })) { + std::set> next_child_consumers; + for (const auto& child_consumer : child_consumers) { + const auto grand_child_consumers = child_consumer.get_node()->get_output_target_inputs(0); + next_child_consumers.insert(grand_child_consumers.begin(), grand_child_consumers.end()); + } + return next_child_consumers; + } + return child_consumers; + }; + + auto child_consumers = get_child_consumers(); for (const auto& child_consumer : child_consumers) { const auto& type_info = child_consumer.get_node()->get_type_info(); @@ -582,12 +579,12 @@ void TransformationsPipeline::apply(std::shared_ptr func) { ov::element::f4e2m1, ov::element::f8e8m0}, !device_info.supports_immad); - // if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(func)) { - // manager.register_pass( - // std::vector{ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1, ov::element::f8e8m0}, - // !device_info.supports_immad, - // !device_info.supports_immad); - // } + if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(*func)) { + manager.register_pass( + std::vector{ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1, ov::element::f8e8m0}, + !device_info.supports_immad, + !device_info.supports_immad); + } const bool is_pa = [&func]() { for (const auto& op : func->get_ops()) { From 721ba4f3d2928cd4efd780c2ecb6b6de36e4b605 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 16 Jun 2026 16:43:21 +0000 Subject: [PATCH 34/48] Bring back reorder code and f8_utils --- .../include/batch_headers/f8_utils.cl | 2466 ++++++++--------- .../cl_kernels/reorder_data.cl | 9 + 2 files changed, 1242 insertions(+), 1233 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl index 230cae04aaff..051fcf380d21 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl @@ -1,1279 +1,1279 @@ -// // Copyright (C) 2026 Intel Corporation -// // SPDX-License-Identifier: Apache-2.0 -// // +// Copyright (C) 2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// -// // TODO: Replace `_intel_convert*` with bultins when ready, current implementations are copied from XeTLA: +// TODO: Replace `_intel_convert*` with bultins when ready, current implementations are copied from XeTLA: -// uchar _f16_to_bf8_universal(half val, bool is_saturation) { -// half val_fp16 = val; -// const ushort *p = (ushort *)&val_fp16; -// const ushort mant = p[0] & 0x3FF; -// const uchar exp_mask = 0x7C; +uchar _f16_to_bf8_universal(half val, bool is_saturation) { + half val_fp16 = val; + const ushort *p = (ushort *)&val_fp16; + const ushort mant = p[0] & 0x3FF; + const uchar exp_mask = 0x7C; -// uchar ret_tmp = p[0] >> 8; -// const bool is_infnan = (ret_tmp & exp_mask) == exp_mask; -// if (is_infnan) { -// ret_tmp |= (mant != 0); // The bit signifying NaNness may have been cut off -// } else { -// ret_tmp += (mant & 0x80) && ((mant & 0x7F) || (mant & 0x100)); // RTE -// if (is_saturation) { -// bool is_overflow = (ret_tmp & exp_mask) == exp_mask; -// ret_tmp -= is_overflow; -// } -// } + uchar ret_tmp = p[0] >> 8; + const bool is_infnan = (ret_tmp & exp_mask) == exp_mask; + if (is_infnan) { + ret_tmp |= (mant != 0); // The bit signifying NaNness may have been cut off + } else { + ret_tmp += (mant & 0x80) && ((mant & 0x7F) || (mant & 0x100)); // RTE + if (is_saturation) { + bool is_overflow = (ret_tmp & exp_mask) == exp_mask; + ret_tmp -= is_overflow; + } + } -// return ret_tmp; -// } + return ret_tmp; +} -// uchar _intel_convert_f16_to_bf8(half val) { -// return _f16_to_bf8_universal(val, false); -// } +uchar _intel_convert_f16_to_bf8(half val) { + return _f16_to_bf8_universal(val, false); +} -// uchar _intel_convert_f16_to_bf8_sat(half val) { -// return _f16_to_bf8_universal(val, true); -// } +uchar _intel_convert_f16_to_bf8_sat(half val) { + return _f16_to_bf8_universal(val, true); +} -// half _intel_convert_bf8_to_f16(uchar val) { -// ushort temp = val; -// temp = temp << 0x8; -// half temp_fp16 = as_half(temp); -// return temp_fp16; -// } +half _intel_convert_bf8_to_f16(uchar val) { + ushort temp = val; + temp = temp << 0x8; + half temp_fp16 = as_half(temp); + return temp_fp16; +} -// char _f16_to_hf8_universal(half val, bool is_saturation) { -// half val_fp16 = val; -// ushort *p = (ushort *)&val_fp16; -// ushort src = p[0]; -// const ushort src_exp_size = 5; -// const ushort src_mant_size = 10; -// const ushort src_exp_bias = (1 << (src_exp_size - 1)) - 1; -// const ushort src_exp_mask = (1 << src_exp_size) - 1; -// const ushort src_mant_mask = (1 << src_mant_size) - 1; -// const short max_exp_unbiased = 8; -// const short min_exp_unbiased = -6; -// const short exp_bias = 7; -// const ushort exp_size = 4; -// const ushort mant_size = 3; -// const uchar nan = 0x7f; -// const uchar max_val = 0x7e; +char _f16_to_hf8_universal(half val, bool is_saturation) { + half val_fp16 = val; + ushort *p = (ushort *)&val_fp16; + ushort src = p[0]; + const ushort src_exp_size = 5; + const ushort src_mant_size = 10; + const ushort src_exp_bias = (1 << (src_exp_size - 1)) - 1; + const ushort src_exp_mask = (1 << src_exp_size) - 1; + const ushort src_mant_mask = (1 << src_mant_size) - 1; + const short max_exp_unbiased = 8; + const short min_exp_unbiased = -6; + const short exp_bias = 7; + const ushort exp_size = 4; + const ushort mant_size = 3; + const uchar nan = 0x7f; + const uchar max_val = 0x7e; -// ushort src_sign = src >> (src_exp_size + src_mant_size); -// ushort src_exp = (src >> src_mant_size) & src_exp_mask; -// short src_exp_unbiased = src_exp - src_exp_bias; -// ushort src_mant = src & src_mant_mask; + ushort src_sign = src >> (src_exp_size + src_mant_size); + ushort src_exp = (src >> src_mant_size) & src_exp_mask; + short src_exp_unbiased = src_exp - src_exp_bias; + ushort src_mant = src & src_mant_mask; -// bool is_src_inf_nan = src_exp == 0x1f; -// bool is_overflow = (src_exp_unbiased > max_exp_unbiased) -// || ((src_exp_unbiased == max_exp_unbiased) && (src_mant > 0x0340)); -// bool is_zero = (src_exp_unbiased < (min_exp_unbiased - mant_size)); -// bool is_denorm = (src_exp_unbiased < min_exp_unbiased) && (!is_zero); + bool is_src_inf_nan = src_exp == 0x1f; + bool is_overflow = (src_exp_unbiased > max_exp_unbiased) + || ((src_exp_unbiased == max_exp_unbiased) && (src_mant > 0x0340)); + bool is_zero = (src_exp_unbiased < (min_exp_unbiased - mant_size)); + bool is_denorm = (src_exp_unbiased < min_exp_unbiased) && (!is_zero); -// uchar dst_val; -// if (is_src_inf_nan) { -// dst_val = nan; -// } else if (is_overflow) { -// dst_val = is_saturation ? max_val : nan; -// } else if (is_zero) { -// dst_val = 0; -// } else if (is_denorm) { -// ushort src_m = src_mant | 0x0400; -// short shift_out_bit = min_exp_unbiased - src_exp_unbiased; -// bool sticky_flag = (src_m & ((1 << shift_out_bit) - 1)) != 0; -// src_m = src_m >> shift_out_bit; -// ushort tail_size = src_mant_size - mant_size; -// sticky_flag -// = sticky_flag || ((src_m & ((1 << (tail_size - 1)) - 1)) != 0); -// bool lsb_bit = src_m & (1 << tail_size); -// bool rnd_bit = src_m & (1 << (tail_size - 1)); -// bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); + uchar dst_val; + if (is_src_inf_nan) { + dst_val = nan; + } else if (is_overflow) { + dst_val = is_saturation ? max_val : nan; + } else if (is_zero) { + dst_val = 0; + } else if (is_denorm) { + ushort src_m = src_mant | 0x0400; + short shift_out_bit = min_exp_unbiased - src_exp_unbiased; + bool sticky_flag = (src_m & ((1 << shift_out_bit) - 1)) != 0; + src_m = src_m >> shift_out_bit; + ushort tail_size = src_mant_size - mant_size; + sticky_flag + = sticky_flag || ((src_m & ((1 << (tail_size - 1)) - 1)) != 0); + bool lsb_bit = src_m & (1 << tail_size); + bool rnd_bit = src_m & (1 << (tail_size - 1)); + bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); -// dst_val = (src_m >> tail_size) + carry; -// } else { -// ushort tail_size = src_mant_size - mant_size; -// bool sticky_flag = (src_mant & ((1 << (tail_size - 1)) - 1)) != 0; -// bool lsb_bit = src_mant & (1 << tail_size); -// bool rnd_bit = src_mant & (1 << (tail_size - 1)); -// bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); -// ushort src_m = (src_mant >> tail_size) + carry; -// ushort src_e = src_exp_unbiased + exp_bias; -// dst_val = (src_e << mant_size) + src_m; -// } -// return src_sign << (exp_size + mant_size) | dst_val; -// } + dst_val = (src_m >> tail_size) + carry; + } else { + ushort tail_size = src_mant_size - mant_size; + bool sticky_flag = (src_mant & ((1 << (tail_size - 1)) - 1)) != 0; + bool lsb_bit = src_mant & (1 << tail_size); + bool rnd_bit = src_mant & (1 << (tail_size - 1)); + bool carry = (lsb_bit && rnd_bit) || (rnd_bit && sticky_flag); + ushort src_m = (src_mant >> tail_size) + carry; + ushort src_e = src_exp_unbiased + exp_bias; + dst_val = (src_e << mant_size) + src_m; + } + return src_sign << (exp_size + mant_size) | dst_val; +} -// char _intel_convert_f16_to_hf8(half val) { -// return _f16_to_hf8_universal(val, false); -// } +char _intel_convert_f16_to_hf8(half val) { + return _f16_to_hf8_universal(val, false); +} -// char _intel_convert_f16_to_hf8_sat(half val) { -// return _f16_to_hf8_universal(val, true); -// } +char _intel_convert_f16_to_hf8_sat(half val) { + return _f16_to_hf8_universal(val, true); +} -// half _intel_convert_hf8_to_f16(char val) { -// const short exp_bias = 7; -// char data = val; -// ushort sign = data >> 7; -// ushort exp = (data >> 3) & 0b1111; -// ushort mant = data & 0x07; -// ushort dst_val; -// if ((exp == 0xf) && (mant == 0x7)) { -// dst_val = 0x7fff; -// } else if ((exp == 0) && (mant == 0)) { -// dst_val = 0; -// } else if ((exp == 0) && (mant != 0)) { -// ushort lz_count = (mant > 3) ? 0 : ((mant > 1) ? 1 : 2); -// ushort dst_exp = exp - exp_bias + 15 - lz_count; -// ushort dst_mant = (mant << (lz_count + 1)) & 0x7; -// dst_val = (dst_exp << 10) | (dst_mant << 7); -// } else { -// ushort dst_exp = exp - exp_bias + 15; -// dst_val = (dst_exp << 10) | (mant << 7); -// } +half _intel_convert_hf8_to_f16(char val) { + const short exp_bias = 7; + char data = val; + ushort sign = data >> 7; + ushort exp = (data >> 3) & 0b1111; + ushort mant = data & 0x07; + ushort dst_val; + if ((exp == 0xf) && (mant == 0x7)) { + dst_val = 0x7fff; + } else if ((exp == 0) && (mant == 0)) { + dst_val = 0; + } else if ((exp == 0) && (mant != 0)) { + ushort lz_count = (mant > 3) ? 0 : ((mant > 1) ? 1 : 2); + ushort dst_exp = exp - exp_bias + 15 - lz_count; + ushort dst_mant = (mant << (lz_count + 1)) & 0x7; + dst_val = (dst_exp << 10) | (dst_mant << 7); + } else { + ushort dst_exp = exp - exp_bias + 15; + dst_val = (dst_exp << 10) | (mant << 7); + } -// ushort temp = (sign << 15) | dst_val; -// half temp_fp16 = as_half(temp); -// return temp_fp16; -// } + ushort temp = (sign << 15) | dst_val; + half temp_fp16 = as_half(temp); + return temp_fp16; +} -// uchar _intel_convert_f32_fo_e8m0(float val) { -// uint val_uint = as_uint(val); -// return (uchar)((val_uint >> 23) & 0xFF); -// } +uchar _intel_convert_f32_fo_e8m0(float val) { + uint val_uint = as_uint(val); + return (uchar)((val_uint >> 23) & 0xFF); +} -// uchar _intel_convert_f32_fo_e8m0_sat(float val) { -// return _intel_convert_f32_fo_e8m0(val); -// } +uchar _intel_convert_f32_fo_e8m0_sat(float val) { + return _intel_convert_f32_fo_e8m0(val); +} -// float _intel_convert_e8m0_to_f32(uchar val) { -// if (val == 0xFF) { -// return as_float(0xFFC00000); // NaN -// } else if (val == 0) { -// return as_float(0x00400000); // 2^(-127) -// } +float _intel_convert_e8m0_to_f32(uchar val) { + if (val == 0xFF) { + return as_float(0xFFC00000); // NaN + } else if (val == 0) { + return as_float(0x00400000); // 2^(-127) + } -// uint temp = (uint)val; -// temp = temp << 23; -// float temp_fp32 = as_float(temp); -// return temp_fp32; -// } + uint temp = (uint)val; + temp = temp << 23; + float temp_fp32 = as_float(temp); + return temp_fp32; +} -// typedef struct fp8e5m2_t { uchar data; } fp8e5m2_t; // bf8 -// typedef struct fp8e5m2_t1 { uchar data; } fp8e5m2_t1; -// typedef struct fp8e5m2_t2 { uchar2 data; } fp8e5m2_t2; -// typedef struct fp8e5m2_t3 { uchar3 data; } fp8e5m2_t3; -// typedef struct fp8e5m2_t4 { uchar4 data; } fp8e5m2_t4; -// typedef struct fp8e5m2_t8 { uchar8 data; } fp8e5m2_t8; -// typedef struct fp8e5m2_t16 { uchar16 data; } fp8e5m2_t16; +typedef struct fp8e5m2_t { uchar data; } fp8e5m2_t; // bf8 +typedef struct fp8e5m2_t1 { uchar data; } fp8e5m2_t1; +typedef struct fp8e5m2_t2 { uchar2 data; } fp8e5m2_t2; +typedef struct fp8e5m2_t3 { uchar3 data; } fp8e5m2_t3; +typedef struct fp8e5m2_t4 { uchar4 data; } fp8e5m2_t4; +typedef struct fp8e5m2_t8 { uchar8 data; } fp8e5m2_t8; +typedef struct fp8e5m2_t16 { uchar16 data; } fp8e5m2_t16; -// typedef struct fp8e4m3_t { char data; } fp8e4m3_t; // hf8 -// typedef struct fp8e4m3_t1 { char data; } fp8e4m3_t1; -// typedef struct fp8e4m3_t2 { char2 data; } fp8e4m3_t2; -// typedef struct fp8e4m3_t3 { char3 data; } fp8e4m3_t3; -// typedef struct fp8e4m3_t4 { char4 data; } fp8e4m3_t4; -// typedef struct fp8e4m3_t8 { char8 data; } fp8e4m3_t8; -// typedef struct fp8e4m3_t16 { char16 data; } fp8e4m3_t16; +typedef struct fp8e4m3_t { char data; } fp8e4m3_t; // hf8 +typedef struct fp8e4m3_t1 { char data; } fp8e4m3_t1; +typedef struct fp8e4m3_t2 { char2 data; } fp8e4m3_t2; +typedef struct fp8e4m3_t3 { char3 data; } fp8e4m3_t3; +typedef struct fp8e4m3_t4 { char4 data; } fp8e4m3_t4; +typedef struct fp8e4m3_t8 { char8 data; } fp8e4m3_t8; +typedef struct fp8e4m3_t16 { char16 data; } fp8e4m3_t16; -// typedef struct fp8e8m0_t { uchar data; } fp8e8m0_t; // e8m0 -// typedef struct fp8e8m0_t1 { uchar data; } fp8e8m0_t1; -// typedef struct fp8e8m0_t2 { uchar2 data; } fp8e8m0_t2; -// typedef struct fp8e8m0_t3 { uchar3 data; } fp8e8m0_t3; -// typedef struct fp8e8m0_t4 { uchar4 data; } fp8e8m0_t4; -// typedef struct fp8e8m0_t8 { uchar8 data; } fp8e8m0_t8; -// typedef struct fp8e8m0_t16 { uchar16 data; } fp8e8m0_t16; +typedef struct fp8e8m0_t { uchar data; } fp8e8m0_t; // e8m0 +typedef struct fp8e8m0_t1 { uchar data; } fp8e8m0_t1; +typedef struct fp8e8m0_t2 { uchar2 data; } fp8e8m0_t2; +typedef struct fp8e8m0_t3 { uchar3 data; } fp8e8m0_t3; +typedef struct fp8e8m0_t4 { uchar4 data; } fp8e8m0_t4; +typedef struct fp8e8m0_t8 { uchar8 data; } fp8e8m0_t8; +typedef struct fp8e8m0_t16 { uchar16 data; } fp8e8m0_t16; -// half __attribute__((overloadable)) _convert_half(fp8e5m2_t val) { -// return _intel_convert_bf8_to_f16(val.data); -// } -// half __attribute__((overloadable)) _convert_half(fp8e5m2_t1 val) { -// return _intel_convert_bf8_to_f16(val.data); -// } -// half2 __attribute__((overloadable)) _convert_half2(fp8e5m2_t2 val) { -// return (half2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); -// } -// half3 __attribute__((overloadable)) _convert_half3(fp8e5m2_t3 val) { -// return (half3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); -// } -// half4 __attribute__((overloadable)) _convert_half4(fp8e5m2_t4 val) { -// return (half4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), -// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); -// } -// half8 __attribute__((overloadable)) _convert_half8(fp8e5m2_t8 val) { -// return (half8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), -// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), -// _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), -// _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); -// } -// half16 __attribute__((overloadable)) _convert_half16(fp8e5m2_t16 val) { -// return (half16)( -// _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), -// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), -// _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), -// _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), -// _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), -// _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), -// _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), -// _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) -// ); -// } +half __attribute__((overloadable)) _convert_half(fp8e5m2_t val) { + return _intel_convert_bf8_to_f16(val.data); +} +half __attribute__((overloadable)) _convert_half(fp8e5m2_t1 val) { + return _intel_convert_bf8_to_f16(val.data); +} +half2 __attribute__((overloadable)) _convert_half2(fp8e5m2_t2 val) { + return (half2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); +} +half3 __attribute__((overloadable)) _convert_half3(fp8e5m2_t3 val) { + return (half3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); +} +half4 __attribute__((overloadable)) _convert_half4(fp8e5m2_t4 val) { + return (half4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); +} +half8 __attribute__((overloadable)) _convert_half8(fp8e5m2_t8 val) { + return (half8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); +} +half16 __attribute__((overloadable)) _convert_half16(fp8e5m2_t16 val) { + return (half16)( + _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), + _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), + _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), + _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), + _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) + ); +} -// float __attribute__((overloadable)) _convert_float(fp8e5m2_t val) { -// return (float)_intel_convert_bf8_to_f16(val.data); -// } -// float __attribute__((overloadable)) _convert_float(fp8e5m2_t1 val) { -// return (float)_intel_convert_bf8_to_f16(val.data); -// } -// float2 __attribute__((overloadable)) _convert_float2(fp8e5m2_t2 val) { -// return (float2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); -// } -// float3 __attribute__((overloadable)) _convert_float3(fp8e5m2_t3 val) { -// return (float3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); -// } -// float4 __attribute__((overloadable)) _convert_float4(fp8e5m2_t4 val) { -// return (float4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), -// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); -// } -// float8 __attribute__((overloadable)) _convert_float8(fp8e5m2_t8 val) { -// return (float8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), -// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), -// _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), -// _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); -// } -// float16 __attribute__((overloadable)) _convert_float16(fp8e5m2_t16 val) { -// return (float16)( -// _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), -// _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), -// _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), -// _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), -// _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), -// _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), -// _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), -// _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) -// ); -// } +float __attribute__((overloadable)) _convert_float(fp8e5m2_t val) { + return (float)_intel_convert_bf8_to_f16(val.data); +} +float __attribute__((overloadable)) _convert_float(fp8e5m2_t1 val) { + return (float)_intel_convert_bf8_to_f16(val.data); +} +float2 __attribute__((overloadable)) _convert_float2(fp8e5m2_t2 val) { + return (float2)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1)); +} +float3 __attribute__((overloadable)) _convert_float3(fp8e5m2_t3 val) { + return (float3)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), _intel_convert_bf8_to_f16(val.data.s2)); +} +float4 __attribute__((overloadable)) _convert_float4(fp8e5m2_t4 val) { + return (float4)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3)); +} +float8 __attribute__((overloadable)) _convert_float8(fp8e5m2_t8 val) { + return (float8)(_intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7)); +} +float16 __attribute__((overloadable)) _convert_float16(fp8e5m2_t16 val) { + return (float16)( + _intel_convert_bf8_to_f16(val.data.s0), _intel_convert_bf8_to_f16(val.data.s1), + _intel_convert_bf8_to_f16(val.data.s2), _intel_convert_bf8_to_f16(val.data.s3), + _intel_convert_bf8_to_f16(val.data.s4), _intel_convert_bf8_to_f16(val.data.s5), + _intel_convert_bf8_to_f16(val.data.s6), _intel_convert_bf8_to_f16(val.data.s7), + _intel_convert_bf8_to_f16(val.data.s8), _intel_convert_bf8_to_f16(val.data.s9), + _intel_convert_bf8_to_f16(val.data.sA), _intel_convert_bf8_to_f16(val.data.sB), + _intel_convert_bf8_to_f16(val.data.sC), _intel_convert_bf8_to_f16(val.data.sD), + _intel_convert_bf8_to_f16(val.data.sE), _intel_convert_bf8_to_f16(val.data.sF) + ); +} -// half __attribute__((overloadable)) _convert_half(fp8e4m3_t val) { -// return _intel_convert_hf8_to_f16(val.data); -// } -// half __attribute__((overloadable)) _convert_half(fp8e4m3_t1 val) { -// return _intel_convert_hf8_to_f16(val.data); -// } -// half2 __attribute__((overloadable)) _convert_half2(fp8e4m3_t2 val) { -// return (half2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); -// } -// half3 __attribute__((overloadable)) _convert_half3(fp8e4m3_t3 val) { -// return (half3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); -// } -// half4 __attribute__((overloadable)) _convert_half4(fp8e4m3_t4 val) { -// return (half4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), -// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); -// } -// half8 __attribute__((overloadable)) _convert_half8(fp8e4m3_t8 val) { -// return (half8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), -// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), -// _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), -// _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); -// } -// half16 __attribute__((overloadable)) _convert_half16(fp8e4m3_t16 val) { -// return (half16)( -// _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), -// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), -// _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), -// _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), -// _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), -// _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), -// _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), -// _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) -// ); -// } +half __attribute__((overloadable)) _convert_half(fp8e4m3_t val) { + return _intel_convert_hf8_to_f16(val.data); +} +half __attribute__((overloadable)) _convert_half(fp8e4m3_t1 val) { + return _intel_convert_hf8_to_f16(val.data); +} +half2 __attribute__((overloadable)) _convert_half2(fp8e4m3_t2 val) { + return (half2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); +} +half3 __attribute__((overloadable)) _convert_half3(fp8e4m3_t3 val) { + return (half3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); +} +half4 __attribute__((overloadable)) _convert_half4(fp8e4m3_t4 val) { + return (half4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); +} +half8 __attribute__((overloadable)) _convert_half8(fp8e4m3_t8 val) { + return (half8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); +} +half16 __attribute__((overloadable)) _convert_half16(fp8e4m3_t16 val) { + return (half16)( + _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), + _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), + _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), + _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), + _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) + ); +} -// float __attribute__((overloadable)) _convert_float(fp8e4m3_t val) { -// return (float)_intel_convert_hf8_to_f16(val.data); -// } -// float __attribute__((overloadable)) _convert_float(fp8e4m3_t1 val) { -// return (float)_intel_convert_hf8_to_f16(val.data); -// } -// float2 __attribute__((overloadable)) _convert_float2(fp8e4m3_t2 val) { -// return (float2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); -// } -// float3 __attribute__((overloadable)) _convert_float3(fp8e4m3_t3 val) { -// return (float3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); -// } -// float4 __attribute__((overloadable)) _convert_float4(fp8e4m3_t4 val) { -// return (float4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), -// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); -// } -// float8 __attribute__((overloadable)) _convert_float8(fp8e4m3_t8 val) { -// return (float8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), -// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), -// _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), -// _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); -// } -// float16 __attribute__((overloadable)) _convert_float16(fp8e4m3_t16 val) { -// return (float16)( -// _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), -// _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), -// _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), -// _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), -// _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), -// _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), -// _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), -// _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) -// ); -// } +float __attribute__((overloadable)) _convert_float(fp8e4m3_t val) { + return (float)_intel_convert_hf8_to_f16(val.data); +} +float __attribute__((overloadable)) _convert_float(fp8e4m3_t1 val) { + return (float)_intel_convert_hf8_to_f16(val.data); +} +float2 __attribute__((overloadable)) _convert_float2(fp8e4m3_t2 val) { + return (float2)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1)); +} +float3 __attribute__((overloadable)) _convert_float3(fp8e4m3_t3 val) { + return (float3)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), _intel_convert_hf8_to_f16(val.data.s2)); +} +float4 __attribute__((overloadable)) _convert_float4(fp8e4m3_t4 val) { + return (float4)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3)); +} +float8 __attribute__((overloadable)) _convert_float8(fp8e4m3_t8 val) { + return (float8)(_intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7)); +} +float16 __attribute__((overloadable)) _convert_float16(fp8e4m3_t16 val) { + return (float16)( + _intel_convert_hf8_to_f16(val.data.s0), _intel_convert_hf8_to_f16(val.data.s1), + _intel_convert_hf8_to_f16(val.data.s2), _intel_convert_hf8_to_f16(val.data.s3), + _intel_convert_hf8_to_f16(val.data.s4), _intel_convert_hf8_to_f16(val.data.s5), + _intel_convert_hf8_to_f16(val.data.s6), _intel_convert_hf8_to_f16(val.data.s7), + _intel_convert_hf8_to_f16(val.data.s8), _intel_convert_hf8_to_f16(val.data.s9), + _intel_convert_hf8_to_f16(val.data.sA), _intel_convert_hf8_to_f16(val.data.sB), + _intel_convert_hf8_to_f16(val.data.sC), _intel_convert_hf8_to_f16(val.data.sD), + _intel_convert_hf8_to_f16(val.data.sE), _intel_convert_hf8_to_f16(val.data.sF) + ); +} -// float __attribute__((overloadable)) _convert_float(fp8e8m0_t val) { -// return (float)_intel_convert_e8m0_to_f32(val.data); -// } -// float __attribute__((overloadable)) _convert_float(fp8e8m0_t1 val) { -// return (float)_intel_convert_e8m0_to_f32(val.data); -// } -// float2 __attribute__((overloadable)) _convert_float2(fp8e8m0_t2 val) { -// return (float2)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1)); -// } -// float3 __attribute__((overloadable)) _convert_float3(fp8e8m0_t3 val) { -// return (float3)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), _intel_convert_e8m0_to_f32(val.data.s2)); -// } -// float4 __attribute__((overloadable)) _convert_float4(fp8e8m0_t4 val) { -// return (float4)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), -// _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3)); -// } -// float8 __attribute__((overloadable)) _convert_float8(fp8e8m0_t8 val) { -// return (float8)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), -// _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), -// _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), -// _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7)); -// } -// float16 __attribute__((overloadable)) _convert_float16(fp8e8m0_t16 val) { -// return (float16)( -// _intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), -// _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), -// _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), -// _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7), -// _intel_convert_e8m0_to_f32(val.data.s8), _intel_convert_e8m0_to_f32(val.data.s9), -// _intel_convert_e8m0_to_f32(val.data.sA), _intel_convert_e8m0_to_f32(val.data.sB), -// _intel_convert_e8m0_to_f32(val.data.sC), _intel_convert_e8m0_to_f32(val.data.sD), -// _intel_convert_e8m0_to_f32(val.data.sE), _intel_convert_e8m0_to_f32(val.data.sF) -// ); -// } +float __attribute__((overloadable)) _convert_float(fp8e8m0_t val) { + return (float)_intel_convert_e8m0_to_f32(val.data); +} +float __attribute__((overloadable)) _convert_float(fp8e8m0_t1 val) { + return (float)_intel_convert_e8m0_to_f32(val.data); +} +float2 __attribute__((overloadable)) _convert_float2(fp8e8m0_t2 val) { + return (float2)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1)); +} +float3 __attribute__((overloadable)) _convert_float3(fp8e8m0_t3 val) { + return (float3)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), _intel_convert_e8m0_to_f32(val.data.s2)); +} +float4 __attribute__((overloadable)) _convert_float4(fp8e8m0_t4 val) { + return (float4)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), + _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3)); +} +float8 __attribute__((overloadable)) _convert_float8(fp8e8m0_t8 val) { + return (float8)(_intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), + _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), + _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), + _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7)); +} +float16 __attribute__((overloadable)) _convert_float16(fp8e8m0_t16 val) { + return (float16)( + _intel_convert_e8m0_to_f32(val.data.s0), _intel_convert_e8m0_to_f32(val.data.s1), + _intel_convert_e8m0_to_f32(val.data.s2), _intel_convert_e8m0_to_f32(val.data.s3), + _intel_convert_e8m0_to_f32(val.data.s4), _intel_convert_e8m0_to_f32(val.data.s5), + _intel_convert_e8m0_to_f32(val.data.s6), _intel_convert_e8m0_to_f32(val.data.s7), + _intel_convert_e8m0_to_f32(val.data.s8), _intel_convert_e8m0_to_f32(val.data.s9), + _intel_convert_e8m0_to_f32(val.data.sA), _intel_convert_e8m0_to_f32(val.data.sB), + _intel_convert_e8m0_to_f32(val.data.sC), _intel_convert_e8m0_to_f32(val.data.sD), + _intel_convert_e8m0_to_f32(val.data.sE), _intel_convert_e8m0_to_f32(val.data.sF) + ); +} -// fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(float val) { -// fp8e5m2_t res; -// res.data = _intel_convert_f16_to_bf8((half)val); -// return res; -// } -// fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(float val[1]) { -// fp8e5m2_t1 res; -// res.data = _intel_convert_f16_to_bf8((half)val[0]); -// return res; -// } -// fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(float2 val) { -// fp8e5m2_t2 res; -// res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); -// res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); -// return res; -// } -// fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(float3 val) { -// fp8e5m2_t3 res; -// res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); -// res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); -// res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); -// return res; -// } -// fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(float4 val) { -// fp8e5m2_t4 res; -// res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); -// res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); -// res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); -// res.data.s3 = _intel_convert_f16_to_bf8((half)val.w); -// return res; -// } -// fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(float8 val) { -// fp8e5m2_t8 res; -// res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); -// res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); -// res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); -// res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); -// res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); -// res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); -// res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); -// res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); -// return res; -// } -// fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(float16 val) { -// fp8e5m2_t16 res; -// res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); -// res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); -// res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); -// res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); -// res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); -// res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); -// res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); -// res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); -// res.data.s8 = _intel_convert_f16_to_bf8((half)val.s8); -// res.data.s9 = _intel_convert_f16_to_bf8((half)val.s9); -// res.data.sA = _intel_convert_f16_to_bf8((half)val.sA); -// res.data.sB = _intel_convert_f16_to_bf8((half)val.sB); -// res.data.sC = _intel_convert_f16_to_bf8((half)val.sC); -// res.data.sD = _intel_convert_f16_to_bf8((half)val.sD); -// res.data.sE = _intel_convert_f16_to_bf8((half)val.sE); -// res.data.sF = _intel_convert_f16_to_bf8((half)val.sF); -// return res; -// } +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(float val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8((half)val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(float val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8((half)val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(float2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(float3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(float4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.z); + res.data.s3 = _intel_convert_f16_to_bf8((half)val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(float8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(float16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8((half)val.s7); + res.data.s8 = _intel_convert_f16_to_bf8((half)val.s8); + res.data.s9 = _intel_convert_f16_to_bf8((half)val.s9); + res.data.sA = _intel_convert_f16_to_bf8((half)val.sA); + res.data.sB = _intel_convert_f16_to_bf8((half)val.sB); + res.data.sC = _intel_convert_f16_to_bf8((half)val.sC); + res.data.sD = _intel_convert_f16_to_bf8((half)val.sD); + res.data.sE = _intel_convert_f16_to_bf8((half)val.sE); + res.data.sF = _intel_convert_f16_to_bf8((half)val.sF); + return res; +} -// fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(half val) { -// fp8e5m2_t res; -// res.data = _intel_convert_f16_to_bf8(val); -// return res; -// } -// fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(half val[1]) { -// fp8e5m2_t1 res; -// res.data = _intel_convert_f16_to_bf8(val[0]); -// return res; -// } -// fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(half2 val) { -// fp8e5m2_t2 res; -// res.data.s0 = _intel_convert_f16_to_bf8(val.x); -// res.data.s1 = _intel_convert_f16_to_bf8(val.y); -// return res; -// } -// fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(half3 val) { -// fp8e5m2_t3 res; -// res.data.s0 = _intel_convert_f16_to_bf8(val.x); -// res.data.s1 = _intel_convert_f16_to_bf8(val.y); -// res.data.s2 = _intel_convert_f16_to_bf8(val.z); -// return res; -// } -// fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(half4 val) { -// fp8e5m2_t4 res; -// res.data.s0 = _intel_convert_f16_to_bf8(val.x); -// res.data.s1 = _intel_convert_f16_to_bf8(val.y); -// res.data.s2 = _intel_convert_f16_to_bf8(val.z); -// res.data.s3 = _intel_convert_f16_to_bf8(val.w); -// return res; -// } -// fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(half8 val) { -// fp8e5m2_t8 res; -// res.data.s0 = _intel_convert_f16_to_bf8(val.s0); -// res.data.s1 = _intel_convert_f16_to_bf8(val.s1); -// res.data.s2 = _intel_convert_f16_to_bf8(val.s2); -// res.data.s3 = _intel_convert_f16_to_bf8(val.s3); -// res.data.s4 = _intel_convert_f16_to_bf8(val.s4); -// res.data.s5 = _intel_convert_f16_to_bf8(val.s5); -// res.data.s6 = _intel_convert_f16_to_bf8(val.s6); -// res.data.s7 = _intel_convert_f16_to_bf8(val.s7); -// return res; -// } -// fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(half16 val) { -// fp8e5m2_t16 res; -// res.data.s0 = _intel_convert_f16_to_bf8(val.s0); -// res.data.s1 = _intel_convert_f16_to_bf8(val.s1); -// res.data.s2 = _intel_convert_f16_to_bf8(val.s2); -// res.data.s3 = _intel_convert_f16_to_bf8(val.s3); -// res.data.s4 = _intel_convert_f16_to_bf8(val.s4); -// res.data.s5 = _intel_convert_f16_to_bf8(val.s5); -// res.data.s6 = _intel_convert_f16_to_bf8(val.s6); -// res.data.s7 = _intel_convert_f16_to_bf8(val.s7); -// res.data.s8 = _intel_convert_f16_to_bf8(val.s8); -// res.data.s9 = _intel_convert_f16_to_bf8(val.s9); -// res.data.sA = _intel_convert_f16_to_bf8(val.sA); -// res.data.sB = _intel_convert_f16_to_bf8(val.sB); -// res.data.sC = _intel_convert_f16_to_bf8(val.sC); -// res.data.sD = _intel_convert_f16_to_bf8(val.sD); -// res.data.sE = _intel_convert_f16_to_bf8(val.sE); -// res.data.sF = _intel_convert_f16_to_bf8(val.sF); -// return res; -// } +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t(half val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8(val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1(half val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8(val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2(half2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.x); + res.data.s1 = _intel_convert_f16_to_bf8(val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3(half3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.x); + res.data.s1 = _intel_convert_f16_to_bf8(val.y); + res.data.s2 = _intel_convert_f16_to_bf8(val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4(half4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.x); + res.data.s1 = _intel_convert_f16_to_bf8(val.y); + res.data.s2 = _intel_convert_f16_to_bf8(val.z); + res.data.s3 = _intel_convert_f16_to_bf8(val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8(half8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8(val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16(half16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8(val.s7); + res.data.s8 = _intel_convert_f16_to_bf8(val.s8); + res.data.s9 = _intel_convert_f16_to_bf8(val.s9); + res.data.sA = _intel_convert_f16_to_bf8(val.sA); + res.data.sB = _intel_convert_f16_to_bf8(val.sB); + res.data.sC = _intel_convert_f16_to_bf8(val.sC); + res.data.sD = _intel_convert_f16_to_bf8(val.sD); + res.data.sE = _intel_convert_f16_to_bf8(val.sE); + res.data.sF = _intel_convert_f16_to_bf8(val.sF); + return res; +} -// fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(float val) { -// fp8e5m2_t res; -// res.data = _intel_convert_f16_to_bf8_sat((half)val); -// return res; -// } -// fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(float val[1]) { -// fp8e5m2_t1 res; -// res.data = _intel_convert_f16_to_bf8_sat((half)val[0]); -// return res; -// } -// fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(float2 val) { -// fp8e5m2_t2 res; -// res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); -// res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); -// return res; -// } -// fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(float3 val) { -// fp8e5m2_t3 res; -// res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); -// res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); -// res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); -// return res; -// } -// fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(float4 val) { -// fp8e5m2_t4 res; -// res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); -// res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); -// res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); -// res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.w); -// return res; -// } -// fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(float8 val) { -// fp8e5m2_t8 res; -// res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); -// res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); -// res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); -// res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); -// res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); -// res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); -// res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); -// res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); -// return res; -// } -// fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(float16 val) { -// fp8e5m2_t16 res; -// res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); -// res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); -// res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); -// res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); -// res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); -// res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); -// res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); -// res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); -// res.data.s8 = _intel_convert_f16_to_bf8_sat((half)val.s8); -// res.data.s9 = _intel_convert_f16_to_bf8_sat((half)val.s9); -// res.data.sA = _intel_convert_f16_to_bf8_sat((half)val.sA); -// res.data.sB = _intel_convert_f16_to_bf8_sat((half)val.sB); -// res.data.sC = _intel_convert_f16_to_bf8_sat((half)val.sC); -// res.data.sD = _intel_convert_f16_to_bf8_sat((half)val.sD); -// res.data.sE = _intel_convert_f16_to_bf8_sat((half)val.sE); -// res.data.sF = _intel_convert_f16_to_bf8_sat((half)val.sF); -// return res; -// } +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(float val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8_sat((half)val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(float val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8_sat((half)val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(float2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(float3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(float4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.z); + res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(float8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(float16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat((half)val.s7); + res.data.s8 = _intel_convert_f16_to_bf8_sat((half)val.s8); + res.data.s9 = _intel_convert_f16_to_bf8_sat((half)val.s9); + res.data.sA = _intel_convert_f16_to_bf8_sat((half)val.sA); + res.data.sB = _intel_convert_f16_to_bf8_sat((half)val.sB); + res.data.sC = _intel_convert_f16_to_bf8_sat((half)val.sC); + res.data.sD = _intel_convert_f16_to_bf8_sat((half)val.sD); + res.data.sE = _intel_convert_f16_to_bf8_sat((half)val.sE); + res.data.sF = _intel_convert_f16_to_bf8_sat((half)val.sF); + return res; +} -// fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(half val) { -// fp8e5m2_t res; -// res.data = _intel_convert_f16_to_bf8_sat(val); -// return res; -// } -// fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(half val[1]) { -// fp8e5m2_t1 res; -// res.data = _intel_convert_f16_to_bf8_sat(val[0]); -// return res; -// } -// fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(half2 val) { -// fp8e5m2_t2 res; -// res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); -// res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); -// return res; -// } -// fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(half3 val) { -// fp8e5m2_t3 res; -// res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); -// res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); -// res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); -// return res; -// } -// fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(half4 val) { -// fp8e5m2_t4 res; -// res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); -// res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); -// res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); -// res.data.s3 = _intel_convert_f16_to_bf8_sat(val.w); -// return res; -// } -// fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(half8 val) { -// fp8e5m2_t8 res; -// res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); -// res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); -// res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); -// res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); -// res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); -// res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); -// res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); -// res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); -// return res; -// } -// fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(half16 val) { -// fp8e5m2_t16 res; -// res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); -// res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); -// res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); -// res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); -// res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); -// res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); -// res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); -// res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); -// res.data.s8 = _intel_convert_f16_to_bf8_sat(val.s8); -// res.data.s9 = _intel_convert_f16_to_bf8_sat(val.s9); -// res.data.sA = _intel_convert_f16_to_bf8_sat(val.sA); -// res.data.sB = _intel_convert_f16_to_bf8_sat(val.sB); -// res.data.sC = _intel_convert_f16_to_bf8_sat(val.sC); -// res.data.sD = _intel_convert_f16_to_bf8_sat(val.sD); -// res.data.sE = _intel_convert_f16_to_bf8_sat(val.sE); -// res.data.sF = _intel_convert_f16_to_bf8_sat(val.sF); -// return res; -// } +fp8e5m2_t __attribute__((overloadable)) _convert_fp8e5m2_t_sat(half val) { + fp8e5m2_t res; + res.data = _intel_convert_f16_to_bf8_sat(val); + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) _convert_fp8e5m2_t1_sat(half val[1]) { + fp8e5m2_t1 res; + res.data = _intel_convert_f16_to_bf8_sat(val[0]); + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) _convert_fp8e5m2_t2_sat(half2 val) { + fp8e5m2_t2 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) _convert_fp8e5m2_t3_sat(half3 val) { + fp8e5m2_t3 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) _convert_fp8e5m2_t4_sat(half4 val) { + fp8e5m2_t4 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.z); + res.data.s3 = _intel_convert_f16_to_bf8_sat(val.w); + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) _convert_fp8e5m2_t8_sat(half8 val) { + fp8e5m2_t8 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) _convert_fp8e5m2_t16_sat(half16 val) { + fp8e5m2_t16 res; + res.data.s0 = _intel_convert_f16_to_bf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_bf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_bf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_bf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_bf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_bf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_bf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_bf8_sat(val.s7); + res.data.s8 = _intel_convert_f16_to_bf8_sat(val.s8); + res.data.s9 = _intel_convert_f16_to_bf8_sat(val.s9); + res.data.sA = _intel_convert_f16_to_bf8_sat(val.sA); + res.data.sB = _intel_convert_f16_to_bf8_sat(val.sB); + res.data.sC = _intel_convert_f16_to_bf8_sat(val.sC); + res.data.sD = _intel_convert_f16_to_bf8_sat(val.sD); + res.data.sE = _intel_convert_f16_to_bf8_sat(val.sE); + res.data.sF = _intel_convert_f16_to_bf8_sat(val.sF); + return res; +} -// fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(float val) { -// fp8e4m3_t res; -// res.data = _intel_convert_f16_to_hf8((half)val); -// return res; -// } -// fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(float val[1]) { -// fp8e4m3_t1 res; -// res.data = _intel_convert_f16_to_hf8((half)val[0]); -// return res; -// } -// fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(float2 val) { -// fp8e4m3_t2 res; -// res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); -// res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); -// return res; -// } -// fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(float3 val) { -// fp8e4m3_t3 res; -// res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); -// res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); -// res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); -// return res; -// } -// fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(float4 val) { -// fp8e4m3_t4 res; -// res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); -// res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); -// res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); -// res.data.s3 = _intel_convert_f16_to_hf8((half)val.w); -// return res; -// } -// fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(float8 val) { -// fp8e4m3_t8 res; -// res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); -// res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); -// res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); -// res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); -// res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); -// res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); -// res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); -// res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); -// return res; -// } -// fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(float16 val) { -// fp8e4m3_t16 res; -// res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); -// res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); -// res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); -// res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); -// res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); -// res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); -// res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); -// res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); -// res.data.s8 = _intel_convert_f16_to_hf8((half)val.s8); -// res.data.s9 = _intel_convert_f16_to_hf8((half)val.s9); -// res.data.sA = _intel_convert_f16_to_hf8((half)val.sA); -// res.data.sB = _intel_convert_f16_to_hf8((half)val.sB); -// res.data.sC = _intel_convert_f16_to_hf8((half)val.sC); -// res.data.sD = _intel_convert_f16_to_hf8((half)val.sD); -// res.data.sE = _intel_convert_f16_to_hf8((half)val.sE); -// res.data.sF = _intel_convert_f16_to_hf8((half)val.sF); -// return res; -// } +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(float val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8((half)val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(float val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8((half)val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(float2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(float3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(float4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.z); + res.data.s3 = _intel_convert_f16_to_hf8((half)val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(float8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(float16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8((half)val.s7); + res.data.s8 = _intel_convert_f16_to_hf8((half)val.s8); + res.data.s9 = _intel_convert_f16_to_hf8((half)val.s9); + res.data.sA = _intel_convert_f16_to_hf8((half)val.sA); + res.data.sB = _intel_convert_f16_to_hf8((half)val.sB); + res.data.sC = _intel_convert_f16_to_hf8((half)val.sC); + res.data.sD = _intel_convert_f16_to_hf8((half)val.sD); + res.data.sE = _intel_convert_f16_to_hf8((half)val.sE); + res.data.sF = _intel_convert_f16_to_hf8((half)val.sF); + return res; +} -// fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(half val) { -// fp8e4m3_t res; -// res.data = _intel_convert_f16_to_hf8(val); -// return res; -// } -// fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(half val[1]) { -// fp8e4m3_t1 res; -// res.data = _intel_convert_f16_to_hf8(val[0]); -// return res; -// } -// fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(half2 val) { -// fp8e4m3_t2 res; -// res.data.s0 = _intel_convert_f16_to_hf8(val.x); -// res.data.s1 = _intel_convert_f16_to_hf8(val.y); -// return res; -// } -// fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(half3 val) { -// fp8e4m3_t3 res; -// res.data.s0 = _intel_convert_f16_to_hf8(val.x); -// res.data.s1 = _intel_convert_f16_to_hf8(val.y); -// res.data.s2 = _intel_convert_f16_to_hf8(val.z); -// return res; -// } -// fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(half4 val) { -// fp8e4m3_t4 res; -// res.data.s0 = _intel_convert_f16_to_hf8(val.x); -// res.data.s1 = _intel_convert_f16_to_hf8(val.y); -// res.data.s2 = _intel_convert_f16_to_hf8(val.z); -// res.data.s3 = _intel_convert_f16_to_hf8(val.w); -// return res; -// } -// fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(half8 val) { -// fp8e4m3_t8 res; -// res.data.s0 = _intel_convert_f16_to_hf8(val.s0); -// res.data.s1 = _intel_convert_f16_to_hf8(val.s1); -// res.data.s2 = _intel_convert_f16_to_hf8(val.s2); -// res.data.s3 = _intel_convert_f16_to_hf8(val.s3); -// res.data.s4 = _intel_convert_f16_to_hf8(val.s4); -// res.data.s5 = _intel_convert_f16_to_hf8(val.s5); -// res.data.s6 = _intel_convert_f16_to_hf8(val.s6); -// res.data.s7 = _intel_convert_f16_to_hf8(val.s7); -// return res; -// } -// fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(half16 val) { -// fp8e4m3_t16 res; -// res.data.s0 = _intel_convert_f16_to_hf8(val.s0); -// res.data.s1 = _intel_convert_f16_to_hf8(val.s1); -// res.data.s2 = _intel_convert_f16_to_hf8(val.s2); -// res.data.s3 = _intel_convert_f16_to_hf8(val.s3); -// res.data.s4 = _intel_convert_f16_to_hf8(val.s4); -// res.data.s5 = _intel_convert_f16_to_hf8(val.s5); -// res.data.s6 = _intel_convert_f16_to_hf8(val.s6); -// res.data.s7 = _intel_convert_f16_to_hf8(val.s7); -// res.data.s8 = _intel_convert_f16_to_hf8(val.s8); -// res.data.s9 = _intel_convert_f16_to_hf8(val.s9); -// res.data.sA = _intel_convert_f16_to_hf8(val.sA); -// res.data.sB = _intel_convert_f16_to_hf8(val.sB); -// res.data.sC = _intel_convert_f16_to_hf8(val.sC); -// res.data.sD = _intel_convert_f16_to_hf8(val.sD); -// res.data.sE = _intel_convert_f16_to_hf8(val.sE); -// res.data.sF = _intel_convert_f16_to_hf8(val.sF); -// return res; -// } +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t(half val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8(val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1(half val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8(val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2(half2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.x); + res.data.s1 = _intel_convert_f16_to_hf8(val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3(half3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.x); + res.data.s1 = _intel_convert_f16_to_hf8(val.y); + res.data.s2 = _intel_convert_f16_to_hf8(val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4(half4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.x); + res.data.s1 = _intel_convert_f16_to_hf8(val.y); + res.data.s2 = _intel_convert_f16_to_hf8(val.z); + res.data.s3 = _intel_convert_f16_to_hf8(val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8(half8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8(val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16(half16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8(val.s7); + res.data.s8 = _intel_convert_f16_to_hf8(val.s8); + res.data.s9 = _intel_convert_f16_to_hf8(val.s9); + res.data.sA = _intel_convert_f16_to_hf8(val.sA); + res.data.sB = _intel_convert_f16_to_hf8(val.sB); + res.data.sC = _intel_convert_f16_to_hf8(val.sC); + res.data.sD = _intel_convert_f16_to_hf8(val.sD); + res.data.sE = _intel_convert_f16_to_hf8(val.sE); + res.data.sF = _intel_convert_f16_to_hf8(val.sF); + return res; +} -// fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(float val) { -// fp8e4m3_t res; -// res.data = _intel_convert_f16_to_hf8_sat((half)val); -// return res; -// } -// fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(float val[1]) { -// fp8e4m3_t1 res; -// res.data = _intel_convert_f16_to_hf8_sat((half)val[0]); -// return res; -// } -// fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(float2 val) { -// fp8e4m3_t2 res; -// res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); -// res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); -// return res; -// } -// fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(float3 val) { -// fp8e4m3_t3 res; -// res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); -// res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); -// res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); -// return res; -// } -// fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(float4 val) { -// fp8e4m3_t4 res; -// res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); -// res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); -// res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); -// res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.w); -// return res; -// } -// fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(float8 val) { -// fp8e4m3_t8 res; -// res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); -// res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); -// res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); -// res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); -// res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); -// res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); -// res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); -// res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); -// return res; -// } -// fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(float16 val) { -// fp8e4m3_t16 res; -// res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); -// res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); -// res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); -// res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); -// res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); -// res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); -// res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); -// res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); -// res.data.s8 = _intel_convert_f16_to_hf8_sat((half)val.s8); -// res.data.s9 = _intel_convert_f16_to_hf8_sat((half)val.s9); -// res.data.sA = _intel_convert_f16_to_hf8_sat((half)val.sA); -// res.data.sB = _intel_convert_f16_to_hf8_sat((half)val.sB); -// res.data.sC = _intel_convert_f16_to_hf8_sat((half)val.sC); -// res.data.sD = _intel_convert_f16_to_hf8_sat((half)val.sD); -// res.data.sE = _intel_convert_f16_to_hf8_sat((half)val.sE); -// res.data.sF = _intel_convert_f16_to_hf8_sat((half)val.sF); -// return res; -// } +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(float val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8_sat((half)val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(float val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8_sat((half)val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(float2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(float3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(float4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.z); + res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(float8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(float16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat((half)val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat((half)val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat((half)val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat((half)val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat((half)val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat((half)val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat((half)val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat((half)val.s7); + res.data.s8 = _intel_convert_f16_to_hf8_sat((half)val.s8); + res.data.s9 = _intel_convert_f16_to_hf8_sat((half)val.s9); + res.data.sA = _intel_convert_f16_to_hf8_sat((half)val.sA); + res.data.sB = _intel_convert_f16_to_hf8_sat((half)val.sB); + res.data.sC = _intel_convert_f16_to_hf8_sat((half)val.sC); + res.data.sD = _intel_convert_f16_to_hf8_sat((half)val.sD); + res.data.sE = _intel_convert_f16_to_hf8_sat((half)val.sE); + res.data.sF = _intel_convert_f16_to_hf8_sat((half)val.sF); + return res; +} -// fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(half val) { -// fp8e4m3_t res; -// res.data = _intel_convert_f16_to_hf8_sat(val); -// return res; -// } -// fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(half val[1]) { -// fp8e4m3_t1 res; -// res.data = _intel_convert_f16_to_hf8_sat(val[0]); -// return res; -// } -// fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(half2 val) { -// fp8e4m3_t2 res; -// res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); -// res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); -// return res; -// } -// fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(half3 val) { -// fp8e4m3_t3 res; -// res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); -// res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); -// res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); -// return res; -// } -// fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(half4 val) { -// fp8e4m3_t4 res; -// res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); -// res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); -// res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); -// res.data.s3 = _intel_convert_f16_to_hf8_sat(val.w); -// return res; -// } -// fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(half8 val) { -// fp8e4m3_t8 res; -// res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); -// res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); -// res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); -// res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); -// res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); -// res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); -// res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); -// res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); -// return res; -// } -// fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(half16 val) { -// fp8e4m3_t16 res; -// res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); -// res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); -// res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); -// res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); -// res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); -// res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); -// res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); -// res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); -// res.data.s8 = _intel_convert_f16_to_hf8_sat(val.s8); -// res.data.s9 = _intel_convert_f16_to_hf8_sat(val.s9); -// res.data.sA = _intel_convert_f16_to_hf8_sat(val.sA); -// res.data.sB = _intel_convert_f16_to_hf8_sat(val.sB); -// res.data.sC = _intel_convert_f16_to_hf8_sat(val.sC); -// res.data.sD = _intel_convert_f16_to_hf8_sat(val.sD); -// res.data.sE = _intel_convert_f16_to_hf8_sat(val.sE); -// res.data.sF = _intel_convert_f16_to_hf8_sat(val.sF); -// return res; -// } +fp8e4m3_t __attribute__((overloadable)) _convert_fp8e4m3_t_sat(half val) { + fp8e4m3_t res; + res.data = _intel_convert_f16_to_hf8_sat(val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) _convert_fp8e4m3_t1_sat(half val[1]) { + fp8e4m3_t1 res; + res.data = _intel_convert_f16_to_hf8_sat(val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) _convert_fp8e4m3_t2_sat(half2 val) { + fp8e4m3_t2 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) _convert_fp8e4m3_t3_sat(half3 val) { + fp8e4m3_t3 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) _convert_fp8e4m3_t4_sat(half4 val) { + fp8e4m3_t4 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.x); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.y); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.z); + res.data.s3 = _intel_convert_f16_to_hf8_sat(val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) _convert_fp8e4m3_t8_sat(half8 val) { + fp8e4m3_t8 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) _convert_fp8e4m3_t16_sat(half16 val) { + fp8e4m3_t16 res; + res.data.s0 = _intel_convert_f16_to_hf8_sat(val.s0); + res.data.s1 = _intel_convert_f16_to_hf8_sat(val.s1); + res.data.s2 = _intel_convert_f16_to_hf8_sat(val.s2); + res.data.s3 = _intel_convert_f16_to_hf8_sat(val.s3); + res.data.s4 = _intel_convert_f16_to_hf8_sat(val.s4); + res.data.s5 = _intel_convert_f16_to_hf8_sat(val.s5); + res.data.s6 = _intel_convert_f16_to_hf8_sat(val.s6); + res.data.s7 = _intel_convert_f16_to_hf8_sat(val.s7); + res.data.s8 = _intel_convert_f16_to_hf8_sat(val.s8); + res.data.s9 = _intel_convert_f16_to_hf8_sat(val.s9); + res.data.sA = _intel_convert_f16_to_hf8_sat(val.sA); + res.data.sB = _intel_convert_f16_to_hf8_sat(val.sB); + res.data.sC = _intel_convert_f16_to_hf8_sat(val.sC); + res.data.sD = _intel_convert_f16_to_hf8_sat(val.sD); + res.data.sE = _intel_convert_f16_to_hf8_sat(val.sE); + res.data.sF = _intel_convert_f16_to_hf8_sat(val.sF); + return res; +} -// fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(fp8e8m0_t val) { -// return val; -// } +fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(fp8e8m0_t val) { + return val; +} -// fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(float val) { -// fp8e8m0_t res; -// res.data = _intel_convert_f32_fo_e8m0(val); -// return res; -// } -// fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1(float val[1]) { -// fp8e8m0_t1 res; -// res.data = _intel_convert_f32_fo_e8m0(val[0]); -// return res; -// } -// fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2(float2 val) { -// fp8e8m0_t2 res; -// res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); -// res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); -// return res; -// } -// fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3(float3 val) { -// fp8e8m0_t3 res; -// res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); -// res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); -// res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); -// return res; -// } -// fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4(float4 val) { -// fp8e8m0_t4 res; -// res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); -// res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); -// res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); -// res.data.s3 = _intel_convert_f32_fo_e8m0(val.w); -// return res; -// } -// fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8(float8 val) { -// fp8e8m0_t8 res; -// res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); -// res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); -// res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); -// res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); -// res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); -// res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); -// res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); -// res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); -// return res; -// } -// fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16(float16 val) { -// fp8e8m0_t16 res; -// res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); -// res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); -// res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); -// res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); -// res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); -// res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); -// res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); -// res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); -// res.data.s8 = _intel_convert_f32_fo_e8m0(val.s8); -// res.data.s9 = _intel_convert_f32_fo_e8m0(val.s9); -// res.data.sA = _intel_convert_f32_fo_e8m0(val.sA); -// res.data.sB = _intel_convert_f32_fo_e8m0(val.sB); -// res.data.sC = _intel_convert_f32_fo_e8m0(val.sC); -// res.data.sD = _intel_convert_f32_fo_e8m0(val.sD); -// res.data.sE = _intel_convert_f32_fo_e8m0(val.sE); -// res.data.sF = _intel_convert_f32_fo_e8m0(val.sF); -// return res; -// } +fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t(float val) { + fp8e8m0_t res; + res.data = _intel_convert_f32_fo_e8m0(val); + return res; +} +fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1(float val[1]) { + fp8e8m0_t1 res; + res.data = _intel_convert_f32_fo_e8m0(val[0]); + return res; +} +fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2(float2 val) { + fp8e8m0_t2 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); + return res; +} +fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3(float3 val) { + fp8e8m0_t3 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); + return res; +} +fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4(float4 val) { + fp8e8m0_t4 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.z); + res.data.s3 = _intel_convert_f32_fo_e8m0(val.w); + return res; +} +fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8(float8 val) { + fp8e8m0_t8 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); + return res; +} +fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16(float16 val) { + fp8e8m0_t16 res; + res.data.s0 = _intel_convert_f32_fo_e8m0(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0(val.s7); + res.data.s8 = _intel_convert_f32_fo_e8m0(val.s8); + res.data.s9 = _intel_convert_f32_fo_e8m0(val.s9); + res.data.sA = _intel_convert_f32_fo_e8m0(val.sA); + res.data.sB = _intel_convert_f32_fo_e8m0(val.sB); + res.data.sC = _intel_convert_f32_fo_e8m0(val.sC); + res.data.sD = _intel_convert_f32_fo_e8m0(val.sD); + res.data.sE = _intel_convert_f32_fo_e8m0(val.sE); + res.data.sF = _intel_convert_f32_fo_e8m0(val.sF); + return res; +} -// fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t_sat(float val) { -// fp8e8m0_t res; -// res.data = _intel_convert_f32_fo_e8m0_sat(val); -// return res; -// } -// fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1_sat(float val[1]) { -// fp8e8m0_t1 res; -// res.data = _intel_convert_f32_fo_e8m0_sat(val[0]); -// return res; -// } -// fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2_sat(float2 val) { -// fp8e8m0_t2 res; -// res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); -// res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); -// return res; -// } -// fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3_sat(float3 val) { -// fp8e8m0_t3 res; -// res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); -// res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); -// res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); -// return res; -// } -// fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4_sat(float4 val) { -// fp8e8m0_t4 res; -// res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); -// res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); -// res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); -// res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.w); -// return res; -// } -// fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8_sat(float8 val) { -// fp8e8m0_t8 res; -// res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); -// res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); -// res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); -// res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); -// res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); -// res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); -// res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); -// res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); -// return res; -// } -// fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16_sat(float16 val) { -// fp8e8m0_t16 res; -// res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); -// res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); -// res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); -// res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); -// res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); -// res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); -// res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); -// res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); -// res.data.s8 = _intel_convert_f32_fo_e8m0_sat(val.s8); -// res.data.s9 = _intel_convert_f32_fo_e8m0_sat(val.s9); -// res.data.sA = _intel_convert_f32_fo_e8m0_sat(val.sA); -// res.data.sB = _intel_convert_f32_fo_e8m0_sat(val.sB); -// res.data.sC = _intel_convert_f32_fo_e8m0_sat(val.sC); -// res.data.sD = _intel_convert_f32_fo_e8m0_sat(val.sD); -// res.data.sE = _intel_convert_f32_fo_e8m0_sat(val.sE); -// res.data.sF = _intel_convert_f32_fo_e8m0_sat(val.sF); -// return res; -// } +fp8e8m0_t __attribute__((overloadable)) _convert_fp8e8m0_t_sat(float val) { + fp8e8m0_t res; + res.data = _intel_convert_f32_fo_e8m0_sat(val); + return res; +} +fp8e8m0_t1 __attribute__((overloadable)) _convert_fp8e8m0_t1_sat(float val[1]) { + fp8e8m0_t1 res; + res.data = _intel_convert_f32_fo_e8m0_sat(val[0]); + return res; +} +fp8e8m0_t2 __attribute__((overloadable)) _convert_fp8e8m0_t2_sat(float2 val) { + fp8e8m0_t2 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); + return res; +} +fp8e8m0_t3 __attribute__((overloadable)) _convert_fp8e8m0_t3_sat(float3 val) { + fp8e8m0_t3 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); + return res; +} +fp8e8m0_t4 __attribute__((overloadable)) _convert_fp8e8m0_t4_sat(float4 val) { + fp8e8m0_t4 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.x); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.y); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.z); + res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.w); + return res; +} +fp8e8m0_t8 __attribute__((overloadable)) _convert_fp8e8m0_t8_sat(float8 val) { + fp8e8m0_t8 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); + return res; +} +fp8e8m0_t16 __attribute__((overloadable)) _convert_fp8e8m0_t16_sat(float16 val) { + fp8e8m0_t16 res; + res.data.s0 = _intel_convert_f32_fo_e8m0_sat(val.s0); + res.data.s1 = _intel_convert_f32_fo_e8m0_sat(val.s1); + res.data.s2 = _intel_convert_f32_fo_e8m0_sat(val.s2); + res.data.s3 = _intel_convert_f32_fo_e8m0_sat(val.s3); + res.data.s4 = _intel_convert_f32_fo_e8m0_sat(val.s4); + res.data.s5 = _intel_convert_f32_fo_e8m0_sat(val.s5); + res.data.s6 = _intel_convert_f32_fo_e8m0_sat(val.s6); + res.data.s7 = _intel_convert_f32_fo_e8m0_sat(val.s7); + res.data.s8 = _intel_convert_f32_fo_e8m0_sat(val.s8); + res.data.s9 = _intel_convert_f32_fo_e8m0_sat(val.s9); + res.data.sA = _intel_convert_f32_fo_e8m0_sat(val.sA); + res.data.sB = _intel_convert_f32_fo_e8m0_sat(val.sB); + res.data.sC = _intel_convert_f32_fo_e8m0_sat(val.sC); + res.data.sD = _intel_convert_f32_fo_e8m0_sat(val.sD); + res.data.sE = _intel_convert_f32_fo_e8m0_sat(val.sE); + res.data.sF = _intel_convert_f32_fo_e8m0_sat(val.sF); + return res; +} -// fp8e5m2_t __attribute__((overloadable)) as_fp8e5m2_t(uchar val) { -// fp8e5m2_t res; -// res.data = val; -// return res; -// } -// fp8e5m2_t1 __attribute__((overloadable)) as_fp8e5m2_t1(uchar val[1]) { -// fp8e5m2_t1 res; -// res.data = val[0]; -// return res; -// } -// fp8e5m2_t2 __attribute__((overloadable)) as_fp8e5m2_t2(uchar2 val) { -// fp8e5m2_t2 res; -// res.data.s0 = val.x; -// res.data.s1 = val.y; -// return res; -// } -// fp8e5m2_t3 __attribute__((overloadable)) as_fp8e5m2_t3(uchar3 val) { -// fp8e5m2_t3 res; -// res.data.s0 = val.x; -// res.data.s1 = val.y; -// res.data.s2 = val.z; -// return res; -// } -// fp8e5m2_t4 __attribute__((overloadable)) as_fp8e5m2_t4(uchar4 val) { -// fp8e5m2_t4 res; -// res.data.s0 = val.x; -// res.data.s1 = val.y; -// res.data.s2 = val.z; -// res.data.s3 = val.w; -// return res; -// } -// fp8e5m2_t8 __attribute__((overloadable)) as_fp8e5m2_t8(uchar8 val) { -// fp8e5m2_t8 res; -// res.data.s0 = val.s0; -// res.data.s1 = val.s1; -// res.data.s2 = val.s2; -// res.data.s3 = val.s3; -// res.data.s4 = val.s4; -// res.data.s5 = val.s5; -// res.data.s6 = val.s6; -// res.data.s7 = val.s7; -// return res; -// } -// fp8e5m2_t16 __attribute__((overloadable)) as_fp8e5m2_t16(uchar16 val) { -// fp8e5m2_t16 res; -// res.data.s0 = val.s0; -// res.data.s1 = val.s1; -// res.data.s2 = val.s2; -// res.data.s3 = val.s3; -// res.data.s4 = val.s4; -// res.data.s5 = val.s5; -// res.data.s6 = val.s6; -// res.data.s7 = val.s7; -// res.data.s8 = val.s8; -// res.data.s9 = val.s9; -// res.data.sA = val.sA; -// res.data.sB = val.sB; -// res.data.sC = val.sC; -// res.data.sD = val.sD; -// res.data.sE = val.sE; -// res.data.sF = val.sF; -// return res; -// } +fp8e5m2_t __attribute__((overloadable)) as_fp8e5m2_t(uchar val) { + fp8e5m2_t res; + res.data = val; + return res; +} +fp8e5m2_t1 __attribute__((overloadable)) as_fp8e5m2_t1(uchar val[1]) { + fp8e5m2_t1 res; + res.data = val[0]; + return res; +} +fp8e5m2_t2 __attribute__((overloadable)) as_fp8e5m2_t2(uchar2 val) { + fp8e5m2_t2 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + return res; +} +fp8e5m2_t3 __attribute__((overloadable)) as_fp8e5m2_t3(uchar3 val) { + fp8e5m2_t3 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + return res; +} +fp8e5m2_t4 __attribute__((overloadable)) as_fp8e5m2_t4(uchar4 val) { + fp8e5m2_t4 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + res.data.s3 = val.w; + return res; +} +fp8e5m2_t8 __attribute__((overloadable)) as_fp8e5m2_t8(uchar8 val) { + fp8e5m2_t8 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + return res; +} +fp8e5m2_t16 __attribute__((overloadable)) as_fp8e5m2_t16(uchar16 val) { + fp8e5m2_t16 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + res.data.s8 = val.s8; + res.data.s9 = val.s9; + res.data.sA = val.sA; + res.data.sB = val.sB; + res.data.sC = val.sC; + res.data.sD = val.sD; + res.data.sE = val.sE; + res.data.sF = val.sF; + return res; +} -// fp8e4m3_t __attribute__((overloadable)) as_fp8e4m3_t(uchar val) { -// fp8e4m3_t res; -// res.data = as_char(val); -// return res; -// } -// fp8e4m3_t1 __attribute__((overloadable)) as_fp8e4m3_t1(uchar val[1]) { -// fp8e4m3_t1 res; -// res.data = as_char(val[0]); -// return res; -// } -// fp8e4m3_t2 __attribute__((overloadable)) as_fp8e4m3_t2(uchar2 val) { -// fp8e4m3_t2 res; -// res.data.s0 = as_char(val.x); -// res.data.s1 = as_char(val.y); -// return res; -// } -// fp8e4m3_t3 __attribute__((overloadable)) as_fp8e4m3_t3(uchar3 val) { -// fp8e4m3_t3 res; -// res.data.s0 = as_char(val.x); -// res.data.s1 = as_char(val.y); -// res.data.s2 = as_char(val.z); -// return res; -// } -// fp8e4m3_t4 __attribute__((overloadable)) as_fp8e4m3_t4(uchar4 val) { -// fp8e4m3_t4 res; -// res.data.s0 = as_char(val.x); -// res.data.s1 = as_char(val.y); -// res.data.s2 = as_char(val.z); -// res.data.s3 = as_char(val.w); -// return res; -// } -// fp8e4m3_t8 __attribute__((overloadable)) as_fp8e4m3_t8(uchar8 val) { -// fp8e4m3_t8 res; -// res.data.s0 = as_char(val.s0); -// res.data.s1 = as_char(val.s1); -// res.data.s2 = as_char(val.s2); -// res.data.s3 = as_char(val.s3); -// res.data.s4 = as_char(val.s4); -// res.data.s5 = as_char(val.s5); -// res.data.s6 = as_char(val.s6); -// res.data.s7 = as_char(val.s7); -// return res; -// } -// fp8e4m3_t16 __attribute__((overloadable)) as_fp8e4m3_t16(uchar16 val) { -// fp8e4m3_t16 res; -// res.data.s0 = as_char(val.s0); -// res.data.s1 = as_char(val.s1); -// res.data.s2 = as_char(val.s2); -// res.data.s3 = as_char(val.s3); -// res.data.s4 = as_char(val.s4); -// res.data.s5 = as_char(val.s5); -// res.data.s6 = as_char(val.s6); -// res.data.s7 = as_char(val.s7); -// res.data.s8 = as_char(val.s8); -// res.data.s9 = as_char(val.s9); -// res.data.sA = as_char(val.sA); -// res.data.sB = as_char(val.sB); -// res.data.sC = as_char(val.sC); -// res.data.sD = as_char(val.sD); -// res.data.sE = as_char(val.sE); -// res.data.sF = as_char(val.sF); -// return res; -// } +fp8e4m3_t __attribute__((overloadable)) as_fp8e4m3_t(uchar val) { + fp8e4m3_t res; + res.data = as_char(val); + return res; +} +fp8e4m3_t1 __attribute__((overloadable)) as_fp8e4m3_t1(uchar val[1]) { + fp8e4m3_t1 res; + res.data = as_char(val[0]); + return res; +} +fp8e4m3_t2 __attribute__((overloadable)) as_fp8e4m3_t2(uchar2 val) { + fp8e4m3_t2 res; + res.data.s0 = as_char(val.x); + res.data.s1 = as_char(val.y); + return res; +} +fp8e4m3_t3 __attribute__((overloadable)) as_fp8e4m3_t3(uchar3 val) { + fp8e4m3_t3 res; + res.data.s0 = as_char(val.x); + res.data.s1 = as_char(val.y); + res.data.s2 = as_char(val.z); + return res; +} +fp8e4m3_t4 __attribute__((overloadable)) as_fp8e4m3_t4(uchar4 val) { + fp8e4m3_t4 res; + res.data.s0 = as_char(val.x); + res.data.s1 = as_char(val.y); + res.data.s2 = as_char(val.z); + res.data.s3 = as_char(val.w); + return res; +} +fp8e4m3_t8 __attribute__((overloadable)) as_fp8e4m3_t8(uchar8 val) { + fp8e4m3_t8 res; + res.data.s0 = as_char(val.s0); + res.data.s1 = as_char(val.s1); + res.data.s2 = as_char(val.s2); + res.data.s3 = as_char(val.s3); + res.data.s4 = as_char(val.s4); + res.data.s5 = as_char(val.s5); + res.data.s6 = as_char(val.s6); + res.data.s7 = as_char(val.s7); + return res; +} +fp8e4m3_t16 __attribute__((overloadable)) as_fp8e4m3_t16(uchar16 val) { + fp8e4m3_t16 res; + res.data.s0 = as_char(val.s0); + res.data.s1 = as_char(val.s1); + res.data.s2 = as_char(val.s2); + res.data.s3 = as_char(val.s3); + res.data.s4 = as_char(val.s4); + res.data.s5 = as_char(val.s5); + res.data.s6 = as_char(val.s6); + res.data.s7 = as_char(val.s7); + res.data.s8 = as_char(val.s8); + res.data.s9 = as_char(val.s9); + res.data.sA = as_char(val.sA); + res.data.sB = as_char(val.sB); + res.data.sC = as_char(val.sC); + res.data.sD = as_char(val.sD); + res.data.sE = as_char(val.sE); + res.data.sF = as_char(val.sF); + return res; +} -// fp8e8m0_t __attribute__((overloadable)) as_fp8e8m0_t(uchar val) { -// fp8e8m0_t res; -// res.data = val; -// return res; -// } -// fp8e8m0_t1 __attribute__((overloadable)) as_fp8e8m0_t1(uchar val[1]) { -// fp8e8m0_t1 res; -// res.data = val[0]; -// return res; -// } -// fp8e8m0_t2 __attribute__((overloadable)) as_fp8e8m0_t2(uchar2 val) { -// fp8e8m0_t2 res; -// res.data.s0 = val.x; -// res.data.s1 = val.y; -// return res; -// } -// fp8e8m0_t3 __attribute__((overloadable)) as_fp8e8m0_t3(uchar3 val) { -// fp8e8m0_t3 res; -// res.data.s0 = val.x; -// res.data.s1 = val.y; -// res.data.s2 = val.z; -// return res; -// } -// fp8e8m0_t4 __attribute__((overloadable)) as_fp8e8m0_t4(uchar4 val) { -// fp8e8m0_t4 res; -// res.data.s0 = val.x; -// res.data.s1 = val.y; -// res.data.s2 = val.z; -// res.data.s3 = val.w; -// return res; -// } -// fp8e8m0_t8 __attribute__((overloadable)) as_fp8e8m0_t8(uchar8 val) { -// fp8e8m0_t8 res; -// res.data.s0 = val.s0; -// res.data.s1 = val.s1; -// res.data.s2 = val.s2; -// res.data.s3 = val.s3; -// res.data.s4 = val.s4; -// res.data.s5 = val.s5; -// res.data.s6 = val.s6; -// res.data.s7 = val.s7; -// return res; -// } -// fp8e8m0_t16 __attribute__((overloadable)) as_fp8e8m0_t16(uchar16 val) { -// fp8e8m0_t16 res; -// res.data.s0 = val.s0; -// res.data.s1 = val.s1; -// res.data.s2 = val.s2; -// res.data.s3 = val.s3; -// res.data.s4 = val.s4; -// res.data.s5 = val.s5; -// res.data.s6 = val.s6; -// res.data.s7 = val.s7; -// res.data.s8 = val.s8; -// res.data.s9 = val.s9; -// res.data.sA = val.sA; -// res.data.sB = val.sB; -// res.data.sC = val.sC; -// res.data.sD = val.sD; -// res.data.sE = val.sE; -// res.data.sF = val.sF; -// return res; -// } +fp8e8m0_t __attribute__((overloadable)) as_fp8e8m0_t(uchar val) { + fp8e8m0_t res; + res.data = val; + return res; +} +fp8e8m0_t1 __attribute__((overloadable)) as_fp8e8m0_t1(uchar val[1]) { + fp8e8m0_t1 res; + res.data = val[0]; + return res; +} +fp8e8m0_t2 __attribute__((overloadable)) as_fp8e8m0_t2(uchar2 val) { + fp8e8m0_t2 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + return res; +} +fp8e8m0_t3 __attribute__((overloadable)) as_fp8e8m0_t3(uchar3 val) { + fp8e8m0_t3 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + return res; +} +fp8e8m0_t4 __attribute__((overloadable)) as_fp8e8m0_t4(uchar4 val) { + fp8e8m0_t4 res; + res.data.s0 = val.x; + res.data.s1 = val.y; + res.data.s2 = val.z; + res.data.s3 = val.w; + return res; +} +fp8e8m0_t8 __attribute__((overloadable)) as_fp8e8m0_t8(uchar8 val) { + fp8e8m0_t8 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + return res; +} +fp8e8m0_t16 __attribute__((overloadable)) as_fp8e8m0_t16(uchar16 val) { + fp8e8m0_t16 res; + res.data.s0 = val.s0; + res.data.s1 = val.s1; + res.data.s2 = val.s2; + res.data.s3 = val.s3; + res.data.s4 = val.s4; + res.data.s5 = val.s5; + res.data.s6 = val.s6; + res.data.s7 = val.s7; + res.data.s8 = val.s8; + res.data.s9 = val.s9; + res.data.sA = val.sA; + res.data.sB = val.sB; + res.data.sC = val.sC; + res.data.sD = val.sD; + res.data.sE = val.sE; + res.data.sF = val.sF; + return res; +} -// uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t val) { -// return val.data; -// } -// uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t1 val) { -// return val.data; -// } -// uchar2 __attribute__((overloadable)) _as_uchar2(fp8e5m2_t2 val) { -// return (uchar2)(val.data.s0, val.data.s1); -// } -// uchar3 __attribute__((overloadable)) _as_uchar3(fp8e5m2_t3 val) { -// return (uchar3)(val.data.s0, val.data.s1, val.data.s2); -// } -// uchar4 __attribute__((overloadable)) _as_uchar4(fp8e5m2_t4 val) { -// return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); -// } -// uchar8 __attribute__((overloadable)) _as_uchar8(fp8e5m2_t8 val) { -// return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, -// val.data.s4, val.data.s5, val.data.s6, val.data.s7); -// } -// uchar16 __attribute__((overloadable)) _as_uchar16(fp8e5m2_t16 val) { -// return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, -// val.data.s4, val.data.s5, val.data.s6, val.data.s7, -// val.data.s8, val.data.s9, val.data.sA, val.data.sB, -// val.data.sC, val.data.sD, val.data.sE, val.data.sF); -// } +uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t val) { + return val.data; +} +uchar __attribute__((overloadable)) _as_uchar(fp8e5m2_t1 val) { + return val.data; +} +uchar2 __attribute__((overloadable)) _as_uchar2(fp8e5m2_t2 val) { + return (uchar2)(val.data.s0, val.data.s1); +} +uchar3 __attribute__((overloadable)) _as_uchar3(fp8e5m2_t3 val) { + return (uchar3)(val.data.s0, val.data.s1, val.data.s2); +} +uchar4 __attribute__((overloadable)) _as_uchar4(fp8e5m2_t4 val) { + return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); +} +uchar8 __attribute__((overloadable)) _as_uchar8(fp8e5m2_t8 val) { + return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7); +} +uchar16 __attribute__((overloadable)) _as_uchar16(fp8e5m2_t16 val) { + return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7, + val.data.s8, val.data.s9, val.data.sA, val.data.sB, + val.data.sC, val.data.sD, val.data.sE, val.data.sF); +} -// uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t val) { -// return as_uchar((char)val.data); -// } -// uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t1 val) { -// return as_uchar((char)val.data); -// } -// uchar2 __attribute__((overloadable)) _as_uchar2(fp8e4m3_t2 val) { -// return (uchar2)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1)); -// } -// uchar3 __attribute__((overloadable)) _as_uchar3(fp8e4m3_t3 val) { -// return (uchar3)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), as_uchar((char)val.data.s2)); -// } -// uchar4 __attribute__((overloadable)) _as_uchar4(fp8e4m3_t4 val) { -// return (uchar4)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), -// as_uchar((char)val.data.s2), as_uchar((char)val.data.s3)); -// } -// uchar8 __attribute__((overloadable)) _as_uchar8(fp8e4m3_t8 val) { -// return (uchar8)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), -// as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), -// as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), -// as_uchar((char)val.data.s6), as_uchar((char)val.data.s7)); -// } -// uchar16 __attribute__((overloadable)) _as_uchar16(fp8e4m3_t16 val) { -// return (uchar16)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), -// as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), -// as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), -// as_uchar((char)val.data.s6), as_uchar((char)val.data.s7), -// as_uchar((char)val.data.s8), as_uchar((char)val.data.s9), -// as_uchar((char)val.data.sA), as_uchar((char)val.data.sB), -// as_uchar((char)val.data.sC), as_uchar((char)val.data.sD), -// as_uchar((char)val.data.sE), as_uchar((char)val.data.sF)); -// } +uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t val) { + return as_uchar((char)val.data); +} +uchar __attribute__((overloadable)) _as_uchar(fp8e4m3_t1 val) { + return as_uchar((char)val.data); +} +uchar2 __attribute__((overloadable)) _as_uchar2(fp8e4m3_t2 val) { + return (uchar2)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1)); +} +uchar3 __attribute__((overloadable)) _as_uchar3(fp8e4m3_t3 val) { + return (uchar3)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), as_uchar((char)val.data.s2)); +} +uchar4 __attribute__((overloadable)) _as_uchar4(fp8e4m3_t4 val) { + return (uchar4)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), + as_uchar((char)val.data.s2), as_uchar((char)val.data.s3)); +} +uchar8 __attribute__((overloadable)) _as_uchar8(fp8e4m3_t8 val) { + return (uchar8)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), + as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), + as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), + as_uchar((char)val.data.s6), as_uchar((char)val.data.s7)); +} +uchar16 __attribute__((overloadable)) _as_uchar16(fp8e4m3_t16 val) { + return (uchar16)(as_uchar((char)val.data.s0), as_uchar((char)val.data.s1), + as_uchar((char)val.data.s2), as_uchar((char)val.data.s3), + as_uchar((char)val.data.s4), as_uchar((char)val.data.s5), + as_uchar((char)val.data.s6), as_uchar((char)val.data.s7), + as_uchar((char)val.data.s8), as_uchar((char)val.data.s9), + as_uchar((char)val.data.sA), as_uchar((char)val.data.sB), + as_uchar((char)val.data.sC), as_uchar((char)val.data.sD), + as_uchar((char)val.data.sE), as_uchar((char)val.data.sF)); +} -// uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t val) { -// return val.data; -// } -// uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t1 val) { -// return val.data; -// } -// uchar2 __attribute__((overloadable)) _as_uchar2(fp8e8m0_t2 val) { -// return (uchar2)(val.data.s0, val.data.s1); -// } -// uchar3 __attribute__((overloadable)) _as_uchar3(fp8e8m0_t3 val) { -// return (uchar3)(val.data.s0, val.data.s1, val.data.s2); -// } -// uchar4 __attribute__((overloadable)) _as_uchar4(fp8e8m0_t4 val) { -// return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); -// } -// uchar8 __attribute__((overloadable)) _as_uchar8(fp8e8m0_t8 val) { -// return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, -// val.data.s4, val.data.s5, val.data.s6, val.data.s7); -// } -// uchar16 __attribute__((overloadable)) _as_uchar16(fp8e8m0_t16 val) { -// return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, -// val.data.s4, val.data.s5, val.data.s6, val.data.s7, -// val.data.s8, val.data.s9, val.data.sA, val.data.sB, -// val.data.sC, val.data.sD, val.data.sE, val.data.sF); -// } +uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t val) { + return val.data; +} +uchar __attribute__((overloadable)) _as_uchar(fp8e8m0_t1 val) { + return val.data; +} +uchar2 __attribute__((overloadable)) _as_uchar2(fp8e8m0_t2 val) { + return (uchar2)(val.data.s0, val.data.s1); +} +uchar3 __attribute__((overloadable)) _as_uchar3(fp8e8m0_t3 val) { + return (uchar3)(val.data.s0, val.data.s1, val.data.s2); +} +uchar4 __attribute__((overloadable)) _as_uchar4(fp8e8m0_t4 val) { + return (uchar4)(val.data.s0, val.data.s1, val.data.s2, val.data.s3); +} +uchar8 __attribute__((overloadable)) _as_uchar8(fp8e8m0_t8 val) { + return (uchar8)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7); +} +uchar16 __attribute__((overloadable)) _as_uchar16(fp8e8m0_t16 val) { + return (uchar16)(val.data.s0, val.data.s1, val.data.s2, val.data.s3, + val.data.s4, val.data.s5, val.data.s6, val.data.s7, + val.data.s8, val.data.s9, val.data.sA, val.data.sB, + val.data.sC, val.data.sD, val.data.sE, val.data.sF); +} diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl index 6dd09d08dc62..b7252b033712 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl @@ -2,6 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // +#define IS_F8 (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT || F8E5M2_OUTPUT || F8E4M3_OUTPUT || F8E8M0_OUTPUT) + +#if IS_F8 +#include "include/batch_headers/common.cl" +#include "include/batch_headers/f8_utils.cl" +#endif + #include "include/reshape_dims.cl" #include "include/fetch_utils.cl" @@ -153,6 +160,8 @@ KERNEL (reorder_data)( #elif defined UINT4_INPUT const uint uint4_idx = input_idx >> 1; OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(convert_as_uint4_float(input[uint4_idx], input_idx)); + #elif (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT) + OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(_convert_float(input[input_idx])); #else CALC_TYPE res = TO_CALC_TYPE(input[input_idx]); #endif From e7c3157bf0f9cbfc9405d840c59a20da074eb79b Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Thu, 18 Jun 2026 05:44:08 +0000 Subject: [PATCH 35/48] Remove just reorder and leave f8_utils --- .../src/kernel_selector/cl_kernels/reorder_data.cl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl index b7252b033712..6dd09d08dc62 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl @@ -2,13 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // -#define IS_F8 (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT || F8E5M2_OUTPUT || F8E4M3_OUTPUT || F8E8M0_OUTPUT) - -#if IS_F8 -#include "include/batch_headers/common.cl" -#include "include/batch_headers/f8_utils.cl" -#endif - #include "include/reshape_dims.cl" #include "include/fetch_utils.cl" @@ -160,8 +153,6 @@ KERNEL (reorder_data)( #elif defined UINT4_INPUT const uint uint4_idx = input_idx >> 1; OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(convert_as_uint4_float(input[uint4_idx], input_idx)); - #elif (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT) - OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(_convert_float(input[input_idx])); #else CALC_TYPE res = TO_CALC_TYPE(input[input_idx]); #endif From a771d0469e59d8aa70521463c337c191b957a86c Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Thu, 18 Jun 2026 12:07:02 +0000 Subject: [PATCH 36/48] Bring back everything and move f8_utils out of batch headers --- .../src/graph/common_utils/kernels_db_gen.py | 11 ++ .../cl_kernels/dynamic_quantize_gpu_opt.cl | 6 +- .../cl_kernels/dynamic_quantize_gpu_ref.cl | 2 +- .../include/batch_headers/common.cl | 170 +++++++++--------- .../batch_headers/sub_group_block_write.cl | 8 +- .../include/{batch_headers => }/f8_utils.cl | 6 + .../cl_kernels/reorder_data.cl | 9 + .../src/kernel_selector/primitive_db_gen.py | 26 ++- .../test_cases/dynamic_quantize_gpu_test.cpp | 94 +++++----- 9 files changed, 186 insertions(+), 146 deletions(-) rename src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/{batch_headers => }/f8_utils.cl (99%) diff --git a/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py b/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py index 414c993f4064..36555e08839e 100644 --- a/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py +++ b/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py @@ -78,7 +78,18 @@ def add_missing_undefs(self, content): defines = set(match.group(1) for match in define_pattern.finditer(content)) undef_directives = [] content_lines = content.split("\n") + + # Detect include guard macros (#ifndef X paired with #define X for the + # same name) to avoid adding #undef for them. Without this, the guard is + # defeated when multiple kernels with the same inlined header are batched + # into a single compilation unit. + ifndef_pattern = re.compile(r'#ifndef\s+(\w+)') + ifndef_names = set(match.group(1) for match in ifndef_pattern.finditer(content)) + for define in defines: + if define in ifndef_names: + continue + # Regex to check if #undef for this define already exists undef_pattern = re.compile(r'#undef\s+' + re.escape(define) + r'\s*(?:\n|$)') diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index 335295d2d355..730198a017fa 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -5,9 +5,9 @@ #define IS_F8 (F8E5M2_OUTPUT || F8E4M3_OUTPUT) #include "include/batch_headers/fetch_data.cl" -// #if IS_F8 -// #include "include/batch_headers/f8_utils.cl" -// #endif +#if IS_F8 +#include "include/f8_utils.cl" +#endif #if OUTPUT_DIMS != 4 && OUTPUT_DIMS != 2 #error "dynamic_quantize_gpu_opt.cl: Unsupported output dimension" diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl index 37e1d48f7230..51c75097a681 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl @@ -6,7 +6,7 @@ #include "include/batch_headers/fetch_data.cl" #if IS_F8 -#include "include/batch_headers/f8_utils.cl" +#include "include/f8_utils.cl" #endif #define UINT64_MAX 0xFFFFFFFFFFFFFFFF diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index 5d8309112861..ea7de660a57b 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -38,21 +38,21 @@ #define MAKE_VECTOR_TYPE_IMPL_16(elem_type) CAT(elem_type, 16) #define MAKE_VECTOR_TYPE(elem_type, size) CAT(MAKE_VECTOR_TYPE_IMPL_, size)(elem_type) -//#define AS_TYPE_PREFIX_uchar as_ -//#define AS_TYPE_PREFIX_char as_ -//#define AS_TYPE_PREFIX_fp8e5m2_t _as_ -//#define AS_TYPE_PREFIX_fp8e4m3_t _as_ -//#define AS_TYPE_PREFIX_fp8e8m0_t _as_ -//#define AS_TYPE_PREFIX_ushort as_ -//#define AS_TYPE_PREFIX_short as_ -//#define AS_TYPE_PREFIX_half as_ -//#define AS_TYPE_PREFIX_int as_ -//#define AS_TYPE_PREFIX_uint as_ -//#define AS_TYPE_PREFIX_float as_ -//#define AS_TYPE_PREFIX_ulong as_ -//#define AS_TYPE_PREFIX_long as_ - -// #define AS_TYPE_EXT(type, val, src_type) CAT(CAT(AS_TYPE_PREFIX_, src_type), type)(val) +#define AS_TYPE_PREFIX_uchar as_ +#define AS_TYPE_PREFIX_char as_ +#define AS_TYPE_PREFIX_fp8e5m2_t _as_ +#define AS_TYPE_PREFIX_fp8e4m3_t _as_ +#define AS_TYPE_PREFIX_fp8e8m0_t _as_ +#define AS_TYPE_PREFIX_ushort as_ +#define AS_TYPE_PREFIX_short as_ +#define AS_TYPE_PREFIX_half as_ +#define AS_TYPE_PREFIX_int as_ +#define AS_TYPE_PREFIX_uint as_ +#define AS_TYPE_PREFIX_float as_ +#define AS_TYPE_PREFIX_ulong as_ +#define AS_TYPE_PREFIX_long as_ + +#define AS_TYPE_EXT(type, val, src_type) CAT(CAT(AS_TYPE_PREFIX_, src_type), type)(val) #define AS_TYPE(type, val) CAT(as_, type)(val) // ==================================================================================================================== @@ -61,9 +61,9 @@ // ==================================================================================================================== #define TYPE_SIZE_uchar 1 #define TYPE_SIZE_char 1 -// #define TYPE_SIZE_fp8e5m2_t 1 -// #define TYPE_SIZE_fp8e4m3_t 1 -// #define TYPE_SIZE_fp8e8m0_t 1 +#define TYPE_SIZE_fp8e5m2_t 1 +#define TYPE_SIZE_fp8e4m3_t 1 +#define TYPE_SIZE_fp8e8m0_t 1 #define TYPE_SIZE_ushort 2 #define TYPE_SIZE_short 2 #define TYPE_SIZE_half 2 @@ -80,70 +80,70 @@ #define REQD_SUB_GROUP_SIZE(sg_size) #endif -// #define DEFINE_VECTOR_CONVERT(target_type, source_type, vector_size) \ -// MAKE_VECTOR_TYPE(target_type, vector_size) __attribute__((overloadable)) _convert_##target_type##vector_size(MAKE_VECTOR_TYPE(source_type, vector_size) val) { \ -// return CAT(convert_, CAT(target_type, vector_size))(val); \ -// } - -// #define DEFINE_VECTOR_IDENTITY(type, vector_size) \ -// MAKE_VECTOR_TYPE(type, vector_size) __attribute__((overloadable)) _convert_##type##vector_size(MAKE_VECTOR_TYPE(type, vector_size) val) { \ -// return val; \ -// } - -// #define DEFINE_ALL_VECTOR_CONVERTS(target_type, source_type) \ -// DEFINE_VECTOR_CONVERT(target_type, source_type, 2) \ -// DEFINE_VECTOR_CONVERT(target_type, source_type, 3) \ -// DEFINE_VECTOR_CONVERT(target_type, source_type, 4) \ -// DEFINE_VECTOR_CONVERT(target_type, source_type, 8) \ -// DEFINE_VECTOR_CONVERT(target_type, source_type, 16) - -// #define DEFINE_ALL_VECTOR_IDENTITY(type) \ -// DEFINE_VECTOR_IDENTITY(type, 2) \ -// DEFINE_VECTOR_IDENTITY(type, 3) \ -// DEFINE_VECTOR_IDENTITY(type, 4) \ -// DEFINE_VECTOR_IDENTITY(type, 8) \ -// DEFINE_VECTOR_IDENTITY(type, 16) - -// float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } -// float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } -// float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } -// float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } -// float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } -// float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } -// float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } -// float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } -// float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } -// float __attribute__((overloadable)) _convert_float(float val) { return val; } - -// DEFINE_ALL_VECTOR_CONVERTS(float, char) -// DEFINE_ALL_VECTOR_CONVERTS(float, uchar) -// DEFINE_ALL_VECTOR_CONVERTS(float, short) -// DEFINE_ALL_VECTOR_CONVERTS(float, ushort) -// DEFINE_ALL_VECTOR_CONVERTS(float, int) -// DEFINE_ALL_VECTOR_CONVERTS(float, uint) -// DEFINE_ALL_VECTOR_CONVERTS(float, long) -// DEFINE_ALL_VECTOR_CONVERTS(float, ulong) -// DEFINE_ALL_VECTOR_CONVERTS(float, half) -// DEFINE_ALL_VECTOR_IDENTITY(float) - -// half __attribute__((overloadable)) _convert_half(char val) { return convert_half(val); } -// half __attribute__((overloadable)) _convert_half(uchar val) { return convert_half(val); } -// half __attribute__((overloadable)) _convert_half(short val) { return convert_half(val); } -// half __attribute__((overloadable)) _convert_half(ushort val) { return convert_half(val); } -// half __attribute__((overloadable)) _convert_half(int val) { return convert_half(val); } -// half __attribute__((overloadable)) _convert_half(uint val) { return convert_half(val); } -// half __attribute__((overloadable)) _convert_half(long val) { return convert_half(val); } -// half __attribute__((overloadable)) _convert_half(ulong val) { return convert_half(val); } -// half __attribute__((overloadable)) _convert_half(half val) { return val; } -// half __attribute__((overloadable)) _convert_half(float val) { return convert_half(val); } - -// DEFINE_ALL_VECTOR_CONVERTS(half, char) -// DEFINE_ALL_VECTOR_CONVERTS(half, uchar) -// DEFINE_ALL_VECTOR_CONVERTS(half, short) -// DEFINE_ALL_VECTOR_CONVERTS(half, ushort) -// DEFINE_ALL_VECTOR_CONVERTS(half, int) -// DEFINE_ALL_VECTOR_CONVERTS(half, uint) -// DEFINE_ALL_VECTOR_CONVERTS(half, long) -// DEFINE_ALL_VECTOR_CONVERTS(half, ulong) -// DEFINE_ALL_VECTOR_CONVERTS(half, float) -// DEFINE_ALL_VECTOR_IDENTITY(half) +#define DEFINE_VECTOR_CONVERT(target_type, source_type, vector_size) \ +MAKE_VECTOR_TYPE(target_type, vector_size) __attribute__((overloadable)) _convert_##target_type##vector_size(MAKE_VECTOR_TYPE(source_type, vector_size) val) { \ + return CAT(convert_, CAT(target_type, vector_size))(val); \ +} + +#define DEFINE_VECTOR_IDENTITY(type, vector_size) \ +MAKE_VECTOR_TYPE(type, vector_size) __attribute__((overloadable)) _convert_##type##vector_size(MAKE_VECTOR_TYPE(type, vector_size) val) { \ + return val; \ +} + +#define DEFINE_ALL_VECTOR_CONVERTS(target_type, source_type) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 2) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 3) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 4) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 8) \ +DEFINE_VECTOR_CONVERT(target_type, source_type, 16) + +#define DEFINE_ALL_VECTOR_IDENTITY(type) \ +DEFINE_VECTOR_IDENTITY(type, 2) \ +DEFINE_VECTOR_IDENTITY(type, 3) \ +DEFINE_VECTOR_IDENTITY(type, 4) \ +DEFINE_VECTOR_IDENTITY(type, 8) \ +DEFINE_VECTOR_IDENTITY(type, 16) + +float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } +float __attribute__((overloadable)) _convert_float(float val) { return val; } + +DEFINE_ALL_VECTOR_CONVERTS(float, char) +DEFINE_ALL_VECTOR_CONVERTS(float, uchar) +DEFINE_ALL_VECTOR_CONVERTS(float, short) +DEFINE_ALL_VECTOR_CONVERTS(float, ushort) +DEFINE_ALL_VECTOR_CONVERTS(float, int) +DEFINE_ALL_VECTOR_CONVERTS(float, uint) +DEFINE_ALL_VECTOR_CONVERTS(float, long) +DEFINE_ALL_VECTOR_CONVERTS(float, ulong) +DEFINE_ALL_VECTOR_CONVERTS(float, half) +DEFINE_ALL_VECTOR_IDENTITY(float) + +half __attribute__((overloadable)) _convert_half(char val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(uchar val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(short val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(ushort val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(int val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(uint val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(long val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(ulong val) { return convert_half(val); } +half __attribute__((overloadable)) _convert_half(half val) { return val; } +half __attribute__((overloadable)) _convert_half(float val) { return convert_half(val); } + +DEFINE_ALL_VECTOR_CONVERTS(half, char) +DEFINE_ALL_VECTOR_CONVERTS(half, uchar) +DEFINE_ALL_VECTOR_CONVERTS(half, short) +DEFINE_ALL_VECTOR_CONVERTS(half, ushort) +DEFINE_ALL_VECTOR_CONVERTS(half, int) +DEFINE_ALL_VECTOR_CONVERTS(half, uint) +DEFINE_ALL_VECTOR_CONVERTS(half, long) +DEFINE_ALL_VECTOR_CONVERTS(half, ulong) +DEFINE_ALL_VECTOR_CONVERTS(half, float) +DEFINE_ALL_VECTOR_IDENTITY(half) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl index 822952765a04..f89ebe1eb02e 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sub_group_block_write.cl @@ -48,16 +48,16 @@ #define BLOCK_WRITEN_FUNC_size8(vector_size) BLOCK_WRITEN_FUNC_SIZE_DEF(8, vector_size) #define BLOCK_WRITEN_FUNC(type_size, vector_size) CAT(BLOCK_WRITEN_FUNC_size, type_size)(vector_size) -#define BLOCK_WRITEN_RAW(type_size, vector_size, addr_space, ptr, offset, val) \ +#define BLOCK_WRITEN_RAW(type_size, vector_size, addr_space, ptr, offset, val, src_type) \ BLOCK_WRITEN_FUNC(type_size, vector_size)( \ (addr_space BLOCK_WRITE_TYPE(type_size)*)(ptr) + (offset), \ - AS_TYPE(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val)) + AS_TYPE_EXT(MAKE_VECTOR_TYPE(BLOCK_WRITE_TYPE(type_size), vector_size), val, src_type)) #define BLOCK_WRITEN(type, vector_size, ptr, offset, val) \ - BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __global, ptr, offset, val) + BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __global, ptr, offset, val, type) #define BLOCK_WRITEN_SLM(type, vector_size, ptr, offset, val) \ - BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __local, ptr, offset, val) + BLOCK_WRITEN_RAW(TYPE_SIZE(type), vector_size, __local, ptr, offset, val, type) #define DT_OUTPUT_BLOCK_WRITE(ptr, offset, val) BLOCK_WRITEN(OUTPUT_TYPE, 1, ptr, offset, val) #define DT_OUTPUT_BLOCK_WRITE2(ptr, offset, val) BLOCK_WRITEN(OUTPUT_TYPE, 2, ptr, offset, val) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/f8_utils.cl similarity index 99% rename from src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl rename to src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/f8_utils.cl index 051fcf380d21..70ae7d37b646 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/f8_utils.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/f8_utils.cl @@ -4,6 +4,9 @@ // TODO: Replace `_intel_convert*` with bultins when ready, current implementations are copied from XeTLA: +#ifndef OV_GPU_OCL_F8_UTILS_H +#define OV_GPU_OCL_F8_UTILS_H + uchar _f16_to_bf8_universal(half val, bool is_saturation) { half val_fp16 = val; const ushort *p = (ushort *)&val_fp16; @@ -1277,3 +1280,6 @@ uchar16 __attribute__((overloadable)) _as_uchar16(fp8e8m0_t16 val) { val.data.s8, val.data.s9, val.data.sA, val.data.sB, val.data.sC, val.data.sD, val.data.sE, val.data.sF); } + +#endif + diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl index 6dd09d08dc62..d3bc565d1977 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/reorder_data.cl @@ -2,6 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // +#define IS_F8 (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT || F8E5M2_OUTPUT || F8E4M3_OUTPUT || F8E8M0_OUTPUT) + +#if IS_F8 +#include "include/batch_headers/common.cl" +#include "include/f8_utils.cl" +#endif + #include "include/reshape_dims.cl" #include "include/fetch_utils.cl" @@ -153,6 +160,8 @@ KERNEL (reorder_data)( #elif defined UINT4_INPUT const uint uint4_idx = input_idx >> 1; OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(convert_as_uint4_float(input[uint4_idx], input_idx)); + #elif (F8E5M2_INPUT || F8E4M3_INPUT || F8E8M0_INPUT) + OUTPUT_TYPE res = TO_OUTPUT_REORDER_TYPE(_convert_float(input[input_idx])); #else CALC_TYPE res = TO_CALC_TYPE(input[input_idx]); #endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py b/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py index e7d2aa4e86f7..e97a54c74952 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py +++ b/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py @@ -88,17 +88,31 @@ def append_undefs(self, filename): content = [] with open(filename) as f: content += f.readlines() + + # Detect include guard macros (#ifndef X paired with #define X for the + # same name) to avoid adding #undef for them. Without this, the guard + # is defeated when multiple kernels with the same inlined header are + # batched into a single compilation unit. + ifndef_names = set() + for line in content: + stripped = line.strip() + if stripped.startswith('#ifndef '): + ifndef_names.add(stripped.split(" ")[1]) + elif stripped.startswith('# ifndef '): + ifndef_names.add(stripped.split(" ")[2]) for line in content: if '#define' in line: name = line.strip().split(" ")[1].split("(")[0] - undefs += "#ifdef " + name + "\n" - undefs += "#undef " + name + "\n" - undefs += "#endif\n" + if name not in ifndef_names: + undefs += "#ifdef " + name + "\n" + undefs += "#undef " + name + "\n" + undefs += "#endif\n" if '# define' in line: name = line.strip().split(" ")[2].split("(")[0] - undefs += "#ifdef " + name + "\n" - undefs += "#undef " + name + "\n" - undefs += "#endif\n" + if name not in ifndef_names: + undefs += "#ifdef " + name + "\n" + undefs += "#undef " + name + "\n" + undefs += "#endif\n" if filename in self.include_files: for include_file in self.include_files[filename]: include_file_undefs = self.append_undefs(include_file) diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp index c671b20a3d78..d13bac29402c 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp @@ -447,53 +447,53 @@ TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_inner_most_dim data_types::i8, data_types::f16, data_types::f16, OutputStorageType::InterleavedScalesZP, "dynamic_quantize_gpu_kv_cache", SetInnerMostDimValuesZero::Yes); } -//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e4m3) { -// this->test_dynamic_quantization(false, -// {1, 1, 4096}, -// {1, 1, 4096}, -// QuantizationType::Symmetric, -// 32, -// data_types::f8e4m3, -// data_types::f8e8m0, -// data_types::dynamic, -// OutputStorageType::Planar); -//} -// -//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e5m2) { -// this->test_dynamic_quantization(false, -// {1, 1, 4096}, -// {1, 1, 4096}, -// QuantizationType::Symmetric, -// 32, -// data_types::f8e5m2, -// data_types::f8e8m0, -// data_types::dynamic, -// OutputStorageType::Planar); -//} -// -//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e4m3) { -// this->test_dynamic_quantization(false, -// {1, 1, 4096}, -// {1, 1, 4096}, -// QuantizationType::Symmetric, -// UINT64_MAX, -// data_types::f8e4m3, -// data_types::f16, -// data_types::dynamic, -// OutputStorageType::Planar); -//} -// -//TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { -// this->test_dynamic_quantization(false, -// {1, 1, 4096}, -// {1, 1, 4096}, -// QuantizationType::Symmetric, -// UINT64_MAX, -// data_types::f8e5m2, -// data_types::f16, -// data_types::dynamic, -// OutputStorageType::Planar); -//} +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e4m3) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + 32, + data_types::f8e4m3, + data_types::f8e8m0, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_mxf8e5m2) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + 32, + data_types::f8e5m2, + data_types::f8e8m0, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e4m3) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + UINT64_MAX, + data_types::f8e4m3, + data_types::f16, + data_types::dynamic, + OutputStorageType::Planar); +} + +TEST_F(dynamic_quantization_gpu_tests, dynamic_quantization_f8e5m2) { + this->test_dynamic_quantization(false, + {1, 1, 4096}, + {1, 1, 4096}, + QuantizationType::Symmetric, + UINT64_MAX, + data_types::f8e5m2, + data_types::f16, + data_types::dynamic, + OutputStorageType::Planar); +} TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_opt_group_size_256) { this->test_dynamic_quantization(false, {1, 1, 8192}, {1, 1, 8192}, QuantizationType::Symmetric, 256, From 451790cab001370e071d2c6a20026fd95df4f3eb Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Fri, 19 Jun 2026 08:45:04 +0000 Subject: [PATCH 37/48] Remove unnecessary changes from transformations_pipeline --- .../intel_gpu/src/plugin/transformations_pipeline.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index e83b7e436176..972a1a89fc26 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -487,7 +487,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { const auto& defaultPrecisions = ov::pass::low_precision::precision_set::get_int8_support(); const ov::element::TypeVector supported_woq_types = - {ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4, ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f8e8m0}; + {ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}; bool enableInt8; bool unroll_loop = config.get_enable_loop_unrolling(); const bool disable_gated_mlp_fusion = GPU_DEBUG_VALUE_OR(config.get_disable_gated_mlp_fusion(), true); @@ -577,8 +577,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { ov::element::u4, ov::element::f8e4m3, ov::element::f8e5m2, - ov::element::f4e2m1, - ov::element::f8e8m0}, + ov::element::f4e2m1}, !device_info.supports_immad); if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(*func)) { manager.register_pass( From de0dfc7dcfff97374b4848b064a89786430c6654 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Fri, 19 Jun 2026 09:50:51 +0000 Subject: [PATCH 38/48] Add more tests --- .../low_precision_dequantize.cpp | 9 +- ...ion_dequantize_decomposition_mark_test.cpp | 128 +++++++++++++++++- .../convert_fc_to_compressed_test.cpp | 76 +++++++++-- 3 files changed, 196 insertions(+), 17 deletions(-) diff --git a/src/common/transformations/src/decompositions/low_precision_dequantize.cpp b/src/common/transformations/src/decompositions/low_precision_dequantize.cpp index fa5b4dc9bec0..1b65434481ed 100644 --- a/src/common/transformations/src/decompositions/low_precision_dequantize.cpp +++ b/src/common/transformations/src/decompositions/low_precision_dequantize.cpp @@ -48,7 +48,8 @@ ov::Output low_precision_dequantize(ov::pass::NodeRegistry& reg, // Multiply(Subtract(Convert(x), zp), scale) [-> Reshape] // or, when zero_point is empty: // Multiply(Convert(x), scale) [-> Reshape] - const auto& dst_type = scale.get_element_type(); + const bool is_mxfp = scale.get_element_type() == element::f8e8m0; + const auto& dst_type = is_mxfp ? element::f16 : scale.get_element_type(); ov::Output result = reg.make(x, dst_type); if (zero_point.get_node_shared_ptr()) { @@ -59,7 +60,11 @@ ov::Output low_precision_dequantize(ov::pass::NodeRegistry& reg, result = reg.make(result, zp); } - result = reg.make(result, scale); + ov::Output mutable_scale = scale; + if (is_mxfp) { + mutable_scale = reg.make(scale, dst_type); + } + result = reg.make(result, mutable_scale); if (output_shape.get_node_shared_ptr() && !reshape_is_noop(result, output_shape)) { result = reg.make(result, output_shape, /*special_zero=*/false); diff --git a/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp b/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp index 650babeabda5..3cf5af31c70d 100644 --- a/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp +++ b/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp @@ -30,7 +30,7 @@ using namespace ov; namespace { -const element::TypeVector kLowPrecisionTypes{element::u8, element::i8, element::u4, element::i4}; +const element::TypeVector kLowPrecisionTypes{element::u8, element::i8, element::u4, element::i4, element::f8e4m3, element::f8e5m2, element::f4e2m1, element::f8e8m0}; } // namespace @@ -231,3 +231,129 @@ TEST_F(TransformationTestsF, LowPrecisionDequantize_NoopReshapeSkipped) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); } + +TEST_F(TransformationTestsF, LowPrecisionDequantize_f8e4m3) { + const Shape weights_shape{4, 16}; + + { + auto weights = op::v0::Constant::create(element::f8e4m3, weights_shape, {-2}); + auto scale = op::v0::Constant::create(element::f16, Shape{}, {0.2f}); + + auto out = decomposition::low_precision_dequantize(weights, scale); + model = std::make_shared(OutputVector{out}, ParameterVector{}); + } + + manager.register_pass(kLowPrecisionTypes); + manager.register_pass(); + + { + auto weights = op::v0::Constant::create(element::f8e4m3, weights_shape, {-2}); + auto convert = std::make_shared(weights, element::f16); + disable_constant_folding(convert); + + auto scale = op::v0::Constant::create(element::f16, Shape{}, {0.2f}); + auto multiply = std::make_shared(convert, scale); + mark_as_dequantization_node(multiply); + + model_ref = std::make_shared(OutputVector{multiply}, ParameterVector{}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); +} + +TEST_F(TransformationTestsF, LowPrecisionDequantize_f8e5m2) { + const Shape weights_shape{4, 16}; + + { + auto weights = op::v0::Constant::create(element::f8e5m2, weights_shape, {-2}); + auto scale = op::v0::Constant::create(element::f16, Shape{}, {0.2f}); + + auto out = decomposition::low_precision_dequantize(weights, scale); + model = std::make_shared(OutputVector{out}, ParameterVector{}); + } + + manager.register_pass(kLowPrecisionTypes); + manager.register_pass(); + + { + auto weights = op::v0::Constant::create(element::f8e5m2, weights_shape, {-2}); + auto convert = std::make_shared(weights, element::f16); + disable_constant_folding(convert); + + auto scale = op::v0::Constant::create(element::f16, Shape{}, {0.2f}); + auto multiply = std::make_shared(convert, scale); + mark_as_dequantization_node(multiply); + + model_ref = std::make_shared(OutputVector{multiply}, ParameterVector{}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); +} + +TEST_F(TransformationTestsF, LowPrecisionDequantize_mxf8e4m3) { + const Shape weights_shape{4, 32}; + + { + auto weights = op::v0::Constant::create(element::f8e4m3, weights_shape, {-2}); + auto scale = op::v0::Constant::create(element::f8e8m0, Shape{}, {0.2f}); + + auto out = decomposition::low_precision_dequantize(weights, scale); + model = std::make_shared(OutputVector{out}, ParameterVector{}); + } + + manager.register_pass(kLowPrecisionTypes, false, false); + manager.register_pass(); + + { + auto weights = op::v0::Constant::create(element::f8e4m3, weights_shape, {-2}); + auto convert_weights = std::make_shared(weights, element::f16); + disable_constant_folding(convert_weights); + + auto scale = op::v0::Constant::create(element::f8e8m0, Shape{}, {0.2f}); + auto convert_scale = std::make_shared(scale, element::f16); + disable_constant_folding(convert_scale); + + auto multiply = std::make_shared(convert_weights, convert_scale); + mark_as_dequantization_node(multiply); + + model_ref = std::make_shared(OutputVector{multiply}, ParameterVector{}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); +} + +TEST_F(TransformationTestsF, LowPrecisionDequantize_mxf8e5m2) { + const Shape weights_shape{4, 32}; + + { + auto weights = op::v0::Constant::create(element::f8e5m2, weights_shape, {-2}); + auto scale = op::v0::Constant::create(element::f8e8m0, Shape{}, {0.2f}); + + auto out = decomposition::low_precision_dequantize(weights, scale); + model = std::make_shared(OutputVector{out}, ParameterVector{}); + } + + manager.register_pass(kLowPrecisionTypes, false, false); + manager.register_pass(); + + { + auto weights = op::v0::Constant::create(element::f8e5m2, weights_shape, {-2}); + auto convert_weights = std::make_shared(weights, element::f16); + disable_constant_folding(convert_weights); + + auto scale = op::v0::Constant::create(element::f8e8m0, Shape{}, {0.2f}); + auto convert_scale = std::make_shared(scale, element::f16); + disable_constant_folding(convert_scale); + + auto multiply = std::make_shared(convert_weights, convert_scale); + mark_as_dequantization_node(multiply); + + model_ref = std::make_shared(OutputVector{multiply}, ParameterVector{}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); +} diff --git a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp index 63d1e72b91ed..3c8f6105cb1f 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/convert_fc_to_compressed_test.cpp @@ -726,23 +726,23 @@ TEST_F(TransformationTestsF, ConvertFCToCompressed18_4dGroupedWith3dWeight) { } } -TEST_F(TransformationTestsF, ConvertFCToCompressed19) { +TEST_F(TransformationTestsF, ConvertFCToCompressed19_mxfp8e4m3) { { - auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); - auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 16 }, { 1 }); + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 32 }, { 1 }); auto weights_convert = std::make_shared(weights_const, ov::element::f16); auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); auto scale_convert = std::make_shared(scale_const, ov::element::f16); - auto scale = std::make_shared(weights_convert, scale_convert); + auto deq_mul = std::make_shared(weights_convert, scale_convert); auto no_bias = std::make_shared(); - auto fc = std::make_shared(input1, scale, no_bias); + auto fc = std::make_shared(input1, deq_mul, no_bias); model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input1}); manager.register_pass(); } { - auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); - auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 16 }, { 1 }); + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 32 }, { 1 }); auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); auto no_bias = std::make_shared(); auto fc_compressed = std::make_shared(input1, weights_const, no_bias, scale_const); @@ -751,23 +751,23 @@ TEST_F(TransformationTestsF, ConvertFCToCompressed19) { } } -TEST_F(TransformationTestsF, ConvertFCToCompressed20) { +TEST_F(TransformationTestsF, ConvertFCToCompressed20_mxfp8_e5m2) { { - auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); - auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 16 }, { 1 }); + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 32 }, { 1 }); auto weights_convert = std::make_shared(weights_const, ov::element::f16); auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); auto scale_convert = std::make_shared(scale_const, ov::element::f16); - auto scale = std::make_shared(weights_convert, scale_convert); + auto deq_mul = std::make_shared(weights_convert, scale_convert); auto no_bias = std::make_shared(); - auto fc = std::make_shared(input1, scale, no_bias); + auto fc = std::make_shared(input1, deq_mul, no_bias); model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input1}); manager.register_pass(); } { - auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); - auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 16 }, { 1 }); + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 32 }, { 1 }); auto scale_const = ov::op::v0::Constant::create(ov::element::f8e8m0, ov::Shape{ 32, 1 }, { 1 }); auto no_bias = std::make_shared(); auto fc_compressed = std::make_shared(input1, weights_const, no_bias, scale_const); @@ -776,6 +776,54 @@ TEST_F(TransformationTestsF, ConvertFCToCompressed20) { } } +TEST_F(TransformationTestsF, ConvertFCToCompressed21_fp8e4m3) { + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 32 }, { 1 }); + auto weights_convert = std::make_shared(weights_const, ov::element::f16); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto deq_mul = std::make_shared(weights_convert, scale_const); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input1, deq_mul, no_bias); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input1}); + manager.register_pass(); + } + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e4m3, ov::Shape{ 32, 32 }, { 1 }); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(input1, weights_const, no_bias, scale_const); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input1}); + } +} + +TEST_F(TransformationTestsF, ConvertFCToCompressed22_fp8_e5m2) { + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 32 }, { 1 }); + auto weights_convert = std::make_shared(weights_const, ov::element::f16); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto deq_mul = std::make_shared(weights_convert, scale_const); + auto no_bias = std::make_shared(); + auto fc = std::make_shared(input1, deq_mul, no_bias); + + model = std::make_shared(ov::OutputVector{fc}, ov::ParameterVector{input1}); + manager.register_pass(); + } + { + auto input1 = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 32 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::f8e5m2, ov::Shape{ 32, 32 }, { 1 }); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto no_bias = std::make_shared(); + auto fc_compressed = std::make_shared(input1, weights_const, no_bias, scale_const); + + model_ref = std::make_shared(ov::OutputVector{fc_compressed}, ov::ParameterVector{input1}); + } +} + } // namespace intel_gpu } // namespace test } // namespace ov From fe594269f49b81484b7edfc88a8ba17032c41d7c Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Fri, 19 Jun 2026 10:09:31 +0000 Subject: [PATCH 39/48] Remove unnecessary common.cl code --- .../cl_kernels/dynamic_quantize_gpu_opt.cl | 4 +- .../cl_kernels/dynamic_quantize_gpu_ref.cl | 4 +- .../include/batch_headers/common.cl | 68 ------------------- 3 files changed, 4 insertions(+), 72 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl index 730198a017fa..43a771fc7d3c 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_opt.cl @@ -17,7 +17,7 @@ #define SCALE_TYPE float #define TO_SCALE_TYPE(x) _convert_float(x) #define ACT_MIN_VAL 0.000000059604645h // min half dtype val - #define TO_TYPE_N_(type, n, x) _convert_##type##n(x) + #define TO_TYPE_N_(type, n, x) convert_##type##n(x) #define TO_TYPE_N_SAT_(type, n, x) _convert_##type##n##_sat(x) #else #define SCALE_TYPE half @@ -94,7 +94,7 @@ KERNEL(dynamic_quantize_gpu_opt)( } #if IS_MXFP - SCALE_TYPE quan_scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / _convert_float(max_value))))); + SCALE_TYPE quan_scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / convert_float(max_value))))); #else SCALE_TYPE quan_scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_value; FOR_PRECOMPUTED_REDUCTION(int precomputed_reduction = 0); diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl index 51c75097a681..dee2976bd61d 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl @@ -14,7 +14,7 @@ #if IS_F8 #define SCALE_TYPE float #define TO_SCALE_TYPE(x) _convert_float(x) - #define TO_SCALE_TYPE_8(x) _convert_float8(x) + #define TO_SCALE_TYPE_8(x) convert_float8(x) #define ACT_MIN_VAL 0.000000059604645h // min half dtype val #else #define SCALE_TYPE half @@ -147,7 +147,7 @@ KERNEL(dynamic_quantize_gpu_ref)( OUTPUT1_TYPE zp = (OUTPUT1_TYPE)(zp_tmp); #else // !ASYMMETRIC_QUANTIZATION #if IS_MXFP - SCALE_TYPE scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / _convert_float(max_val))))); + SCALE_TYPE scale = (SCALE_TYPE)(exp2(floor(log2(_convert_float(OUTPUT_VAL_MAX) / convert_float(max_val))))); #else SCALE_TYPE scale = TO_SCALE_TYPE(OUTPUT_VAL_MAX) / max_val; #endif // IS_FP8 diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl index ea7de660a57b..5bcd72ef1ff8 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/common.cl @@ -79,71 +79,3 @@ #else #define REQD_SUB_GROUP_SIZE(sg_size) #endif - -#define DEFINE_VECTOR_CONVERT(target_type, source_type, vector_size) \ -MAKE_VECTOR_TYPE(target_type, vector_size) __attribute__((overloadable)) _convert_##target_type##vector_size(MAKE_VECTOR_TYPE(source_type, vector_size) val) { \ - return CAT(convert_, CAT(target_type, vector_size))(val); \ -} - -#define DEFINE_VECTOR_IDENTITY(type, vector_size) \ -MAKE_VECTOR_TYPE(type, vector_size) __attribute__((overloadable)) _convert_##type##vector_size(MAKE_VECTOR_TYPE(type, vector_size) val) { \ - return val; \ -} - -#define DEFINE_ALL_VECTOR_CONVERTS(target_type, source_type) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 2) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 3) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 4) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 8) \ -DEFINE_VECTOR_CONVERT(target_type, source_type, 16) - -#define DEFINE_ALL_VECTOR_IDENTITY(type) \ -DEFINE_VECTOR_IDENTITY(type, 2) \ -DEFINE_VECTOR_IDENTITY(type, 3) \ -DEFINE_VECTOR_IDENTITY(type, 4) \ -DEFINE_VECTOR_IDENTITY(type, 8) \ -DEFINE_VECTOR_IDENTITY(type, 16) - -float __attribute__((overloadable)) _convert_float(char val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(uchar val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(short val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(ushort val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(int val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(uint val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(long val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(ulong val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(half val) { return convert_float(val); } -float __attribute__((overloadable)) _convert_float(float val) { return val; } - -DEFINE_ALL_VECTOR_CONVERTS(float, char) -DEFINE_ALL_VECTOR_CONVERTS(float, uchar) -DEFINE_ALL_VECTOR_CONVERTS(float, short) -DEFINE_ALL_VECTOR_CONVERTS(float, ushort) -DEFINE_ALL_VECTOR_CONVERTS(float, int) -DEFINE_ALL_VECTOR_CONVERTS(float, uint) -DEFINE_ALL_VECTOR_CONVERTS(float, long) -DEFINE_ALL_VECTOR_CONVERTS(float, ulong) -DEFINE_ALL_VECTOR_CONVERTS(float, half) -DEFINE_ALL_VECTOR_IDENTITY(float) - -half __attribute__((overloadable)) _convert_half(char val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(uchar val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(short val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(ushort val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(int val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(uint val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(long val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(ulong val) { return convert_half(val); } -half __attribute__((overloadable)) _convert_half(half val) { return val; } -half __attribute__((overloadable)) _convert_half(float val) { return convert_half(val); } - -DEFINE_ALL_VECTOR_CONVERTS(half, char) -DEFINE_ALL_VECTOR_CONVERTS(half, uchar) -DEFINE_ALL_VECTOR_CONVERTS(half, short) -DEFINE_ALL_VECTOR_CONVERTS(half, ushort) -DEFINE_ALL_VECTOR_CONVERTS(half, int) -DEFINE_ALL_VECTOR_CONVERTS(half, uint) -DEFINE_ALL_VECTOR_CONVERTS(half, long) -DEFINE_ALL_VECTOR_CONVERTS(half, ulong) -DEFINE_ALL_VECTOR_CONVERTS(half, float) -DEFINE_ALL_VECTOR_IDENTITY(half) From f0006bbefe2d18ed1c65533a71339e578c6512e6 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Fri, 19 Jun 2026 11:48:58 +0000 Subject: [PATCH 40/48] Merge correction --- .../tests/unit/test_cases/dynamic_quantize_gpu_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp index 945c649543e9..f65dd9d15c69 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp @@ -515,7 +515,7 @@ TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_group_size_8192_with_pre TEST_F(dynamic_quantization_gpu_tests, dynamic_quantize_opt_gs128_K2560_sym_precompute_sum) { this->test_dynamic_quantization(false, {1, 1, 2560}, {1, 1, 2560}, QuantizationType::Symmetric, 128, - data_types::i8, data_types::i8, OutputStorageType::Planar, + data_types::i8, data_types::f16, data_types::i8, OutputStorageType::Planar, "dynamic_quantize_gpu_opt", SetInnerMostDimValuesZero::No, PrecomputeSum::Enabled); } From 59dba855241000aad5be00647c4f520ea83a6c1a Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Fri, 19 Jun 2026 12:03:14 +0000 Subject: [PATCH 41/48] Code style & copyright --- .../low_precision_dequantize_decomposition_mark_test.cpp | 9 ++++++++- .../intel_gpu/src/graph/common_utils/kernels_db_gen.py | 1 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp b/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp index 3cf5af31c70d..23f4c25344ee 100644 --- a/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp +++ b/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp @@ -30,7 +30,14 @@ using namespace ov; namespace { -const element::TypeVector kLowPrecisionTypes{element::u8, element::i8, element::u4, element::i4, element::f8e4m3, element::f8e5m2, element::f4e2m1, element::f8e8m0}; +const element::TypeVector kLowPrecisionTypes{element::u8, + element::i8, + element::u4, + element::i4, + element::f8e4m3, + element::f8e5m2, + element::f4e2m1, + element::f8e8m0}; } // namespace diff --git a/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py b/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py index 36555e08839e..66aea02e6aea 100644 --- a/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py +++ b/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 # Copyright (C) 2018-2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 From 712e6e04d2ccd4d3400294bba69ec8441e41665c Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Tue, 23 Jun 2026 13:15:17 +0000 Subject: [PATCH 42/48] Implement review suggestions --- .../src/graph/common_utils/kernels_db_gen.py | 9 ++++++--- .../src/kernel_selector/primitive_db_gen.py | 15 ++++++++------- .../dynamic_quantize_fully_connected.hpp | 1 - .../dynamic/matmul_weights_decompression.cpp | 17 ++++++++++------- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py b/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py index 66aea02e6aea..a3bb7ff51ac3 100644 --- a/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py +++ b/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py @@ -76,14 +76,17 @@ def add_missing_undefs(self, content): defines = set(match.group(1) for match in define_pattern.finditer(content)) undef_directives = [] - content_lines = content.split("\n") # Detect include guard macros (#ifndef X paired with #define X for the # same name) to avoid adding #undef for them. Without this, the guard is # defeated when multiple kernels with the same inlined header are batched # into a single compilation unit. - ifndef_pattern = re.compile(r'#ifndef\s+(\w+)') - ifndef_names = set(match.group(1) for match in ifndef_pattern.finditer(content)) + ifndef_pattern = re.compile(r'#\s*ifndef\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_pattern2 = re.compile(r'#\s*if\s+!\s*defined\s*\(\s*(\w+)\s*\)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_pattern3 = re.compile(r'#\s*if\s+!\s*defined\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_names = set(match.group(1) for match in ifndef_pattern.finditer(content) if match.group(1) == match.group(2)) + ifndef_names.update(match.group(1) for match in ifndef_pattern2.finditer(content) if match.group(1) == match.group(2)) + ifndef_names.update(match.group(1) for match in ifndef_pattern3.finditer(content) if match.group(1) == match.group(2)) for define in defines: if define in ifndef_names: diff --git a/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py b/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py index e97a54c74952..dc9fafec2ebd 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py +++ b/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py @@ -93,13 +93,14 @@ def append_undefs(self, filename): # same name) to avoid adding #undef for them. Without this, the guard # is defeated when multiple kernels with the same inlined header are # batched into a single compilation unit. - ifndef_names = set() - for line in content: - stripped = line.strip() - if stripped.startswith('#ifndef '): - ifndef_names.add(stripped.split(" ")[1]) - elif stripped.startswith('# ifndef '): - ifndef_names.add(stripped.split(" ")[2]) + ifndef_pattern = re.compile(r'#\s*ifndef\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_pattern2 = re.compile(r'#\s*if\s+!\s*defined\s*\(\s*(\w+)\s*\)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_pattern3 = re.compile(r'#\s*if\s+!\s*defined\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') + content_str = "".join(content) + ifndef_names = set(match.group(1) for match in ifndef_pattern.finditer(content_str) if match.group(1) == match.group(2)) + ifndef_names.update(match.group(1) for match in ifndef_pattern2.finditer(content_str) if match.group(1) == match.group(2)) + ifndef_names.update(match.group(1) for match in ifndef_pattern3.finditer(content_str) if match.group(1) == match.group(2)) + for line in content: if '#define' in line: name = line.strip().split(" ")[1].split("(")[0] diff --git a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.hpp b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.hpp index a6a9693b7711..8f214c24fc71 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.hpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.hpp @@ -5,7 +5,6 @@ #pragma once #include "openvino/pass/graph_rewrite.hpp" -#include "openvino/runtime/properties.hpp" namespace ov::intel_gpu { diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp index 8462948306a1..cb09427315f1 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp @@ -209,9 +209,11 @@ class MatmulWeightsDecompression : public testing::WithParamInterface(weights)); } else { ov::test::utils::InputGenerateData wei_data; - wei_data.start_from = -0.05; - wei_data.range = 0.1; - wei_data.resolution = 30000; + if (weights_precision.is_real()) { + wei_data.start_from = -0.5; + wei_data.range = 1; + wei_data.resolution = 30000; + } // else for integer types, the default values are used auto weights_tensor = ov::test::utils::create_and_fill_tensor(weights_precision, transformed_weights_shape, wei_data); weights = std::make_shared(weights_tensor); } @@ -664,10 +666,12 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(false), ::testing::Values(false), ::testing::Values(32), - ::testing::Values(2.0f), + ::testing::Values(2.5f), ::testing::Values(true)), MatmulWeightsDecompression::get_test_case_name); + +const std::vector group_size_all_modes = {32, 128, std::numeric_limits::max()}; INSTANTIATE_TEST_SUITE_P( smoke_MatMulCompressedWeights_dyn_quan_fp8e4m3_fp8e4m3, MatmulWeightsDecompression, @@ -680,7 +684,7 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(true), ::testing::Values(false), ::testing::Values(false), - ::testing::Values(std::numeric_limits::max()), + ::testing::ValuesIn(group_size_all_modes), ::testing::Values(0.2f), ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); @@ -697,12 +701,11 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(true), ::testing::Values(false), ::testing::Values(false), - ::testing::Values(std::numeric_limits::max()), + ::testing::ValuesIn(group_size_all_modes), ::testing::Values(0.2f), ::testing::Values(false)), MatmulWeightsDecompression::get_test_case_name); - INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_scalar_wzp, MatmulWeightsDecompressionScalarWeightZp, ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 1024}, {{1024, 1, 1024}}}, From 6af3fa7cd2fec58681f1fbe5b09ac2c02fd8e8dc Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Wed, 24 Jun 2026 09:07:12 +0000 Subject: [PATCH 43/48] Move detection of guard patterns to a common function --- .../src/graph/common_utils/kernels_db_gen.py | 35 ++++++++++++------- .../src/kernel_selector/primitive_db_gen.py | 20 ++++------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py b/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py index a3bb7ff51ac3..83015585a0eb 100644 --- a/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py +++ b/src/plugins/intel_gpu/src/graph/common_utils/kernels_db_gen.py @@ -7,6 +7,27 @@ import ntpath import re +def detect_guard_patterns(content): + # Detect include guard macros (#ifndef X paired with bodyless #define X for the + # same name) to avoid adding #undef for them. Without this, the guard is + # defeated when multiple kernels with the same inlined header are batched + # into a single compilation unit. + # The following patterns are supported: + # 1. #ifndef X + # #define X + # 2. #if !defined(X) + # #define X + # 3. #if !defined X + # #define X + # All patterns allowed to have arbitrary whitespace between the tokens + ifndef_pattern = re.compile(r'#\s*ifndef\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_pattern2 = re.compile(r'#\s*if\s+!\s*defined\s*\(\s*(\w+)\s*\)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_pattern3 = re.compile(r'#\s*if\s+!\s*defined\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') + ifndef_names = set(match.group(1) for match in ifndef_pattern.finditer(content) if match.group(1) == match.group(2)) + ifndef_names.update(match.group(1) for match in ifndef_pattern2.finditer(content) if match.group(1) == match.group(2)) + ifndef_names.update(match.group(1) for match in ifndef_pattern3.finditer(content) if match.group(1) == match.group(2)) + return ifndef_names + class Code2CHeaders(object): def __init__(self, kernels_folder, headers_folder, lang): self.kernels_folder = os.path.abspath(kernels_folder) @@ -76,20 +97,10 @@ def add_missing_undefs(self, content): defines = set(match.group(1) for match in define_pattern.finditer(content)) undef_directives = [] - - # Detect include guard macros (#ifndef X paired with #define X for the - # same name) to avoid adding #undef for them. Without this, the guard is - # defeated when multiple kernels with the same inlined header are batched - # into a single compilation unit. - ifndef_pattern = re.compile(r'#\s*ifndef\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') - ifndef_pattern2 = re.compile(r'#\s*if\s+!\s*defined\s*\(\s*(\w+)\s*\)\s*\n\s*#\s*define\s+(\w+)\s*\n') - ifndef_pattern3 = re.compile(r'#\s*if\s+!\s*defined\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') - ifndef_names = set(match.group(1) for match in ifndef_pattern.finditer(content) if match.group(1) == match.group(2)) - ifndef_names.update(match.group(1) for match in ifndef_pattern2.finditer(content) if match.group(1) == match.group(2)) - ifndef_names.update(match.group(1) for match in ifndef_pattern3.finditer(content) if match.group(1) == match.group(2)) + guard_patterns = detect_guard_patterns(content) for define in defines: - if define in ifndef_names: + if define in guard_patterns: continue # Regex to check if #undef for this define already exists diff --git a/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py b/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py index dc9fafec2ebd..6310f71424a1 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py +++ b/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py @@ -12,6 +12,10 @@ import glob import ntpath import re +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "graph", "common_utils")) +from kernels_db_gen import detect_guard_patterns class KernelLang(Enum): OCLC = 0 @@ -89,28 +93,18 @@ def append_undefs(self, filename): with open(filename) as f: content += f.readlines() - # Detect include guard macros (#ifndef X paired with #define X for the - # same name) to avoid adding #undef for them. Without this, the guard - # is defeated when multiple kernels with the same inlined header are - # batched into a single compilation unit. - ifndef_pattern = re.compile(r'#\s*ifndef\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') - ifndef_pattern2 = re.compile(r'#\s*if\s+!\s*defined\s*\(\s*(\w+)\s*\)\s*\n\s*#\s*define\s+(\w+)\s*\n') - ifndef_pattern3 = re.compile(r'#\s*if\s+!\s*defined\s+(\w+)\s*\n\s*#\s*define\s+(\w+)\s*\n') - content_str = "".join(content) - ifndef_names = set(match.group(1) for match in ifndef_pattern.finditer(content_str) if match.group(1) == match.group(2)) - ifndef_names.update(match.group(1) for match in ifndef_pattern2.finditer(content_str) if match.group(1) == match.group(2)) - ifndef_names.update(match.group(1) for match in ifndef_pattern3.finditer(content_str) if match.group(1) == match.group(2)) + guard_patterns = detect_guard_patterns("".join(content)) for line in content: if '#define' in line: name = line.strip().split(" ")[1].split("(")[0] - if name not in ifndef_names: + if name not in guard_patterns: undefs += "#ifdef " + name + "\n" undefs += "#undef " + name + "\n" undefs += "#endif\n" if '# define' in line: name = line.strip().split(" ")[2].split("(")[0] - if name not in ifndef_names: + if name not in guard_patterns: undefs += "#ifdef " + name + "\n" undefs += "#undef " + name + "\n" undefs += "#endif\n" From 22758b16a84ec8413ee8829d1f00d851cc42c8b2 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Wed, 24 Jun 2026 15:15:41 +0000 Subject: [PATCH 44/48] Apply review suggestions --- .../include/ov_ops/dynamic_quantize.hpp | 60 ++++--- .../src/ov_ops/dynamic_quantize.cpp | 47 ++++++ .../shared_ops_optimization.cpp | 2 +- src/core/include/openvino/core/graph_util.hpp | 3 + src/core/src/graph_util.cpp | 49 ++++-- .../dynamic_quantize_fully_connected.cpp | 30 +--- .../src/plugin/transformations_pipeline.cpp | 22 +-- .../dynamic/matmul_weights_decompression.cpp | 152 +++++++----------- 8 files changed, 190 insertions(+), 175 deletions(-) diff --git a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp index 306a65d8d297..eff88e1c4610 100644 --- a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp +++ b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp @@ -6,6 +6,7 @@ #include "openvino/op/op.hpp" #include "transformations_visibility.hpp" +#include "openvino/core/attribute_adapter.hpp" namespace ov { namespace op { @@ -56,6 +57,8 @@ class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; + const Attributes& get_attrs() const { return m_attrs; } @@ -83,43 +86,34 @@ class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { static std::vector shape_infer(const DynamicQuantize* op, const std::vector& input_shapes); - bool is_config_equal(const Attributes& attr) const { - bool is_equal = - m_attrs.quantization_type == attr.quantization_type && m_attrs.quantization_dt == attr.quantization_dt && - m_attrs.scale_dt == attr.scale_dt && m_attrs.zp_dt == attr.zp_dt && - m_attrs.precomputed_reduction_dt == attr.precomputed_reduction_dt && - m_attrs.precomputed_reduction == attr.precomputed_reduction && m_attrs.group_sizes == attr.group_sizes && - m_attrs.output_storage_type == attr.output_storage_type; - - if (!is_equal) { - return false; - } - - if (m_attrs.scales_zp_output_order != attr.scales_zp_output_order) { - return false; - } - - // Edge case: if zp_output_order is empty, DynamicQuantize constructor generates it with std::iota(). - // Check if this is the case here. - if (attr.scales_zp_output_order.empty()) { - for (size_t i = 0; i < m_attrs.scales_zp_output_order.size(); ++i) { - if (m_attrs.scales_zp_output_order[i] != i) { - return false; - } - } - } - - return true; - } - - bool operator==(const DynamicQuantize& rhs) const { - return is_config_equal(rhs.m_attrs); - } - protected: Attributes m_attrs; }; } // namespace internal } // namespace op + +std::ostream& operator<<(std::ostream& s, const ov::op::internal::DynamicQuantize::QuantizationType& quantization_type); +std::ostream& operator<<(std::ostream& s, const ov::op::internal::DynamicQuantize::OutputStorageType& output_storage_type); + +template <> +class OPENVINO_API AttributeAdapter + : public EnumAttributeAdapterBase { +public: + AttributeAdapter(ov::op::internal::DynamicQuantize::QuantizationType& value) + : EnumAttributeAdapterBase(value) {} + + OPENVINO_RTTI("AttributeAdapter"); +}; + +template <> +class OPENVINO_API AttributeAdapter + : public EnumAttributeAdapterBase { +public: + AttributeAdapter(ov::op::internal::DynamicQuantize::OutputStorageType& value) + : EnumAttributeAdapterBase(value) {} + + OPENVINO_RTTI("AttributeAdapter"); +}; + } // namespace ov diff --git a/src/common/transformations/src/ov_ops/dynamic_quantize.cpp b/src/common/transformations/src/ov_ops/dynamic_quantize.cpp index 247bfb374fd2..5e1f0fa59b95 100644 --- a/src/common/transformations/src/ov_ops/dynamic_quantize.cpp +++ b/src/common/transformations/src/ov_ops/dynamic_quantize.cpp @@ -9,6 +9,8 @@ #include "openvino/op/variadic_split.hpp" #include "openvino/util/common_util.hpp" +#include "itt.hpp" + namespace ov { namespace op { namespace internal { @@ -134,6 +136,51 @@ std::vector DynamicQuantize::shape_infer(const DynamicQuantize return out_shapes; } +bool DynamicQuantize::visit_attributes(AttributeVisitor& visitor) { + INTERNAL_OP_SCOPE(internal_DynamicQuantize_visit_attributes); + visitor.on_attribute("quantization_type", m_attrs.quantization_type); + visitor.on_attribute("quantization_dt", m_attrs.quantization_dt); + visitor.on_attribute("scale_dt", m_attrs.scale_dt); + visitor.on_attribute("zp_dt", m_attrs.zp_dt); + visitor.on_attribute("precomputed_reduction_dt", m_attrs.precomputed_reduction_dt); + visitor.on_attribute("precomputed_reduction", m_attrs.precomputed_reduction); + visitor.on_attribute("group_sizes", m_attrs.group_sizes); + visitor.on_attribute("scales_zp_output_order", m_attrs.scales_zp_output_order); + visitor.on_attribute("output_storage_type", m_attrs.output_storage_type); + return true; +} + } // namespace internal } // namespace op + +std::ostream& operator<<(std::ostream& s, const ov::op::internal::DynamicQuantize::QuantizationType& quantization_type) { + return s << ov::as_string(quantization_type); +} + +std::ostream& operator<<(std::ostream& s, const ov::op::internal::DynamicQuantize::OutputStorageType& output_storage_type) { + return s << ov::as_string(output_storage_type); +} + +template <> +OPENVINO_API EnumNames& EnumNames::get() { + static auto enum_names = EnumNames( + "ov::op::internal::DynamicQuantize::QuantizationType", + { + {"symmetric", ov::op::internal::DynamicQuantize::QuantizationType::Symmetric}, + {"asymmetric", ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric}, + }); + return enum_names; +} + +template <> +OPENVINO_API EnumNames& EnumNames::get() { + static auto enum_names = EnumNames( + "ov::op::internal::DynamicQuantize::OutputStorageType", + { + {"planar", ov::op::internal::DynamicQuantize::OutputStorageType::Planar}, + {"interleaved_scales_zp", ov::op::internal::DynamicQuantize::OutputStorageType::InterleavedScalesZP}, + }); + return enum_names; +} + } // namespace ov diff --git a/src/common/transformations/src/transformations/common_optimizations/shared_ops_optimization.cpp b/src/common/transformations/src/transformations/common_optimizations/shared_ops_optimization.cpp index 281a4dcf9d29..585be3ceee12 100644 --- a/src/common/transformations/src/transformations/common_optimizations/shared_ops_optimization.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/shared_ops_optimization.cpp @@ -176,7 +176,7 @@ bool shared_node_optimization(const shared_ptr& model) { if (nodes_are_equal(root_op, child_op, node_attributes_cache)) { rewritten = - replace_output_update_name(child_op->output(0), root_op->output(0)) || rewritten; + replace_outputs_update_name(child_op->outputs(), root_op->outputs()) || rewritten; visited_nodes[j] = true; } } diff --git a/src/core/include/openvino/core/graph_util.hpp b/src/core/include/openvino/core/graph_util.hpp index f465c77fc571..7c8500ad57a8 100644 --- a/src/core/include/openvino/core/graph_util.hpp +++ b/src/core/include/openvino/core/graph_util.hpp @@ -292,6 +292,9 @@ bool compare_constants(const std::shared_ptr& n1, const std::shared_ptr node, const Output& node_input); +OPENVINO_API +bool replace_outputs_update_name(OutputVector nodes, const OutputVector& node_inputs); + OPENVINO_API bool replace_node_update_name(const std::shared_ptr& target, const std::shared_ptr& replacement); diff --git a/src/core/src/graph_util.cpp b/src/core/src/graph_util.cpp index ba031ae29c4d..4c95ecc8a359 100644 --- a/src/core/src/graph_util.cpp +++ b/src/core/src/graph_util.cpp @@ -65,6 +65,23 @@ void clone_ov_nodes(const std::vector>& nodes, } } +bool has_result_consumers(const ov::Output& port) { + const auto& consumers = port.get_target_inputs(); + return std::any_of(consumers.cbegin(), consumers.cend(), [](const ov::Input& consumer) { + return ov::is_type(consumer.get_node()); + }); +} + +bool can_output_be_replaced(const ov::Output& output, const ov::Output& replacement) { + if (has_result_consumers(output)) { + if (output.get_node()->get_output_size() != 1 || replacement.get_node()->get_output_size() != 1 || + ov::is_type(replacement.get_node()) || has_result_consumers(replacement)) { + return false; + } + } + return true; +} + } // namespace namespace ov { @@ -268,18 +285,11 @@ bool replace_output_update_name(Output output, const Output& replace // and has exactly one output port // In all other cases output name will be lost or changed, so we don't perform the replacement - auto has_result_consumers = [](const Output& port) { - const auto& consumers = port.get_target_inputs(); - return std::any_of(consumers.cbegin(), consumers.cend(), [](const Input& consumer) { - return ov::is_type(consumer.get_node()); - }); - }; + if (!can_output_be_replaced(output, replacement)) { + return false; + } if (has_result_consumers(output)) { - if (output.get_node()->get_output_size() != 1 || replacement.get_node()->get_output_size() != 1 || - is_type(replacement.get_node()) || has_result_consumers(replacement)) { - return false; - } replacement.get_node()->set_friendly_name(output.get_node()->get_friendly_name()); } @@ -290,6 +300,25 @@ bool replace_output_update_name(Output output, const Output& replace return true; } +bool replace_outputs_update_name(OutputVector outputs, const OutputVector& replacements) { + if (outputs.size() != replacements.size()) { + return false; + } + + for (size_t i = 0; i < outputs.size(); ++i) { + if (!can_output_be_replaced(outputs[i], replacements[i])) { + return false; + } + } + + for (size_t i = 0; i < outputs.size(); ++i) { + bool replaced = replace_output_update_name(outputs[i], replacements[i]); + OPENVINO_ASSERT(replaced, "Failed to replace output ", i, " of node ", outputs[i].get_node()->get_friendly_name()); + } + + return true; +} + bool replace_node_update_name(const std::shared_ptr& target, const std::shared_ptr& replacement) { for (auto& output : target->output(0).get_target_inputs()) { if (replacement->get_input_size() > 0 && diff --git a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp index 3def49560cd5..e368ddc59161 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp @@ -99,14 +99,10 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size auto scale_dtype = m_fc->get_input_element_type(3); if (weight_dtype.is_integral()) { config.quantization_dt = element::i8; - } else if (weight_dtype == element::f8e4m3) { - config.quantization_dt = element::f8e4m3; - } else if (weight_dtype == element::f8e5m2) { - config.quantization_dt = element::f8e5m2; - } else if (weight_dtype == element::f4e2m1) { - config.quantization_dt = element::f4e2m1; + } else if (cldnn::one_of(weight_dtype, {element::f8e4m3, element::f8e5m2, element::f4e2m1})) { + config.quantization_dt = weight_dtype; } else { - OPENVINO_THROW("Unexpected weight data type: " + weight_dtype.to_string()); + OPENVINO_THROW("Unexpected weight data type: ", weight_dtype); } const bool is_mxfp = scale_dtype == element::f8e8m0; @@ -120,25 +116,7 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size config.zp_dt = element::u8; // it supports u8 only now } - std::shared_ptr dyn_quan = nullptr; - - // If the parent already has an identical dynamic quantize as a user, reuse it instead of creating a new one. - const auto siblings = m_fc->get_input_node_shared_ptr(0)->get_users(); - for (const auto& sibling : siblings) { - if (auto dyn_quan_sibling = as_type_ptr(sibling)) { - if (dyn_quan_sibling->is_config_equal(config)) { - dyn_quan = dyn_quan_sibling; - } - } - - if (dyn_quan) { - break; - } - } - - if (!dyn_quan) { - dyn_quan = std::make_shared(m_fc->input_value(0), config); - } + std::shared_ptr dyn_quan = std::make_shared(m_fc->input_value(0), config); int dyn_quan_output_idx = 2; auto optional_a_zp = config.quantization_type == QuantizationType::Symmetric ? diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 972a1a89fc26..c5159c90e261 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -571,19 +571,18 @@ void TransformationsPipeline::apply(std::shared_ptr func) { } manager.register_pass(); - manager.register_pass(std::vector{ov::element::i8, - ov::element::u8, - ov::element::i4, - ov::element::u4, - ov::element::f8e4m3, - ov::element::f8e5m2, - ov::element::f4e2m1}, - !device_info.supports_immad); - if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(*func)) { + + const bool can_use_lp_fp_systolic = config.get_use_onednn() && m_context->get_engine().get_device_info().arch >= cldnn::gpu_arch::xe3p; + ov::element::TypeVector dequantization_dtypes = {ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}; + if (can_use_lp_fp_systolic) { + dequantization_dtypes.insert(dequantization_dtypes.end(), {ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1}); + } + manager.register_pass(std::vector{dequantization_dtypes}, !device_info.supports_immad); + if (can_use_lp_fp_systolic && ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(*func)) { manager.register_pass( std::vector{ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1, ov::element::f8e8m0}, - !device_info.supports_immad, - !device_info.supports_immad); + true, + false); } const bool is_pa = [&func]() { @@ -1734,6 +1733,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { precomputed_reduction, use_gs128_for_int8_per_token, use_gs128_for_linear_attention); + manager.register_pass(); } } diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp index cb09427315f1..6ff892aadc3a 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp @@ -58,6 +58,7 @@ struct ShapeParams { using MatmulWeightsDecompressionParams = std::tuple; class MatmulWeightsDecompression : public testing::WithParamInterface, @@ -76,6 +76,7 @@ class MatmulWeightsDecompression : public testing::WithParamInterface(weights_convert, shift_convert); } - const ov::element::Type scale_data_precision = is_mxfp ? ov::element::f8e8m0 : data_precision; + const bool is_mxfp = scale_precision == ov::element::f8e8m0; ov::test::utils::InputGenerateData in_data; in_data.start_from = is_mxfp ? 0 : -0.5; in_data.range = 1; in_data.resolution = 30000; - auto scale_tensor = ov::test::utils::create_and_fill_tensor(scale_data_precision, scaleshift_const_shape, in_data); + auto scale_tensor = ov::test::utils::create_and_fill_tensor(scale_precision, scaleshift_const_shape, in_data); for (size_t i = 0; i < scale_tensor.get_size(); i++) { - if (scale_data_precision == ov::element::f16) + if (scale_precision == ov::element::f16) scale_tensor.data()[i] /= ov::float16(16.f); - else if (scale_data_precision == ov::element::f32) + else if (scale_precision == ov::element::f32) scale_tensor.data()[i] /= 16.f; } std::shared_ptr scale_const = std::make_shared(scale_tensor); - if (scale_data_precision != data_precision) { + if (scale_precision != data_precision) { auto scale_convert = std::make_shared(scale_const, data_precision); scale_const = scale_convert; } @@ -311,6 +311,7 @@ class MatmulWeightsDecompression : public testing::WithParamInterface input_shapes_corner_cases_basic = { @@ -518,6 +518,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_corner_cases_basic, ::testing::Combine(::testing::ValuesIn(input_shapes_corner_cases_basic), ::testing::ValuesIn(weights_precisions), ::testing::ValuesIn(activations_precisions), + ::testing::Values(ov::element::dynamic), ::testing::ValuesIn(transpose_weights), ::testing::ValuesIn(add_decompression_sub), ::testing::ValuesIn(reshape_on_decompression), @@ -525,8 +526,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_corner_cases_basic, ::testing::ValuesIn(per_tensor_zp), ::testing::ValuesIn(param_weights), ::testing::Values(0), - ::testing::Values(1.0f), - ::testing::Values(false)), + ::testing::Values(1.0f)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(MatMulCompressedWeights_corner_cases_big, @@ -534,6 +534,7 @@ INSTANTIATE_TEST_SUITE_P(MatMulCompressedWeights_corner_cases_big, ::testing::Combine(::testing::ValuesIn(input_shapes_corner_cases_big), ::testing::ValuesIn(weights_precisions), ::testing::ValuesIn(activations_precisions), + ::testing::Values(ov::element::dynamic), ::testing::ValuesIn(transpose_weights), ::testing::ValuesIn(add_decompression_sub), ::testing::ValuesIn(reshape_on_decompression), @@ -541,8 +542,7 @@ INSTANTIATE_TEST_SUITE_P(MatMulCompressedWeights_corner_cases_big, ::testing::ValuesIn(per_tensor_zp), ::testing::ValuesIn(param_weights), ::testing::Values(0), - ::testing::Values(1.0f), - ::testing::Values(false)), + ::testing::Values(1.0f)), MatmulWeightsDecompression::get_test_case_name); @@ -556,6 +556,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan, {1024, 1024}, 128}), // shape ::testing::Values(ov::element::u4), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::ValuesIn(add_decompression_sub), ::testing::Values(true), @@ -563,8 +564,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan, ::testing::Values(true), // per_tensor_zp ::testing::Values(false), ::testing::ValuesIn(group_size), - ::testing::Values(2.0f), // Note: this is because of potential cldnn accuracy issue - ::testing::Values(false)), + ::testing::Values(2.0f)), // Note: this is because of potential cldnn accuracy issue MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_reduction, @@ -573,6 +573,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_redu {1024, 1024}, 128}), // shape ::testing::Values(ov::element::u4), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::ValuesIn(add_decompression_sub), ::testing::Values(true), @@ -580,8 +581,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_redu ::testing::Values(false), // per_tensor_zp ::testing::Values(false), ::testing::Values(128), - ::testing::Values(2.0f), // Note: this is because of potential cldnn accuracy issue - ::testing::Values(false)), + ::testing::Values(2.0f)), // Note: this is because of potential cldnn accuracy issue MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_unaligned, // dyn_quan is turned off because of innermost-shape @@ -591,6 +591,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_unaligned, / }), // shape ::testing::Values(ov::element::u4), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::Values(false), ::testing::Values(true), @@ -598,8 +599,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_unaligned, / ::testing::Values(true), // per_tensor_zp ::testing::Values(false), ::testing::Values(std::numeric_limits::max()), - ::testing::Values(2.0f), // Note: this is because of potential cldnn accuracy issue - ::testing::Values(false)), + ::testing::Values(2.0f)), // Note: this is because of potential cldnn accuracy issue MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_no_slm, @@ -608,6 +608,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_no_slm, {1024, 1}, 128}), // shape ::testing::Values(ov::element::u8), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::ValuesIn(add_decompression_sub), ::testing::Values(true), @@ -615,8 +616,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_no_slm, ::testing::Values(true), // per_tensor_zp ::testing::Values(false), ::testing::ValuesIn(group_size), - ::testing::Values(2.0f), // Note: this is because of potential cldnn accuracy issue - ::testing::Values(false)), + ::testing::Values(2.0f)), // Note: this is because of potential cldnn accuracy issue MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_3D_weight, @@ -625,6 +625,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_3D_weight, {32, 32, 1024}, 32}), ::testing::Values(ov::element::u4), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::Values(false), ::testing::Values(false), @@ -632,16 +633,16 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_3D_weight, ::testing::Values(true), ::testing::Values(false), ::testing::Values(0), - ::testing::Values(2.0f), - ::testing::Values(false)), + ::testing::Values(2.0f)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P( - smoke_MatMulCompressedWeights_dyn_quan_mxfp8_e4m3, + smoke_MatMulCompressedWeights_dyn_quan_mxfp8, MatmulWeightsDecompression, ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 4096}, {{1, 1, 4096}, {8, 1, 4096}}}, {4096, 1024}, 32}), // shape - ::testing::Values(ov::element::f8e4m3), + ::testing::ValuesIn({ov::element::f8e4m3, ov::element::f8e5m2}), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f8e8m0), ::testing::Values(true), ::testing::Values(false), ::testing::Values(false), @@ -649,51 +650,15 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(false), ::testing::Values(false), ::testing::Values(32), - ::testing::Values(2.0f), - ::testing::Values(true)), + ::testing::Values(2.5f)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P( - smoke_MatMulCompressedWeights_dyn_quan_mxfp8_e5m2, - MatmulWeightsDecompression, - ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 4096}, {{1, 1, 4096}, {8, 1, 4096}}}, {4096, 1024}, 32}), // shape - ::testing::Values(ov::element::f8e5m2), - ::testing::Values(ov::element::f16), - ::testing::Values(true), - ::testing::Values(false), - ::testing::Values(false), - ::testing::Values(false), - ::testing::Values(false), - ::testing::Values(false), - ::testing::Values(32), - ::testing::Values(2.5f), - ::testing::Values(true)), - MatmulWeightsDecompression::get_test_case_name); - - -const std::vector group_size_all_modes = {32, 128, std::numeric_limits::max()}; -INSTANTIATE_TEST_SUITE_P( - smoke_MatMulCompressedWeights_dyn_quan_fp8e4m3_fp8e4m3, + smoke_MatMulCompressedWeights_dyn_quan_fp8, MatmulWeightsDecompression, ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape - ::testing::Values(ov::element::f8e4m3), + ::testing::ValuesIn({ov::element::f8e4m3, ov::element::f8e5m2}), ::testing::Values(ov::element::f16), - ::testing::Values(true), - ::testing::Values(false), - ::testing::Values(false), - ::testing::Values(true), - ::testing::Values(false), - ::testing::Values(false), - ::testing::ValuesIn(group_size_all_modes), - ::testing::Values(0.2f), - ::testing::Values(false)), - MatmulWeightsDecompression::get_test_case_name); - -INSTANTIATE_TEST_SUITE_P( - smoke_MatMulCompressedWeights_dyn_quan_fp8e5m2_fp8e5m2, - MatmulWeightsDecompression, - ::testing::Combine(::testing::Values(ShapeParams{{{-1, -1, 128}, {{2, 1, 128}, {1, 1, 128}, {2, 1, 128}}}, {128, 16}, 128}), // shape - ::testing::Values(ov::element::f8e5m2), ::testing::Values(ov::element::f16), ::testing::Values(true), ::testing::Values(false), @@ -701,9 +666,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(true), ::testing::Values(false), ::testing::Values(false), - ::testing::ValuesIn(group_size_all_modes), - ::testing::Values(0.2f), - ::testing::Values(false)), + ::testing::ValuesIn(std::vector{32, 128, std::numeric_limits::max()}), + ::testing::Values(0.2f)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_scalar_wzp, @@ -712,6 +676,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_scalar_wzp, {1024, 1024}, 128}), ::testing::Values(ov::element::u4), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::Values(true), ::testing::Values(true), @@ -719,8 +684,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_scalar_wzp, ::testing::Values(true), ::testing::Values(false), ::testing::Values(128), - ::testing::Values(2.0f), - ::testing::Values(false)), + ::testing::Values(2.0f)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_reduction_with_gs16, @@ -729,6 +693,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_redu {128, 128}, 16}), // shape ::testing::Values(ov::element::u8), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::ValuesIn(add_decompression_sub), ::testing::Values(true), @@ -736,8 +701,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_precomputed_redu ::testing::Values(false), ::testing::Values(false), ::testing::Values(128), - ::testing::Values(2.0f), - ::testing::Values(false)), + ::testing::Values(2.0f)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_input_4d, @@ -746,6 +710,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_input_4d, {2048, 512}, 32}), // shape ::testing::Values(ov::element::u4), ::testing::Values(ov::element::f16), + ::testing::Values(ov::element::f16), ::testing::Values(false), ::testing::Values(false), ::testing::Values(false), @@ -753,7 +718,6 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_input_4d, ::testing::Values(true), ::testing::Values(false), ::testing::Values(0), - ::testing::Values(1.0f), - ::testing::Values(false)), + ::testing::Values(1.0f)), MatmulWeightsDecompression::get_test_case_name); } // namespace From abf2d2349fd081a1a02658978bf76f7f0349907c Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Wed, 24 Jun 2026 15:40:42 +0000 Subject: [PATCH 45/48] Code style --- .../include/ov_ops/dynamic_quantize.hpp | 5 +++-- .../src/ov_ops/dynamic_quantize.cpp | 15 +++++++++------ src/core/src/graph_util.cpp | 6 +++++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp index eff88e1c4610..ef1cf24d6ba0 100644 --- a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp +++ b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp @@ -4,9 +4,9 @@ #pragma once +#include "openvino/core/attribute_adapter.hpp" #include "openvino/op/op.hpp" #include "transformations_visibility.hpp" -#include "openvino/core/attribute_adapter.hpp" namespace ov { namespace op { @@ -94,7 +94,8 @@ class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { } // namespace op std::ostream& operator<<(std::ostream& s, const ov::op::internal::DynamicQuantize::QuantizationType& quantization_type); -std::ostream& operator<<(std::ostream& s, const ov::op::internal::DynamicQuantize::OutputStorageType& output_storage_type); +std::ostream& operator<<(std::ostream& s, + const ov::op::internal::DynamicQuantize::OutputStorageType& output_storage_type); template <> class OPENVINO_API AttributeAdapter diff --git a/src/common/transformations/src/ov_ops/dynamic_quantize.cpp b/src/common/transformations/src/ov_ops/dynamic_quantize.cpp index 5e1f0fa59b95..6798b4c2d066 100644 --- a/src/common/transformations/src/ov_ops/dynamic_quantize.cpp +++ b/src/common/transformations/src/ov_ops/dynamic_quantize.cpp @@ -4,13 +4,12 @@ #include "ov_ops/dynamic_quantize.hpp" +#include "itt.hpp" #include "openvino/core/partial_shape.hpp" #include "openvino/core/validation_util.hpp" #include "openvino/op/variadic_split.hpp" #include "openvino/util/common_util.hpp" -#include "itt.hpp" - namespace ov { namespace op { namespace internal { @@ -153,16 +152,19 @@ bool DynamicQuantize::visit_attributes(AttributeVisitor& visitor) { } // namespace internal } // namespace op -std::ostream& operator<<(std::ostream& s, const ov::op::internal::DynamicQuantize::QuantizationType& quantization_type) { +std::ostream& operator<<(std::ostream& s, + const ov::op::internal::DynamicQuantize::QuantizationType& quantization_type) { return s << ov::as_string(quantization_type); } -std::ostream& operator<<(std::ostream& s, const ov::op::internal::DynamicQuantize::OutputStorageType& output_storage_type) { +std::ostream& operator<<(std::ostream& s, + const ov::op::internal::DynamicQuantize::OutputStorageType& output_storage_type) { return s << ov::as_string(output_storage_type); } template <> -OPENVINO_API EnumNames& EnumNames::get() { +OPENVINO_API EnumNames& +EnumNames::get() { static auto enum_names = EnumNames( "ov::op::internal::DynamicQuantize::QuantizationType", { @@ -173,7 +175,8 @@ OPENVINO_API EnumNames& Enu } template <> -OPENVINO_API EnumNames& EnumNames::get() { +OPENVINO_API EnumNames& +EnumNames::get() { static auto enum_names = EnumNames( "ov::op::internal::DynamicQuantize::OutputStorageType", { diff --git a/src/core/src/graph_util.cpp b/src/core/src/graph_util.cpp index 4c95ecc8a359..b64800d21f51 100644 --- a/src/core/src/graph_util.cpp +++ b/src/core/src/graph_util.cpp @@ -313,7 +313,11 @@ bool replace_outputs_update_name(OutputVector outputs, const OutputVector& repla for (size_t i = 0; i < outputs.size(); ++i) { bool replaced = replace_output_update_name(outputs[i], replacements[i]); - OPENVINO_ASSERT(replaced, "Failed to replace output ", i, " of node ", outputs[i].get_node()->get_friendly_name()); + OPENVINO_ASSERT(replaced, + "Failed to replace output ", + i, + " of node ", + outputs[i].get_node()->get_friendly_name()); } return true; From 8b7ac919402eeeb70979b7a39b3c164cc01b876a Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Thu, 25 Jun 2026 06:09:03 +0000 Subject: [PATCH 46/48] Apply review suggestions --- .../low_precision_dequantize.hpp | 25 ++++++++++++------- .../low_precision_dequantize.cpp | 20 ++++++++------- ...ion_dequantize_decomposition_mark_test.cpp | 4 +-- .../dynamic/matmul_weights_decompression.cpp | 20 ++++++--------- 4 files changed, 36 insertions(+), 33 deletions(-) diff --git a/src/common/transformations/include/openvino/decompositions/low_precision_dequantize.hpp b/src/common/transformations/include/openvino/decompositions/low_precision_dequantize.hpp index 84a1a4e23f04..85c27dee7433 100644 --- a/src/common/transformations/include/openvino/decompositions/low_precision_dequantize.hpp +++ b/src/common/transformations/include/openvino/decompositions/low_precision_dequantize.hpp @@ -56,19 +56,26 @@ namespace decomposition { /// \param output_shape Optional shape constant. When provided a Reshape with /// special_zero=false is appended after the Multiply /// (skipped if the Multiply output already has that shape). -ov::Output TRANSFORMATIONS_API low_precision_dequantize(ov::pass::NodeRegistry& reg, - const ov::Output& x, - const ov::Output& scale, - const ov::Output& zero_point = {}, - const ov::Output& output_shape = {}); +/// \param scale_decompression_precision Optional element type for the Convert node +/// that converts the quantized scale and zp to the output element type. If not provided, the +/// Convert node is omitted and the scale is used as-is. +ov::Output TRANSFORMATIONS_API +low_precision_dequantize(ov::pass::NodeRegistry& reg, + const ov::Output& x, + const ov::Output& scale, + const ov::Output& zero_point = {}, + const ov::Output& output_shape = {}, + const ov::element::Type& decompression_precision = ov::element::dynamic); /// \brief Convenience overload for callers that do not need access to the /// intermediate nodes. Internally allocates a NodeRegistry and /// forwards to the registry-based overload. -ov::Output TRANSFORMATIONS_API low_precision_dequantize(const ov::Output& x, - const ov::Output& scale, - const ov::Output& zero_point = {}, - const ov::Output& output_shape = {}); +ov::Output TRANSFORMATIONS_API +low_precision_dequantize(const ov::Output& x, + const ov::Output& scale, + const ov::Output& zero_point = {}, + const ov::Output& output_shape = {}, + const ov::element::Type& decompression_precision = ov::element::dynamic); } // namespace decomposition } // namespace ov diff --git a/src/common/transformations/src/decompositions/low_precision_dequantize.cpp b/src/common/transformations/src/decompositions/low_precision_dequantize.cpp index 1b65434481ed..ef9550ad2da9 100644 --- a/src/common/transformations/src/decompositions/low_precision_dequantize.cpp +++ b/src/common/transformations/src/decompositions/low_precision_dequantize.cpp @@ -43,13 +43,14 @@ ov::Output low_precision_dequantize(ov::pass::NodeRegistry& reg, const ov::Output& x, const ov::Output& scale, const ov::Output& zero_point, - const ov::Output& output_shape) { + const ov::Output& output_shape, + const ov::element::Type& decompression_precision) { // Decomposition shape (matches ov::pass::MarkDequantization): // Multiply(Subtract(Convert(x), zp), scale) [-> Reshape] // or, when zero_point is empty: // Multiply(Convert(x), scale) [-> Reshape] - const bool is_mxfp = scale.get_element_type() == element::f8e8m0; - const auto& dst_type = is_mxfp ? element::f16 : scale.get_element_type(); + const auto& dst_type = + decompression_precision == ov::element::dynamic ? scale.get_element_type() : decompression_precision; ov::Output result = reg.make(x, dst_type); if (zero_point.get_node_shared_ptr()) { @@ -60,11 +61,11 @@ ov::Output low_precision_dequantize(ov::pass::NodeRegistry& reg, result = reg.make(result, zp); } - ov::Output mutable_scale = scale; - if (is_mxfp) { - mutable_scale = reg.make(scale, dst_type); + ov::Output scale_convert = scale; + if (scale.get_element_type() != dst_type) { + scale_convert = reg.make(scale, dst_type); } - result = reg.make(result, mutable_scale); + result = reg.make(result, scale_convert); if (output_shape.get_node_shared_ptr() && !reshape_is_noop(result, output_shape)) { result = reg.make(result, output_shape, /*special_zero=*/false); @@ -76,9 +77,10 @@ ov::Output low_precision_dequantize(ov::pass::NodeRegistry& reg, ov::Output low_precision_dequantize(const ov::Output& x, const ov::Output& scale, const ov::Output& zero_point, - const ov::Output& output_shape) { + const ov::Output& output_shape, + const ov::element::Type& decompression_precision) { ov::pass::NodeRegistry reg; - return low_precision_dequantize(reg, x, scale, zero_point, output_shape); + return low_precision_dequantize(reg, x, scale, zero_point, output_shape, decompression_precision); } } // namespace decomposition diff --git a/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp b/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp index 23f4c25344ee..634e42e85e42 100644 --- a/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp +++ b/src/common/transformations/tests/decompositions/low_precision_dequantize_decomposition_mark_test.cpp @@ -306,7 +306,7 @@ TEST_F(TransformationTestsF, LowPrecisionDequantize_mxf8e4m3) { auto weights = op::v0::Constant::create(element::f8e4m3, weights_shape, {-2}); auto scale = op::v0::Constant::create(element::f8e8m0, Shape{}, {0.2f}); - auto out = decomposition::low_precision_dequantize(weights, scale); + auto out = decomposition::low_precision_dequantize(weights, scale, {}, {}, element::f16); model = std::make_shared(OutputVector{out}, ParameterVector{}); } @@ -339,7 +339,7 @@ TEST_F(TransformationTestsF, LowPrecisionDequantize_mxf8e5m2) { auto weights = op::v0::Constant::create(element::f8e5m2, weights_shape, {-2}); auto scale = op::v0::Constant::create(element::f8e8m0, Shape{}, {0.2f}); - auto out = decomposition::low_precision_dequantize(weights, scale); + auto out = decomposition::low_precision_dequantize(weights, scale, {}, {}, element::f16); model = std::make_shared(OutputVector{out}, ParameterVector{}); } diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp index 6ff892aadc3a..3d51f606e833 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp @@ -208,13 +208,7 @@ class MatmulWeightsDecompression : public testing::WithParamInterface(weights_precision, transformed_weights_shape); params.push_back(ov::as_type_ptr(weights)); } else { - ov::test::utils::InputGenerateData wei_data; - if (weights_precision.is_real()) { - wei_data.start_from = -0.5; - wei_data.range = 1; - wei_data.resolution = 30000; - } // else for integer types, the default values are used - auto weights_tensor = ov::test::utils::create_and_fill_tensor(weights_precision, transformed_weights_shape, wei_data); + auto weights_tensor = ov::test::utils::create_and_fill_tensor(weights_precision, transformed_weights_shape); weights = std::make_shared(weights_tensor); } weights->set_friendly_name("Compressed_weights"); @@ -255,10 +249,8 @@ class MatmulWeightsDecompression : public testing::WithParamInterface(weights_convert, shift_convert); } - const bool is_mxfp = scale_precision == ov::element::f8e8m0; - ov::test::utils::InputGenerateData in_data; - in_data.start_from = is_mxfp ? 0 : -0.5; + in_data.start_from = -0.5; in_data.range = 1; in_data.resolution = 30000; auto scale_tensor = ov::test::utils::create_and_fill_tensor(scale_precision, scaleshift_const_shape, in_data); @@ -267,6 +259,8 @@ class MatmulWeightsDecompression : public testing::WithParamInterface()[i] /= ov::float16(16.f); else if (scale_precision == ov::element::f32) scale_tensor.data()[i] /= 16.f; + else if (scale_precision == ov::element::f8e8m0) + scale_tensor.data()[i] /= ov::float8_e8m0(16.f); } std::shared_ptr scale_const = std::make_shared(scale_tensor); @@ -325,12 +319,13 @@ class MatmulWeightsDecompression : public testing::WithParamInterface{32, 128, std::numeric_limits::max()}), - ::testing::Values(0.2f)), + ::testing::Values(1.0f)), MatmulWeightsDecompression::get_test_case_name); INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_dyn_quan_scalar_wzp, From c475d81024cca1c0c88d3031227d4338d25f2089 Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Thu, 25 Jun 2026 08:00:33 +0000 Subject: [PATCH 47/48] Remove unnecessary check --- .../include/low_precision/low_precision.hpp | 2 -- .../src/low_precision.cpp | 27 ------------------- .../moc_transformations.cpp | 8 +++--- src/core/src/preprocess/pre_post_process.cpp | 4 +-- .../src/plugin/transformations_pipeline.cpp | 10 +++---- 5 files changed, 7 insertions(+), 44 deletions(-) diff --git a/src/common/low_precision_transformations/include/low_precision/low_precision.hpp b/src/common/low_precision_transformations/include/low_precision/low_precision.hpp index 5dddc0c68572..421d0643eace 100644 --- a/src/common/low_precision_transformations/include/low_precision/low_precision.hpp +++ b/src/common/low_precision_transformations/include/low_precision/low_precision.hpp @@ -76,8 +76,6 @@ class LP_TRANSFORMATIONS_API ov::pass::low_precision::LowPrecision : public ov:: const std::set& supported_levels = all_levels, bool check_fake_convert = false); - static bool does_model_contain_mxfp_patterns(const ov::Model& model); - template std::shared_ptr add_main(Args&&... args) { const auto tr = std::make_shared(std::forward(args)...); diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index ae76c8d6592c..4f71b0af8b0a 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -344,30 +344,3 @@ bool LowPrecision::isFunctionQuantized(const std::shared_ptr& m } return false; } - -bool ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns( - const ov::Model& model) { - using namespace ov::op; - using namespace ov::pass::pattern; - - auto weight_pattern = any_input( - type_matches_any({ov::element::Type_t::f8e4m3, ov::element::Type_t::f8e5m2, ov::element::Type_t::f4e2m1}) && - shape_matches("[Dims..., 32]")); - auto weight_cvt_pattern = wrap_type({weight_pattern}); - - auto scale_pattern = any_input(type_matches(ov::element::Type_t::f8e8m0) && shape_matches("[Dims..., 1]")); - auto scale_cvt_pattern = wrap_type({scale_pattern}); - - auto mult_pattern = wrap_type({weight_cvt_pattern, scale_cvt_pattern}); - - auto m = std::make_shared(mult_pattern, "does_model_contain_mxfp_patterns"); - - const auto ops = model.get_ops(); - for (const auto& n : ops) { - if (m->match(n)) { - return true; - } - } - - return false; -} diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index f07156dd58c3..edc1e1d45fe2 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -148,11 +148,9 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr manager.register_pass(TypeVector{f8e4m3}, TypeVector{u4}); manager.register_pass( TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); - if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(*f)) { - manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, - false, - false); - } + manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, + true, + false); } if (!m_use_shapes) { manager.register_pass(); diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index 92149cf8ba9f..c912d104aea8 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -118,9 +118,7 @@ void transformation_pipeline(std::shared_ptr& model) { manager, MarkDequantization, TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); - if (ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(model)) { - REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); - } + REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, true, false); REGISTER_PASS(manager, DisableShapeOfConstantFolding, false); REGISTER_PASS(manager, DisableRandomUniformConstantFolding) // Mark quantized and f16/bf16 compressed constants to prevent CF for them, diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index c5159c90e261..9b7f30d9a879 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -572,13 +572,9 @@ void TransformationsPipeline::apply(std::shared_ptr func) { manager.register_pass(); - const bool can_use_lp_fp_systolic = config.get_use_onednn() && m_context->get_engine().get_device_info().arch >= cldnn::gpu_arch::xe3p; - ov::element::TypeVector dequantization_dtypes = {ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}; - if (can_use_lp_fp_systolic) { - dequantization_dtypes.insert(dequantization_dtypes.end(), {ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1}); - } - manager.register_pass(std::vector{dequantization_dtypes}, !device_info.supports_immad); - if (can_use_lp_fp_systolic && ov::pass::low_precision::LowPrecision::does_model_contain_mxfp_patterns(*func)) { + manager.register_pass(std::vector{ov::element::i8, ov::element::u8, ov::element::i4, ov::element::u4}, + !device_info.supports_immad); + if (config.get_use_onednn() && m_context->get_engine().get_device_info().arch >= cldnn::gpu_arch::xe3p) { manager.register_pass( std::vector{ov::element::f8e4m3, ov::element::f8e5m2, ov::element::f4e2m1, ov::element::f8e8m0}, true, From 850ea41067bb84e72b3e06f456e92cd0a6ef732f Mon Sep 17 00:00:00 2001 From: Tomasz Krupa Date: Thu, 25 Jun 2026 09:23:47 +0000 Subject: [PATCH 48/48] Fix zp handling --- .../common_optimizations/moc_transformations.cpp | 4 +--- src/core/src/preprocess/pre_post_process.cpp | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index edc1e1d45fe2..4520c65664c0 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -148,9 +148,7 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr manager.register_pass(TypeVector{f8e4m3}, TypeVector{u4}); manager.register_pass( TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, nf4, u3, u2, u1, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); - manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, - true, - false); + manager.register_pass(TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); } if (!m_use_shapes) { manager.register_pass(); diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index c912d104aea8..b325f201fee5 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -118,7 +118,7 @@ void transformation_pipeline(std::shared_ptr& model) { manager, MarkDequantization, TypeVector{i32, u32, i16, u16, i8, u8, u6, i4, u4, u3, u2, u1, nf4, f8e4m3, f8e5m2, f4e2m1, f8e8m0}); - REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, true, false); + REGISTER_PASS(manager, MarkDequantization, TypeVector{f8e4m3, f8e5m2, f4e2m1, f8e8m0}, false, false); REGISTER_PASS(manager, DisableShapeOfConstantFolding, false); REGISTER_PASS(manager, DisableRandomUniformConstantFolding) // Mark quantized and f16/bf16 compressed constants to prevent CF for them,