Revert "[GPU] Fix GPU DynamicQuantize batch consistency (#36078)" (#36927)

### Details

Reverts #36078.

#36078 narrowed the runtime skip of `DynamicQuantize` in front of
`FullyConnected`. On discrete GPU the FC runs through oneDNN, which then
consumes the i8 activation on every token instead of the previous f16
path. This caused a decode throughput regression on discrete GPUs (~+57%
2nd-token latency on Arc A770 for 4-bit LLMs, reported on multiple
platforms/models). Integrated GPU was unaffected.

Reverting to restore the previous behavior while a targeted fix is
worked out.

Author: allnes

src/plugins/intel_gpu/include/intel_gpu/runtime/options.inl

@@ -58,7 +58,7 @@ OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, max_kernels_per_batch, 8, "Cont
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, impls_cache_capacity, 300, "Controls capacity of LRU implementations cache that is created for each program object for dynamic models")
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, asym_dynamic_quantization, false, "Enforce asymmetric mode for dynamically quantized activations")
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, could_use_flashattn_v2, true, "Enable/Disable SDPA primitive executing with FlashAttenV2 online softmax tricks.")
-OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, dynamic_quantization_threshold, 64, "Skip runtime DynamicQuantize for batch sizes <= this value only when integrated-GPU FC can execute an equivalent internal dynamic-quantized path. 0 disables threshold-based skipping")
+OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, dynamic_quantization_threshold, 64, "Apply dynamic quantization only when batch size is larger than this value in OneDNN")
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, dynamic_quantization_precomputed_reduction, true, "Precompute reduction of activation for faster dynamic quantization in case of asymmetric weight")
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, allow_bypass_xattn, true, "Allow bypass xattn execution if threshold >= 1.0.")
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, weightless_attr, nullptr, "Used to configure ov::WeightlessCacheAttribute for constants that are not loaded from a .bin file. This typically applies to non-IR inputs (e.g., ORT)")

src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp

@@ -4,167 +4,17 @@
 
 #include "ov_ops/dynamic_quantize.hpp"
 #include "dynamic_quantize_inst.h"
-#include "fully_connected/fully_connected_kernel_bf_tiled.h"
 #include "fully_connected_inst.h"
 
 #include "primitive_type_base.h"
 #include "json_object.h"
-#include <limits>
 #include <string>
 
 namespace cldnn {
 GPU_DEFINE_PRIMITIVE_TYPE_ID(dynamic_quantize);
 
-static size_t get_static_last_dim(const layout& layout) {
-    const auto& shape = layout.get_partial_shape();
-    if (shape.rank().is_dynamic() || shape.size() == 0 || shape[shape.size() - 1].is_dynamic()) {
-        return 0;
-    }
-
-    return static_cast<size_t>(shape[shape.size() - 1].get_length());
-}
-
-static bool has_fc_decompression_zp(const fully_connected_node& fc_node) {
-    const auto& fc_prim = fc_node.get_primitive();
-    return fc_prim->decompression_zero_point.is_valid() ||
-           fc_prim->decompression_zero_point_scalar.has_value();
-}
-
-static size_t get_fc_scale_group_size(const fully_connected_node& fc_node) {
-    const auto& fc_prim = fc_node.get_primitive();
-    if (!fc_prim->decompression_scale.is_valid()) {
-        return 0;
-    }
-
-    const size_t scale_dep_idx = 2 + (fc_prim->bias.is_valid() ? 1 : 0);
-    if (fc_node.get_dependencies().size() <= scale_dep_idx) {
-        return 0;
-    }
-
-    const auto ifm_size = get_static_last_dim(fc_node.weights().get_output_layout());
-    const auto scale_groups = get_static_last_dim(fc_node.get_dependency(scale_dep_idx).get_output_layout());
-    if (ifm_size == 0 || scale_groups == 0 || ifm_size % scale_groups != 0) {
-        return 0;
-    }
-
-    return ifm_size / scale_groups;
-}
-
-static size_t get_fc_zp_group_size(const fully_connected_node& fc_node) {
-    const auto& fc_prim = fc_node.get_primitive();
-    if (fc_prim->decompression_zero_point_scalar.has_value()) {
-        return get_static_last_dim(fc_node.weights().get_output_layout());
-    }
-
-    if (!fc_prim->decompression_zero_point.is_valid()) {
-        return 0;
-    }
-
-    const size_t zp_dep_idx = 2 + (fc_prim->bias.is_valid() ? 1 : 0) + 1;
-    if (fc_node.get_dependencies().size() <= zp_dep_idx) {
-        return 0;
-    }
-
-    const auto ifm_size = get_static_last_dim(fc_node.weights().get_output_layout());
-    const auto zp_groups = get_static_last_dim(fc_node.get_dependency(zp_dep_idx).get_output_layout());
-    if (ifm_size == 0 || zp_groups == 0 || ifm_size % zp_groups != 0) {
-        return 0;
-    }
-
-    return ifm_size / zp_groups;
-}
-
-static bool is_fc_bf_tiled_kernel(const std::string& kernel_name) {
-    return kernel_name == "fully_connected_gpu_bf_tiled";
-}
-
-static kernel_selector::dev_type to_kernel_selector_device_type(device_type type) {
-    return type == device_type::discrete_gpu ? kernel_selector::dev_type::discrete_gpu
-                                             : kernel_selector::dev_type::integrated_gpu;
-}
-
-static bool can_skip_for_fully_connected(const dynamic_quantize_node& node,
-                                         const fully_connected_node& fc_node,
-                                         const layout& act_layout,
-                                         size_t input_batch) {
-    const auto& attrs = node.get_primitive()->attrs;
-
-    if (attrs.quantization_type != ov::op::internal::DynamicQuantize::QuantizationType::Symmetric ||
-        attrs.quantization_dt != ov::element::i8 ||
-        attrs.scale_dt != ov::element::f16 ||
-        attrs.precomputed_reduction ||
-        attrs.group_sizes.empty() ||
-        attrs.group_sizes.back() == std::numeric_limits<uint64_t>::max()) {
-        return false;
-    }
-
-    for (size_t i = 0; i + 1 < attrs.group_sizes.size(); ++i) {
-        if (attrs.group_sizes[i] != 1) {
-            return false;
-        }
-    }
-
-    if (act_layout.data_type != data_types::f16) {
-        return false;
-    }
-
-    bool has_equivalent_fc_fast_path = false;
-    const auto& forced_impls = fc_node.get_program().get_config().get_force_implementations();
-    auto forced_impl = forced_impls.find(fc_node.id());
-    if (forced_impl != forced_impls.end()) {
-        if (forced_impl->second.impl_type != impl_types::ocl) {
-            return false;
-        }
-        if (!forced_impl->second.kernel_name.empty() && !is_fc_bf_tiled_kernel(forced_impl->second.kernel_name)) {
-            return false;
-        }
-        has_equivalent_fc_fast_path = is_fc_bf_tiled_kernel(forced_impl->second.kernel_name);
-    }
-
-    const auto* fc_impl = fc_node.get_selected_impl();
-    if (fc_impl != nullptr) {
-        if (fc_impl->is_onednn()) {
-            return false;
-        }
-        has_equivalent_fc_fast_path |= is_fc_bf_tiled_kernel(fc_impl->get_kernel_name());
-    }
-
-    if (fc_node.get_preferred_impl_type() == impl_types::onednn && !has_equivalent_fc_fast_path) {
-        return false;
-    }
-
-    const auto input_features = get_static_last_dim(act_layout);
-    const auto weight_layout = fc_node.weights().get_output_layout();
-    const auto weight_ifm = get_static_last_dim(weight_layout);
-    if (input_features == 0 || weight_ifm == 0 || input_features != weight_ifm) {
-        return false;
-    }
-
-    const auto weight_type = weight_layout.data_type;
-    const bool has_zp = has_fc_decompression_zp(fc_node);
-    const bool is_4bit_weight = weight_type == data_types::i4 || weight_type == data_types::u4;
-    const bool is_8bit_asym_weight = weight_type == data_types::u8 && has_zp;
-    const auto& device_info = fc_node.get_program().get_engine().get_device_info();
-
-    kernel_selector::fc_kernel_bf_tiled_utils::slm_dq_eligibility_params eligibility;
-    eligibility.device_type = to_kernel_selector_device_type(device_info.dev_type);
-    eligibility.max_local_mem_size = device_info.max_local_mem_size;
-    eligibility.is_4bit_weight = is_4bit_weight;
-    eligibility.is_8bit_asym_weight = is_8bit_asym_weight;
-    eligibility.weight_ifm = weight_ifm;
-    eligibility.scale_group_size = get_fc_scale_group_size(fc_node);
-    eligibility.zp_group_size = get_fc_zp_group_size(fc_node);
-    eligibility.has_decompression_zp = has_zp;
-    eligibility.dq_group_size = fc_node.get_program().get_config().get_dynamic_quantization_group_size();
-
-    return has_equivalent_fc_fast_path &&
-           kernel_selector::fc_kernel_bf_tiled_utils::would_use_slm_with_internal_dq(eligibility,
-                                                                                     input_batch,
-                                                                                     attrs.group_sizes.back());
-}
-
-// can_be_optimized flag will be turned on from primitive_inst::update_shape function only when the
-// following primitive can reproduce DynamicQuantize semantics internally for the current shape.
+// We should skip dynamic_quantization execution for 2nd token of LLM because it does not show performance gain.
+// can_be_optimized flag will be turned on from primitive_inst::update_shape function
 static bool should_skip_execution(const dynamic_quantize_node& node, const layout& act_layout, const dynamic_quantize::Attributes& attrs) {
     if (cldnn::data_type_traits::is_floating_point(attrs.quantization_dt)) {
         return false; // Execute unconditionally for floating point types.
@@ -180,22 +30,16 @@ static bool should_skip_execution(const dynamic_quantize_node& node, const layou
     if (!(*node.get_users().begin())->is_type<fully_connected>())
         return false;
 
+    // If batch size is 1, dynamic_quantize is disabled for performance reason
     size_t input_batch = act_layout.batch();
     if (act_layout.format == format::bfyx && act_layout.get_partial_shape().size() != 2) {
         // 3D input
         input_batch = act_layout.batch() * act_layout.feature();
     }
 
-    auto& fc_user = (*node.get_users().begin())->as<fully_connected>();
-    if (!can_skip_for_fully_connected(node, fc_user, act_layout, input_batch)) {
-        GPU_DEBUG_TRACE << node.id() << "  dyn_quan is not runtime-skipped: no equivalent FC fast path" << std::endl;
-        return false;
-    }
-
-    const auto dynamic_quantization_threshold = node.get_program().get_config().get_dynamic_quantization_threshold();
-    if (dynamic_quantization_threshold != 0 && dynamic_quantization_threshold >= input_batch) {
+    if (node.get_program().get_config().get_dynamic_quantization_threshold() >= input_batch) {
         GPU_DEBUG_TRACE << node.id() << "  dyn_quan is turned off: input batch size is too small - " << input_batch << " / "
-                        << dynamic_quantization_threshold << std::endl;
+                        << node.get_program().get_config().get_dynamic_quantization_threshold() << std::endl;
         return true;
     }
 

src/plugins/intel_gpu/src/kernel_selector/cl_kernels/fully_connected_gpu_bf_tiled.cl

@@ -36,28 +36,29 @@ KERNEL(quantize_input)(
         max[i] = fmax(fmax(fabs(input_0[0]), fabs(input_0[1])), fmax(fabs(input_0[2]), fabs(input_0[3])));
     }
 
-    INPUT0_TYPE max_value = 0.003h;
+    INPUT0_TYPE max_value = 0.001h;
     for (uint i = 0 ; i < quantize_block ; i+=8) {
         INPUT0_TYPE temp = fmax(fmax(fmax(max[i], max[i+1]), fmax(max[i+2], max[i+3])),
                                 fmax(fmax(max[i+4], max[i+5]), fmax(max[i+6], max[i+7])));
         max_value = fmax(max_value, temp);
     }
 
-    INPUT0_TYPE inv_quan_scale = 127.0h / max_value;
+    float quan_scale = (float)max_value / 127.f;
     #if COMPRESSED_WEIGHTS_INT8
         int quantized_sum = 0;
     #endif
     for (uint i = 0 ; i < quantize_block ; ++i) {
         input_0 = vload4(0, &input[input_offset + i * 4]);
-        quantized_value = CAT(CAT(convert_, MAKE_VECTOR_TYPE(DQ_TYPE, INPUT_LOAD_SIZE)), _rte)(input_0 * inv_quan_scale);
+        float4 buff = convert_float4(input_0) / quan_scale;
+        quantized_value = CAT(CAT(convert_, MAKE_VECTOR_TYPE(DQ_TYPE, INPUT_LOAD_SIZE)), _rte)(buff);
         #if COMPRESSED_WEIGHTS_INT8
             quantized_sum += quantized_value[0] + quantized_value[1] + quantized_value[2] + quantized_value[3];
         #endif
         vstore4(quantized_value, 0, &quantized_input[input_offset + i * 4]);
     }
 
-    // Pair of dequantization scale and quantized activation_sum for each group
-    quan_var[offset * 2] = convert_half(1.0h / inv_quan_scale);
+    // Pair of quantizing_scale and quantized activation_sum for each group
+    quan_var[offset * 2] = convert_half(quan_scale);
     #if COMPRESSED_WEIGHTS_INT8
         quan_var[(offset * 2) + 1] = convert_half(quantized_sum);
     #endif

src/plugins/intel_gpu/src/kernel_selector/kernels/fully_connected/fully_connected_kernel_bf_tiled.cpp

@@ -15,6 +15,7 @@ static constexpr size_t simd = 16;
 static constexpr size_t input_load_size = 4;
 static constexpr size_t min_quantize_grp_size = (simd * 2); // SIMD * (min value of tile_ifm)
 static constexpr size_t min_slm_size = 256;
+static std::vector<size_t> available_quantize_grp_size = {128, 64, 32};
 
 // Per-direction bitmasks of batch sizes where dyn_b is beneficial (based on MTL profiling).
 // IFM << OFM (e.g. 3584→18944): exclude {20, 24}
@@ -95,14 +96,13 @@ bool is_8bit_asym_wei(const fully_connected_params& params) {
 
 bool is_weight_dyn_quantizable(const fully_connected_params& params) {
     auto weight_type = params.weights.GetDType();
-    const bool is_4bit_weight = weight_type == WeightsType::INT4 || weight_type == WeightsType::UINT4;
-
-    return is_weight_dyn_quantizable(is_4bit_weight, is_8bit_asym_wei(params));
-}
+    if (weight_type == WeightsType::INT4 || weight_type == WeightsType::UINT4)
+        return true;
+    // No validated case of sym 8bit weight
+    if (is_8bit_asym_wei(params))
+        return true;
 
-bool is_weight_dyn_quantizable(bool is_4bit_weight, bool is_8bit_asym_weight) {
-    // No validated case of sym 8bit weight.
-    return is_4bit_weight || is_8bit_asym_weight;
+    return false;
 }
 
 bool is_per_token_dynamic_quantize(const fully_connected_params& params) {
@@ -114,90 +114,54 @@ bool is_per_token_dynamic_quantize(const fully_connected_params& params) {
 }
 
 // DYNAMIC_QUANTIZE
-size_t get_dynamic_quantize_group_size(size_t requested_group_size,
-                                       size_t scale_group_size,
-                                       size_t zp_group_size,
-                                       bool has_decompression_zp,
-                                       bool is_8bit_asym_weight) {
-    if (requested_group_size == UINT64_MAX) {
+size_t get_dynamic_quantize_group_size(const fully_connected_params& params) {
+    auto dynamic_quantization_group_size = params.dynamic_quantization_group_size;
+
+    size_t scale_group_size = get_scale_group_size(params);
+    size_t zp_group_num = params.decompression_zero_point.Feature().v;
+    size_t zp_group_size = 0;
+    if (params.has_decompression_zp)
+        zp_group_size = params.weights.IFM().v / params.decompression_zero_point.Feature().v;
+
+    // Per-token dyn-quan
+    if (dynamic_quantization_group_size >= min_quantize_grp_size && is_per_token_dynamic_quantize(params)) {
+        // Validate size to fit dyn-quan group to the size of weight-scale and weight-zp
         if ((scale_group_size % min_quantize_grp_size) == 0 && scale_group_size > min_quantize_grp_size) {
-            auto group_size = scale_group_size;
+            dynamic_quantization_group_size = scale_group_size;
 
             // For int8 ASYM model, activation_sum should fit to weight zp
-            if (is_8bit_asym_weight && has_decompression_zp &&
-                group_size > zp_group_size && (zp_group_size % input_load_size) == 0) {
-                group_size = zp_group_size;
+            if (is_8bit_asym_wei(params) && params.has_decompression_zp == true &&
+                dynamic_quantization_group_size > zp_group_size && (zp_group_size % input_load_size) == 0) {
+                dynamic_quantization_group_size = zp_group_size;
             }
 
-            return group_size;
+            GPU_DEBUG_LOG << "FC dyn-quantize by per-token. Actual dyn_quan_group_size("
+                          << dynamic_quantization_group_size << ") : From scale_group_size (" << scale_group_size
+                          << ", zp_group_size(" << zp_group_size << "), zp_group_num(" << zp_group_num
+                          << "), ifm_size (" << get_input_bf_size(params).second << ")" << std::endl;
+            return (size_t)dynamic_quantization_group_size;
         }
     }
 
-    // Grouped-size dyn-quan : use aligned sizes which are in descending priority order.
-    constexpr size_t available_quantize_grp_size[] = {128, 64, 32};
+    // Grouped-size dyn-quan : use aligned sizes which are in 'available_quantize_grp_size'
     for (auto group_size : available_quantize_grp_size) {
-        if (requested_group_size >= group_size && (scale_group_size % group_size) == 0) {
-            if (group_size > scale_group_size) {
-                return scale_group_size;
+        if (dynamic_quantization_group_size >= group_size && (scale_group_size % group_size) == 0) {
+            dynamic_quantization_group_size = group_size;
+
+            if (dynamic_quantization_group_size > scale_group_size) {
+                GPU_DEBUG_TRACE_DETAIL << " Scale group size " << scale_group_size
+                                       << " is smaller than FC dyn-quan group size " << dynamic_quantization_group_size
+                                       << ". Reduce FC dyn-quan group size to scale size." << std::endl;
+                dynamic_quantization_group_size = scale_group_size;
             }
 
-            return group_size;
+            return (size_t)dynamic_quantization_group_size;
         }
     }
 
     return 0;
 }
 
-size_t get_dynamic_quantize_group_size(const fully_connected_params& params) {
-    size_t scale_group_size = get_scale_group_size(params);
-    size_t zp_group_num = params.decompression_zero_point.Feature().v;
-    size_t zp_group_size = 0;
-    if (params.has_decompression_zp)
-        zp_group_size = params.weights.IFM().v / params.decompression_zero_point.Feature().v;
-
-    const auto dynamic_quantization_group_size = get_dynamic_quantize_group_size(params.dynamic_quantization_group_size,
-                                                                                scale_group_size,
-                                                                                zp_group_size,
-                                                                                params.has_decompression_zp,
-                                                                                is_8bit_asym_wei(params));
-    if (is_per_token_dynamic_quantize(params) && dynamic_quantization_group_size != 0) {
-        GPU_DEBUG_LOG << "FC dyn-quantize by per-token. Actual dyn_quan_group_size("
-                      << dynamic_quantization_group_size << ") : From scale_group_size (" << scale_group_size
-                      << ", zp_group_size(" << zp_group_size << "), zp_group_num(" << zp_group_num
-                      << "), ifm_size (" << get_input_bf_size(params).second << ")" << std::endl;
-    }
-
-    return dynamic_quantization_group_size;
-}
-
-bool would_use_slm_with_internal_dq(const slm_dq_eligibility_params& params,
-                                    size_t runtime_batch,
-                                    size_t standalone_dq_group_size) {
-    if (params.device_type != dev_type::integrated_gpu)
-        return false;
-
-    if (!is_weight_dyn_quantizable(params.is_4bit_weight, params.is_8bit_asym_weight))
-        return false;
-
-    constexpr size_t min_required_slm = 2 * simd * 1 * simd * 2;
-    if (params.max_local_mem_size < min_required_slm)
-        return false;
-
-    constexpr size_t default_alignment = 16;
-    if (runtime_batch + default_alignment <= min_slm_size)
-        return false;
-
-    const auto fc_internal_group_size = get_dynamic_quantize_group_size(params.dq_group_size,
-                                                                        params.scale_group_size,
-                                                                        params.zp_group_size,
-                                                                        params.has_decompression_zp,
-                                                                        params.is_8bit_asym_weight);
-    return fc_internal_group_size != 0 &&
-           params.weight_ifm != 0 &&
-           params.weight_ifm % fc_internal_group_size == 0 &&
-           fc_internal_group_size == standalone_dq_group_size;
-}
-
 bool should_dynamic_quantize(const fully_connected_params& params) {
     size_t dynamic_quantization_group_size = get_dynamic_quantize_group_size(params);