Skip to content

[GPU] Support non-transposed INT4 compressed weights in FullyConnected for shared weights scenario #36845

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kwieloch-intel wants to merge 4 commits into
base: master
Choose a base branch
from
Open

[GPU] Support non-transposed INT4 compressed weights in FullyConnected for shared weights scenario #36845

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3c5f063
[GPU] INT4 Transpose Crash Fix
kwieloch-intel Jul 10, 2026
1c7d059
[GPU] Sub-byte types copy support
kwieloch-intel Jul 10, 2026
1dfec72
[GPU] Fix IFM dim index for non-transposed
kwieloch-intel Jul 10, 2026
f35396b
[GPU] Add test for INT4 compressed weights
kwieloch-intel Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
int idx = !arg.bias_term() ? 1 : 2;
int per_oc = PER_OC << shift_size;
int grouped = (1 << prim->input_size) - 1;
auto ifm_dim_idx = prim->weights_transposed ? 1 : 0;

bool has_decompression_scale = prim->decompression_scale.is_valid();
if (has_decompression_scale) {
Expand All @@ -279,7 +280,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {

auto decompression_scale_idx = ++idx;
auto scale_layout = arg.get_dependency(decompression_scale_idx).get_output_layout();
auto ngroups = scale_layout.get_dim(1);
auto ngroups = scale_layout.get_dim(ifm_dim_idx);
if (scale_layout.count() == 1) {
_attrs->set_scales(DNNL_ARG_WEIGHTS, COMMON, dnnl::memory::dims{}, _ds_data_type);
} else if (ngroups == 1) {
Expand All @@ -299,7 +300,7 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
if (dzp_layout.count() == 1) {
_attrs->set_zero_points(DNNL_ARG_WEIGHTS, COMMON, dnnl::memory::dims{}, _dzp_data_type);
} else {
auto ngroups = dzp_layout.get_dim(1);
auto ngroups = dzp_layout.get_dim(ifm_dim_idx);
if (ngroups == 1) {
_attrs->set_zero_points(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, _dzp_data_type);
} else {
Expand Down Expand Up @@ -380,8 +381,11 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
auto decompression_scale_idx = ++idx;
auto scale_layout = arg.get_dependency(decompression_scale_idx).get_output_layout();
ds_data_type = convert_data_type(scale_layout.data_type);
auto ifm = arg.get_dependency(1).get_output_layout().get_dim(weight_rank - 1);
auto ngroups = scale_layout.get_dim(weight_rank - 1);
// For transposed weights [N, K], IFM (K) is the last dimension.
// For non-transposed weights [K, N], IFM (K) is the second-to-last dimension.
auto ifm_dim_idx = prim->weights_transposed ? (weight_rank - 1) : (weight_rank - 2);
auto ifm = arg.get_dependency(1).get_output_layout().get_dim(ifm_dim_idx);
auto ngroups = scale_layout.get_dim(ifm_dim_idx);
group_size = static_cast<int>(ifm / ngroups);
OPENVINO_ASSERT((group_size == 1 || ngroups == 1 || group_size % 16 == 0),
"[GPU] group_size should be aligned to 16 if it is not a single scale group or the group_size is not one.");
Expand Down Expand Up @@ -410,7 +414,8 @@ struct fully_connected_onednn : typed_primitive_onednn_impl<fully_connected> {
auto dzp_rank = std::count_if(dzp_shape.begin(), dzp_shape.end(), [](ov::Dimension d) { return d.get_length() > 1; });
dzp_rank = std::max(static_cast<int64_t>(2), dzp_rank);

auto ngroups = dzp_layout.get_dim(dzp_rank - 1);
auto dzp_ifm_dim_idx = prim->weights_transposed ? (dzp_rank - 1) : (dzp_rank - 2);
auto ngroups = dzp_layout.get_dim(dzp_ifm_dim_idx);
if (ngroups == 1 && dzp_rank <= 2) {
attr->set_zero_points(DNNL_ARG_WEIGHTS, per_oc, dnnl::memory::dims{}, dzp_data_type);
} else {
Expand Down
36 changes: 35 additions & 1 deletion src/plugins/intel_gpu/src/plugin/remote_tensor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,9 @@ static void validate_and_check_shapes(const std::shared_ptr<const ov::ITensor>&
" != dst: ",
dst->get_element_type(),
")");
OPENVINO_ASSERT(src->get_element_type().bitwidth() >= 8, "[GPU] Unsupported element type for copying: ", src->get_element_type());
// Sub-byte types (i4/u4) are allowed but only for contiguous (non-ROI) copies
OPENVINO_ASSERT(src->get_element_type().bitwidth() >= 8 || roi_shape.empty(),
"[GPU] ROI copy is not supported for sub-byte element type: ", src->get_element_type());

// If it's a simple copy_to/copy_from call, then change dst shape
if (roi_shape.empty()) {
Expand Down Expand Up @@ -215,6 +217,22 @@ void RemoteTensorImpl::copy_to(const std::shared_ptr<ov::ITensor>& dst,
auto dst_remote_tensor = std::dynamic_pointer_cast<RemoteTensorImpl>(dst);
auto shape = roi_shape.empty() ? get_shape() : roi_shape;

// Sub-byte types (i4/u4) don't support strided copy — use flat contiguous copy
if (m_element_type.bitwidth() < 8) {
const auto byte_size = get_byte_size();
if (dst_remote_tensor != nullptr) {
auto src_mem = MemWrapper(stream, get_memory(), nullptr);
auto dst_mem = MemWrapper(stream, dst_remote_tensor->get_memory(), nullptr);
src_mem.copy_to(dst_mem, src_offset, dst_offset, byte_size);
} else {
OPENVINO_ASSERT(!std::dynamic_pointer_cast<ov::IRemoteTensor>(dst), "[GPU] Unsupported Remote Tensor type");
auto src_mem = MemWrapper(stream, get_memory(), nullptr);
auto dst_mem = MemWrapper(stream, nullptr, dst->data());
src_mem.copy_to(dst_mem, src_offset, dst_offset, byte_size);
}
return;
}

ov::Strides roi_strides = calculate_strides(shape, m_element_type);
if (dst_remote_tensor != nullptr) {
GPU_DEBUG_TRACE_DETAIL << "Copying from RemoteTensor (" << get_memory()->get_allocation_type() << ") to RemoteTensor ("
Expand Down Expand Up @@ -249,6 +267,22 @@ void RemoteTensorImpl::copy_from(const std::shared_ptr<const ov::ITensor>& src,
auto& stream = m_context->get_engine().get_service_stream();
auto src_remote_tensor = std::dynamic_pointer_cast<const RemoteTensorImpl>(src);

// Sub-byte types (i4/u4) don't support strided copy — use flat contiguous copy
if (m_element_type.bitwidth() < 8) {
const auto byte_size = get_byte_size();
if (src_remote_tensor != nullptr) {
auto src_mem = MemWrapper(stream, src_remote_tensor->get_memory(), nullptr);
auto dst_mem = MemWrapper(stream, get_memory(), nullptr);
src_mem.copy_to(dst_mem, src_offset, dst_offset, byte_size);
} else {
OPENVINO_ASSERT(!std::dynamic_pointer_cast<const ov::IRemoteTensor>(src), "[GPU] Unsupported Remote Tensor type");
auto src_mem = MemWrapper(stream, nullptr, const_cast<void*>(src->data()));
auto dst_mem = MemWrapper(stream, get_memory(), nullptr);
src_mem.copy_to(dst_mem, src_offset, dst_offset, byte_size);
}
return;
}

ov::Strides roi_strides = calculate_strides(shape, m_element_type);
if (src_remote_tensor != nullptr) {
GPU_DEBUG_TRACE_DETAIL << "Copying from RemoteTensor (" << src_remote_tensor->get_memory()->get_allocation_type() << ") to RemoteTensor ("
Expand Down
14 changes: 13 additions & 1 deletion src/plugins/intel_gpu/src/plugin/transformations/convert_matmul_to_fc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,19 @@ ConvertMatMulToFullyConnected::ConvertMatMulToFullyConnected(bool supports_immad

// Weights normalization
bool is_small_matmul = true;
if (shape_a.is_static() && shape_b.is_static()) {

// When compressed weights come from a Parameter (shared/external weights) and
// the original matmul does not transpose B, we must avoid inserting a runtime
// Transpose node because the GPU permute kernel does not support INT4 data type.
// Instead, use the non-transposed weight layout and let oneDNN handle it natively.
bool is_parameter_compressed_weight = is_compressed_weight &&
!matmul->get_transpose_b() &&
pattern_map.count(weights_param_m) &&
pattern_map.at(weights_param_m).get_node_shared_ptr() != nullptr;

if (is_parameter_compressed_weight) {
is_small_matmul = false;
} else if (shape_a.is_static() && shape_b.is_static()) {
auto output_shape = matmul->get_output_shape(0);
size_t k = 0;
if (matmul->get_transpose_a())
Expand Down
27 changes: 27 additions & 0 deletions src/plugins/intel_gpu/tests/unit/transformations/convert_matmul_to_fc_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1000,3 +1000,30 @@ TEST_F(TransformationTestsF, ConvertMatMulToFullyConnected_LargeF16_NoXMX_Transp
model_ref = std::make_shared<ov::Model>(ov::OutputVector{fc}, ov::ParameterVector{input});
}
}

// Verify that Parameter INT4 compressed weights skip the Transpose and produce FC with transpose_b=false
TEST_F(TransformationTestsF, ConvertMatMulToFullyConnected_ParameterCompressedWeights_NoTranspose) {
{
auto input = std::make_shared<ov::opset1::Parameter>(ov::element::f32, ov::Shape{128, 2048});
auto weights = std::make_shared<ov::opset1::Parameter>(ov::element::u4, ov::Shape{2048, 8192});
auto convert = std::make_shared<ov::opset1::Convert>(weights, ov::element::f32);
auto scale = ov::opset1::Constant::create(ov::element::f32, ov::Shape{1, 8192}, {0.1f});
auto mul = std::make_shared<ov::opset1::Multiply>(convert, scale);
auto matmul = std::make_shared<ov::opset1::MatMul>(input, mul, false, false);

model = std::make_shared<ov::Model>(ov::OutputVector{matmul}, ov::ParameterVector{input, weights});
manager.register_pass<ConvertMatMulToFullyConnected>();
}
{
auto input = std::make_shared<ov::opset1::Parameter>(ov::element::f32, ov::Shape{128, 2048});
auto weights = std::make_shared<ov::opset1::Parameter>(ov::element::u4, ov::Shape{2048, 8192});
auto convert = std::make_shared<ov::opset1::Convert>(weights, ov::element::f32);
auto scale = ov::opset1::Constant::create(ov::element::f32, ov::Shape{1, 8192}, {0.1f});
auto mul = std::make_shared<ov::opset1::Multiply>(convert, scale);
auto no_bias = std::make_shared<ov::intel_gpu::op::Placeholder>();
// transpose_b=false: weights stay in [K, N] layout, no Transpose inserted
auto fc = std::make_shared<op::FullyConnected>(input, mul, no_bias, ov::element::f32, /*transpose_b=*/false);

model_ref = std::make_shared<ov::Model>(ov::OutputVector{fc}, ov::ParameterVector{input, weights});
}
}
Loading