Skip to content

Commit 02023e2

Browse files
committed
[DRAFT: DONT COMMIT] Enable zero-copy subbuffers across all alloc types for iGPUs
#### Details: Description of the issue (symptom, root-cause, how it was resolved) - Eliminates redundant weight copies during model loading for integrated GPUs by binding mmap'd cache file memory directly to GPU without intermediate usm_device allocation - Reduces memory footprint during inference by reusing host memory as GPU-accessible buffers - Zero-copy enabled when: iGPU, XE2+ architecture, USM host support available - Falls back gracefully to copy-based path when zero-copy requirements not met Implementation changes: - Add ocl_engine::supports_hostbuffer() to validate zero-copy capability (checks for integrated GPU, XE2+ arch, USM host allocation support) - Add ocl_engine::create_hostbuffer_impl() to bind host memory with CL_MEM_USE_HOST_PTR - Simplify zero-copy detection in data.hpp load path (remove redundant capability checks) - Use ib.seek_current_ptr() instead of dummy reads for padding alignment - Remove is_mmap_tensor_4K_aligned() helpers Changed files: src/plugins/intel_gpu/include/intel_gpu/graph/serialization/binary_buffer.hpp src/plugins/intel_gpu/include/intel_gpu/primitives/data.hpp src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp src/plugins/intel_gpu/src/graph/program.cpp src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp src/plugins/intel_gpu/tests/unit/test_cases/cache_serialization_test.cpp Reproduction step and snapshot (if applicable. Do not attach for customer model) NA Problematic graph NA #### Checklist - [x] Is it a proper fix? Yes (memory optimization, not a workaround) - [x] Did you include test case for this fix, if necessary? NA - [x] Did you review existing test that can be extended to cover this scenario? NA #### Tickets: - CVS-176160 #### AI Assistance: - AI assistance used: yes - AI was used for code implementation, naming suggestions, and comment refinement.
1 parent bbb6398 commit 02023e2

7 files changed

Lines changed: 48 additions & 59 deletions

File tree

src/plugins/intel_gpu/include/intel_gpu/graph/serialization/binary_buffer.hpp

Lines changed: 12 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,6 @@
1414

1515
namespace cldnn {
1616

17-
/// @brief Page size alignment constant for cache blob serialization.
18-
/// This value (4KB) matches the typical OS page size on Windows, Linux, and macOS.
19-
inline constexpr uint32_t CACHE_PAGE_SIZE = 4096;
20-
21-
/// @brief Alignment requirement for individual weight buffers within cache blobs.
22-
/// This alignment (128 bytes) ensures optimal GPU memory access patterns for weight data.
23-
inline constexpr uint32_t CACHE_SUB_BUFFER_ALIGNMENT = 128;
2417
struct memory;
2518

2619
class BinaryOutputBuffer : public OutputBuffer<BinaryOutputBuffer> {
@@ -61,20 +54,12 @@ class BinaryOutputBuffer : public OutputBuffer<BinaryOutputBuffer> {
6154
size_t get_offset() const {
6255
return _offset;
6356
}
64-
bool is_offset_page_aligned() const {
65-
return (get_offset() % CACHE_PAGE_SIZE == 0);
66-
}
67-
68-
size_t get_bytes_to_page_boundary() const {
69-
return CACHE_PAGE_SIZE - (get_offset() % CACHE_PAGE_SIZE);
70-
}
71-
72-
bool is_offset_sub_buffer_aligned() const {
73-
return (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT == 0);
57+
bool is_offset_aligned(const size_t alignment) const {
58+
return (get_offset() % alignment == 0);
7459
}
7560

76-
size_t get_bytes_to_sub_buffer_boundary() const {
77-
return CACHE_SUB_BUFFER_ALIGNMENT - (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT);
61+
size_t get_bytes_to_aligned_boundary(const size_t alignment) const {
62+
return alignment - (get_offset() % alignment);
7863
}
7964

8065
private:
@@ -117,15 +102,15 @@ class BinaryInputBuffer : public InputBuffer<BinaryInputBuffer> {
117102
std::streambuf* get_streambuf() const {
118103
return _stream.rdbuf();
119104
}
120-
bool has_mmap_tensor() const {
105+
bool is_tensor_valid() const {
121106
return _tensor_base_ptr != nullptr;
122107
}
123108

124-
bool is_mmap_tensor_4K_aligned() const {
125-
return has_mmap_tensor() && (reinterpret_cast<std::uintptr_t>(_tensor_base_ptr) % CACHE_PAGE_SIZE == 0);
109+
bool is_tensor_aligned(const size_t alignment) const {
110+
return is_tensor_valid() && (reinterpret_cast<std::uintptr_t>(_tensor_base_ptr) % alignment == 0);
126111
}
127112

128-
const size_t* get_mmap_tensor() const {
113+
const size_t* get_tensor() const {
129114
return _tensor_base_ptr;
130115
}
131116

@@ -150,20 +135,11 @@ class BinaryInputBuffer : public InputBuffer<BinaryInputBuffer> {
150135
_stream.seekg(current_pos + static_cast<std::streampos>(size));
151136
_offset += size;
152137
}
153-
bool is_offset_page_aligned() const {
154-
return (get_offset() % CACHE_PAGE_SIZE == 0);
138+
bool is_offset_aligned(const size_t alignment) const {
139+
return (get_offset() % alignment == 0);
155140
}
156-
157-
size_t get_bytes_to_page_boundary() const {
158-
return CACHE_PAGE_SIZE - (get_offset() % CACHE_PAGE_SIZE);
159-
}
160-
161-
bool is_offset_sub_buffer_aligned() const {
162-
return (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT == 0);
163-
}
164-
165-
size_t get_bytes_to_sub_buffer_boundary() const {
166-
return CACHE_SUB_BUFFER_ALIGNMENT - (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT);
141+
size_t get_bytes_to_aligned_boundary(const size_t alignment) const {
142+
return alignment - (get_offset() % alignment);
167143
}
168144
void setKernelImplParams(void* impl_params) {
169145
_impl_params = impl_params;

src/plugins/intel_gpu/include/intel_gpu/primitives/data.hpp

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "openvino/op/util/op_types.hpp"
1919
#include "openvino/pass/constant_folding.hpp"
2020
#include "openvino/pass/manager.hpp"
21+
#include "openvino/util/memory.hpp"
2122
#include "openvino/runtime/shared_buffer.hpp"
2223
#include "openvino/util/mmap_object.hpp"
2324
#include "primitive.hpp"
@@ -395,8 +396,8 @@ struct data : public primitive_base<data> {
395396
bool do_weightless_caching = cache_info->save(ob, data_size);
396397
if (!do_weightless_caching) {
397398
if (is_alloc_host_accessible(_allocation_type)) {
398-
if (!ob.is_encrypted() && !ob.is_offset_sub_buffer_aligned()) {
399-
std::vector<uint8_t> pad(ob.get_bytes_to_sub_buffer_boundary(), 0);
399+
if (!ob.is_encrypted() && !ob.is_offset_aligned(mem->get_engine()->get_device_info().sub_buffer_base_alignment)) {
400+
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(mem->get_engine()->get_device_info().sub_buffer_base_alignment), 0);
400401
ob << make_data(pad.data(), pad.size());
401402
}
402403
ob << make_data(mem->buffer_ptr(), data_size);
@@ -431,7 +432,8 @@ struct data : public primitive_base<data> {
431432
bool weightless_caching = false;
432433
ib >> weightless_caching;
433434

434-
bool enable_zero_copy_mode = ib.is_mmap_tensor_4K_aligned() && ib.get_engine().get_device_info().arch >= gpu_arch::xe2 &&
435+
bool enable_zero_copy_mode = ib.is_tensor_aligned(ov::util::min_page_alignment) &&
436+
ib.get_engine().get_device_info().arch >= gpu_arch::xe2 &&
435437
ib.get_engine().get_device_info().dev_type == device_type::integrated_gpu &&
436438
_allocation_type == allocation_type::usm_host && !weightless_caching &&
437439
model_tensor_base != nullptr;
@@ -442,17 +444,15 @@ struct data : public primitive_base<data> {
442444
bool is_weightless_caching = cache_info->load(ib, mem, weights_memory, weightless_caching);
443445

444446
if (!is_weightless_caching) {
445-
if (is_alloc_host_accessible(_allocation_type)) {
446-
if (!ib.is_encrypted() && !ib.is_offset_sub_buffer_aligned()) {
447-
std::vector<uint8_t> pad(ib.get_bytes_to_sub_buffer_boundary(), 0);
448-
ib >> make_data(pad.data(), pad.size());
449-
}
450-
if (enable_zero_copy_mode) {
451-
mem = ib.get_engine().create_subbuffer(*model_tensor_base, output_layout, ib.get_offset());
452-
ib.seek_current_ptr(data_size);
453-
} else {
447+
if (!ib.is_encrypted() && !ib.is_offset_aligned(ib.get_engine().get_device_info().sub_buffer_base_alignment)) {
448+
std::vector<uint8_t> pad(ib.get_bytes_to_aligned_boundary(ib.get_engine().get_device_info().sub_buffer_base_alignment), 0);
449+
ib >> make_data(pad.data(), pad.size());
450+
}
451+
if (enable_zero_copy_mode) {
452+
mem = ib.get_engine().create_subbuffer(*model_tensor_base, output_layout, ib.get_offset());
453+
ib.seek_current_ptr(data_size);
454+
} else if (is_alloc_host_accessible(_allocation_type)) {
454455
ib >> make_data(std::move(mem->buffer_ptr()), data_size);
455-
}
456456
} else {
457457
const size_t DATA_BLOCK_SIZE = 4 * 1024 * 1024;
458458
auto& eng = ib.get_engine();

src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ struct device_info {
145145
uint32_t num_ccs; ///< Number of compute command streamers
146146
uint32_t sub_device_idx; ///< Index of sub-device
147147
int32_t cacheline_size;
148+
uint32_t sub_buffer_base_alignment;
148149

149150
pci_bus_info pci_info; ///< PCI bus information for the device
150151

src/plugins/intel_gpu/src/graph/program.cpp

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "openvino/runtime/threading/cpu_streams_info.hpp"
1313
#include "openvino/util/file_util.hpp"
1414
#include "openvino/util/parallel_read_streambuf.hpp"
15+
#include "openvino/util/memory.hpp"
1516
#include "common_utils/parallel_mem_streambuf.hpp"
1617

1718
#include "intel_gpu/runtime/memory.hpp"
@@ -1985,9 +1986,8 @@ void program::save(cldnn::BinaryOutputBuffer& ob) const {
19851986
ob << state_initializer.first;
19861987
ob << state_initializer.second;
19871988
}
1988-
1989-
if (!ob.is_encrypted() && !ob.is_offset_page_aligned()) {
1990-
std::vector<uint8_t> pad(ob.get_bytes_to_page_boundary(), 0);
1989+
if (!ob.is_encrypted() && !ob.is_offset_aligned(get_engine().get_device_info().cacheline_size)) {
1990+
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(get_engine().get_device_info().cacheline_size), 0);
19911991
ob << make_data(pad.data(), pad.size());
19921992
}
19931993
}
@@ -2017,12 +2017,13 @@ void program::load(cldnn::BinaryInputBuffer& ib,
20172017
}
20182018
}
20192019

2020-
const bool can_use_mmap_zero_copy = ib.is_mmap_tensor_4K_aligned() && _engine.get_device_info().arch >= gpu_arch::xe2 &&
2020+
const bool can_use_mmap_zero_copy = ib.is_tensor_aligned(ov::util::min_page_alignment) &&
2021+
_engine.get_device_info().arch >= gpu_arch::xe2 &&
20212022
_engine.get_device_info().dev_type == device_type::integrated_gpu && !_config.get_enable_weightless();
20222023
memory_ptr model_tensor_base_ptr = nullptr;
20232024
if (can_use_mmap_zero_copy) {
20242025
model_tensor_base_ptr =
2025-
ib.get_engine().create_hostbuffer(ib.get_mmap_tensor(),
2026+
ib.get_engine().create_hostbuffer(ib.get_tensor(),
20262027
ib.get_stream_size(),
20272028
allocation_type::cl_mem,
20282029
layout({{static_cast<tensor::value_type>(ib.get_stream_size()), 1, 1, 1}, data_types::u8, format::bfyx}));
@@ -2215,8 +2216,8 @@ void program::load(cldnn::BinaryInputBuffer& ib,
22152216
}
22162217

22172218
// At the end of load
2218-
if (!ib.is_encrypted() && !ib.is_offset_page_aligned()) {
2219-
std::vector<uint8_t> pad(ib.get_bytes_to_page_boundary(), 0);
2219+
if (!ib.is_encrypted() && !ib.is_offset_aligned(get_engine().get_device_info().cacheline_size)) {
2220+
std::vector<uint8_t> pad(ib.get_bytes_to_aligned_boundary(get_engine().get_device_info().cacheline_size), 0);
22202221
ib >> make_data(pad.data(), pad.size());
22212222
}
22222223
}

src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,9 @@ device_info init_device_info(const cl::Device& device, const cl::Context& contex
209209
info.sub_device_idx = std::numeric_limits<uint32_t>::max();
210210

211211
info.cacheline_size = device.getInfo<CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE>();
212+
// Alignment requirement (in bits) for sub-buffer offsets, converting to bytes and ensuring at least 1 byte alignment.
213+
auto bits = device.getInfo<CL_DEVICE_MEM_BASE_ADDR_ALIGN>();
214+
info.sub_buffer_base_alignment = std::max<uint32_t>(1, bits / 8);
212215
info.execution_units_count = device.getInfo<CL_DEVICE_MAX_COMPUTE_UNITS>();
213216

214217
info.gpu_frequency = static_cast<uint32_t>(device.getInfo<CL_DEVICE_MAX_CLOCK_FREQUENCY>());

src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
#include "ocl_engine.hpp"
66
#include "intel_gpu/runtime/utils.hpp"
7-
#include "intel_gpu/graph/serialization/binary_buffer.hpp" // For CACHE_PAGE_SIZE
87
#include "openvino/runtime/intel_gpu/remote_properties.hpp"
98

109
#include "ocl_kernel.hpp"

src/plugins/intel_gpu/tests/unit/test_cases/cache_serialization_test.cpp

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -329,9 +329,18 @@ membuf make_oversized_data_blob(const layout& output_layout, size_t malicious_da
329329
bool weightless_caching = false; // take the plain (non-weightless) path
330330
ob << weightless_caching;
331331

332+
// Checks if zero-copy host memory binding is supported.
333+
// Requires: integrated GPU, XE2+ architecture, USM host allocation
334+
const auto supports_hostbuffer = [&](const cldnn::engine& eng) {
335+
const auto& info = eng.get_device_info();
336+
return info.dev_type == cldnn::device_type::integrated_gpu && info.arch >= cldnn::gpu_arch::xe2 &&
337+
eng.supports_allocation(cldnn::allocation_type::usm_host);
338+
};
339+
auto& engine = get_test_engine();
332340
// Mirror the reader's alignment padding so stream offsets stay in sync.
333-
if (!ob.is_encrypted() && !ob.is_offset_sub_buffer_aligned()) {
334-
std::vector<uint8_t> pad(ob.get_bytes_to_sub_buffer_boundary(), 0);
341+
auto sub_buffer_base_alignment = engine.get_device_info().sub_buffer_base_alignment;
342+
if (supports_hostbuffer(engine) && !ob.is_encrypted() && !ob.is_offset_aligned(sub_buffer_base_alignment)) {
343+
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(sub_buffer_base_alignment), 0);
335344
ob << make_data(pad.data(), pad.size());
336345
}
337346

0 commit comments

Comments
 (0)