Skip to content

Commit 5f3a5e3

Browse files
committed
[DRAFT: DONT COMMIT] Enable zero-copy subbuffers across all alloc types for iGPUs
#### Details: Description of the issue (symptom, root-cause, how it was resolved) - Eliminates redundant weight copies during model loading for integrated GPUs by binding mmap'd cache file memory directly to GPU without intermediate usm_device allocation - Reduces memory footprint during inference by reusing host memory as GPU-accessible buffers - Zero-copy enabled when: iGPU, XE2+ architecture, USM host support available - Falls back gracefully to copy-based path when zero-copy requirements not met Implementation changes: - Add ocl_engine::supports_hostbuffer() to validate zero-copy capability (checks for integrated GPU, XE2+ arch, USM host allocation support) - Add ocl_engine::create_hostbuffer_impl() to bind host memory with CL_MEM_USE_HOST_PTR - Simplify zero-copy detection in data.hpp load path (remove redundant capability checks) - Use ib.seek_current_ptr() instead of dummy reads for padding alignment - Remove is_mmap_tensor_4K_aligned() helpers Changed files: src/plugins/intel_gpu/include/intel_gpu/graph/serialization/binary_buffer.hpp src/plugins/intel_gpu/include/intel_gpu/primitives/data.hpp src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp src/plugins/intel_gpu/src/graph/program.cpp src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp src/plugins/intel_gpu/tests/unit/test_cases/cache_serialization_test.cpp Reproduction step and snapshot (if applicable. Do not attach for customer model) NA Problematic graph NA #### Checklist - [x] Is it a proper fix? Yes (memory optimization, not a workaround) - [x] Did you include test case for this fix, if necessary? NA - [x] Did you review existing test that can be extended to cover this scenario? NA #### Tickets: - CVS-176160 #### AI Assistance: - AI assistance used: yes - AI was used for code implementation, naming suggestions, and comment refinement.
1 parent 654ba6f commit 5f3a5e3

7 files changed

Lines changed: 52 additions & 73 deletions

File tree

src/plugins/intel_gpu/include/intel_gpu/graph/serialization/binary_buffer.hpp

Lines changed: 11 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,6 @@
1414

1515
namespace cldnn {
1616

17-
/// @brief Page size alignment constant for cache blob serialization.
18-
/// This value (4KB) matches the typical OS page size on Windows, Linux, and macOS.
19-
inline constexpr uint32_t CACHE_PAGE_SIZE = 4096;
20-
21-
/// @brief Alignment requirement for individual weight buffers within cache blobs.
22-
/// This alignment (128 bytes) ensures optimal GPU memory access patterns for weight data.
23-
inline constexpr uint32_t CACHE_SUB_BUFFER_ALIGNMENT = 128;
2417
struct memory;
2518

2619
class BinaryOutputBuffer : public OutputBuffer<BinaryOutputBuffer> {
@@ -61,20 +54,12 @@ class BinaryOutputBuffer : public OutputBuffer<BinaryOutputBuffer> {
6154
size_t get_offset() const {
6255
return _offset;
6356
}
64-
bool is_offset_page_aligned() const {
65-
return (get_offset() % CACHE_PAGE_SIZE == 0);
66-
}
6757

68-
size_t get_bytes_to_page_boundary() const {
69-
return CACHE_PAGE_SIZE - (get_offset() % CACHE_PAGE_SIZE);
58+
bool is_offset_aligned(const size_t alignment) const {
59+
return (get_offset() % alignment == 0);
7060
}
71-
72-
bool is_offset_sub_buffer_aligned() const {
73-
return (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT == 0);
74-
}
75-
76-
size_t get_bytes_to_sub_buffer_boundary() const {
77-
return CACHE_SUB_BUFFER_ALIGNMENT - (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT);
61+
size_t get_bytes_to_aligned_boundary(const size_t alignment) const {
62+
return alignment - (get_offset() % alignment);
7863
}
7964

8065
private:
@@ -117,13 +102,6 @@ class BinaryInputBuffer : public InputBuffer<BinaryInputBuffer> {
117102
std::streambuf* get_streambuf() const {
118103
return _stream.rdbuf();
119104
}
120-
bool has_mmap_tensor() const {
121-
return _tensor_base_ptr != nullptr;
122-
}
123-
124-
bool is_mmap_tensor_4K_aligned() const {
125-
return has_mmap_tensor() && (reinterpret_cast<std::uintptr_t>(_tensor_base_ptr) % CACHE_PAGE_SIZE == 0);
126-
}
127105

128106
const size_t* get_mmap_tensor() const {
129107
return _tensor_base_ptr;
@@ -142,28 +120,22 @@ class BinaryInputBuffer : public InputBuffer<BinaryInputBuffer> {
142120
size_t get_offset() const {
143121
return _offset;
144122
}
145-
123+
bool has_mmap_tensor() const {
124+
return _tensor_base_ptr != nullptr;
125+
}
146126
void seek_current_ptr(std::streamsize size) {
147127
// Get current stream position
148128
std::streampos current_pos = _stream.tellg();
149129
// Advance stream position
150130
_stream.seekg(current_pos + static_cast<std::streampos>(size));
151131
_offset += size;
152132
}
153-
bool is_offset_page_aligned() const {
154-
return (get_offset() % CACHE_PAGE_SIZE == 0);
155-
}
156-
157-
size_t get_bytes_to_page_boundary() const {
158-
return CACHE_PAGE_SIZE - (get_offset() % CACHE_PAGE_SIZE);
159-
}
160133

161-
bool is_offset_sub_buffer_aligned() const {
162-
return (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT == 0);
134+
bool is_offset_aligned(const size_t alignment) const {
135+
return (get_offset() % alignment == 0);
163136
}
164-
165-
size_t get_bytes_to_sub_buffer_boundary() const {
166-
return CACHE_SUB_BUFFER_ALIGNMENT - (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT);
137+
size_t get_bytes_to_aligned_boundary(const size_t alignment) const {
138+
return alignment - (get_offset() % alignment);
167139
}
168140
void setKernelImplParams(void* impl_params) {
169141
_impl_params = impl_params;

src/plugins/intel_gpu/include/intel_gpu/primitives/data.hpp

Lines changed: 17 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -394,11 +394,12 @@ struct data : public primitive_base<data> {
394394

395395
bool do_weightless_caching = cache_info->save(ob, data_size);
396396
if (!do_weightless_caching) {
397+
auto sub_buffer_base_alignment = mem->get_engine()->get_device_info().sub_buffer_base_alignment;
398+
if (!ob.is_encrypted() && !ob.is_offset_aligned(sub_buffer_base_alignment)) {
399+
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(sub_buffer_base_alignment), 0);
400+
ob << make_data(pad.data(), pad.size());
401+
}
397402
if (is_alloc_host_accessible(_allocation_type)) {
398-
if (!ob.is_encrypted() && !ob.is_offset_sub_buffer_aligned()) {
399-
std::vector<uint8_t> pad(ob.get_bytes_to_sub_buffer_boundary(), 0);
400-
ob << make_data(pad.data(), pad.size());
401-
}
402403
ob << make_data(mem->buffer_ptr(), data_size);
403404
} else {
404405
std::vector<uint8_t> _buf;
@@ -430,29 +431,24 @@ struct data : public primitive_base<data> {
430431

431432
bool weightless_caching = false;
432433
ib >> weightless_caching;
433-
434-
bool enable_zero_copy_mode = ib.is_mmap_tensor_4K_aligned() && ib.get_engine().get_device_info().arch >= gpu_arch::xe2 &&
435-
ib.get_engine().get_device_info().dev_type == device_type::integrated_gpu &&
436-
_allocation_type == allocation_type::usm_host && !weightless_caching &&
437-
model_tensor_base != nullptr;
438-
if (!enable_zero_copy_mode) {
434+
bool zero_copy_mode = !weightless_caching && model_tensor_base != nullptr && ib.has_mmap_tensor();
435+
if (!zero_copy_mode) {
439436
mem = ib.get_engine().allocate_memory(output_layout, _allocation_type, false);
440437
}
441438

442439
bool is_weightless_caching = cache_info->load(ib, mem, weights_memory, weightless_caching);
443440

444441
if (!is_weightless_caching) {
445-
if (is_alloc_host_accessible(_allocation_type)) {
446-
if (!ib.is_encrypted() && !ib.is_offset_sub_buffer_aligned()) {
447-
std::vector<uint8_t> pad(ib.get_bytes_to_sub_buffer_boundary(), 0);
448-
ib >> make_data(pad.data(), pad.size());
449-
}
450-
if (enable_zero_copy_mode) {
451-
mem = ib.get_engine().create_subbuffer(*model_tensor_base, output_layout, ib.get_offset());
452-
ib.seek_current_ptr(data_size);
453-
} else {
454-
ib >> make_data(std::move(mem->buffer_ptr()), data_size);
455-
}
442+
auto sub_buffer_base_alignment = ib.get_engine().get_device_info().sub_buffer_base_alignment;
443+
if (!ib.is_encrypted() && !ib.is_offset_aligned(sub_buffer_base_alignment)) {
444+
ib.seek_current_ptr(ib.get_bytes_to_aligned_boundary(sub_buffer_base_alignment));
445+
}
446+
// Zero-copy: create subbuffer referencing mmap'd cache without host-to-device transfer
447+
if (zero_copy_mode) {
448+
mem = ib.get_engine().create_subbuffer(*model_tensor_base, output_layout, ib.get_offset());
449+
ib.seek_current_ptr(data_size);
450+
} else if (is_alloc_host_accessible(_allocation_type)) {
451+
ib >> make_data(std::move(mem->buffer_ptr()), data_size);
456452
} else {
457453
const size_t DATA_BLOCK_SIZE = 4 * 1024 * 1024;
458454
auto& eng = ib.get_engine();

src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ struct device_info {
145145
uint32_t num_ccs; ///< Number of compute command streamers
146146
uint32_t sub_device_idx; ///< Index of sub-device
147147
int32_t cacheline_size;
148-
148+
uint32_t sub_buffer_base_alignment;
149149
pci_bus_info pci_info; ///< PCI bus information for the device
150150

151151
uint64_t timer_resolution; ///< [ZE] Resolution of device timer used for profiling in cycles/sec

src/plugins/intel_gpu/src/graph/program.cpp

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1985,9 +1985,10 @@ void program::save(cldnn::BinaryOutputBuffer& ob) const {
19851985
ob << state_initializer.first;
19861986
ob << state_initializer.second;
19871987
}
1988-
1989-
if (!ob.is_encrypted() && !ob.is_offset_page_aligned()) {
1990-
std::vector<uint8_t> pad(ob.get_bytes_to_page_boundary(), 0);
1988+
1989+
auto minimal_alignment = get_engine().get_device_info().cacheline_size;
1990+
if (!ob.is_encrypted() && !ob.is_offset_aligned(minimal_alignment)) {
1991+
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(minimal_alignment), 0);
19911992
ob << make_data(pad.data(), pad.size());
19921993
}
19931994
}
@@ -2016,10 +2017,17 @@ void program::load(cldnn::BinaryInputBuffer& ib,
20162017
OPENVINO_THROW("Weights path or model is required for cache mode OPTIMIZE_SIZE");
20172018
}
20182019
}
2019-
2020-
const bool can_use_mmap_zero_copy = ib.is_mmap_tensor_4K_aligned() && _engine.get_device_info().arch >= gpu_arch::xe2 &&
2020+
const bool can_use_mmap_zero_copy = ib.has_mmap_tensor() && _engine.get_device_info().arch >= gpu_arch::xe2 &&
20212021
_engine.get_device_info().dev_type == device_type::integrated_gpu && !_config.get_enable_weightless();
20222022
memory_ptr model_tensor_base_ptr = nullptr;
2023+
// Checks if zero-copy host memory binding is supported.
2024+
// Requires: integrated GPU, XE2+ architecture, USM host allocation
2025+
const auto supports_hostbuffer = [&](const cldnn::engine& eng) {
2026+
const auto& info = eng.get_device_info();
2027+
return info.dev_type == cldnn::device_type::integrated_gpu && info.arch >= cldnn::gpu_arch::xe2 &&
2028+
eng.supports_allocation(cldnn::allocation_type::usm_host) && ib.has_mmap_tensor();
2029+
};
2030+
20232031
if (can_use_mmap_zero_copy) {
20242032
model_tensor_base_ptr =
20252033
ib.get_engine().create_hostbuffer(ib.get_mmap_tensor(),
@@ -2214,10 +2222,9 @@ void program::load(cldnn::BinaryInputBuffer& ib,
22142222
state_initializers[variable_id] = initializers;
22152223
}
22162224

2217-
// At the end of load
2218-
if (!ib.is_encrypted() && !ib.is_offset_page_aligned()) {
2219-
std::vector<uint8_t> pad(ib.get_bytes_to_page_boundary(), 0);
2220-
ib >> make_data(pad.data(), pad.size());
2225+
auto minimal_alignment = get_engine().get_device_info().cacheline_size;
2226+
if (!ib.is_encrypted() && !ib.is_offset_aligned(minimal_alignment)) {
2227+
ib.seek_current_ptr(ib.get_bytes_to_aligned_boundary(minimal_alignment));
22212228
}
22222229
}
22232230

src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,9 @@ device_info init_device_info(const cl::Device& device, const cl::Context& contex
209209
info.sub_device_idx = std::numeric_limits<uint32_t>::max();
210210

211211
info.cacheline_size = device.getInfo<CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE>();
212+
// Alignment requirement (in bits) for sub-buffer offsets, converting to bytes and ensuring at least 1 byte alignment.
213+
auto bits = device.getInfo<CL_DEVICE_MEM_BASE_ADDR_ALIGN>();
214+
info.sub_buffer_base_alignment = std::max<uint32_t>(1, bits / 8);
212215
info.execution_units_count = device.getInfo<CL_DEVICE_MAX_COMPUTE_UNITS>();
213216

214217
info.gpu_frequency = static_cast<uint32_t>(device.getInfo<CL_DEVICE_MAX_CLOCK_FREQUENCY>());

src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
#include "ocl_engine.hpp"
66
#include "intel_gpu/runtime/utils.hpp"
7-
#include "intel_gpu/graph/serialization/binary_buffer.hpp" // For CACHE_PAGE_SIZE
87
#include "openvino/runtime/intel_gpu/remote_properties.hpp"
98

109
#include "ocl_kernel.hpp"

src/plugins/intel_gpu/tests/unit/test_cases/cache_serialization_test.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -330,8 +330,10 @@ membuf make_oversized_data_blob(const layout& output_layout, size_t malicious_da
330330
ob << weightless_caching;
331331

332332
// Mirror the reader's alignment padding so stream offsets stay in sync.
333-
if (!ob.is_encrypted() && !ob.is_offset_sub_buffer_aligned()) {
334-
std::vector<uint8_t> pad(ob.get_bytes_to_sub_buffer_boundary(), 0);
333+
auto& engine = get_test_engine();
334+
auto sub_buffer_base_alignment = engine.get_device_info().sub_buffer_base_alignment;
335+
if (!ob.is_encrypted() && !ob.is_offset_aligned(sub_buffer_base_alignment)) {
336+
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(sub_buffer_base_alignment), 0);
335337
ob << make_data(pad.data(), pad.size());
336338
}
337339

0 commit comments

Comments
 (0)