Skip to content

Commit f9b3e5b

Browse files
committed
[GPU] Enable zero-copy subbuffers for usm_device on iGPUs
#### Details: Description of the issue (symptom, root-cause, how it was resolved) - Zero-copy handling for `usm_host` and `usm_shared` was already implemented in earlier PRs (openvinotoolkit#34494, openvinotoolkit#36539); this PR focuses on closing the `usm_device` gap. - Previously, for `usm_device` allocations, a separate GPU `usm_device` buffer was allocated and filled from mapped host buffer data during cache load, causing duplicate allocation/copy overhead and increasing inference memory footprint. - This change enables zero-copy subbuffer usage for supported XE2+ iGPUs in the `usm_device` flow, removing extra staging/copy in applicable cases. - Blob offset alignment handling in deserialization is kept correct for subbuffer creation. Implementation changes: - Extend zero-copy eligibility in `data.hpp` to include `usm_device` while preserving existing `usm_host`/`usm_shared` behavior. - Add/use helper check for zero-copy-capable device (XE2+ integrated GPU). - Use `seek_current_ptr()` for padding skip during deserialization alignment. - Refine zero-copy condition naming/readability in load path. Changed files: src/plugins/intel_gpu/include/intel_gpu/graph/serialization/binary_buffer.hpp src/plugins/intel_gpu/include/intel_gpu/primitives/data.hpp src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp src/plugins/intel_gpu/src/graph/program.cpp src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp src/plugins/intel_gpu/tests/unit/test_cases/cache_serialization_test.cpp Reproduction step and snapshot (if applicable. Do not attach for customer model) NA Problematic graph NA #### Checklist - [x] Is it a proper fix? Yes (removes duplicate `usm_device` staging/copy in supported zero-copy cases) - [x] Did you include test case for this fix, if necessary? NA - [x] Did you review existing test that can be extended to cover this scenario? NA #### Tickets: - CVS-176160 #### AI Assistance: - AI assistance used: yes - AI was used for wording refinement and commit message cleanup.
1 parent ea600b7 commit f9b3e5b

7 files changed

Lines changed: 77 additions & 79 deletions

File tree

src/plugins/intel_gpu/include/intel_gpu/graph/serialization/binary_buffer.hpp

Lines changed: 15 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,6 @@
1414

1515
namespace cldnn {
1616

17-
/// @brief Page size alignment constant for cache blob serialization.
18-
/// This value (4KB) matches the typical OS page size on Windows, Linux, and macOS.
19-
inline constexpr uint32_t CACHE_PAGE_SIZE = 4096;
20-
21-
/// @brief Alignment requirement for individual weight buffers within cache blobs.
22-
/// This alignment (128 bytes) ensures optimal GPU memory access patterns for weight data.
23-
inline constexpr uint32_t CACHE_SUB_BUFFER_ALIGNMENT = 128;
2417
struct memory;
2518

2619
class BinaryOutputBuffer : public OutputBuffer<BinaryOutputBuffer> {
@@ -61,20 +54,12 @@ class BinaryOutputBuffer : public OutputBuffer<BinaryOutputBuffer> {
6154
size_t get_offset() const {
6255
return _offset;
6356
}
64-
bool is_offset_page_aligned() const {
65-
return (get_offset() % CACHE_PAGE_SIZE == 0);
66-
}
6757

68-
size_t get_bytes_to_page_boundary() const {
69-
return CACHE_PAGE_SIZE - (get_offset() % CACHE_PAGE_SIZE);
58+
bool is_offset_aligned(const size_t alignment) const {
59+
return (get_offset() % alignment == 0);
7060
}
71-
72-
bool is_offset_sub_buffer_aligned() const {
73-
return (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT == 0);
74-
}
75-
76-
size_t get_bytes_to_sub_buffer_boundary() const {
77-
return CACHE_SUB_BUFFER_ALIGNMENT - (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT);
61+
size_t get_bytes_to_aligned_boundary(const size_t alignment) const {
62+
return alignment - (get_offset() % alignment);
7863
}
7964

8065
private:
@@ -117,15 +102,8 @@ class BinaryInputBuffer : public InputBuffer<BinaryInputBuffer> {
117102
std::streambuf* get_streambuf() const {
118103
return _stream.rdbuf();
119104
}
120-
bool has_mmap_tensor() const {
121-
return _tensor_base_ptr != nullptr;
122-
}
123-
124-
bool is_mmap_tensor_4K_aligned() const {
125-
return has_mmap_tensor() && (reinterpret_cast<std::uintptr_t>(_tensor_base_ptr) % CACHE_PAGE_SIZE == 0);
126-
}
127105

128-
const size_t* get_mmap_tensor() const {
106+
const size_t* get_tensor() const {
129107
return _tensor_base_ptr;
130108
}
131109

@@ -142,28 +120,25 @@ class BinaryInputBuffer : public InputBuffer<BinaryInputBuffer> {
142120
size_t get_offset() const {
143121
return _offset;
144122
}
145-
123+
bool is_valid_tensor() const {
124+
return _tensor_base_ptr != nullptr;
125+
}
126+
bool is_tensor_aligned(const size_t alignment) const {
127+
return reinterpret_cast<std::uintptr_t>(_tensor_base_ptr) % alignment == 0;
128+
}
146129
void seek_current_ptr(std::streamsize size) {
147130
// Get current stream position
148131
std::streampos current_pos = _stream.tellg();
149132
// Advance stream position
150133
_stream.seekg(current_pos + static_cast<std::streampos>(size));
151134
_offset += size;
152135
}
153-
bool is_offset_page_aligned() const {
154-
return (get_offset() % CACHE_PAGE_SIZE == 0);
155-
}
156-
157-
size_t get_bytes_to_page_boundary() const {
158-
return CACHE_PAGE_SIZE - (get_offset() % CACHE_PAGE_SIZE);
159-
}
160136

161-
bool is_offset_sub_buffer_aligned() const {
162-
return (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT == 0);
137+
bool is_offset_aligned(const size_t alignment) const {
138+
return (get_offset() % alignment == 0);
163139
}
164-
165-
size_t get_bytes_to_sub_buffer_boundary() const {
166-
return CACHE_SUB_BUFFER_ALIGNMENT - (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT);
140+
size_t get_bytes_to_aligned_boundary(const size_t alignment) const {
141+
return alignment - (get_offset() % alignment);
167142
}
168143
void setKernelImplParams(void* impl_params) {
169144
_impl_params = impl_params;

src/plugins/intel_gpu/include/intel_gpu/primitives/data.hpp

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ bool is_alloc_host_accessible(const cldnn::allocation_type& alloc_type) {
3333
return alloc_type == cldnn::allocation_type::usm_host || alloc_type == cldnn::allocation_type::usm_shared;
3434
}
3535

36+
bool is_zero_copy_capable_device(const cldnn::gpu_arch arch, const cldnn::device_type dev_type) {
37+
return dev_type == cldnn::device_type::integrated_gpu && arch >= cldnn::gpu_arch::xe2;
38+
}
39+
3640
void copy_to_dst_mem(cldnn::memory::ptr mem_ptr, const uint8_t* data_ptr) {
3741
if (is_alloc_host_accessible(mem_ptr->get_allocation_type())) {
3842
size_t data_size = mem_ptr->size();
@@ -393,12 +397,15 @@ struct data : public primitive_base<data> {
393397
ob << make_data(&data_size, sizeof(size_t));
394398

395399
bool do_weightless_caching = cache_info->save(ob, data_size);
400+
const auto& engine = mem->get_engine();
401+
const auto& dev_info = engine->get_device_info();
396402
if (!do_weightless_caching) {
403+
if (is_zero_copy_capable_device(dev_info.arch, dev_info.dev_type) && engine->supports_allocation(cldnn::allocation_type::usm_host) &&
404+
!ob.is_encrypted() && !ob.is_offset_aligned(dev_info.sub_buffer_base_alignment)) {
405+
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(dev_info.sub_buffer_base_alignment), 0);
406+
ob << make_data(pad.data(), pad.size());
407+
}
397408
if (is_alloc_host_accessible(_allocation_type)) {
398-
if (!ob.is_encrypted() && !ob.is_offset_sub_buffer_aligned()) {
399-
std::vector<uint8_t> pad(ob.get_bytes_to_sub_buffer_boundary(), 0);
400-
ob << make_data(pad.data(), pad.size());
401-
}
402409
ob << make_data(mem->buffer_ptr(), data_size);
403410
} else {
404411
std::vector<uint8_t> _buf;
@@ -414,7 +421,7 @@ struct data : public primitive_base<data> {
414421
primitive_base<data>::load(ib);
415422
}
416423

417-
void load_weights(BinaryInputBuffer& ib, std::shared_ptr<WeightsMemory> weights_memory, memory_ptr model_tensor_base) {
424+
void load_weights(BinaryInputBuffer& ib, std::shared_ptr<WeightsMemory> weights_memory, memory_ptr host_buffer_base_ptr) {
418425
layout output_layout = layout();
419426
ib >> output_layout;
420427

@@ -430,29 +437,26 @@ struct data : public primitive_base<data> {
430437

431438
bool weightless_caching = false;
432439
ib >> weightless_caching;
433-
434-
bool enable_zero_copy_mode = ib.is_mmap_tensor_4K_aligned() && ib.get_engine().get_device_info().arch >= gpu_arch::xe2 &&
435-
ib.get_engine().get_device_info().dev_type == device_type::integrated_gpu &&
436-
_allocation_type == allocation_type::usm_host && !weightless_caching &&
437-
model_tensor_base != nullptr;
438-
if (!enable_zero_copy_mode) {
440+
const auto& dev_info = ib.get_engine().get_device_info();
441+
const bool can_use_zero_copy_mode = is_zero_copy_capable_device(dev_info.arch, dev_info.dev_type) &&
442+
ib.get_engine().supports_allocation(cldnn::allocation_type::usm_host) && host_buffer_base_ptr != nullptr && !weightless_caching;
443+
if (!can_use_zero_copy_mode) {
439444
mem = ib.get_engine().allocate_memory(output_layout, _allocation_type, false);
440445
}
441446

442447
bool is_weightless_caching = cache_info->load(ib, mem, weights_memory, weightless_caching);
443448

444449
if (!is_weightless_caching) {
445-
if (is_alloc_host_accessible(_allocation_type)) {
446-
if (!ib.is_encrypted() && !ib.is_offset_sub_buffer_aligned()) {
447-
std::vector<uint8_t> pad(ib.get_bytes_to_sub_buffer_boundary(), 0);
448-
ib >> make_data(pad.data(), pad.size());
449-
}
450-
if (enable_zero_copy_mode) {
451-
mem = ib.get_engine().create_subbuffer(*model_tensor_base, output_layout, ib.get_offset());
452-
ib.seek_current_ptr(data_size);
453-
} else {
454-
ib >> make_data(std::move(mem->buffer_ptr()), data_size);
450+
// Zero-copy: create subbuffer referencing mmap'd cache without host-to-device transfer.
451+
if (can_use_zero_copy_mode) {
452+
// Advance input stream past padding so subbuffer base offset meets use_host_ptr() alignment requirements.
453+
if (!ib.is_encrypted() && !ib.is_offset_aligned(dev_info.sub_buffer_base_alignment)) {
454+
ib.seek_current_ptr(ib.get_bytes_to_aligned_boundary(dev_info.sub_buffer_base_alignment));
455455
}
456+
mem = ib.get_engine().create_subbuffer(*host_buffer_base_ptr, output_layout, ib.get_offset());
457+
ib.seek_current_ptr(data_size);
458+
} else if (is_alloc_host_accessible(_allocation_type)) {
459+
ib >> make_data(std::move(mem->buffer_ptr()), data_size);
456460
} else {
457461
const size_t DATA_BLOCK_SIZE = 4 * 1024 * 1024;
458462
auto& eng = ib.get_engine();

src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ struct device_info {
145145
uint32_t num_ccs; ///< Number of compute command streamers
146146
uint32_t sub_device_idx; ///< Index of sub-device
147147
int32_t cacheline_size;
148-
148+
uint32_t sub_buffer_base_alignment; ///< Alignment requirement (in bytes) for sub-buffer offsets
149149
pci_bus_info pci_info; ///< PCI bus information for the device
150150

151151
uint64_t timer_resolution; ///< [ZE] Resolution of device timer used for profiling in cycles/sec

src/plugins/intel_gpu/src/graph/program.cpp

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "openvino/runtime/system_conf.hpp"
1212
#include "openvino/runtime/threading/cpu_streams_info.hpp"
1313
#include "openvino/util/file_util.hpp"
14+
#include "openvino/util/memory.hpp"
1415
#include "openvino/util/parallel_read_streambuf.hpp"
1516
#include "common_utils/parallel_mem_streambuf.hpp"
1617

@@ -1985,9 +1986,10 @@ void program::save(cldnn::BinaryOutputBuffer& ob) const {
19851986
ob << state_initializer.first;
19861987
ob << state_initializer.second;
19871988
}
1988-
1989-
if (!ob.is_encrypted() && !ob.is_offset_page_aligned()) {
1990-
std::vector<uint8_t> pad(ob.get_bytes_to_page_boundary(), 0);
1989+
1990+
const auto& dev_info = get_engine().get_device_info();
1991+
if (!ob.is_encrypted() && !ob.is_offset_aligned(dev_info.cacheline_size)) {
1992+
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(dev_info.cacheline_size), 0);
19911993
ob << make_data(pad.data(), pad.size());
19921994
}
19931995
}
@@ -2017,12 +2019,18 @@ void program::load(cldnn::BinaryInputBuffer& ib,
20172019
}
20182020
}
20192021

2020-
const bool can_use_mmap_zero_copy = ib.is_mmap_tensor_4K_aligned() && _engine.get_device_info().arch >= gpu_arch::xe2 &&
2021-
_engine.get_device_info().dev_type == device_type::integrated_gpu && !_config.get_enable_weightless();
2022-
memory_ptr model_tensor_base_ptr = nullptr;
2023-
if (can_use_mmap_zero_copy) {
2024-
model_tensor_base_ptr =
2025-
ib.get_engine().create_hostbuffer(ib.get_mmap_tensor(),
2022+
memory_ptr host_buffer_base_ptr = nullptr;
2023+
// Checks if zero-copy host memory binding is supported.
2024+
// Requires: integrated GPU, XE2+ architecture, USM host allocation
2025+
const auto supports_hostbuffer = [&](const cldnn::engine& eng) {
2026+
const auto& info = eng.get_device_info();
2027+
return info.dev_type == cldnn::device_type::integrated_gpu && info.arch >= cldnn::gpu_arch::xe2 &&
2028+
eng.supports_allocation(cldnn::allocation_type::usm_host) && ib.is_valid_tensor() && ib.is_tensor_aligned(ov::util::min_page_alignment);
2029+
};
2030+
2031+
if (!_config.get_enable_weightless() && supports_hostbuffer(ib.get_engine())) {
2032+
host_buffer_base_ptr =
2033+
ib.get_engine().create_hostbuffer(ib.get_tensor(),
20262034
ib.get_stream_size(),
20272035
allocation_type::cl_mem,
20282036
layout({{static_cast<tensor::value_type>(ib.get_stream_size()), 1, 1, 1}, data_types::u8, format::bfyx}));
@@ -2055,7 +2063,7 @@ void program::load(cldnn::BinaryInputBuffer& ib,
20552063
std::shared_ptr<cldnn::primitive> prim;
20562064
ib >> prim;
20572065
if (auto data_prim = dynamic_cast<cldnn::data*>(prim.get())) {
2058-
data_prim->load_weights(ib, weights_memory, model_tensor_base_ptr);
2066+
data_prim->load_weights(ib, weights_memory, host_buffer_base_ptr);
20592067
}
20602068
get_or_create(prim);
20612069
}
@@ -2214,10 +2222,9 @@ void program::load(cldnn::BinaryInputBuffer& ib,
22142222
state_initializers[variable_id] = initializers;
22152223
}
22162224

2217-
// At the end of load
2218-
if (!ib.is_encrypted() && !ib.is_offset_page_aligned()) {
2219-
std::vector<uint8_t> pad(ib.get_bytes_to_page_boundary(), 0);
2220-
ib >> make_data(pad.data(), pad.size());
2225+
const auto& dev_info = get_engine().get_device_info();
2226+
if (!ib.is_encrypted() && !ib.is_offset_aligned(dev_info.cacheline_size)) {
2227+
ib.seek_current_ptr(ib.get_bytes_to_aligned_boundary(dev_info.cacheline_size));
22212228
}
22222229
}
22232230

src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,9 @@ device_info init_device_info(const cl::Device& device, const cl::Context& contex
209209
info.sub_device_idx = std::numeric_limits<uint32_t>::max();
210210

211211
info.cacheline_size = device.getInfo<CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE>();
212+
// Alignment requirement (in bits) for sub-buffer offsets, converting to bytes and ensuring at least 1 byte alignment.
213+
auto bits = device.getInfo<CL_DEVICE_MEM_BASE_ADDR_ALIGN>();
214+
info.sub_buffer_base_alignment = std::max<uint32_t>(1, bits / 8);
212215
info.execution_units_count = device.getInfo<CL_DEVICE_MAX_COMPUTE_UNITS>();
213216

214217
info.gpu_frequency = static_cast<uint32_t>(device.getInfo<CL_DEVICE_MAX_CLOCK_FREQUENCY>());

src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
#include "ocl_engine.hpp"
66
#include "intel_gpu/runtime/utils.hpp"
7-
#include "intel_gpu/graph/serialization/binary_buffer.hpp" // For CACHE_PAGE_SIZE
87
#include "openvino/runtime/intel_gpu/remote_properties.hpp"
98

109
#include "ocl_kernel.hpp"
@@ -381,6 +380,7 @@ memory_ptr ocl_engine::create_hostbuffer_impl(void* cpu_address, size_t data_siz
381380
#ifdef CL_MEM_FORCE_HOST_MEMORY_INTEL
382381
const size_t minimal_alignment = static_cast<size_t>(get_device_info().cacheline_size);
383382
OPENVINO_ASSERT(minimal_alignment > 0, "[GPU] cacheline_size must be > 0 for host pointer import");
383+
OPENVINO_ASSERT(cpu_address != nullptr, "[GPU] shared buffer pointer is invalid");
384384
OPENVINO_ASSERT((reinterpret_cast<std::uintptr_t>(cpu_address) % minimal_alignment) == 0,
385385
"[GPU] shared buffer pointer must be ", minimal_alignment, "-byte aligned");
386386
OPENVINO_ASSERT((data_size % minimal_alignment) == 0,

src/plugins/intel_gpu/tests/unit/test_cases/cache_serialization_test.cpp

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -330,8 +330,17 @@ membuf make_oversized_data_blob(const layout& output_layout, size_t malicious_da
330330
ob << weightless_caching;
331331

332332
// Mirror the reader's alignment padding so stream offsets stay in sync.
333-
if (!ob.is_encrypted() && !ob.is_offset_sub_buffer_aligned()) {
334-
std::vector<uint8_t> pad(ob.get_bytes_to_sub_buffer_boundary(), 0);
333+
// Checks if zero-copy host memory binding is supported.
334+
// Requires: integrated GPU, XE2+ architecture, USM host allocation
335+
const auto supports_hostbuffer = [&](const cldnn::engine& eng) {
336+
const auto& info = eng.get_device_info();
337+
return info.dev_type == cldnn::device_type::integrated_gpu && info.arch >= cldnn::gpu_arch::xe2 &&
338+
eng.supports_allocation(cldnn::allocation_type::usm_host);
339+
};
340+
341+
const auto& dev_info = get_engine().get_device_info();
342+
if (supports_hostbuffer(get_test_engine()) && !ob.is_encrypted() && !ob.is_offset_aligned(dev_info.sub_buffer_base_alignment)) {
343+
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(dev_info.sub_buffer_base_alignment), 0);
335344
ob << make_data(pad.data(), pad.size());
336345
}
337346

0 commit comments

Comments
 (0)