Skip to content

[DRAFT: DONT COMMIT] Enable zero-copy subbuffers across all alloc typ… #36862

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Prithviraj-R wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[DRAFT: DONT COMMIT] Enable zero-copy subbuffers across all alloc typ… #36862

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 12 additions & 36 deletions src/plugins/intel_gpu/include/intel_gpu/graph/serialization/binary_buffer.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,6 @@

namespace cldnn {

/// @brief Page size alignment constant for cache blob serialization.
/// This value (4KB) matches the typical OS page size on Windows, Linux, and macOS.
inline constexpr uint32_t CACHE_PAGE_SIZE = 4096;

/// @brief Alignment requirement for individual weight buffers within cache blobs.
/// This alignment (128 bytes) ensures optimal GPU memory access patterns for weight data.
inline constexpr uint32_t CACHE_SUB_BUFFER_ALIGNMENT = 128;
struct memory;

class BinaryOutputBuffer : public OutputBuffer<BinaryOutputBuffer> {
Expand Down Expand Up @@ -61,20 +54,12 @@ class BinaryOutputBuffer : public OutputBuffer<BinaryOutputBuffer> {
size_t get_offset() const {
return _offset;
}
bool is_offset_page_aligned() const {
return (get_offset() % CACHE_PAGE_SIZE == 0);
}

size_t get_bytes_to_page_boundary() const {
return CACHE_PAGE_SIZE - (get_offset() % CACHE_PAGE_SIZE);
}

bool is_offset_sub_buffer_aligned() const {
return (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT == 0);
bool is_offset_aligned(const size_t alignment) const {
return (get_offset() % alignment == 0);
}

size_t get_bytes_to_sub_buffer_boundary() const {
return CACHE_SUB_BUFFER_ALIGNMENT - (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT);
size_t get_bytes_to_aligned_boundary(const size_t alignment) const {
return alignment - (get_offset() % alignment);
}

private:
Expand Down Expand Up @@ -117,15 +102,15 @@ class BinaryInputBuffer : public InputBuffer<BinaryInputBuffer> {
std::streambuf* get_streambuf() const {
return _stream.rdbuf();
}
bool has_mmap_tensor() const {
bool is_tensor_valid() const {
return _tensor_base_ptr != nullptr;
}

bool is_mmap_tensor_4K_aligned() const {
return has_mmap_tensor() && (reinterpret_cast<std::uintptr_t>(_tensor_base_ptr) % CACHE_PAGE_SIZE == 0);
bool is_tensor_aligned(const size_t alignment) const {
return is_tensor_valid() && (reinterpret_cast<std::uintptr_t>(_tensor_base_ptr) % alignment == 0);
}

const size_t* get_mmap_tensor() const {
const size_t* get_tensor() const {
return _tensor_base_ptr;
}

Expand All @@ -150,20 +135,11 @@ class BinaryInputBuffer : public InputBuffer<BinaryInputBuffer> {
_stream.seekg(current_pos + static_cast<std::streampos>(size));
_offset += size;
}
bool is_offset_page_aligned() const {
return (get_offset() % CACHE_PAGE_SIZE == 0);
bool is_offset_aligned(const size_t alignment) const {
return (get_offset() % alignment == 0);
}

size_t get_bytes_to_page_boundary() const {
return CACHE_PAGE_SIZE - (get_offset() % CACHE_PAGE_SIZE);
}

bool is_offset_sub_buffer_aligned() const {
return (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT == 0);
}

size_t get_bytes_to_sub_buffer_boundary() const {
return CACHE_SUB_BUFFER_ALIGNMENT - (get_offset() % CACHE_SUB_BUFFER_ALIGNMENT);
size_t get_bytes_to_aligned_boundary(const size_t alignment) const {
return alignment - (get_offset() % alignment);
}
void setKernelImplParams(void* impl_params) {
_impl_params = impl_params;
Expand Down
31 changes: 16 additions & 15 deletions src/plugins/intel_gpu/include/intel_gpu/primitives/data.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "openvino/op/util/op_types.hpp"
#include "openvino/pass/constant_folding.hpp"
#include "openvino/pass/manager.hpp"
#include "openvino/util/memory.hpp"
#include "openvino/runtime/shared_buffer.hpp"
#include "openvino/util/mmap_object.hpp"
#include "primitive.hpp"
Expand Down Expand Up @@ -394,11 +395,13 @@ struct data : public primitive_base<data> {

bool do_weightless_caching = cache_info->save(ob, data_size);
if (!do_weightless_caching) {
const bool enable_zero_copy_mode =
mem->get_engine()->get_device_info().arch >= gpu_arch::xe2 && mem->get_engine()->get_device_info().dev_type == device_type::integrated_gpu;
if (enable_zero_copy_mode && !ob.is_encrypted() && !ob.is_offset_aligned(mem->get_engine()->get_device_info().sub_buffer_base_alignment)) {
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(mem->get_engine()->get_device_info().sub_buffer_base_alignment), 0);
ob << make_data(pad.data(), pad.size());
}
if (is_alloc_host_accessible(_allocation_type)) {
if (!ob.is_encrypted() && !ob.is_offset_sub_buffer_aligned()) {
std::vector<uint8_t> pad(ob.get_bytes_to_sub_buffer_boundary(), 0);
ob << make_data(pad.data(), pad.size());
}
ob << make_data(mem->buffer_ptr(), data_size);
} else {
std::vector<uint8_t> _buf;
Expand Down Expand Up @@ -431,9 +434,9 @@ struct data : public primitive_base<data> {
bool weightless_caching = false;
ib >> weightless_caching;

bool enable_zero_copy_mode = ib.is_mmap_tensor_4K_aligned() && ib.get_engine().get_device_info().arch >= gpu_arch::xe2 &&
ib.get_engine().get_device_info().dev_type == device_type::integrated_gpu &&
_allocation_type == allocation_type::usm_host && !weightless_caching &&
bool enable_zero_copy_mode = ib.is_tensor_aligned(ov::util::min_page_alignment) &&
ib.get_engine().get_device_info().arch >= gpu_arch::xe2 &&
ib.get_engine().get_device_info().dev_type == device_type::integrated_gpu && !weightless_caching &&
model_tensor_base != nullptr;
if (!enable_zero_copy_mode) {
mem = ib.get_engine().allocate_memory(output_layout, _allocation_type, false);
Expand All @@ -442,17 +445,15 @@ struct data : public primitive_base<data> {
bool is_weightless_caching = cache_info->load(ib, mem, weights_memory, weightless_caching);

if (!is_weightless_caching) {
if (is_alloc_host_accessible(_allocation_type)) {
if (!ib.is_encrypted() && !ib.is_offset_sub_buffer_aligned()) {
std::vector<uint8_t> pad(ib.get_bytes_to_sub_buffer_boundary(), 0);
if (enable_zero_copy_mode) {
if (!ib.is_encrypted() && !ib.is_offset_aligned(ib.get_engine().get_device_info().sub_buffer_base_alignment)) {
std::vector<uint8_t> pad(ib.get_bytes_to_aligned_boundary(ib.get_engine().get_device_info().sub_buffer_base_alignment), 0);
ib >> make_data(pad.data(), pad.size());
}
if (enable_zero_copy_mode) {
mem = ib.get_engine().create_subbuffer(*model_tensor_base, output_layout, ib.get_offset());
ib.seek_current_ptr(data_size);
} else {
mem = ib.get_engine().create_subbuffer(*model_tensor_base, output_layout, ib.get_offset());
ib.seek_current_ptr(data_size);
} else if (is_alloc_host_accessible(_allocation_type)) {
ib >> make_data(std::move(mem->buffer_ptr()), data_size);
}
} else {
const size_t DATA_BLOCK_SIZE = 4 * 1024 * 1024;
auto& eng = ib.get_engine();
Expand Down
1 change: 1 addition & 0 deletions src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ struct device_info {
uint32_t num_ccs; ///< Number of compute command streamers
uint32_t sub_device_idx; ///< Index of sub-device
int32_t cacheline_size;
uint32_t sub_buffer_base_alignment;

pci_bus_info pci_info; ///< PCI bus information for the device

Expand Down
15 changes: 8 additions & 7 deletions src/plugins/intel_gpu/src/graph/program.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include "openvino/runtime/threading/cpu_streams_info.hpp"
#include "openvino/util/file_util.hpp"
#include "openvino/util/parallel_read_streambuf.hpp"
#include "openvino/util/memory.hpp"
#include "common_utils/parallel_mem_streambuf.hpp"

#include "intel_gpu/runtime/memory.hpp"
Expand Down Expand Up @@ -1985,9 +1986,8 @@ void program::save(cldnn::BinaryOutputBuffer& ob) const {
ob << state_initializer.first;
ob << state_initializer.second;
}

if (!ob.is_encrypted() && !ob.is_offset_page_aligned()) {
std::vector<uint8_t> pad(ob.get_bytes_to_page_boundary(), 0);
if (!ob.is_encrypted() && !ob.is_offset_aligned(get_engine().get_device_info().cacheline_size)) {
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(get_engine().get_device_info().cacheline_size), 0);
ob << make_data(pad.data(), pad.size());
}
}
Expand Down Expand Up @@ -2017,12 +2017,13 @@ void program::load(cldnn::BinaryInputBuffer& ib,
}
}

const bool can_use_mmap_zero_copy = ib.is_mmap_tensor_4K_aligned() && _engine.get_device_info().arch >= gpu_arch::xe2 &&
const bool can_use_mmap_zero_copy = ib.is_tensor_aligned(ov::util::min_page_alignment) &&
_engine.get_device_info().arch >= gpu_arch::xe2 &&
_engine.get_device_info().dev_type == device_type::integrated_gpu && !_config.get_enable_weightless();
memory_ptr model_tensor_base_ptr = nullptr;
if (can_use_mmap_zero_copy) {
model_tensor_base_ptr =
ib.get_engine().create_hostbuffer(ib.get_mmap_tensor(),
ib.get_engine().create_hostbuffer(ib.get_tensor(),
ib.get_stream_size(),
allocation_type::cl_mem,
layout({{static_cast<tensor::value_type>(ib.get_stream_size()), 1, 1, 1}, data_types::u8, format::bfyx}));
Expand Down Expand Up @@ -2215,8 +2216,8 @@ void program::load(cldnn::BinaryInputBuffer& ib,
}

// At the end of load
if (!ib.is_encrypted() && !ib.is_offset_page_aligned()) {
std::vector<uint8_t> pad(ib.get_bytes_to_page_boundary(), 0);
if (!ib.is_encrypted() && !ib.is_offset_aligned(get_engine().get_device_info().cacheline_size)) {
std::vector<uint8_t> pad(ib.get_bytes_to_aligned_boundary(get_engine().get_device_info().cacheline_size), 0);
ib >> make_data(pad.data(), pad.size());
}
}
Expand Down
3 changes: 3 additions & 0 deletions src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,9 @@ device_info init_device_info(const cl::Device& device, const cl::Context& contex
info.sub_device_idx = std::numeric_limits<uint32_t>::max();

info.cacheline_size = device.getInfo<CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE>();
// Alignment requirement (in bits) for sub-buffer offsets, converting to bytes and ensuring at least 1 byte alignment.
auto bits = device.getInfo<CL_DEVICE_MEM_BASE_ADDR_ALIGN>();
info.sub_buffer_base_alignment = std::max<uint32_t>(1, bits / 8);
info.execution_units_count = device.getInfo<CL_DEVICE_MAX_COMPUTE_UNITS>();

info.gpu_frequency = static_cast<uint32_t>(device.getInfo<CL_DEVICE_MAX_CLOCK_FREQUENCY>());
Expand Down
1 change: 0 additions & 1 deletion src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

#include "ocl_engine.hpp"
#include "intel_gpu/runtime/utils.hpp"
#include "intel_gpu/graph/serialization/binary_buffer.hpp" // For CACHE_PAGE_SIZE
#include "openvino/runtime/intel_gpu/remote_properties.hpp"

#include "ocl_kernel.hpp"
Expand Down
13 changes: 11 additions & 2 deletions src/plugins/intel_gpu/tests/unit/test_cases/cache_serialization_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,9 +329,18 @@ membuf make_oversized_data_blob(const layout& output_layout, size_t malicious_da
bool weightless_caching = false; // take the plain (non-weightless) path
ob << weightless_caching;

// Checks if zero-copy host memory binding is supported.
// Requires: integrated GPU, XE2+ architecture, USM host allocation
const auto supports_hostbuffer = [&](const cldnn::engine& eng) {
const auto& info = eng.get_device_info();
return info.dev_type == cldnn::device_type::integrated_gpu && info.arch >= cldnn::gpu_arch::xe2 &&
eng.supports_allocation(cldnn::allocation_type::usm_host);
};
auto& engine = get_test_engine();
// Mirror the reader's alignment padding so stream offsets stay in sync.
if (!ob.is_encrypted() && !ob.is_offset_sub_buffer_aligned()) {
std::vector<uint8_t> pad(ob.get_bytes_to_sub_buffer_boundary(), 0);
auto sub_buffer_base_alignment = engine.get_device_info().sub_buffer_base_alignment;
if (supports_hostbuffer(engine) && !ob.is_encrypted() && !ob.is_offset_aligned(sub_buffer_base_alignment)) {
std::vector<uint8_t> pad(ob.get_bytes_to_aligned_boundary(sub_buffer_base_alignment), 0);
ob << make_data(pad.data(), pad.size());
}

Expand Down
Loading