diff --git a/src/inference/dev_api/openvino/runtime/shared_context_buffer_descriptor.hpp b/src/inference/dev_api/openvino/runtime/shared_context_buffer_descriptor.hpp new file mode 100644 index 00000000000000..701ce0fddd8f37 --- /dev/null +++ b/src/inference/dev_api/openvino/runtime/shared_context_buffer_descriptor.hpp @@ -0,0 +1,59 @@ +// Copyright (C) 2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "openvino/runtime/aligned_buffer.hpp" +#include "openvino/runtime/iremote_context.hpp" +#include "openvino/runtime/iremote_tensor.hpp" +#include "openvino/runtime/so_ptr.hpp" + +namespace ov { + +/// \brief Buffer descriptor that carries the set of remote contexts which co-own +/// (or should receive) the underlying allocation. +/// +/// Extends the base IBufferDescriptor interface so that consumers can +/// retrieve the associated remote contexts via get_remote_contexts(). + +class OPENVINO_RUNTIME_API SharedContextBufferDescriptor : public ov::IBufferDescriptor { +public: + struct RemoteContextPtrLess { + bool operator()(const ov::SoPtr& lhs, + const ov::SoPtr& rhs) const noexcept { + return std::less{}(lhs._ptr.get(), rhs._ptr.get()); + } + }; + + using RemoteContextsMap = std::map, + ov::SoPtr, + RemoteContextPtrLess>; + + SharedContextBufferDescriptor(size_t id, + size_t offset, + size_t real_buffer_size, + const std::shared_ptr& source_buffer, + RemoteContextsMap remote_contexts); + + size_t get_id() const override; + size_t get_offset() const override; + size_t get_real_buffer_size() const; + std::shared_ptr get_source_buffer() const override; + + const RemoteContextsMap& get_remote_contexts() const; + + ~SharedContextBufferDescriptor() override; + +private: + size_t m_id = 0; + size_t m_offset = 0; + size_t m_real_buffer_size = 0; + std::weak_ptr m_source_buffer; + RemoteContextsMap m_remote_contexts; +}; + +} // namespace ov diff --git a/src/inference/src/dev/shared_context_buffer_descriptor.cpp b/src/inference/src/dev/shared_context_buffer_descriptor.cpp new file mode 100644 index 00000000000000..027b97f2ca89ba --- /dev/null +++ b/src/inference/src/dev/shared_context_buffer_descriptor.cpp @@ -0,0 +1,45 @@ +// Copyright (C) 2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/runtime/shared_context_buffer_descriptor.hpp" + +#include + +namespace ov { + +SharedContextBufferDescriptor::SharedContextBufferDescriptor( + size_t id, + size_t offset, + size_t real_buffer_size, + const std::shared_ptr& source_buffer, + RemoteContextsMap remote_contexts) + : m_id(id), + m_offset(offset), + m_real_buffer_size(real_buffer_size), + m_source_buffer(source_buffer), + m_remote_contexts(std::move(remote_contexts)) {} + +SharedContextBufferDescriptor::~SharedContextBufferDescriptor() = default; + +size_t SharedContextBufferDescriptor::get_id() const { + return m_id; +} + +size_t SharedContextBufferDescriptor::get_offset() const { + return m_offset; +} + +size_t SharedContextBufferDescriptor::get_real_buffer_size() const { + return m_real_buffer_size; +} + +std::shared_ptr SharedContextBufferDescriptor::get_source_buffer() const { + return m_source_buffer.lock(); +} + +const SharedContextBufferDescriptor::RemoteContextsMap& SharedContextBufferDescriptor::get_remote_contexts() const { + return m_remote_contexts; +} + +} // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/ops/constant.cpp b/src/plugins/intel_gpu/src/plugin/ops/constant.cpp index 027e70f8ac5871..b05a867389a534 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/constant.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/constant.cpp @@ -4,6 +4,7 @@ #include "intel_gpu/plugin/program_builder.hpp" #include "intel_gpu/plugin/common_utils.hpp" +#include "intel_gpu/plugin/remote_tensor.hpp" #include "intel_gpu/op/convolution.hpp" #include "openvino/core/weight_sharing_util.hpp" @@ -28,6 +29,7 @@ #include "openvino/op/matmul.hpp" #include "openvino/op/moe.hpp" #include "openvino/op/util/binary_elementwise_bitwise.hpp" +#include "openvino/runtime/shared_context_buffer_descriptor.hpp" #include "ov_ops/moe_compressed.hpp" @@ -110,8 +112,40 @@ static void create_data(ProgramBuilder& p, const ov::Shape& const_shape, const s p.primitive_ids[initialconstPrimID] = constPrimID; p.profiling_ids.push_back(initialconstPrimID); } else { + GPU_DEBUG_LOG << "[" << initialconstPrimID << ": constant] check whether the constant is a shared weight or not" << std::endl; + const auto shared_buffer = weight_sharing::Extension::get_constant_source_buffer(*op); + if (shared_buffer) { + GPU_DEBUG_LOG << "[" << initialconstPrimID << ": constant] check whether the constant memory has been already allocated" << std::endl; + auto shared_context_buffer = std::dynamic_pointer_cast(shared_buffer->get_descriptor()); + if (shared_context_buffer) { + const auto& remote_contexts = shared_context_buffer->get_remote_contexts(); + GPU_DEBUG_LOG << "[" << initialconstPrimID << ": constant] shared between contexts:" << remote_contexts.size() << std::endl; + std::shared_ptr gpu_remote_tensor; + for (const auto& [remote_context, remote_tensor_ptr] : remote_contexts) { + GPU_DEBUG_LOG << "[" << initialconstPrimID << ": constant] check whether weight from the remote context: " << remote_context->get_device_name() << " is available on device: " << p.get_engine().get_device_info().dev_name << std::endl; + if (remote_context->get_device_name().find("GPU") != std::string::npos) { + gpu_remote_tensor = remote_tensor_ptr._ptr; + break; + } + } + if (gpu_remote_tensor) { + GPU_DEBUG_LOG << "[" << initialconstPrimID << ": constant] layout " + << constLayout.to_short_string() << ", weight_sharing(" << static_cast(data) << ", " << constLayout.bytes_count() <<" bytes)" << std::endl; + auto iremote_tensor = std::dynamic_pointer_cast(gpu_remote_tensor); + auto mem = iremote_tensor->get_original_memory(); + p.add_primitive(*op, cldnn::data(initialconstPrimID, mem)); + p.blobMemCache[cache_key] = initialconstPrimID; + constPrimID = initialconstPrimID; + return; + } + } else { + GPU_DEBUG_LOG << "[" << initialconstPrimID << ": constant] layout " + << constLayout.to_short_string() << ", ignore weight sharing, UNEXPECTED buffer type (" << static_cast(data) << ", " << constLayout.bytes_count() <<" bytes)" << std::endl; + std::cout << "[" << initialconstPrimID << ": constant] layout " + << constLayout.to_short_string() << ", ignore weight sharing, UNEXPECTED buffer type (" << static_cast(data) << ", " << constLayout.bytes_count() <<" bytes)" << std::endl; + } + } auto partial_upload = try_prepare_partial_upload(p, op, const_shape, out_dtype, constFormat, constLayout); - cldnn::memory::ptr mem = nullptr; if (partial_upload.enabled) { diff --git a/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp b/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp index 53b9059d03f7a5..59182da25aaa70 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp +++ b/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp @@ -10,6 +10,7 @@ #include "llm_compiled_model_utils.hpp" #include "llm_infer_request.hpp" #include "logging.hpp" +#include "shared_context_buffer.hpp" #include "moe_transformations/apply_moe_device_routed_transforms.hpp" #include "npuw_transformations/add_position_ids_param.hpp" #include "npuw_transformations/convert_kvcache_to_precision.hpp" @@ -22,6 +23,8 @@ #include "npuw_transformations/reshape_to_static.hpp" #include "npuw_transformations/slice_out_embeds.hpp" #include "npuw_transformations/split_kvcache_into_blocks.hpp" +#include "openvino/core/weight_sharing_util.hpp" +#include "openvino/core/rt_info/weightless_caching_attributes.hpp" #include "openvino/op/convert.hpp" #include "openvino/op/greater.hpp" #include "openvino/op/ops.hpp" @@ -36,8 +39,11 @@ #include "openvino/pass/pattern/op/wrap_type.hpp" #include "openvino/pass/stateful_to_stateless.hpp" #include "openvino/pass/validate.hpp" +#include "openvino/runtime/device_id_parser.hpp" #include "openvino/runtime/iasync_infer_request.hpp" #include "openvino/runtime/properties.hpp" +#include "openvino/runtime/shared_context_buffer_descriptor.hpp" +#include "openvino/util/mmap_object.hpp" #include "partitioning/patterns/fold_const.hpp" #include "partitioning/patterns/moe.hpp" #include "partitioning/patterns/pre_compute.hpp" @@ -668,6 +674,78 @@ void ov::npuw::LLMCompiledModel::compile_generate_model_variants( m_kvcache_compiled = m_generate_compiled_variants.back(); } +std::string dump_shared_ctx(const ov::weight_sharing::Context& shared_ctx) { + std::ostringstream oss; + oss << "TODO print me" << std::endl; + (void)shared_ctx; + return oss.str(); +} + +void ov::npuw::LLMCompiledModel::assign_shared_weight_to_model_if_possible(const std::shared_ptr model, const std::shared_ptr& plugin, +const ov::AnyMap& properties) { + NPUW_ASSERT(model && "Model for assigning shared weights must not be null"); + NPUW_ASSERT(plugin && "Plugin for assigning shared weights must not be null"); + auto shared_weight_property_it = properties.find("SHARED_WEIGHTS"); + if (shared_weight_property_it == properties.end()) { + return; + } + std::vector> remote_ctxs; + auto shared_device_contexts = ov::DeviceIDParser::get_hetero_devices(shared_weight_property_it->second.as()); + LOG_DEBUG("Create shared context over devices: " << shared_device_contexts.size()); + for (const auto& ctx_name : shared_device_contexts) { + try { + LOG_DEBUG("trying to get context '" << ctx_name << "'"); + remote_ctxs.push_back(plugin->get_core()->get_default_context(ctx_name)); + } catch (const std::exception& e) { + LOG_WARN("SHARED_WEIGHTS: context '" << ctx_name << "' unavailable (" << e.what() + << "); weights will NOT be shared for this context."); + } + } + LOG_INFO("Created shared contexts: " << remote_ctxs.size() << " out of " << shared_device_contexts.size()); + if (remote_ctxs.empty() || remote_ctxs.size() == 1) { + LOG_INFO("Not enough shared contexts available, skipping shared weights assignment."); + return; + } + // TODO + // think about allocating a one large buffer with individual weights located sequentially + // and distinguished by a pair of offset and size. + // It may reduce fragmentation and improve locality. + // On the other hand, allocation of individual weights can be done in parallel and may be faster. + const size_t kMinRelocateBytes = ov::util::get_system_page_size(); // page-sized+; skip tiny scalar constants + NPUW_ASSERT(!m_shared_ctx_ptr && "NPU shared context must be empty before shared weight assignment."); + m_shared_ctx_ptr = std::make_unique(); + for (const auto& op : model->get_ops()) { + auto not_shared_constant = std::dynamic_pointer_cast(op); + // TODO devise better conditions to determine whether shared weights applicable or not + if (!not_shared_constant || not_shared_constant->get_byte_size() < kMinRelocateBytes) { + continue; + } + const size_t bytes = not_shared_constant->get_byte_size(); + LOG_DEBUG("Allocating shared buffer for weight" << not_shared_constant->get_friendly_name() << ", size: " << bytes); + auto buffer = std::make_shared(bytes, remote_ctxs, kMinRelocateBytes); + ov::weight_sharing::set_weight_source(*m_shared_ctx_ptr, buffer); + auto shared_constant = + std::make_shared(not_shared_constant->get_element_type(), + not_shared_constant->get_shape(), buffer); + ov::weight_sharing::set_constant(*m_shared_ctx_ptr, *shared_constant); + shared_constant->set_friendly_name(not_shared_constant->get_friendly_name()); + ov::copy_runtime_info(not_shared_constant, shared_constant); + std::memcpy(buffer->get_ptr(), not_shared_constant->get_data_ptr(), bytes); + + // Preserve the weightless-cache attribute: copy_runtime_info drops it (is_copyable()==false), + // and without it NPUW's Const wrapper treats every relocated weight as a brand-new Constant + // and eagerly copies it to host (a full second ~2 GB copy at partition time). + // CAVEAT (under investigation): keeping the weightless attr may make NPUW reconstruct the GPU + // prefill submodel's constants from the original .bin mmap (off our malloc) -> GPU share_usm + // sees in_range=0. Toggle via NO_WEIGHTLESS_ATTR=1 to test the GPU-share path. + if (!std::getenv("NO_WEIGHTLESS_ATTR")) { + ov::copy_weightless_cache_attr(not_shared_constant, shared_constant); + } + ov::replace_node(not_shared_constant, shared_constant); + LOG_INFO("[NPUW] SHARED_WEIGHTS: " << dump_shared_ctx(*m_shared_ctx_ptr)); + } +} + ov::npuw::LLMCompiledModel::LLMCompiledModel(const std::shared_ptr& model, const std::shared_ptr& plugin, const ov::AnyMap& properties, @@ -809,6 +887,9 @@ ov::npuw::LLMCompiledModel::LLMCompiledModel(const std::shared_ptr& m const uint32_t seq_len_dim = m_cfg.get<::intel_npu::NPUW_LLM_SEQ_LEN_DIM>(); KVAxesPosition axes{batch_dim, seq_len_dim}; + // Assign shared buffers + assign_shared_weight_to_model_if_possible(model, plugin, properties); + LOG_DEBUG("Creating kvcache model as clone of passed one."); auto kvcache_model = model->clone(); diff --git a/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.hpp b/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.hpp index d9796d2d444ae2..f26a5b52026b15 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.hpp +++ b/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.hpp @@ -10,6 +10,9 @@ #include "npuw_transformations/kv_axes_position.hpp" namespace ov { +namespace weight_sharing { + struct Context; +} namespace test { namespace npuw { struct LLMVariantSwitchTestAccess; @@ -153,8 +156,10 @@ class LLMCompiledModel : public ov::npuw::ICompiledModel { void compile_generate_model_variants(const std::vector>& generate_model_variants, const std::shared_ptr& plugin, const ov::AnyMap& generate_config); - + void assign_shared_weight_to_model_if_possible(const std::shared_ptr model, const std::shared_ptr& plugin, +const ov::AnyMap& properties); bool m_is_eagle = false; + std::unique_ptr m_shared_ctx_ptr; }; } // namespace npuw diff --git a/src/plugins/intel_npu/src/plugin/npuw/shared_context_buffer.cpp b/src/plugins/intel_npu/src/plugin/npuw/shared_context_buffer.cpp new file mode 100644 index 00000000000000..f858436665b904 --- /dev/null +++ b/src/plugins/intel_npu/src/plugin/npuw/shared_context_buffer.cpp @@ -0,0 +1,118 @@ +// Copyright (C) 2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shared_context_buffer.hpp" +#include "openvino/runtime/intel_npu/remote_properties.hpp" +#include "openvino/runtime/intel_gpu/remote_properties.hpp" + +namespace { +std::atomic_size_t g_shared_context_buffer_id{1}; +} + +namespace { + +ov::SoPtr bind_gpu_remote_ctx(ov::AlignedBuffer& buffer, size_t aligned_size, + const ov::SoPtr& remote_ctx) { + OPENVINO_ASSERT(buffer.get_ptr(), "Cannot bind_gpu_remote_ctx: buffer is null"); + OPENVINO_ASSERT(remote_ctx, "Cannot bind_gpu_remote_ctx: remote context is null"); + + ov::SoPtr gpu_remote_tensor; + try { + ov::AnyMap params = {{ov::intel_gpu::shared_mem_type.name(), ov::intel_gpu::SharedMemType::CPU_VA}, + {ov::intel_gpu::cpu_va.name(), buffer.get_ptr()}, + {ov::intel_gpu::cpu_va_size.name(), aligned_size}}; + + gpu_remote_tensor = remote_ctx->create_tensor(ov::element::u8, ov::Shape{buffer.size()}, params); + } catch (const std::exception& e) { + throw std::runtime_error(std::string("Cannot bind_gpu_remote_ctx: failed to create remote tensor: ") + e.what()); + } + return gpu_remote_tensor; +} + +ov::SoPtr bind_npu_remote_ctx(ov::AlignedBuffer& buffer, + size_t aligned_size, + const ov::SoPtr& remote_ctx) { + OPENVINO_ASSERT(buffer.get_ptr(), "Cannot bind_npu_remote_ctx: buffer is null"); + OPENVINO_ASSERT(remote_ctx, "Cannot bind_npu_remote_ctx: remote context is null"); + + // TODO user real shape and element type for the remote tensor, for now we use u8 and shape of {buffer.size()} + try { + ov::AnyMap params = { + {ov::intel_npu::mem_type.name(), ov::intel_npu::MemType::CPU_VA}, + {ov::intel_npu::mem_handle.name(), buffer.get_ptr()}, + {ov::intel_npu::tensor_type.name(), ov::intel_npu::TensorType::INPUT}, + }; + return remote_ctx->create_tensor(ov::element::u8, ov::Shape{aligned_size}, params); + } catch (const std::exception& e) { + throw std::runtime_error(std::string("Cannot bind_npu_remote_ctx: failed to create remote tensor: ") + e.what()); + } +} +} + +ov::npuw::SharedContextBuffer::SharedContextBuffer(size_t byte_size, + std::vector> remote_contexts, + size_t alignment) + : ov::AlignedBuffer(((byte_size + alignment - 1) / alignment) * alignment, alignment), + m_id(g_shared_context_buffer_id.fetch_add(1, std::memory_order_relaxed)) { + OPENVINO_ASSERT(remote_contexts.size() > 0, "SharedContextBuffer: remote_contexts must not be empty"); + m_real_buffer_size = this->size(); // Set the actual byte size, as AlignedBuffer constructor may have rounded it up to alignment + memset(m_aligned_buffer, 0, m_real_buffer_size); // Fill by zeros + + m_byte_size = byte_size; // Set the actual constant size to AlignedBuffer, not aligned + for (const auto& remote_ctx : remote_contexts) { + auto remote_tensor = bind_remote_context(remote_ctx); + if (remote_tensor) { + m_remote_contexts.emplace(remote_ctx, remote_tensor); + } + } +} + +ov::npuw::SharedContextBuffer::~SharedContextBuffer() = default; + +size_t ov::npuw::SharedContextBuffer::get_real_buffer_size() const { + return m_real_buffer_size; +} + +std::shared_ptr ov::npuw::SharedContextBuffer::get_descriptor() const { + auto self = std::const_pointer_cast(shared_from_this()); + return std::make_shared( + m_id, + 0, + m_real_buffer_size, + std::static_pointer_cast(std::move(self)), + m_remote_contexts); +} + +ov::SoPtr ov::npuw::SharedContextBuffer::get_remote_tensors_if_exist(ov::SoPtr remote_ctx) const { + // TODO maybe use device name instead of remote_ctx pointer for lookup? since remote_ctx pointer may be different for the same device (or not?) + auto it = m_remote_contexts.find(remote_ctx); + if (it != m_remote_contexts.end()) { + return it->second; + } + return nullptr; +} + +ov::SoPtr ov::npuw::SharedContextBuffer::bind_remote_context(const ov::SoPtr& remote_ctx) { + if (!remote_ctx) { + return nullptr; + } + + const auto& device_name = remote_ctx->get_device_name(); + if (device_name.find("GPU") != std::string::npos) { + return bind_gpu_remote_ctx(*this, get_real_buffer_size(), remote_ctx); + } else if (device_name.find("NPU") != std::string::npos) { + return bind_npu_remote_ctx(*this, get_real_buffer_size(), remote_ctx); + } else { + OPENVINO_THROW("Unsupported remote context device: " + device_name); + } +} diff --git a/src/plugins/intel_npu/src/plugin/npuw/shared_context_buffer.hpp b/src/plugins/intel_npu/src/plugin/npuw/shared_context_buffer.hpp new file mode 100644 index 00000000000000..5bacaf31671d45 --- /dev/null +++ b/src/plugins/intel_npu/src/plugin/npuw/shared_context_buffer.hpp @@ -0,0 +1,39 @@ +// Copyright (C) 2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +#include "openvino/runtime/aligned_buffer.hpp" +#include "openvino/runtime/iremote_context.hpp" +#include "openvino/runtime/shared_context_buffer_descriptor.hpp" +#include "openvino/runtime/so_ptr.hpp" + +namespace ov { +namespace npuw { +class SharedContextBuffer : public ov::AlignedBuffer, public std::enable_shared_from_this { +public: + using RemoteContextsMap = SharedContextBufferDescriptor::RemoteContextsMap; + SharedContextBuffer(size_t byte_size, + std::vector> remote_contexts, + size_t alignment = 64); + + ~SharedContextBuffer() override; + + std::shared_ptr get_descriptor() const override; + + ov::SoPtr get_remote_tensors_if_exist(ov::SoPtr remote_ctx) const; + + size_t get_real_buffer_size() const; +private: + size_t m_id = 0; + RemoteContextsMap m_remote_contexts; + size_t m_real_buffer_size = 0; + ov::SoPtr bind_remote_context(const ov::SoPtr& remote_ctx); +}; +} // namespace npuw +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/CMakeLists.txt b/src/plugins/intel_npu/tests/functional/CMakeLists.txt index 1b33742934a119..efb45b616e5db5 100644 --- a/src/plugins/intel_npu/tests/functional/CMakeLists.txt +++ b/src/plugins/intel_npu/tests/functional/CMakeLists.txt @@ -20,6 +20,7 @@ set(SKIP_CONFIG_PATH ${CMAKE_CURRENT_SOURCE_DIR}/shared_tests_instances/${SKIP_C list(APPEND OPTIONAL_FUNC_TESTS_INCLUDES "${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/compiler_adapter/include" "${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/include" + "${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/npuw" ) list(APPEND OPTIONAL_FUNC_TESTS_LIBS openvino_npu_common openvino_npu_compiler_adapter @@ -52,6 +53,7 @@ ov_add_test_target( "${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/src/transformations.cpp" "${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/src/metrics.cpp" "${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/src/executor.cpp" + "${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/npuw/shared_context_buffer.cpp" LINK_LIBRARIES ${OPTIONAL_FUNC_TESTS_LIBS} openvino::func_test_utils diff --git a/src/plugins/intel_npu/tests/functional/behavior/npuw/cross_context_tests.cpp b/src/plugins/intel_npu/tests/functional/behavior/npuw/cross_context_tests.cpp new file mode 100644 index 00000000000000..dffdb4205b8d1b --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/npuw/cross_context_tests.cpp @@ -0,0 +1,151 @@ +// Copyright (C) 2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include + +#include "openvino/core/weight_sharing_util.hpp" +#include "cross_context_tests.hpp" +#include "shared_context_buffer.hpp" +#include "openvino/util/mmap_object.hpp" +#include "openvino/runtime/shared_buffer.hpp" +#include "shared_context_buffer.hpp" + +using namespace ov::npuw::tests; +std::shared_ptr CrossContextTestsNPUW::create_shared_context_model(size_t elem_count) { + auto s = std::make_shared(ov::element::u8, ov::Shape{elem_count}); + auto one = ov::opset8::Constant::create(ov::element::u8, ov::Shape{elem_count}, {1}); + auto add = std::make_shared(s, one); // [1,1] = s+1 + auto res = std::make_shared(add); + auto m = std::make_shared(ov::ResultVector{res}, ov::ParameterVector{s}, "producer"); + m->input(0).set_names({"input"}); + m->output(0).set_names({"output"}); + return m; +} + +std::pair +CrossContextTestsNPUW::create_shared_compiled_models(std::shared_ptr model, + const ov::SoPtr& remote_context_gpu, + const ov::SoPtr& remote_context_npu) { + ov::CompiledModel gpu_compiled_model = core.compile_model(model, remote_context_gpu->get_device_name()); + ov::CompiledModel npu_compiled_model = core.compile_model(model, remote_context_npu->get_device_name()); + return {gpu_compiled_model, npu_compiled_model}; +} + +using SharedContextBufferPtr = std::shared_ptr; +std::tuple execute_inference_on_shared_context(ov::CompiledModel& gpu_compiled_model, ov::CompiledModel& npu_compiled_model, + const ov::SoPtr& remote_context_gpu, + const ov::SoPtr& remote_context_npu, + size_t model_elem_count, + size_t iteration_num) { + std::vector> ctx = {remote_context_gpu, remote_context_npu}; + const size_t kMinRelocateBytes = ov::util::get_system_page_size(); + size_t shared_mem_bytes_count = model_elem_count * sizeof(char); + auto shared_context_input_output = std::make_shared(shared_mem_bytes_count, ctx, kMinRelocateBytes); + auto remote_gpu_tensor_input = shared_context_input_output->get_remote_tensors_if_exist(remote_context_gpu); + assert(remote_gpu_tensor_input != nullptr && "Failed to get input remote tensor for GPU context"); + auto remote_npu_tensor_output = shared_context_input_output->get_remote_tensors_if_exist(remote_context_npu); + assert(remote_npu_tensor_output != nullptr && "Failed to get output remote tensor for NPU context"); + auto gpu_tensor_input = ov::make_tensor(remote_gpu_tensor_input); + auto npu_tensor_output = ov::make_tensor(remote_npu_tensor_output); + assert(gpu_tensor_input.get_byte_size() == npu_tensor_output.get_byte_size() && "GPU input and NPU output tensor byte sizes do not match"); + + auto shared_context_output_input = std::make_shared(shared_mem_bytes_count, ctx, kMinRelocateBytes); + auto remote_gpu_tensor_output = shared_context_output_input->get_remote_tensors_if_exist(remote_context_gpu); + assert(remote_gpu_tensor_output != nullptr && "Failed to get output remote tensor for GPU context"); + auto remote_npu_tensor_input = shared_context_output_input->get_remote_tensors_if_exist(remote_context_npu); + assert(remote_npu_tensor_input != nullptr && "Failed to get input remote tensor for NPU context"); + auto npu_tensor_input = ov::make_tensor(remote_npu_tensor_input); + auto gpu_tensor_output = ov::make_tensor(remote_gpu_tensor_output); + assert(gpu_tensor_output.get_byte_size() == npu_tensor_input.get_byte_size() && "GPU output and NPU input tensor byte sizes do not match"); + + auto gireq = gpu_compiled_model.create_infer_request(); + auto nireq = npu_compiled_model.create_infer_request(); + gireq.set_input_tensor(gpu_tensor_input); // the same as NPU output + gireq.set_output_tensor(gpu_tensor_output); + nireq.set_input_tensor(npu_tensor_input); + nireq.set_output_tensor(npu_tensor_output); // the same as GPU input + + // execute inference on GPU and NPU for a number of iterations + for (size_t i = 0; i < iteration_num; ++i) { + gireq.infer(); + nireq.infer(); + } + + return {std::move(shared_context_input_output), std::move(shared_context_output_input)}; +} + +TEST_F(CrossContextTestsNPUW, CanMemoryShareBetweenContexts) { + ASSERT_TRUE(remote_context_gpu != nullptr); + ASSERT_TRUE(remote_context_npu != nullptr); + size_t iteration_num = 13; + auto [shared_context_input_output, shared_context_output_input] = execute_inference_on_shared_context(gpu_compiled_model, npu_compiled_model, remote_context_gpu, remote_context_npu, model_elem_count, iteration_num); + // NPU and GPU must have the same data pointer as parts of the shared context buffer + // the data must be valid and equal to the expected value after the iterations. + // A value of the shared_context_input_output buffer must be greater than a value of the shared_context_output_input buffer by 1, + // as NPU adds additional 1 to the output which is the input for GPU + for (size_t i = 0; i get_ptr()[i], shared_context_output_input->get_ptr()[i] + 1); + ASSERT_EQ(shared_context_input_output->get_ptr()[i], iteration_num * 2); + ASSERT_EQ(shared_context_output_input->get_ptr()[i], iteration_num * 2 - 1); + } +} + +std::shared_ptr make_shared_weight_from_weight( + ov::weight_sharing::Context &shared_weight_ctx, + std::vector> remote_contexts, + std::shared_ptr not_shared_weight) { + const size_t kMinRelocateBytes = ov::util::get_system_page_size(); + const size_t bytes = not_shared_weight->get_byte_size(); + auto buffer = std::make_shared(bytes, remote_contexts, kMinRelocateBytes); + ov::weight_sharing::set_weight_source(shared_weight_ctx, buffer); + auto shared_weight = + std::make_shared(not_shared_weight->get_element_type(), + not_shared_weight->get_shape(), buffer); + ov::weight_sharing::set_constant(shared_weight_ctx, *shared_weight); + shared_weight->set_friendly_name(not_shared_weight->get_friendly_name()); + ov::copy_runtime_info(not_shared_weight, shared_weight); + std::memcpy(buffer->get_ptr(), not_shared_weight->get_data_ptr(), bytes); + + return shared_weight; + +} + +CrossContextTestsWeightSharingNPUW::CrossContextTestsWeightSharingNPUW() = default; +std::pair +CrossContextTestsWeightSharingNPUW::create_shared_compiled_models(std::shared_ptr model, + const ov::SoPtr& remote_context_gpu, + const ov::SoPtr& remote_context_npu) { + if (!m_shared_ctx_ptr) { + m_shared_ctx_ptr = std::make_unique(); + } + // substitute all weights in the model with shared weights + for (const auto& op : model->get_ops()) { + auto c = std::dynamic_pointer_cast(op); + if (!c) { + continue; + } + auto shared_weight = make_shared_weight_from_weight(*m_shared_ctx_ptr, {remote_context_gpu, remote_context_npu}, c); + ov::replace_node(c, shared_weight); + } + ov::CompiledModel gpu_compiled_model = core.compile_model(model, remote_context_gpu->get_device_name()); + ov::CompiledModel npu_compiled_model = core.compile_model(model, remote_context_npu->get_device_name()); + return {gpu_compiled_model, npu_compiled_model}; +} + +TEST_F(CrossContextTestsWeightSharingNPUW, CanMemoryShareBetweenContextsAsWeights) { + ASSERT_TRUE(remote_context_gpu != nullptr); + ASSERT_TRUE(remote_context_npu != nullptr); + size_t iteration_num = 17; + auto [shared_context_input_output, shared_context_output_input] = execute_inference_on_shared_context(gpu_compiled_model, npu_compiled_model, remote_context_gpu, remote_context_npu, model_elem_count, iteration_num); + // NPU and GPU must have the same data pointer as parts of the shared context buffer + // the data must be valid and equal to the expected value after the iterations. + // A value of the shared_context_input_output buffer must be greater than a value of the shared_context_output_input buffer by 1, + // as NPU adds additional 1 to the output which is the input for GPU + for (size_t i = 0; i get_ptr()[i], shared_context_output_input->get_ptr()[i] + 1); + ASSERT_EQ(shared_context_input_output->get_ptr()[i], iteration_num * 2); + ASSERT_EQ(shared_context_output_input->get_ptr()[i], iteration_num * 2 - 1); + } +} diff --git a/src/plugins/intel_npu/tests/functional/behavior/npuw/cross_context_tests.hpp b/src/plugins/intel_npu/tests/functional/behavior/npuw/cross_context_tests.hpp new file mode 100644 index 00000000000000..f627818065e877 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/npuw/cross_context_tests.hpp @@ -0,0 +1,87 @@ +// Copyright (C) 2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include + +#include +#include + +#include "test_engine/models/model_builder.hpp" +#include "test_engine/mocks/mock_plugins.hpp" +#include "test_engine/mocks/register_in_ov.hpp" +#include "intel_npu/npuw_private_properties.hpp" +#include "openvino/runtime/core.hpp" +#include "openvino/runtime/iplugin.hpp" +#include "openvino/runtime/make_tensor.hpp" +#include "openvino/runtime/remote_context.hpp" + +namespace ov { +namespace weight_sharing { + struct Context; +} +} +namespace ov { +namespace npuw { +namespace tests { + +class RemoteContextAccessor : public ov::RemoteContext { +public: + static ov::SoPtr as_so_ptr(const ov::RemoteContext& context) { + RemoteContextAccessor accessor; + accessor.ov::RemoteContext::operator=(context); + return ov::SoPtr(accessor._impl, accessor._so); + } +}; + +class CrossContextTestsNPUW : public ::testing::Test { +public: + ov::Core core; + ov::SoPtr remote_context_gpu; + ov::SoPtr remote_context_npu; + + std::shared_ptr model; + ov::CompiledModel gpu_compiled_model; + ov::CompiledModel npu_compiled_model; + + size_t model_elem_count = 4096; // NPU requires a pagesize aligned buffer for shared memory, so 4096 is a good default size. + void SetUp() override { + try { + remote_context_gpu = RemoteContextAccessor::as_so_ptr(core.get_default_context("GPU")); + remote_context_npu = RemoteContextAccessor::as_so_ptr(core.get_default_context("NPU")); + } catch (const std::exception& e) { + GTEST_SKIP() << "Skipping test due to exception while getting remote contexts: " << e.what(); + } + model = create_shared_context_model(model_elem_count); + std::tie(gpu_compiled_model, npu_compiled_model) = create_shared_compiled_models(model, remote_context_gpu, remote_context_npu); + } +protected: + std::shared_ptr create_shared_context_model(size_t model_elem_count); + virtual std::pair create_shared_compiled_models(std::shared_ptr model, + const ov::SoPtr& remote_context_gpu, + const ov::SoPtr& remote_context_npu); +}; + +class CrossContextTestsWeightSharingNPUW : public CrossContextTestsNPUW { +public: + CrossContextTestsWeightSharingNPUW(); + void SetUp() override { + try { + remote_context_gpu = RemoteContextAccessor::as_so_ptr(core.get_default_context("GPU")); + remote_context_npu = RemoteContextAccessor::as_so_ptr(core.get_default_context("NPU")); + } catch (const std::exception& e) { + GTEST_SKIP() << "Skipping test due to exception while getting remote contexts: " << e.what(); + } + model = create_shared_context_model(model_elem_count); + std::tie(gpu_compiled_model, npu_compiled_model) = create_shared_compiled_models(model, remote_context_gpu, remote_context_npu); + } +private: + std::unique_ptr m_shared_ctx_ptr; + std::pair create_shared_compiled_models(std::shared_ptr model, + const ov::SoPtr& remote_context_gpu, + const ov::SoPtr& remote_context_npu) override; +}; +} // namespace tests +} // namespace npuw +} // namespace ov diff --git a/src/plugins/intel_npu/tests/unit/CMakeLists.txt b/src/plugins/intel_npu/tests/unit/CMakeLists.txt index 659cd90851d594..5b35adc03ded2f 100644 --- a/src/plugins/intel_npu/tests/unit/CMakeLists.txt +++ b/src/plugins/intel_npu/tests/unit/CMakeLists.txt @@ -46,6 +46,7 @@ ov_add_test_target( ${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/npuw/compiled_model.cpp ${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/npuw/gqa_compiled_model.cpp ${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp + ${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/npuw/shared_context_buffer.cpp ${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model_utils.cpp ${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/npuw/llm_infer_request.cpp ${OpenVINO_SOURCE_DIR}/src/plugins/intel_npu/src/plugin/npuw/llm_block_kvcache_strategy.cpp @@ -160,6 +161,7 @@ target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/npuw/subgraph_pipeline.cpp ${CMAKE_CURRENT_SOURCE_DIR}/npuw/subgraph_behavior_infer_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/npuw/preserve_const_matmul_pattern_test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/npuw/shared_context_buffer_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/npuw/pipeline_passes/add_position_ids_param_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/npuw/pipeline_passes/collapse_unqdq_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/npuw/pipeline_passes/conv_to_matmul_test.cpp diff --git a/src/plugins/intel_npu/tests/unit/npuw/shared_context_buffer_test.cpp b/src/plugins/intel_npu/tests/unit/npuw/shared_context_buffer_test.cpp new file mode 100644 index 00000000000000..a19b8bac3a9051 --- /dev/null +++ b/src/plugins/intel_npu/tests/unit/npuw/shared_context_buffer_test.cpp @@ -0,0 +1,163 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "shared_context_buffer.hpp" + +#include + +#include +#include +#include + +#include "openvino/core/any.hpp" +#include "openvino/core/shape.hpp" +#include "openvino/core/type/element_type.hpp" +#include "openvino/runtime/iremote_context.hpp" +#include "openvino/runtime/iremote_tensor.hpp" +#include "openvino/runtime/so_ptr.hpp" + +namespace stubs { + +class RemoteTensor : public ov::IRemoteTensor { +public: + explicit RemoteTensor(std::string device_name) : m_device_name(std::move(device_name)) {} + + const ov::AnyMap& get_properties() const override { + return m_properties; + } + + const std::string& get_device_name() const override { + return m_device_name; + } + + void set_shape(ov::Shape) override { + OPENVINO_NOT_IMPLEMENTED; + } + + const ov::element::Type& get_element_type() const override { + OPENVINO_NOT_IMPLEMENTED; + } + + const ov::Shape& get_shape() const override { + OPENVINO_NOT_IMPLEMENTED; + } + + const ov::Strides& get_strides() const override { + OPENVINO_NOT_IMPLEMENTED; + } + +private: + std::string m_device_name; + ov::AnyMap m_properties; +}; + +class RemoteContext : public ov::IRemoteContext { +public: + explicit RemoteContext(std::string device_name) : m_device_name(std::move(device_name)) {} + + const std::string& get_device_name() const override { + return m_device_name; + } + + const ov::AnyMap& get_property() const override { + return m_properties; + } + + ov::SoPtr create_tensor(const ov::element::Type&, + const ov::Shape&, + const ov::AnyMap&) override { + return {std::make_shared(m_device_name), nullptr}; + } + +private: + std::string m_device_name; + ov::AnyMap m_properties; +}; + +} // namespace stubs + +namespace { + +std::vector> get_required_contexts(const std::vector& required_devices) { + std::vector> contexts; + contexts.reserve(required_devices.size()); + for (const auto& device_prefix : required_devices) { + contexts.push_back({std::make_shared(device_prefix), nullptr}); + } + return contexts; +} + +TEST(SharedContextBufferTests, ProvidesRemoteTensorsForGpuAndNpuContexts) { + const size_t byte_size = 4096; + const size_t alignment = 4096; + + const std::vector required_devices = {"GPU", "NPU"}; + auto contexts = get_required_contexts(required_devices); + if (contexts.size() != required_devices.size()) { + GTEST_SKIP() << "Required contexts are unavailable. Requested: " << required_devices.size() + << ", created: " << contexts.size(); + } + + const auto& gpu_ctx = contexts[0]; + const auto& npu_ctx = contexts[1]; + std::vector> remote_contexts = {gpu_ctx, npu_ctx}; + + auto buffer = std::make_shared(byte_size, std::move(remote_contexts), alignment); + ASSERT_NE(buffer, nullptr); + + auto gpu_tensor = buffer->get_remote_tensors_if_exist(gpu_ctx); + auto npu_tensor = buffer->get_remote_tensors_if_exist(npu_ctx); + + ASSERT_NE(gpu_tensor, nullptr); + ASSERT_NE(npu_tensor, nullptr); + EXPECT_EQ(gpu_tensor->get_device_name().rfind("GPU", 0), 0u); + EXPECT_EQ(npu_tensor->get_device_name().rfind("NPU", 0), 0u); +} + +TEST(SharedContextBufferTests, ProvidesUsableDescriptorWithRemoteContexts) { + const size_t byte_size = 8192; + const size_t alignment = 4096; + + const std::vector required_devices = {"GPU", "NPU"}; + auto contexts = get_required_contexts(required_devices); + if (contexts.size() != required_devices.size()) { + GTEST_SKIP() << "Required contexts are unavailable. Requested: " << required_devices.size() + << ", created: " << contexts.size(); + } + + const auto& gpu_ctx = contexts[0]; + const auto& npu_ctx = contexts[1]; + std::vector> remote_contexts = {gpu_ctx, npu_ctx}; + + auto buffer = std::make_shared(byte_size, std::move(remote_contexts), alignment); + auto descriptor = buffer->get_descriptor(); + + ASSERT_NE(descriptor, nullptr); + auto shared_descriptor = std::dynamic_pointer_cast(descriptor); + ASSERT_NE(shared_descriptor, nullptr); + + EXPECT_GT(shared_descriptor->get_id(), 0u); + EXPECT_EQ(shared_descriptor->get_offset(), 0u); + + auto source_buffer = shared_descriptor->get_source_buffer(); + ASSERT_NE(source_buffer, nullptr); + EXPECT_EQ(source_buffer->get_ptr(), buffer->get_ptr()); + EXPECT_EQ(source_buffer->size(), buffer->size()); + + const auto& remote_context_map = shared_descriptor->get_remote_contexts(); + ASSERT_EQ(remote_context_map.size(), 2u); + + const auto gpu_it = remote_context_map.find(gpu_ctx); + const auto npu_it = remote_context_map.find(npu_ctx); + ASSERT_NE(gpu_it, remote_context_map.end()); + ASSERT_NE(npu_it, remote_context_map.end()); + ASSERT_NE(gpu_it->second, nullptr); + ASSERT_NE(npu_it->second, nullptr); + EXPECT_EQ(gpu_it->second->get_device_name().rfind("GPU", 0), 0u); + EXPECT_EQ(npu_it->second->get_device_name().rfind("NPU", 0), 0u); +} + +} // namespace + +