remove Runtimecontent from DynamicArguments and split memref and ovshape from infer and dynmaic

DanLiu2Intel · DanLiu2Intel · commit 2be36d38d1b2 · 2026-07-14T14:07:00.000+08:00
diff --git a/src/plugins/intel_npu/src/backend/include/zero_dynamic_infer_request.hpp b/src/plugins/intel_npu/src/backend/include/zero_dynamic_infer_request.hpp
@@ -32,16 +32,16 @@ class ZeroDynamicInferRequest final : public ZeroInferRequest {
                                       const std::vector<ov::SoPtr<ov::ITensor>>& tensors,
                                       const std::optional<size_t>& batchSize = std::nullopt) override;
 
-    void predict_output_shapes(std::vector<MemRefType>& outputMemRef);
-    void check_tensor_and_predicted_shapes(const std::vector<MemRefType>& outputMemRef);
+    void predict_output_shapes(std::vector<ov::Shape>& predictedShapes);
+    void check_tensor_and_predicted_shapes(const std::vector<ov::Shape>& predictedShapes);
 
-    void update_tensor(const std::vector<MemRefType>& outputMemRef);
+    void update_tensor(const std::vector<ov::Shape>& predictedShapes);
 
     bool _isTensorChanged = false;
 
 private:
-    // Store the vm context before predicted output shapes
-    std::shared_ptr<DynamicArguments> _arguments;
+    // VM execution context shared with the pipeline; created lazily, reused across inferences.
+    std::shared_ptr<VMExecutionContext> _executionContext = std::make_shared<VMExecutionContext>();
 };
 
 }  //  namespace intel_npu
diff --git a/src/plugins/intel_npu/src/backend/include/zero_dynamic_pipeline.hpp b/src/plugins/intel_npu/src/backend/include/zero_dynamic_pipeline.hpp
@@ -10,26 +10,36 @@
 
 namespace intel_npu {
 
+// VM runtime execution context owner. The handle is created lazily and shared between
+// shape prediction (infer request) and execution (pipeline) so the same context is reused.
+struct VMExecutionContext {
+    npu_vm_runtime_execution_context_handle_t _handle = nullptr;
+
+    VMExecutionContext() = default;
+    VMExecutionContext(const VMExecutionContext&) = delete;
+    VMExecutionContext& operator=(const VMExecutionContext&) = delete;
+    VMExecutionContext(VMExecutionContext&&) = delete;
+    VMExecutionContext& operator=(VMExecutionContext&&) = delete;
+    ~VMExecutionContext();
+
+    // Create the context for vmRuntime if not created yet; returns the handle.
+    npu_vm_runtime_execution_context_handle_t ensure(npu_vm_runtime_handle_t vmRuntime);
+};
+
 struct DynamicArguments {
     std::vector<MemRefType> _inputsMemRef;
     std::vector<MemRefType> _outputsMemRef;
 
-    // Save the memref handle to a vector for VM execution and prediction to extend its lifetime.
-    std::vector<npu_vm_runtime_mem_ref_handle_t> _inputMemRefHandles;
-    std::vector<npu_vm_runtime_mem_ref_handle_t> _outputMemRefHandles;
-
-    // Share runtime_execution_context in param during VM execution and forecasting
-    npu_vm_runtime_execute_params_t _executeParams = {};
+    // True once the command lists have been recorded by a first npuVMRuntimeExecute call. Subsequent
+    // executions can be replayed without re-recording when no tensor changed (see execute_vm_runtime).
+    bool _commandListsRecorded = false;
 
     DynamicArguments() = default;
     DynamicArguments(const DynamicArguments&) = delete;
     DynamicArguments& operator=(const DynamicArguments&) = delete;
     DynamicArguments(DynamicArguments&&) = delete;
     DynamicArguments& operator=(DynamicArguments&&) = delete;
-    ~DynamicArguments();
-
-    // Create the VM execution context for vmRuntime. No-op if already created.
-    void ensureExecutionContext(npu_vm_runtime_handle_t vmRuntime);
+    ~DynamicArguments() = default;
 
     void setArgumentProperties(uint32_t argi,
                                const void* argv,
@@ -45,9 +55,7 @@ class DynamicPipeline final : public IPipeline {
         // Store command list handles to pass it to ExecutionEngine
         std::vector<ze_command_list_handle_t> _commandListHandles;
 
-        PipelinedCommandLists(size_t numCommandLists,
-                              const std::shared_ptr<ZeroInitStructsHolder>& init_structs,
-                              std::shared_ptr<DynamicArguments> requestArguments) {
+        PipelinedCommandLists(size_t numCommandLists, const std::shared_ptr<ZeroInitStructsHolder>& init_structs) {
             _commandLists.reserve(numCommandLists);
             for (size_t i = 0; i < numCommandLists; i++) {
                 _commandLists.emplace_back(std::make_unique<CommandList>(init_structs));
@@ -57,11 +65,7 @@ class DynamicPipeline final : public IPipeline {
                 _commandListHandles.push_back(_commandLists[i]->handle());
             }
 
-            if (requestArguments != nullptr) {
-                _arguments = requestArguments;
-            } else {
-                _arguments = std::make_shared<DynamicArguments>();
-            }
+            _arguments = std::make_shared<DynamicArguments>();
         }
 
         size_t size() const {
@@ -137,7 +141,7 @@ class DynamicPipeline final : public IPipeline {
                     const Config& config,
                     const std::vector<std::vector<std::shared_ptr<ZeroTensor>>>& input_tensors,
                     const std::vector<std::shared_ptr<ZeroTensor>>& output_tensors,
-                    std::shared_ptr<DynamicArguments> requestArguments,
+                    std::shared_ptr<VMExecutionContext> executionContext,
                     size_t batch_size = 1);
 
     DynamicPipeline(const DynamicPipeline&) = delete;
@@ -155,12 +159,13 @@ class DynamicPipeline final : public IPipeline {
                                 size_t batch_index,
                                 const std::shared_ptr<ov::ITensor>& userTensor = nullptr) override;
 
-    // Predicts VM runtime output shape. Independent of pipeline instance state, this depends only on the VM runtime
+    // Predicts VM runtime output shapes for the given input/output tensors. A nullptr tensor entry falls
+    // back to the graph metadata max shape. Independent of pipeline instance state, this depends only on the VM runtime
     // handle and argument-provided context, making it a static method.
-    static void predict_output_shape(const IGraph& graph,
-                                     DynamicArguments& args,
-                                     std::vector<MemRefType>& inputsMemRef,
-                                     std::vector<MemRefType>& outputsMemRef);
+    static std::vector<ov::Shape> predict_output_shapes(const IGraph& graph,
+                                                        VMExecutionContext& executionContext,
+                                                        const std::vector<std::shared_ptr<ov::ITensor>>& inputTensors,
+                                                        const std::vector<std::shared_ptr<ov::ITensor>>& outputTensors);
 
 private:
     void execute_vm_runtime(npu_vm_runtime_handle_t vmRuntime,
@@ -170,6 +175,8 @@ class DynamicPipeline final : public IPipeline {
                             ze_fence_handle_t fence,
                             ze_event_handle_t event);
 
+    // Shared VM execution context (owned by the infer request), reused across all command lists.
+    std::shared_ptr<VMExecutionContext> _executionContext;
     std::vector<std::unique_ptr<PipelinedCommandLists>> _command_lists;
 };
 
diff --git a/src/plugins/intel_npu/src/backend/src/zero_dynamic_infer_request.cpp b/src/plugins/intel_npu/src/backend/src/zero_dynamic_infer_request.cpp
@@ -28,7 +28,7 @@ void ZeroDynamicInferRequest::create_pipeline_impl() {
                                           _config,
                                           _levelZeroInputTensors,
                                           _levelZeroOutputTensors,
-                                          _arguments,
+                                          _executionContext,
                                           batchSize.has_value() ? batchSize.value() : utils::DEFAULT_BATCH_SIZE);
 
     _logger.debug("create_pipeline_impl - completed");
@@ -203,102 +203,61 @@ void ZeroDynamicInferRequest::infer_async() {
     _logger.debug("infer_async - started");
     OV_ITT_TASK_CHAIN(ZERO_INFER, itt::domains::LevelZeroBackend, "infer_async", "start");
     // Store the predicted output shapes
-    std::vector<MemRefType> outputsMemRef;
-    predict_output_shapes(outputsMemRef);
-    check_tensor_and_predicted_shapes(outputsMemRef);
+    std::vector<ov::Shape> predictedShapes;
+    predict_output_shapes(predictedShapes);
+    check_tensor_and_predicted_shapes(predictedShapes);
     prepare_inputs();
     prepare_outputs();
-    update_tensor(outputsMemRef);
+    update_tensor(predictedShapes);
 
     OV_ITT_TASK_NEXT(ZERO_INFER, "push");
     _pipeline->push();
 }
 
-void ZeroDynamicInferRequest::predict_output_shapes(std::vector<MemRefType>& outputsMemRef) {
+void ZeroDynamicInferRequest::predict_output_shapes(std::vector<ov::Shape>& predictedShapes) {
     // TODO: If current output tensor is not large enough to be compatible with input tensor, need recreate pipeline
     // But reshape ZeroTensor can be used to avoid recreate pipeline now
-    // bool reCreatePipeline = false;
-    // Predict output shapes based on current inputs
 
-    if (_arguments == nullptr) {
-        _arguments = std::make_shared<DynamicArguments>();
-    }
+    // Predict output shapes based on current inputs. The infer request only deals with tensors/OV shapes;
+    // MemRef packing is done inside the pipeline layer.
+    predictedShapes.clear();
 
     if (_graph->get_handle() != nullptr && _isTensorChanged) {
-        std::vector<MemRefType> inputsMemRef(_metadata.inputs.size());
-        outputsMemRef.clear();
-        outputsMemRef.resize(_metadata.outputs.size());
+        std::vector<std::shared_ptr<ov::ITensor>> inputTensors(_metadata.inputs.size());
+        std::vector<std::shared_ptr<ov::ITensor>> outputTensors(_metadata.outputs.size());
 
         // TODO: Support Batch later
         // Update input Info
-        // TENTATIVE CODE TO ALLOCATE a memref handle
-        for (size_t i = 0; i < inputsMemRef.size(); ++i) {
-            auto& levelZeroTensor = get_level_zero_input(i);
-            auto& userTensor = get_user_input(i);
+        // A null entry lets the pipeline fall back to the graph metadata max shape.
+        for (size_t i = 0; i < inputTensors.size(); ++i) {
+            const auto& userTensor = get_user_input(i);
             if (userTensor != nullptr) {
-                // If userTensor is set, use userTensor to update memref handle
-                const auto userTensorPtr = userTensor._ptr;
-                OPENVINO_ASSERT(userTensorPtr != nullptr, "Input user tensor pointer is null");
-                inputsMemRef[i].set(get_tensor_data_ptr(userTensorPtr), 0, userTensorPtr);
-            } else if (levelZeroTensor != nullptr) {
-                // If userTensor is not set, use levelZeroTensor to update memref handle
-                inputsMemRef[i].set(get_tensor_data_ptr(levelZeroTensor), 0, levelZeroTensor);
+                // If userTensor is set, use userTensor to update memref handle in prediction
+                inputTensors[i] = userTensor._ptr;
             } else {
-                // If all tensors are not set, use metadata
-                inputsMemRef[i].setArg(nullptr);
-                inputsMemRef[i]._offset = 0;
-                // TODO : BatchSize not checked here
-                inputsMemRef[i].setSize(_metadata.inputs.at(i).shapeFromCompiler.get_max_shape());
-                inputsMemRef[i].updateStride();
+                // If userTensor is not set, use levelZeroTensor
+                inputTensors[i] = get_level_zero_input(i);
             }
         }
-
         // Update output Info
-        for (size_t i = 0; i < outputsMemRef.size(); ++i) {
-            auto& levelZeroTensor = _levelZeroOutputTensors.at(i);
-            auto& userTensor = _userOutputTensors.at(i);
+        for (size_t i = 0; i < outputTensors.size(); ++i) {
+            const auto& userTensor = _userOutputTensors.at(i);
             if (userTensor != nullptr) {
-                // If userTensor is set, use userTensor to update memref handle
-                const auto userTensorPtr = userTensor._ptr;
-                OPENVINO_ASSERT(userTensorPtr != nullptr, "Output user tensor pointer is null");
-                outputsMemRef[i].set(get_tensor_data_ptr(userTensorPtr), 0, userTensorPtr);
-            } else if (levelZeroTensor != nullptr) {
-                // If userTensor is not set, use levelZeroTensor to update memref handle
-                outputsMemRef[i].set(get_tensor_data_ptr(levelZeroTensor), 0, levelZeroTensor);
+                // If userTensor is set, use userTensor to update memref handle in prediction
+                outputTensors[i] = userTensor._ptr;
             } else {
-                // If all tensors are not set, use metadata
-                outputsMemRef[i].setArg(nullptr);
-                outputsMemRef[i]._offset = 0;
-                // TODO : BatchSize not checked here
-                outputsMemRef[i].setSize(_metadata.outputs.at(i).shapeFromCompiler.get_max_shape());
-                outputsMemRef[i].updateStride();
+                // If userTensor is not set, use levelZeroTensor
+                outputTensors[i] = _levelZeroOutputTensors.at(i);
             }
         }
 
-        std::vector<MemRefType> originalOutputMemRef;
-        originalOutputMemRef.resize(outputsMemRef.size());
-
-        for (size_t i = 0; i < outputsMemRef.size(); ++i) {
-            originalOutputMemRef[i]._dimsCount = outputsMemRef[i]._dimsCount;
-            originalOutputMemRef[i]._sizes = outputsMemRef[i]._sizes;
-            originalOutputMemRef[i]._strides = outputsMemRef[i]._strides;
-        }
-
-        // Get VM context before invoking VM shape prediction.
-        DynamicArguments& dynamicArguments = *_arguments;
-        DynamicPipeline::predict_output_shape(*_graph, dynamicArguments, inputsMemRef, outputsMemRef);
-
-        for (size_t i = 0; i < outputsMemRef.size(); i++) {
-            if (!originalOutputMemRef[i].compare(outputsMemRef[i])) {
-                _logger.debug("predict_shapes - output shape change detected");
-                break;
-            }
-        }
+        predictedShapes =
+            DynamicPipeline::predict_output_shapes(*_graph, *_executionContext, inputTensors, outputTensors);
     }
 }
 
-void ZeroDynamicInferRequest::check_tensor_and_predicted_shapes(const std::vector<MemRefType>& outputsMemRef) {
-    if (outputsMemRef.empty()) {
+void ZeroDynamicInferRequest::check_tensor_and_predicted_shapes(const std::vector<ov::Shape>& predictedShapes) {
+    if (predictedShapes.empty()) {
         _logger.debug("check_tensor_and_predicted_shapes - no output props to check, skip check");
         return;
     }
@@ -316,10 +275,7 @@ void ZeroDynamicInferRequest::check_tensor_and_predicted_shapes(const std::vecto
             continue;
         }
 
-        ov::Shape predictedShape;
-        for (int64_t j = 0; j < outputsMemRef[i]._dimsCount; j++) {
-            predictedShape.push_back(outputsMemRef[i]._sizes[j]);
-        }
+        const ov::Shape& predictedShape = predictedShapes[i];
         if (userTensor != nullptr) {
             // User set output tensor, need check size and throw exception if not large enough
             if (shape_size(userTensor->get_shape()) < shape_size(predictedShape)) {
@@ -345,19 +301,16 @@ void ZeroDynamicInferRequest::check_tensor_and_predicted_shapes(const std::vecto
     }
 }
 
-void ZeroDynamicInferRequest::update_tensor(const std::vector<MemRefType>& outputsMemRef) {
+void ZeroDynamicInferRequest::update_tensor(const std::vector<ov::Shape>& predictedShapes) {
     // Update local level zero buffer shape with predicted shape to prepare for comparasion
-    if (outputsMemRef.size() > 0 && _isTensorChanged) {
+    if (predictedShapes.size() > 0 && _isTensorChanged) {
         for (size_t i = 0; i < _levelZeroOutputTensors.size(); i++) {
             auto& levelZeroTensor = _levelZeroOutputTensors.at(i);
             if (levelZeroTensor == nullptr) {
                 // Do not need to update user output tensor
                 continue;
             }
-            ov::Shape predictedShape;
-            for (int64_t j = 0; j < outputsMemRef[i]._dimsCount; j++) {
-                predictedShape.push_back(outputsMemRef[i]._sizes[j]);
-            }
+            const ov::Shape& predictedShape = predictedShapes[i];
             if (levelZeroTensor->get_shape() != predictedShape) {
                 _logger.info("update_tensor - reshape output tensor %d from %s to predicted shape %s",
                              i,
diff --git a/src/plugins/intel_npu/src/backend/src/zero_dynamic_pipeline.cpp b/src/plugins/intel_npu/src/backend/src/zero_dynamic_pipeline.cpp
