issue/343 fix batching

Author: PanZezhong1725

csrc/models/minicpmv/minicpmv_model.cpp

@@ -39,18 +39,15 @@ MiniCPMVModel::MiniCPMVModel(std::shared_ptr<infinilm::config::ModelConfig> mode
                               device);
 }
 
-infinicore::Tensor MiniCPMVModel::replace_embeddings(const infinicore::Tensor &inputs_embeds,
-                                                     const infinicore::Tensor &vision_hidden,
-                                                     const infinicore::Tensor &image_bound) const {
-    auto out = infinicore::Tensor::empty(inputs_embeds->shape(), inputs_embeds->dtype(), inputs_embeds->device());
-    out->copy_from(inputs_embeds);
-
+void MiniCPMVModel::replace_embeddings(infinicore::Tensor inputs_embeds,
+                                       const infinicore::Tensor &vision_hidden,
+                                       const infinicore::Tensor &image_bound) const {
     auto bounds_cpu = image_bound->to(infinicore::Device::cpu());
     auto batch_size = inputs_embeds->size(0);
 
     ASSERT_EQ(batch_size, 1);
     ASSERT_EQ(bounds_cpu->size(0), 1);
-    auto out_slice = out->squeeze(0);
+    auto out_slice = inputs_embeds->squeeze(0);
     auto bound_slice = bounds_cpu->squeeze(0);
     auto vision_len = vision_hidden->size(0);
     for (size_t patch = 0; patch < vision_len; ++patch) {
@@ -62,8 +59,6 @@ infinicore::Tensor MiniCPMVModel::replace_embeddings(const infinicore::Tensor &i
 
         out_slice->narrow({{0, size_t(start), size_t(end - start)}})->copy_from(patch_embed);
     }
-
-    return out;
 }
 
 InfinilmModel::Output MiniCPMVModel::forward(const InfinilmModel::Input &input) const {
@@ -90,7 +85,7 @@ InfinilmModel::Output MiniCPMVModel::forward(const InfinilmModel::Input &input)
             auto pixel_values = input.pixel_values.value().at(i);
             auto vision_embedding = vpm_->forward(pixel_values, input.tgt_sizes.value().at(i));
             auto vision_hidden = resampler_->forward(vision_embedding, input.tgt_sizes.value().at(i));
-            inputs_embeds = replace_embeddings(inputs_embeds->narrow({{1, size_t(offsets[i]), size_t(offsets[i + 1] - offsets[i])}}), vision_hidden, input.image_bound.value().at(i));
+            replace_embeddings(inputs_embeds->narrow({{1, size_t(offsets[i]), size_t(offsets[i + 1] - offsets[i])}}), vision_hidden, input.image_bound.value().at(i));
         }
 
         auto hidden_states = llm_->model().forward_embeds(

csrc/models/minicpmv/minicpmv_model.hpp

@@ -23,9 +23,9 @@ class MiniCPMVModel : public InfinilmModel {
     void reset_cache(const cache::CacheConfig *cache_config) override;
 
 private:
-    infinicore::Tensor replace_embeddings(const infinicore::Tensor &inputs_embeds,
-                                          const infinicore::Tensor &vision_hidden,
-                                          const infinicore::Tensor &image_bound) const;
+    void replace_embeddings(infinicore::Tensor inputs_embeds,
+                            const infinicore::Tensor &vision_hidden,
+                            const infinicore::Tensor &image_bound) const;
 
     std::shared_ptr<infinilm::config::ModelConfig> config_;
 

csrc/pybind11/engine/engine.hpp

@@ -72,7 +72,11 @@ inline void bind_infer_engine(py::module &m) {
         })
         .def("process_weights_after_loading", &InferEngine::process_weights_after_loading, "Process the weights after loading on all workers (e.g., for quantization)")
         .def(
-            "forward", [](InferEngine &self, const InferEngine::Input &input) -> InferEngine::Output { return self.forward(input); }, "Run inference on all ranks with arbitrary arguments")
+            "forward", [](InferEngine &self, const InferEngine::Input &input) -> InferEngine::Output {
+                py::gil_scoped_release release;
+                return self.forward(input);
+            },
+            "Run inference on all ranks with arbitrary arguments")
         .def(
             "reset_cache", [](InferEngine &self, std::shared_ptr<cache::CacheConfig> cfg) { self.reset_cache(cfg ? cfg.get() : nullptr); }, py::arg("cache_config") = py::none())
         .def("get_cache_config", [](const InferEngine &self) -> std::shared_ptr<cache::CacheConfig> {

python/infinilm/processors/minicpmv_processor.py

@@ -155,10 +155,6 @@ def build_model_inputs(
                 ):
                     import torch
 
-                    assert len(scheduler_output.scheduled_requests) == 1, (
-                        "Batching is not supported for image inputs yet"
-                    )
-
                     num_cached_patch = (
                         (req.processed_inputs["image_bound"][0][:, 1] <= num_cached)
                         .sum()

test/service/request.py

@@ -39,10 +39,10 @@ def get_args():
     )
 
     parser.add_argument(
-        "--model-name",
+        "--model",
         type=str,
         default="default",
-        help="Name of the model being served, needed by vllm",
+        help="Name or path of the model being served, needed by vllm",
     )
 
     return parser.parse_args()