ImageManip GPU backend (#1812)

* RVC4 FW: Use DmaHeapBuffer for resizers

* Add GPU backend to ImageManip

* WIP: refactor imagemanip

* WIP: Simplified image manip Warp impl

* Refactored ImageManip

* Bugfixes

* Add cast method to ConvertedOffsetMemory

* Fix ConvertedOffsetMemory

* RVC4 FW: Implement GPU backend for ImageManip

* Set manip examples to use the GPU backend

* PR fixes

* clangformat

* Add image manip GPU backend tests

* PR fixes

* Split offset memory from memory

* Fix examples to only select GPU if available

* Example fix

* Add comments

* Fix build

* Clangformat

* Update rvc4 fw

* Guard ConvertedOffsetMemory from nullptr

* RVC4 FW: Merge develop

Author: asahtik

bindings/python/src/pipeline/node/ImageManipBindings.cpp

@@ -29,7 +29,7 @@ void bind_imagemanip(pybind11::module& m, void* pCallstack) {
         .value("LOW_POWER", ImageManip::PerformanceMode::LOW_POWER);
 
     py::enum_<ImageManip::Backend> backend(imageManip, "Backend");
-    backend.value("HW", ImageManip::Backend::HW).value("CPU", ImageManip::Backend::CPU);
+    backend.value("HW", ImageManip::Backend::HW).value("CPU", ImageManip::Backend::CPU).value("GPU", ImageManip::Backend::GPU);
 
     // ImageManip Node
     imageManip.def_readonly("inputConfig", &ImageManip::inputConfig, DOC(dai, node, ImageManip, inputConfig))

cmake/Depthai/DepthaiDeviceRVC4Config.cmake

@@ -3,4 +3,4 @@
 set(DEPTHAI_DEVICE_RVC4_MATURITY "snapshot")
 
 # "version if applicable"
-set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+6e02a76617ed920eee465c4702fa07f06b6c191e")
+set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+ffaaa351f971c291311f8ab5d8d0753f327799b3")

examples/cpp/ImageManip/image_manip_remap.cpp

@@ -49,16 +49,20 @@ int main() {
         auto manip1 = pipeline.create<dai::node::ImageManip>();
         auto manip2 = pipeline.create<dai::node::ImageManip>();
 
+        // GPU is not available on RVC2 and some RVC4 devices
+        auto backend = pipeline.getDefaultDevice()->hasGPU() ? dai::node::ImageManip::Backend::GPU : dai::node::ImageManip::Backend::CPU;
+        manip1->setBackend(backend);
+        manip2->setBackend(backend);
+
         // Configure camera output
-        auto camOut = cam->requestOutput(std::make_pair(640, 400), dai::ImgFrame::Type::BGR888i, dai::ImgResizeMode::LETTERBOX, 20, 20);
+        auto* camOut = cam->requestOutput(std::make_pair(640, 400), dai::ImgFrame::Type::BGR888i, dai::ImgResizeMode::LETTERBOX, 20, 20);
 
         // Configure image manipulators
         manip1->initialConfig->addRotateDeg(90);
         manip1->initialConfig->setOutputSize(200, 320);
 
         manip2->initialConfig->addRotateDeg(90);
         manip2->initialConfig->setOutputSize(320, 200);
-        manip2->setRunOnHost(true);
 
         // Linking
         camOut->link(manip1->inputImage);
@@ -123,4 +127,4 @@ int main() {
     }
 
     return 0;
-}
+}

examples/cpp/ImageManip/image_manip_resize.cpp

@@ -27,6 +27,9 @@ int main(int argc, char** argv) {
     auto camRgb = pipeline.create<dai::node::Camera>()->build(dai::CameraBoardSocket::CAM_A);
     auto manip = pipeline.create<dai::node::ImageManip>();
 
+    // GPU is not available on RVC2 and some RVC4 devices
+    manip->setBackend(device->hasGPU() ? dai::node::ImageManip::Backend::GPU : dai::node::ImageManip::Backend::CPU);
+
     // Resize to 400x400 and avoid stretching by cropping from the center
     manip->initialConfig->setOutputSize(400, 400, dai::ImageManipConfig::ResizeMode::CENTER_CROP);
     // Set output frame type

examples/python/ImageManip/image_manip_remap.py

@@ -30,6 +30,11 @@ def draw_rotated_rectangle(frame, center, size, angle, color, thickness=2):
     manip1 = pipeline.create(dai.node.ImageManip)
     manip2 = pipeline.create(dai.node.ImageManip)
 
+    # GPU is not available on RVC2 and some RVC4 devices
+    backend = dai.node.ImageManip.Backend.GPU if pipeline.getDefaultDevice().hasGPU() else dai.node.ImageManip.Backend.CPU
+    manip1.setBackend(backend)
+    manip2.setBackend(backend)
+
     camOut.link(manip1.inputImage)
     manip1.out.link(manip2.inputImage)
 
@@ -38,7 +43,6 @@ def draw_rotated_rectangle(frame, center, size, angle, color, thickness=2):
 
     manip2.initialConfig.addRotateDeg(90)
     manip2.initialConfig.setOutputSize(320, 200)
-    manip2.setRunOnHost(True)
 
     outQcam = camOut.createOutputQueue()
     outQ1 = manip1.out.createOutputQueue()

examples/python/ImageManip/image_manip_resize.py

@@ -2,10 +2,15 @@
 import cv2
 
 pipeline = dai.Pipeline()
+device = pipeline.getDefaultDevice()
 
 camRgb = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A)
 manip = pipeline.create(dai.node.ImageManip)
 
+# GPU is not available on RVC2 and some RVC4 devices
+manip.setBackend(dai.node.ImageManip.Backend.GPU if device.hasGPU() else
+                 dai.node.ImageManip.Backend.CPU)
+
 manip.initialConfig.setOutputSize(300, 300, dai.ImageManipConfig.ResizeMode.STRETCH)
 
 camOut = camRgb.requestOutput((1920, 1080))

include/depthai/properties/ImageManipProperties.hpp

@@ -15,7 +15,7 @@ struct ImageManipProperties : PropertiesSerializable<Properties, ImageManipPrope
      * This can cause some unexpected behavior when using multiple ImageManip nodes in series.
      * Currently, the only operation affected is downscaling.
      */
-    enum class Backend : uint8_t { CPU, HW };
+    enum class Backend : uint8_t { CPU, HW, GPU };
     /**
      * Set performance mode for ImageManip with a tradeoff between performance and power consumption. Only applied on RVC4.
      * This only affects scaling NV12 and GRAY images.