Feat: Add abstract TransformableBuffer class (#1778)

* Add transformable buffer and use it in ImgDetections

* make TransformableBuffer abstratract only and add placeholder in SpatialImgDetections.

* Stop out of line virtual definition of Transformalble Buffer.

* Add keypoint TODO

* Add ImgDetections bindings and test case for transformTo

* Adjust test

* Implement transformTo in SpatialImgDetections

* Add transformTo to Tracklets

* Implement transformTo in AprilTags

* PointCloudData now inherits from TransformableBuffer

* SegmentationMask inherits from TransformableBuffer

* Update docstrings

* Add clone overload for each message

* Enumerate TransformableBuffer and adjust bindings and includes

* override TransformableBuffer::getDatatype

* bump RVC4 FW

* Expose ImgTransformations to AprilTags and properly roundtrip TransformableBuffer datatypes.

* Fix import

* Fix includes

* Bump RVC2 fw

* Fix clangformat

* Fix MSVC compilation error

* clangformat

* Fix MSVC

* Protobuf field renumbering breaks wire compatibility.

* Implement PR suggestions.

* Bump FW

* Improve error messages for "image like" TransformableBuffer classes

* Fix typo in comment

* Pivot to Multi inheritance of Buffer and Transformable + add pybind overloads for transformTo

* revert ImageAlign addition

* Serialize transformable buffer and improve tests

* Update bindings and docstrings

* Update SegmentationMask docstrings and bindings

* SegmentationMask missing bindings

* Some Pr suggestions

* Clangformat & RVC2 build issues

* Bump FW

* clangformat

* Update transformableBuffer docstrings and add throw statement

* Implement suggestions

* Bump Fw

* MSVC build issues

* Bump fw

---------

Co-authored-by: Matevz Morato <matevz.morato@gmail.com>

Author: aljazkonec1

CMakeLists.txt

@@ -344,6 +344,7 @@ set(TARGET_CORE_SOURCES
     src/pipeline/node/NeuralAssistedStereo.cpp
     src/pipeline/node/Rectification.cpp
     src/pipeline/datatype/Buffer.cpp
+    src/pipeline/datatype/Transformable.cpp
     src/pipeline/datatype/ImgFrame.cpp
     src/pipeline/datatype/ImgTransformations.cpp
     src/pipeline/datatype/Extrinsics.cpp

bindings/python/CMakeLists.txt

@@ -119,6 +119,7 @@ set(SOURCE_LIST
     src/pipeline/datatype/AprilTagConfigBindings.cpp
     src/pipeline/datatype/AprilTagsBindings.cpp
     src/pipeline/datatype/BufferBindings.cpp
+    src/pipeline/datatype/TransformableBindings.cpp
     src/pipeline/datatype/CameraControlBindings.cpp
     src/pipeline/datatype/EdgeDetectorConfigBindings.cpp
     src/pipeline/datatype/FeatureTrackerConfigBindings.cpp

bindings/python/src/DatatypeBindings.cpp

@@ -7,6 +7,7 @@ void bind_adatatype(pybind11::module& m, void* pCallstack);
 void bind_apriltagconfig(pybind11::module& m, void* pCallstack);
 void bind_apriltags(pybind11::module& m, void* pCallstack);
 void bind_buffer(pybind11::module& m, void* pCallstack);
+void bind_transformable(pybind11::module& m, void* pCallstack);
 void bind_cameracontrol(pybind11::module& m, void* pCallstack);
 void bind_edgedetectorconfig(pybind11::module& m, void* pCallstack);
 void bind_featuretrackerconfig(pybind11::module& m, void* pCallstack);
@@ -57,6 +58,7 @@ void DatatypeBindings::addToCallstack(std::deque<StackFunction>& callstack) {
     // Bind all datatypes (order matters)
     callstack.push_front(bind_adatatype);
     callstack.push_front(bind_buffer);
+    callstack.push_front(bind_transformable);
     callstack.push_front(bind_apriltagconfig);
     callstack.push_front(bind_apriltags);
     callstack.push_front(bind_cameracontrol);
@@ -125,6 +127,7 @@ void DatatypeBindings::bind(pybind11::module& m, void* pCallstack) {
 
     datatypeEnum.value("ADatatype", DatatypeEnum::ADatatype)
         .value("Buffer", DatatypeEnum::Buffer)
+        .value("Transformable", DatatypeEnum::Transformable)
         .value("ImgFrame", DatatypeEnum::ImgFrame)
         .value("EncodedFrame", DatatypeEnum::EncodedFrame)
         .value("NNData", DatatypeEnum::NNData)

bindings/python/src/DatatypeBindings.hpp

@@ -1,8 +1,11 @@
 #pragma once
 
 // pybind
+#include <type_traits>
+
 #include "depthai/pipeline/datatype/ImgAnnotations.hpp"
 #include "depthai/pipeline/datatype/ImgFrame.hpp"
+#include "depthai/pipeline/datatype/Transformable.hpp"
 #include "pybind11_common.hpp"
 
 struct DatatypeBindings {
@@ -13,7 +16,7 @@ struct DatatypeBindings {
 };
 
 namespace dai {
-template <typename T>
+template <typename T, typename Enable = void>
 // This is used so pybind detects the classes as overridable in python,
 // which trigers the holders to keep the python part of the object alive
 class Py : public T {
@@ -26,4 +29,14 @@ class Py : public T {
         );
     }
 };
-}  // namespace dai
+
+template <typename T>
+class Py<T, std::enable_if_t<std::is_base_of_v<dai::TransformableBuffer, T>>> : public T {
+   public:
+    using T::T;
+
+    std::shared_ptr<dai::TransformableBuffer> transformTo(const dai::ImgTransformation& target) const override {
+        PYBIND11_OVERLOAD(std::shared_ptr<dai::TransformableBuffer>, T, transformTo, target);
+    }
+};
+}  // namespace dai

bindings/python/src/pipeline/CommonBindings.cpp

@@ -222,6 +222,7 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack) {
     spatialKeypointsList.def(py::init<>())
         .def(py::init<std::vector<SpatialKeypoint>, std::vector<Edge>>(), py::arg("keypoints"), py::arg("edges"), DOC(dai, KeypointsListT, KeypointsListT))
         .def(py::init<std::vector<SpatialKeypoint>>(), py::arg("keypoints"), DOC(dai, KeypointsListT, KeypointsListT))
+        .def_readwrite("unit", &SpatialKeypointsList::unit, DOC(dai, SpatialKeypointsList, unit))
         .def(
             "setKeypoints",
             [](SpatialKeypointsList& self, const std::vector<SpatialKeypoint>& kps) { self.Base::setKeypoints(kps); },
@@ -248,6 +249,7 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack) {
         .def("setSpatialCoordinates",
              &SpatialKeypointsList::setSpatialCoordinates,
              py::arg("spatialCoordinates"),
+             py::arg("spatialUnit") = LengthUnit::MILLIMETER,
              DOC(dai, SpatialKeypointsList, setSpatialCoordinates))
         .def("getSpatialCoordinates", &SpatialKeypointsList::getSpatialCoordinates, DOC(dai, SpatialKeypointsList, getSpatialCoordinates));
 

bindings/python/src/pipeline/datatype/AprilTagsBindings.cpp

@@ -17,7 +17,7 @@ void bind_apriltags(pybind11::module& m, void* pCallstack) {
     using namespace dai;
 
     py::class_<AprilTag> aprilTag(m, "AprilTag", DOC(dai, AprilTag));
-    py::class_<AprilTags, Py<AprilTags>, Buffer, std::shared_ptr<AprilTags>> aprilTags(m, "AprilTags", DOC(dai, AprilTags));
+    py::class_<AprilTags, Py<AprilTags>, Buffer, Transformable, std::shared_ptr<AprilTags>> aprilTags(m, "AprilTags", DOC(dai, AprilTags));
 
     ///////////////////////////////////////////////////////////////////////
     ///////////////////////////////////////////////////////////////////////
@@ -50,6 +50,7 @@ void bind_apriltags(pybind11::module& m, void* pCallstack) {
         .def("getTimestamp", &AprilTags::Buffer::getTimestamp, DOC(dai, Buffer, getTimestamp))
         .def("getTimestampDevice", &AprilTags::Buffer::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice))
         .def("getSequenceNum", &AprilTags::Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum))
+        .def("transformTo", &AprilTags::transformTo, py::arg("target"), DOC(dai, AprilTags, transformTo))
         // .def("setTimestamp", &AprilTags::setTimestamp, DOC(dai, Buffer, setTimestamp))
         // .def("setTimestampDevice", &AprilTags::setTimestampDevice, DOC(dai, Buffer, setTimestampDevice))
         // .def("setSequenceNum", &AprilTags::setSequenceNum, DOC(dai, Buffer, setSequenceNum))

bindings/python/src/pipeline/datatype/ImgDetectionsBindings.cpp

@@ -20,7 +20,8 @@ void bind_imgdetections(pybind11::module& m, void* pCallstack) {
     using namespace dai;
 
     // py::class_<RawImgDetections, RawBuffer, std::shared_ptr<RawImgDetections>> rawImgDetections(m, "RawImgDetections", DOC(dai, RawImgDetections));
-    py::class_<ImgDetections, Py<ImgDetections>, Buffer, std::shared_ptr<ImgDetections>> imgDetections(m, "ImgDetections", DOC(dai, ImgDetections));
+    py::class_<ImgDetections, Py<ImgDetections>, Buffer, Transformable, std::shared_ptr<ImgDetections>> imgDetections(
+        m, "ImgDetections", DOC(dai, ImgDetections));
     py::class_<ImgDetection> imgDetection(m, "ImgDetection", DOC(dai, ImgDetection));
 
     ///////////////////////////////////////////////////////////////////////
@@ -84,7 +85,8 @@ void bind_imgdetections(pybind11::module& m, void* pCallstack) {
         .def("getCenterY", &dai::ImgDetection::getCenterY)
         .def("getWidth", &dai::ImgDetection::getWidth)
         .def("getHeight", &dai::ImgDetection::getHeight)
-        .def("getAngle", &dai::ImgDetection::getAngle);
+        .def("getAngle", &dai::ImgDetection::getAngle)
+        .def("transform", &dai::ImgDetection::transform, py::arg("source"), py::arg("target"), DOC(dai, ImgDetection, transform));
 
     // rawImgDetections
     //     .def(py::init<>())
@@ -152,6 +154,7 @@ void bind_imgdetections(pybind11::module& m, void* pCallstack) {
              py::return_value_policy::reference_internal)
         .def("getMaskData", &ImgDetections::getMaskData, DOC(dai, ImgDetectionsT, getMaskData))
         .def("getSegmentationMask", &ImgDetections::getSegmentationMask, DOC(dai, ImgDetectionsT, getSegmentationMask))
+        .def("transformTo", &ImgDetections::transformTo, py::arg("target"), DOC(dai, ImgDetections, transformTo))
 #ifdef DEPTHAI_HAVE_OPENCV_SUPPORT
         .def("setCvSegmentationMask", &ImgDetections::setCvSegmentationMask, py::arg("mask"), DOC(dai, ImgDetectionsT, setCvSegmentationMask))
         .def(

bindings/python/src/pipeline/datatype/PointCloudDataBindings.cpp

@@ -17,7 +17,8 @@ void bind_pointclouddata(pybind11::module& m, void* pCallstack) {
     using namespace dai;
 
     // py::class_<RawPointCloudData, RawBuffer, std::shared_ptr<RawPointCloudData>> rawPointCloudData(m, "RawPointCloudData", DOC(dai, RawPointCloudData));
-    py::class_<PointCloudData, Py<PointCloudData>, Buffer, std::shared_ptr<PointCloudData>> pointCloudData(m, "PointCloudData", DOC(dai, PointCloudData));
+    py::class_<PointCloudData, Py<PointCloudData>, Buffer, Transformable, std::shared_ptr<PointCloudData>> pointCloudData(
+        m, "PointCloudData", DOC(dai, PointCloudData));
 
     ///////////////////////////////////////////////////////////////////////
     ///////////////////////////////////////////////////////////////////////

bindings/python/src/pipeline/datatype/SegmentationMaskBindings.cpp

@@ -39,7 +39,7 @@ py::array_t<std::uint8_t> toNumpyMask(const dai::SegmentationMask& mask, const s
 void bind_segmentationmask(pybind11::module& m, void* pCallstack) {
     using namespace dai;
 
-    py::class_<SegmentationMask, Py<SegmentationMask>, Buffer, std::shared_ptr<SegmentationMask>> segmentationMask(
+    py::class_<SegmentationMask, Py<SegmentationMask>, Buffer, Transformable, std::shared_ptr<SegmentationMask>> segmentationMask(
         m, "SegmentationMask", DOC(dai, SegmentationMask));
 
     ///////////////////////////////////////////////////////////////////////
@@ -112,6 +112,7 @@ void bind_segmentationmask(pybind11::module& m, void* pCallstack) {
             py::arg("label"),
             DOC(dai, SegmentationMask, getMaskByLabel))
         .def("hasValidMask", &SegmentationMask::hasValidMask, DOC(dai, SegmentationMask, hasValidMask))
+        .def("transformTo", &SegmentationMask::transformTo, py::arg("target"), DOC(dai, SegmentationMask, transformTo))
 #ifdef DEPTHAI_HAVE_OPENCV_SUPPORT
         .def("setCvMask", &SegmentationMask::setCvMask, py::arg("mask"), DOC(dai, SegmentationMask, setCvMask))
         .def(

bindings/python/src/pipeline/datatype/SpatialImgDetectionsBindings.cpp

@@ -20,7 +20,7 @@ void bind_spatialimgdetections(pybind11::module& m, void* pCallstack) {
     // py::class_<RawSpatialImgDetections, RawBuffer, std::shared_ptr<RawSpatialImgDetections>> rawSpatialImgDetections(m, "RawSpatialImgDetections", DOC(dai,
     // RawSpatialImgDetections));
     py::class_<SpatialImgDetection> spatialImgDetection(m, "SpatialImgDetection", DOC(dai, SpatialImgDetection));
-    py::class_<SpatialImgDetections, Py<SpatialImgDetections>, Buffer, std::shared_ptr<SpatialImgDetections>> spatialImgDetections(
+    py::class_<SpatialImgDetections, Py<SpatialImgDetections>, Buffer, Transformable, std::shared_ptr<SpatialImgDetections>> spatialImgDetections(
         m, "SpatialImgDetections", DOC(dai, SpatialImgDetections));
 
     ///////////////////////////////////////////////////////////////////////
@@ -121,6 +121,7 @@ void bind_spatialimgdetections(pybind11::module& m, void* pCallstack) {
     // Message
     spatialImgDetections.def(py::init<>(), DOC(dai, ImgDetectionsT, ImgDetectionsT))
         .def("__repr__", &SpatialImgDetections::str)
+        .def_readwrite("unit", &SpatialImgDetections::unit, DOC(dai, SpatialImgDetections, unit))
         .def_property(
             "detections",
             [](SpatialImgDetections& det) { return &det.detections; },
@@ -142,6 +143,7 @@ void bind_spatialimgdetections(pybind11::module& m, void* pCallstack) {
              py::return_value_policy::reference_internal)
         .def("getMaskData", &SpatialImgDetections::getMaskData, DOC(dai, ImgDetectionsT, getMaskData))
         .def("getSegmentationMask", &SpatialImgDetections::getSegmentationMask, DOC(dai, ImgDetectionsT, getSegmentationMask))
+        .def("transformTo", &SpatialImgDetections::transformTo, py::arg("target"), DOC(dai, SpatialImgDetections, transformTo))
 #ifdef DEPTHAI_HAVE_OPENCV_SUPPORT
         .def("setCvSegmentationMask", &SpatialImgDetections::setCvSegmentationMask, py::arg("mask"), DOC(dai, ImgDetectionsT, setCvSegmentationMask))
         .def(