Add Mateusz's patch

Author: benITo47

apps/computer-vision/app/vision_camera/index.tsx

@@ -74,14 +74,6 @@ const TASKS: Task[] = [
       { id: 'segmentationSelfie', label: 'Selfie' },
     ],
   },
-  {
-    id: 'objectDetection',
-    label: 'Detect',
-    variants: [
-      { id: 'objectDetectionSsdlite', label: 'SSDLite MobileNet' },
-      { id: 'objectDetectionRfdetr', label: 'RF-DETR Nano' },
-    ],
-  },
   {
     id: 'instanceSegmentation',
     label: 'Inst Seg',
@@ -90,6 +82,14 @@ const TASKS: Task[] = [
       { id: 'instanceSegmentation_rfdetr', label: 'RF-DETR Nano Seg' },
     ],
   },
+  {
+    id: 'objectDetection',
+    label: 'Detect',
+    variants: [
+      { id: 'objectDetectionSsdlite', label: 'SSDLite MobileNet' },
+      { id: 'objectDetectionRfdetr', label: 'RF-DETR Nano' },
+    ],
+  },
 ];
 
 // Module-level const so worklets in task components can always reference the same stable object.

packages/react-native-executorch/common/rnexecutorch/models/instance_segmentation/BaseInstanceSegmentation.cpp

@@ -43,6 +43,16 @@ cv::Size BaseInstanceSegmentation::modelInputSize() const {
   return {shape[shape.size() - 2], shape[shape.size() - 1]};
 }
 
+TensorPtr BaseInstanceSegmentation::buildInputTensor(const cv::Mat &image) {
+  cv::Mat preprocessed = preprocess(image);
+  return (normMean_.has_value() && normStd_.has_value())
+             ? image_processing::getTensorFromMatrix(
+                   modelInputShape_, preprocessed, normMean_.value(),
+                   normStd_.value())
+             : image_processing::getTensorFromMatrix(modelInputShape_,
+                                                     preprocessed);
+}
+
 std::vector<types::Instance> BaseInstanceSegmentation::runInference(
     const cv::Mat &image, double confidenceThreshold, double iouThreshold,
     int32_t maxInstances, const std::vector<int32_t> &classIndices,
@@ -64,16 +74,8 @@ std::vector<types::Instance> BaseInstanceSegmentation::runInference(
   cv::Size modelInputSize(shape[shape.size() - 2], shape[shape.size() - 1]);
   cv::Size originalSize(image.cols, image.rows);
 
-  cv::Mat preprocessed = preprocess(image);
-
-  auto inputTensor = (normMean_.has_value() && normStd_.has_value())
-                         ? image_processing::getTensorFromMatrix(
-                               modelInputShape_, preprocessed,
-                               normMean_.value(), normStd_.value())
-                         : image_processing::getTensorFromMatrix(
-                               modelInputShape_, preprocessed);
-
-  auto forwardResult = BaseModel::execute(methodName, {inputTensor});
+  auto forwardResult =
+      BaseModel::execute(methodName, {buildInputTensor(image)});
   if (!forwardResult.ok()) {
     throw RnExecutorchError(
         forwardResult.error(),
@@ -82,9 +84,13 @@ std::vector<types::Instance> BaseInstanceSegmentation::runInference(
             methodName + "' is valid.");
   }
 
-  return postprocess(forwardResult.get(), originalSize, modelInputSize,
-                     confidenceThreshold, iouThreshold, maxInstances,
-                     classIndices, returnMaskAtOriginalResolution);
+  validateThresholds(confidenceThreshold, iouThreshold);
+  validateOutputTensors(forwardResult.get());
+
+  auto instances = collectInstances(
+      forwardResult.get(), originalSize, modelInputSize, confidenceThreshold,
+      classIndices, returnMaskAtOriginalResolution);
+  return finalizeInstances(std::move(instances), iouThreshold, maxInstances);
 }
 
 std::vector<types::Instance> BaseInstanceSegmentation::generateFromString(
@@ -259,37 +265,16 @@ void BaseInstanceSegmentation::ensureMethodLoaded(
                                 methodName + "'");
   }
 
-  if (currentlyLoadedMethod_ != methodName) {
-    if (!currentlyLoadedMethod_.empty()) {
-      module_->unload_method(currentlyLoadedMethod_);
-    }
-    currentlyLoadedMethod_ = methodName;
-    auto loadResult = module_->load_method(methodName);
-    if (loadResult != executorch::runtime::Error::Ok) {
-      throw RnExecutorchError(
-          loadResult, "Failed to load method '" + methodName +
-                          "'. Ensure the method exists in the exported model.");
-    }
+  if (!currentlyLoadedMethod_.empty()) {
+    module_->unload_method(currentlyLoadedMethod_);
   }
-}
-
-cv::Size BaseInstanceSegmentation::getInputSize(const std::string &methodName) {
-  auto inputShapes = getAllInputShapes(methodName);
-  if (inputShapes.empty()) {
-    throw RnExecutorchError(RnExecutorchErrorCode::UnexpectedNumInputs,
-                            "Method '" + methodName +
-                                "' has no input tensors.");
-  }
-
-  const auto &inputShape = inputShapes[0];
-  if (inputShape.empty()) {
-    throw RnExecutorchError(RnExecutorchErrorCode::UnexpectedNumInputs,
-                            "Method '" + methodName +
-                                "' input tensor has no dimensions.");
+  currentlyLoadedMethod_ = methodName;
+  auto loadResult = module_->load_method(methodName);
+  if (loadResult != executorch::runtime::Error::Ok) {
+    throw RnExecutorchError(
+        loadResult, "Failed to load method '" + methodName +
+                        "'. Ensure the method exists in the exported model.");
   }
-
-  int32_t inputSize = inputShape[inputShape.size() - 1];
-  return cv::Size(inputSize, inputSize);
 }
 
 std::vector<types::Instance> BaseInstanceSegmentation::finalizeInstances(
@@ -308,15 +293,12 @@ std::vector<types::Instance> BaseInstanceSegmentation::finalizeInstances(
   return instances;
 }
 
-std::vector<types::Instance> BaseInstanceSegmentation::postprocess(
+std::vector<types::Instance> BaseInstanceSegmentation::collectInstances(
     const std::vector<EValue> &tensors, cv::Size originalSize,
-    cv::Size modelInputSize, double confidenceThreshold, double iouThreshold,
-    int32_t maxInstances, const std::vector<int32_t> &classIndices,
+    cv::Size modelInputSize, double confidenceThreshold,
+    const std::vector<int32_t> &classIndices,
     bool returnMaskAtOriginalResolution) {
 
-  validateThresholds(confidenceThreshold, iouThreshold);
-  validateOutputTensors(tensors);
-
   float widthRatio =
       static_cast<float>(originalSize.width) / modelInputSize.width;
   float heightRatio =
@@ -371,7 +353,7 @@ std::vector<types::Instance> BaseInstanceSegmentation::postprocess(
                            binaryMask.cols, binaryMask.rows, labelIdx, score);
   }
 
-  return finalizeInstances(std::move(instances), iouThreshold, maxInstances);
+  return instances;
 }
 
 } // namespace rnexecutorch::models::instance_segmentation

packages/react-native-executorch/common/rnexecutorch/models/instance_segmentation/BaseInstanceSegmentation.h

@@ -53,12 +53,13 @@ class BaseInstanceSegmentation : public VisionModel {
       int32_t maxInstances, const std::vector<int32_t> &classIndices,
       bool returnMaskAtOriginalResolution, const std::string &methodName);
 
+  TensorPtr buildInputTensor(const cv::Mat &image);
+
   std::vector<types::Instance>
-  postprocess(const std::vector<EValue> &tensors, cv::Size originalSize,
-              cv::Size modelInputSize, double confidenceThreshold,
-              double iouThreshold, int32_t maxInstances,
-              const std::vector<int32_t> &classIndices,
-              bool returnMaskAtOriginalResolution);
+  collectInstances(const std::vector<EValue> &tensors, cv::Size originalSize,
+                   cv::Size modelInputSize, double confidenceThreshold,
+                   const std::vector<int32_t> &classIndices,
+                   bool returnMaskAtOriginalResolution);
 
   void validateThresholds(double confidenceThreshold,
                           double iouThreshold) const;
@@ -69,7 +70,6 @@ class BaseInstanceSegmentation : public VisionModel {
 
   // Model loading and input helpers
   void ensureMethodLoaded(const std::string &methodName);
-  cv::Size getInputSize(const std::string &methodName);
 
   std::tuple<utils::computer_vision::BBox, float, int32_t>
   extractDetectionData(const float *bboxData, const float *scoresData,