feat: add tests for generateFromPixels method

Author: NorbertKlockiewicz

apps/computer-vision/app/object_detection/index.tsx

@@ -1,18 +1,16 @@
 import Spinner from '../../components/Spinner';
+import { BottomBar } from '../../components/BottomBar';
 import { getImage } from '../../utils';
 import {
   Detection,
   useObjectDetection,
   SSDLITE_320_MOBILENET_V3_LARGE,
-  ScalarType,
-  PixelData,
 } from 'react-native-executorch';
-import { View, StyleSheet, Image, TouchableOpacity, Text } from 'react-native';
+import { View, StyleSheet, Image } from 'react-native';
 import ImageWithBboxes from '../../components/ImageWithBboxes';
 import React, { useContext, useEffect, useState } from 'react';
 import { GeneratingContext } from '../../context';
 import ScreenWrapper from '../../ScreenWrapper';
-import ColorPalette from '../../colors';
 
 export default function ObjectDetectionScreen() {
   const [imageUri, setImageUri] = useState('');
@@ -44,59 +42,14 @@ export default function ObjectDetectionScreen() {
   const runForward = async () => {
     if (imageUri) {
       try {
-        console.log('Running forward with string URI...');
-        const output = await ssdLite.forward(imageUri, 0.5);
-        console.log('String URI result:', output.length, 'detections');
+        const output = await ssdLite.forward(imageUri);
         setResults(output);
       } catch (e) {
-        console.error('Error in runForward:', e);
+        console.error(e);
       }
     }
   };
 
-  const runForwardPixels = async () => {
-    try {
-      console.log('Testing with hardcoded pixel data...');
-
-      // Create a simple 320x320 test image (all zeros - black image)
-      // In a real scenario, you would load actual image pixel data here
-      const width = 320;
-      const height = 320;
-      const channels = 3; // RGB
-
-      // Create a black image (you can replace this with actual pixel data)
-      const rgbData = new Uint8Array(width * height * channels);
-
-      // Optionally, add some test pattern (e.g., white square in center)
-      for (let y = 100; y < 220; y++) {
-        for (let x = 100; x < 220; x++) {
-          const idx = (y * width + x) * 3;
-          rgbData[idx + 0] = 255; // R
-          rgbData[idx + 1] = 255; // G
-          rgbData[idx + 2] = 255; // B
-        }
-      }
-
-      const pixelData: PixelData = {
-        dataPtr: rgbData,
-        sizes: [height, width, channels],
-        scalarType: ScalarType.BYTE,
-      };
-
-      console.log('Running forward with hardcoded pixel data...', {
-        sizes: pixelData.sizes,
-        dataSize: pixelData.dataPtr.byteLength,
-      });
-
-      // Run inference using unified forward() API
-      const output = await ssdLite.forward(pixelData, 0.3);
-      console.log('Pixel data result:', output.length, 'detections');
-      setResults(output);
-    } catch (e) {
-      console.error('Error in runForwardPixels:', e);
-    }
-  };
-
   if (!ssdLite.isReady) {
     return (
       <Spinner
@@ -128,41 +81,10 @@ export default function ObjectDetectionScreen() {
           )}
         </View>
       </View>
-
-      {/* Custom bottom bar with two buttons */}
-      <View style={styles.bottomContainer}>
-        <View style={styles.bottomIconsContainer}>
-          <TouchableOpacity onPress={() => handleCameraPress(false)}>
-            <Text style={styles.iconText}>📷 Gallery</Text>
-          </TouchableOpacity>
-        </View>
-
-        <View style={styles.buttonsRow}>
-          <TouchableOpacity
-            style={[
-              styles.button,
-              styles.halfButton,
-              !imageUri && styles.buttonDisabled,
-            ]}
-            onPress={runForward}
-            disabled={!imageUri}
-          >
-            <Text style={styles.buttonText}>Run (String)</Text>
-          </TouchableOpacity>
-
-          <TouchableOpacity
-            style={[
-              styles.button,
-              styles.halfButton,
-              !imageUri && styles.buttonDisabled,
-            ]}
-            onPress={runForwardPixels}
-            disabled={!imageUri}
-          >
-            <Text style={styles.buttonText}>Run (Pixels)</Text>
-          </TouchableOpacity>
-        </View>
-      </View>
+      <BottomBar
+        handleCameraPress={handleCameraPress}
+        runForward={runForward}
+      />
     </ScreenWrapper>
   );
 }
@@ -207,43 +129,4 @@ const styles = StyleSheet.create({
     width: '100%',
     height: '100%',
   },
-  bottomContainer: {
-    width: '100%',
-    gap: 15,
-    alignItems: 'center',
-    padding: 16,
-    flex: 1,
-  },
-  bottomIconsContainer: {
-    flexDirection: 'row',
-    justifyContent: 'center',
-    width: '100%',
-  },
-  iconText: {
-    fontSize: 16,
-    color: ColorPalette.primary,
-  },
-  buttonsRow: {
-    flexDirection: 'row',
-    width: '100%',
-    gap: 10,
-  },
-  button: {
-    height: 50,
-    justifyContent: 'center',
-    alignItems: 'center',
-    backgroundColor: ColorPalette.primary,
-    color: '#fff',
-    borderRadius: 8,
-  },
-  halfButton: {
-    flex: 1,
-  },
-  buttonDisabled: {
-    opacity: 0.5,
-  },
-  buttonText: {
-    color: '#fff',
-    fontSize: 16,
-  },
 });

packages/react-native-executorch/common/rnexecutorch/models/object_detection/ObjectDetection.cpp

@@ -106,6 +106,10 @@ ObjectDetection::postprocess(const std::vector<EValue> &tensors,
 
 std::vector<types::Detection>
 ObjectDetection::runInference(cv::Mat image, double detectionThreshold) {
+  if (detectionThreshold < 0.0 || detectionThreshold > 1.0) {
+    throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
+                            "detectionThreshold must be in range [0, 1]");
+  }
   std::scoped_lock lock(inference_mutex_);
 
   cv::Size originalSize = image.size();

packages/react-native-executorch/common/rnexecutorch/tests/CMakeLists.txt

@@ -156,8 +156,11 @@ add_rn_test(ObjectDetectionTests integration/ObjectDetectionTest.cpp
     SOURCES
         ${RNEXECUTORCH_DIR}/models/object_detection/ObjectDetection.cpp
         ${RNEXECUTORCH_DIR}/models/object_detection/Utils.cpp
+        ${RNEXECUTORCH_DIR}/models/VisionModel.cpp
+        ${RNEXECUTORCH_DIR}/utils/FrameProcessor.cpp
+        ${RNEXECUTORCH_DIR}/utils/FrameExtractor.cpp
         ${IMAGE_UTILS_SOURCES}
-    LIBS opencv_deps
+    LIBS opencv_deps android
 )
 
 add_rn_test(ImageEmbeddingsTests integration/ImageEmbeddingsTest.cpp

packages/react-native-executorch/common/rnexecutorch/tests/integration/ObjectDetectionTest.cpp

@@ -1,6 +1,8 @@
 #include "BaseModelTests.h"
+#include <executorch/extension/tensor/tensor.h>
 #include <gtest/gtest.h>
 #include <rnexecutorch/Error.h>
+#include <rnexecutorch/host_objects/JSTensorViewIn.h>
 #include <rnexecutorch/models/object_detection/Constants.h>
 #include <rnexecutorch/models/object_detection/ObjectDetection.h>
 
@@ -115,6 +117,73 @@ TEST(ObjectDetectionGenerateTests, DetectionsHaveValidLabels) {
   }
 }
 
+// ============================================================================
+// generateFromPixels tests
+// ============================================================================
+TEST(ObjectDetectionPixelTests, ValidPixelDataReturnsResults) {
+  ObjectDetection model(kValidObjectDetectionModelPath, nullptr);
+  constexpr int width = 4, height = 4, channels = 3;
+  std::vector<uint8_t> pixelData(width * height * channels, 128);
+  JSTensorViewIn tensorView{pixelData.data(),
+                            {height, width, channels},
+                            executorch::aten::ScalarType::Byte};
+  auto results = model.generateFromPixels(tensorView, 0.3);
+  EXPECT_GE(results.size(), 0u);
+}
+
+TEST(ObjectDetectionPixelTests, WrongSizesLengthThrows) {
+  ObjectDetection model(kValidObjectDetectionModelPath, nullptr);
+  std::vector<uint8_t> pixelData(16, 0);
+  JSTensorViewIn tensorView{
+      pixelData.data(), {4, 4}, executorch::aten::ScalarType::Byte};
+  EXPECT_THROW((void)model.generateFromPixels(tensorView, 0.5),
+               RnExecutorchError);
+}
+
+TEST(ObjectDetectionPixelTests, WrongChannelCountThrows) {
+  ObjectDetection model(kValidObjectDetectionModelPath, nullptr);
+  constexpr int width = 4, height = 4, channels = 4;
+  std::vector<uint8_t> pixelData(width * height * channels, 0);
+  JSTensorViewIn tensorView{pixelData.data(),
+                            {height, width, channels},
+                            executorch::aten::ScalarType::Byte};
+  EXPECT_THROW((void)model.generateFromPixels(tensorView, 0.5),
+               RnExecutorchError);
+}
+
+TEST(ObjectDetectionPixelTests, WrongScalarTypeThrows) {
+  ObjectDetection model(kValidObjectDetectionModelPath, nullptr);
+  constexpr int width = 4, height = 4, channels = 3;
+  std::vector<uint8_t> pixelData(width * height * channels, 0);
+  JSTensorViewIn tensorView{pixelData.data(),
+                            {height, width, channels},
+                            executorch::aten::ScalarType::Float};
+  EXPECT_THROW((void)model.generateFromPixels(tensorView, 0.5),
+               RnExecutorchError);
+}
+
+TEST(ObjectDetectionPixelTests, NegativeThresholdThrows) {
+  ObjectDetection model(kValidObjectDetectionModelPath, nullptr);
+  constexpr int width = 4, height = 4, channels = 3;
+  std::vector<uint8_t> pixelData(width * height * channels, 128);
+  JSTensorViewIn tensorView{pixelData.data(),
+                            {height, width, channels},
+                            executorch::aten::ScalarType::Byte};
+  EXPECT_THROW((void)model.generateFromPixels(tensorView, -0.1),
+               RnExecutorchError);
+}
+
+TEST(ObjectDetectionPixelTests, ThresholdAboveOneThrows) {
+  ObjectDetection model(kValidObjectDetectionModelPath, nullptr);
+  constexpr int width = 4, height = 4, channels = 3;
+  std::vector<uint8_t> pixelData(width * height * channels, 128);
+  JSTensorViewIn tensorView{pixelData.data(),
+                            {height, width, channels},
+                            executorch::aten::ScalarType::Byte};
+  EXPECT_THROW((void)model.generateFromPixels(tensorView, 1.1),
+               RnExecutorchError);
+}
+
 TEST(ObjectDetectionInheritedTests, GetInputShapeWorks) {
   ObjectDetection model(kValidObjectDetectionModelPath, nullptr);
   auto shape = model.getInputShape("forward", 0);

packages/react-native-executorch/common/rnexecutorch/tests/integration/stubs/jsi_stubs.cpp

@@ -14,6 +14,14 @@ namespace facebook::jsi {
 MutableBuffer::~MutableBuffer() {}
 Value::~Value() {}
 Value::Value(Value &&other) noexcept {}
+
+// Needed to link ObjectDetectionTests: generateFromFrame and FrameProcessor
+// pull in these JSI symbols, but they are never called in tests.
+Object Value::asObject(Runtime &) const & { __builtin_unreachable(); }
+BigInt Value::asBigInt(Runtime &) const & { __builtin_unreachable(); }
+
+uint64_t BigInt::asUint64(Runtime &) const { return 0; }
+
 } // namespace facebook::jsi
 
 namespace facebook::react {

packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts

@@ -1,5 +1,5 @@
 import { ResourceFetcher } from '../../utils/ResourceFetcher';
-import { ResourceSource } from '../../types/common';
+import { ResourceSource, PixelData } from '../../types/common';
 import { Detection } from '../../types/objectDetection';
 import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
 import { parseUnknownError, RnExecutorchError } from '../../errors/errorUtils';
@@ -41,4 +41,11 @@ export class ObjectDetectionModule extends VisionModule<Detection[]> {
       throw parseUnknownError(error);
     }
   }
+
+  async forward(
+    input: string | PixelData,
+    detectionThreshold: number = 0.5
+  ): Promise<Detection[]> {
+    return super.forward(input, detectionThreshold);
+  }
 }

packages/react-native-executorch/src/types/objectDetection.ts

@@ -179,7 +179,7 @@ export interface ObjectDetectionType {
    * **Note**: For VisionCamera frame processing, use `processFrame` instead.
    *
    * @param input - Image source (string or PixelData object)
-   * @param detectionThreshold - An optional number between 0 and 1 representing the minimum confidence score. Default is 0.7.
+   * @param detectionThreshold - An optional number between 0 and 1 representing the minimum confidence score. Default is 0.5.
    * @returns A Promise that resolves to an array of `Detection` objects.
    * @throws {RnExecutorchError} If the model is not loaded or is currently processing another image.
    *
@@ -225,7 +225,7 @@ export interface ObjectDetectionType {
    * ```
    *
    * @param frame - VisionCamera Frame object
-   * @param detectionThreshold - The threshold for detection sensitivity. Default is 0.7.
+   * @param detectionThreshold - The threshold for detection sensitivity. Default is 0.5.
    * @returns Array of Detection objects representing detected items in the frame.
    */
   runOnFrame: