Merge branch 'main' into @nk/vision-camera-orientation

Author: NorbertKlockiewicz

README.md

@@ -8,6 +8,10 @@
   </h1>
 </div>
 
+[![Ad](https://revive-adserver.swmansion.com/www/images/zone-gh-react-native-executorch-1?n=1)](https://revive-adserver.swmansion.com/www/delivery/ck.php?zoneid=zone-gh-react-native-executorch-1&n=1)
+[![Ad](https://revive-adserver.swmansion.com/www/images/zone-gh-react-native-executorch-2?n=1)](https://revive-adserver.swmansion.com/www/delivery/ck.php?zoneid=zone-gh-react-native-executorch-2&n=1)
+[![Ad](https://revive-adserver.swmansion.com/www/images/zone-gh-react-native-executorch-3?n=1)](https://revive-adserver.swmansion.com/www/delivery/ck.php?zoneid=zone-gh-react-native-executorch-3&n=1)
+
 <div align="center">
   <a href="https://github.com/software-mansion/react-native-executorch/graphs/contributors"><img src="https://img.shields.io/github/contributors/software-mansion/react-native-executorch?style=for-the-badge&color=00008B" alt="GitHub - Contributors"></a>
   <a href="https://github.com/software-mansion/react-native-executorch/stargazers"><img src="https://img.shields.io/github/stars/software-mansion/react-native-executorch?style=for-the-badge&color=00008B" alt="GitHub - Stars"></a>
@@ -60,7 +64,7 @@ React Native ExecuTorch bridges the gap between React Native and native platform
 The minimal supported version are:
 * iOS 17.0
 * Android 13
-* React Native 0.81
+* React Native - see [compatibility table](https://docs.swmansion.com/react-native-executorch/docs/next/other/compatibility)
 
 > [!IMPORTANT]
 > React Native ExecuTorch supports only the [New React Native architecture](https://reactnative.dev/architecture/landing-page).

apps/computer-vision/app/object_detection/index.tsx

@@ -1,30 +1,40 @@
 import Spinner from '../../components/Spinner';
 import { BottomBar } from '../../components/BottomBar';
+import { ModelPicker, ModelOption } from '../../components/ModelPicker';
 import { getImage } from '../../utils';
 import {
   Detection,
   useObjectDetection,
   RF_DETR_NANO,
+  SSDLITE_320_MOBILENET_V3_LARGE,
+  ObjectDetectionModelSources,
 } from 'react-native-executorch';
 import { View, StyleSheet, Image } from 'react-native';
 import ImageWithBboxes from '../../components/ImageWithBboxes';
 import React, { useContext, useEffect, useState } from 'react';
 import { GeneratingContext } from '../../context';
 import ScreenWrapper from '../../ScreenWrapper';
 
+const MODELS: ModelOption<ObjectDetectionModelSources>[] = [
+  { label: 'RF-DeTR Nano', value: RF_DETR_NANO },
+  { label: 'SSDLite MobileNet', value: SSDLITE_320_MOBILENET_V3_LARGE },
+];
+
 export default function ObjectDetectionScreen() {
   const [imageUri, setImageUri] = useState('');
   const [results, setResults] = useState<Detection[]>([]);
   const [imageDimensions, setImageDimensions] = useState<{
     width: number;
     height: number;
   }>();
+  const [selectedModel, setSelectedModel] =
+    useState<ObjectDetectionModelSources>(RF_DETR_NANO);
 
-  const rfDetr = useObjectDetection({ model: RF_DETR_NANO });
+  const model = useObjectDetection({ model: selectedModel });
   const { setGlobalGenerating } = useContext(GeneratingContext);
   useEffect(() => {
-    setGlobalGenerating(rfDetr.isGenerating);
-  }, [rfDetr.isGenerating, setGlobalGenerating]);
+    setGlobalGenerating(model.isGenerating);
+  }, [model.isGenerating, setGlobalGenerating]);
 
   const handleCameraPress = async (isCamera: boolean) => {
     const image = await getImage(isCamera);
@@ -42,19 +52,19 @@ export default function ObjectDetectionScreen() {
   const runForward = async () => {
     if (imageUri) {
       try {
-        const output = await rfDetr.forward(imageUri);
+        const output = await model.forward(imageUri);
         setResults(output);
       } catch (e) {
         console.error(e);
       }
     }
   };
 
-  if (!rfDetr.isReady) {
+  if (!model.isReady) {
     return (
       <Spinner
-        visible={!rfDetr.isReady}
-        textContent={`Loading the model ${(rfDetr.downloadProgress * 100).toFixed(0)} %`}
+        visible={!model.isReady}
+        textContent={`Loading the model ${(model.downloadProgress * 100).toFixed(0)} %`}
       />
     );
   }
@@ -81,6 +91,15 @@ export default function ObjectDetectionScreen() {
           )}
         </View>
       </View>
+      <ModelPicker
+        models={MODELS}
+        selectedModel={selectedModel}
+        disabled={model.isGenerating}
+        onSelect={(m) => {
+          setSelectedModel(m);
+          setResults([]);
+        }}
+      />
       <BottomBar
         handleCameraPress={handleCameraPress}
         runForward={runForward}
@@ -100,31 +119,6 @@ const styles = StyleSheet.create({
     borderRadius: 8,
     width: '100%',
   },
-  results: {
-    flex: 1,
-    alignItems: 'center',
-    justifyContent: 'center',
-    gap: 4,
-    padding: 4,
-  },
-  resultHeader: {
-    fontSize: 18,
-    color: 'navy',
-  },
-  resultsList: {
-    flex: 1,
-  },
-  resultRecord: {
-    flexDirection: 'row',
-    width: '100%',
-    justifyContent: 'space-between',
-    padding: 8,
-    borderBottomWidth: 1,
-  },
-  resultLabel: {
-    flex: 1,
-    marginRight: 4,
-  },
   fullSizeImage: {
     width: '100%',
     height: '100%',

apps/computer-vision/app/ocr/index.tsx

@@ -1,23 +1,48 @@
 import Spinner from '../../components/Spinner';
 import { BottomBar } from '../../components/BottomBar';
+import { ModelPicker, ModelOption } from '../../components/ModelPicker';
 import { getImage } from '../../utils';
-import { useOCR, OCR_ENGLISH } from 'react-native-executorch';
+import {
+  useOCR,
+  OCR_ENGLISH,
+  OCR_GERMAN,
+  OCR_FRENCH,
+  OCR_SPANISH,
+  OCR_ITALIAN,
+  OCR_JAPANESE,
+  OCR_KOREAN,
+  OCRProps,
+} from 'react-native-executorch';
 import { View, StyleSheet, Image, Text, ScrollView } from 'react-native';
 import ImageWithBboxes2 from '../../components/ImageWithOCRBboxes';
 import React, { useContext, useEffect, useState } from 'react';
 import { GeneratingContext } from '../../context';
 import ScreenWrapper from '../../ScreenWrapper';
 
+type OCRModelSources = OCRProps['model'];
+
+const MODELS: ModelOption<OCRModelSources>[] = [
+  { label: 'English', value: OCR_ENGLISH },
+  { label: 'German', value: OCR_GERMAN },
+  { label: 'French', value: OCR_FRENCH },
+  { label: 'Spanish', value: OCR_SPANISH },
+  { label: 'Italian', value: OCR_ITALIAN },
+  { label: 'Japanese', value: OCR_JAPANESE },
+  { label: 'Korean', value: OCR_KOREAN },
+];
+
 export default function OCRScreen() {
   const [imageUri, setImageUri] = useState('');
   const [results, setResults] = useState<any[]>([]);
   const [imageDimensions, setImageDimensions] = useState<{
     width: number;
     height: number;
   }>();
+  const [selectedModel, setSelectedModel] =
+    useState<OCRModelSources>(OCR_ENGLISH);
 
   const model = useOCR({
-    model: OCR_ENGLISH,
+    model: selectedModel,
   });
   const { setGlobalGenerating } = useContext(GeneratingContext);
   useEffect(() => {
@@ -89,6 +114,15 @@ export default function OCRScreen() {
           </View>
         )}
       </View>
+      <ModelPicker
+        models={MODELS}
+        selectedModel={selectedModel}
+        disabled={model.isGenerating}
+        onSelect={(m) => {
+          setSelectedModel(m);
+          setResults([]);
+        }}
+      />
       <BottomBar
         handleCameraPress={handleCameraPress}
         runForward={runForward}

apps/computer-vision/app/semantic_segmentation/index.tsx

@@ -1,9 +1,16 @@
 import Spinner from '../../components/Spinner';
 import { BottomBar } from '../../components/BottomBar';
+import { ModelPicker, ModelOption } from '../../components/ModelPicker';
 import { getImage } from '../../utils';
 import {
   DEEPLAB_V3_MOBILENET_V3_LARGE_QUANTIZED,
+  DEEPLAB_V3_RESNET50_QUANTIZED,
+  DEEPLAB_V3_RESNET101_QUANTIZED,
+  LRASPP_MOBILENET_V3_LARGE_QUANTIZED,
+  FCN_RESNET50_QUANTIZED,
+  FCN_RESNET101_QUANTIZED,
   useSemanticSegmentation,
+  SemanticSegmentationModelSources,
 } from 'react-native-executorch';
 import {
   Canvas,
@@ -42,12 +49,28 @@ const numberToColor: number[][] = [
   [162, 51, 255], // 20 Amethyst
 ];
 
+const MODELS: ModelOption<SemanticSegmentationModelSources>[] = [
+  {
+    label: 'DeepLab MobileNet',
+    value: DEEPLAB_V3_MOBILENET_V3_LARGE_QUANTIZED,
+  },
+  { label: 'DeepLab ResNet50', value: DEEPLAB_V3_RESNET50_QUANTIZED },
+  { label: 'DeepLab ResNet101', value: DEEPLAB_V3_RESNET101_QUANTIZED },
+  { label: 'LRASPP MobileNet', value: LRASPP_MOBILENET_V3_LARGE_QUANTIZED },
+  { label: 'FCN ResNet50', value: FCN_RESNET50_QUANTIZED },
+  { label: 'FCN ResNet101', value: FCN_RESNET101_QUANTIZED },
+];
+
 export default function SemanticSegmentationScreen() {
   const { setGlobalGenerating } = useContext(GeneratingContext);
+  const [selectedModel, setSelectedModel] =
+    useState<SemanticSegmentationModelSources>(
+      DEEPLAB_V3_MOBILENET_V3_LARGE_QUANTIZED
+    );
+
   const { isReady, isGenerating, downloadProgress, forward } =
-    useSemanticSegmentation({
-      model: DEEPLAB_V3_MOBILENET_V3_LARGE_QUANTIZED,
-    });
+    useSemanticSegmentation({ model: selectedModel });
+
   const [imageUri, setImageUri] = useState('');
   const [imageSize, setImageSize] = useState({ width: 0, height: 0 });
   const [segImage, setSegImage] = useState<SkImage | null>(null);
@@ -61,10 +84,7 @@ export default function SemanticSegmentationScreen() {
     const image = await getImage(isCamera);
     if (!image?.uri) return;
     setImageUri(image.uri);
-    setImageSize({
-      width: image.width ?? 0,
-      height: image.height ?? 0,
-    });
+    setImageSize({ width: image.width ?? 0, height: image.height ?? 0 });
     setSegImage(null);
   };
 
@@ -150,6 +170,15 @@ export default function SemanticSegmentationScreen() {
           </View>
         )}
       </View>
+      <ModelPicker
+        models={MODELS}
+        selectedModel={selectedModel}
+        disabled={isGenerating}
+        onSelect={(m) => {
+          setSelectedModel(m);
+          setSegImage(null);
+        }}
+      />
       <BottomBar
         handleCameraPress={handleCameraPress}
         runForward={runForward}
@@ -159,29 +188,15 @@ export default function SemanticSegmentationScreen() {
 }
 
 const styles = StyleSheet.create({
-  imageCanvasContainer: {
-    flex: 6,
-    width: '100%',
-    padding: 16,
-  },
-  imageContainer: {
-    flex: 1,
-    width: '100%',
-  },
-  image: {
-    flex: 1,
-    borderRadius: 8,
-    width: '100%',
-  },
+  imageCanvasContainer: { flex: 6, width: '100%', padding: 16 },
+  imageContainer: { flex: 1, width: '100%' },
+  image: { flex: 1, borderRadius: 8, width: '100%' },
   canvasContainer: {
     flex: 1,
     justifyContent: 'center',
     alignItems: 'center',
     gap: 4,
     padding: 4,
   },
-  canvas: {
-    width: '100%',
-    height: '100%',
-  },
+  canvas: { width: '100%', height: '100%' },
 });

apps/computer-vision/app/style_transfer/index.tsx

@@ -1,17 +1,40 @@
 import Spinner from '../../components/Spinner';
 import { BottomBar } from '../../components/BottomBar';
+import { ModelPicker, ModelOption } from '../../components/ModelPicker';
 import { getImage } from '../../utils';
 import {
   useStyleTransfer,
   STYLE_TRANSFER_CANDY_QUANTIZED,
+  STYLE_TRANSFER_MOSAIC_QUANTIZED,
+  STYLE_TRANSFER_RAIN_PRINCESS_QUANTIZED,
+  STYLE_TRANSFER_UDNIE_QUANTIZED,
+  StyleTransferModelName,
+  ResourceSource,
 } from 'react-native-executorch';
+
 import { View, StyleSheet, Image } from 'react-native';
 import React, { useContext, useEffect, useState } from 'react';
 import { GeneratingContext } from '../../context';
 import ScreenWrapper from '../../ScreenWrapper';
 
+type StyleTransferModelSources = {
+  modelName: StyleTransferModelName;
+  modelSource: ResourceSource;
+};
+
+const MODELS: ModelOption<StyleTransferModelSources>[] = [
+  { label: 'Candy', value: STYLE_TRANSFER_CANDY_QUANTIZED },
+  { label: 'Mosaic', value: STYLE_TRANSFER_MOSAIC_QUANTIZED },
+  { label: 'Rain Princess', value: STYLE_TRANSFER_RAIN_PRINCESS_QUANTIZED },
+  { label: 'Udnie', value: STYLE_TRANSFER_UDNIE_QUANTIZED },
+];
+
 export default function StyleTransferScreen() {
-  const model = useStyleTransfer({ model: STYLE_TRANSFER_CANDY_QUANTIZED });
+  const [selectedModel, setSelectedModel] = useState<StyleTransferModelSources>(
+    STYLE_TRANSFER_CANDY_QUANTIZED
+  );
+
+  const model = useStyleTransfer({ model: selectedModel });
   const { setGlobalGenerating } = useContext(GeneratingContext);
   useEffect(() => {
     setGlobalGenerating(model.isGenerating);
@@ -64,6 +87,15 @@ export default function StyleTransferScreen() {
           }
         />
       </View>
+      <ModelPicker
+        models={MODELS}
+        selectedModel={selectedModel}
+        disabled={model.isGenerating}
+        onSelect={(m) => {
+          setSelectedModel(m);
+          setStyledUri('');
+        }}
+      />
       <BottomBar
         handleCameraPress={handleCameraPress}
         runForward={runForward}
@@ -73,14 +105,6 @@ export default function StyleTransferScreen() {
 }
 
 const styles = StyleSheet.create({
-  imageContainer: {
-    flex: 6,
-    width: '100%',
-    padding: 16,
-  },
-  image: {
-    flex: 1,
-    borderRadius: 8,
-    width: '100%',
-  },
+  imageContainer: { flex: 6, width: '100%', padding: 16 },
+  image: { flex: 1, borderRadius: 8, width: '100%' },
 });