feat: working rotations on both ios and android

NorbertKlockiewicz · NorbertKlockiewicz · commit b3d71b6c69d1 · 2026-03-18T12:50:43.000+01:00
diff --git a/apps/computer-vision/app/vision_camera/index.tsx b/apps/computer-vision/app/vision_camera/index.tsx
@@ -77,9 +77,10 @@ const TASKS: Task[] = [
   },
 ];
 
-// Module-level const so worklets in task components can always reference the same stable object.
+// Module-level consts so worklets in task components can always reference the same stable objects.
 // Never replaced — only mutated via setBlocking to avoid closure staleness.
 const frameKillSwitch = createSynchronizable(false);
+const cameraPositionSync = createSynchronizable<'front' | 'back'>('back');
 
 export default function VisionCameraScreen() {
   const insets = useSafeAreaInsets();
@@ -120,6 +121,10 @@ export default function VisionCameraScreen() {
     return () => clearTimeout(id);
   }, [activeModel]);
 
+  useEffect(() => {
+    cameraPositionSync.setBlocking(cameraPosition);
+  }, [cameraPosition]);
+
   const handleFpsChange = useCallback((newFps: number, newMs: number) => {
     setFps(newFps);
     setFrameMs(newMs);
@@ -163,6 +168,7 @@ export default function VisionCameraScreen() {
     activeModel,
     canvasSize,
     cameraPosition,
+    cameraPositionSync,
     frameKillSwitch,
     onFrameOutputChange: setFrameOutput,
     onReadyChange: setIsReady,
diff --git a/apps/computer-vision/components/vision_camera/tasks/ClassificationTask.tsx b/apps/computer-vision/components/vision_camera/tasks/ClassificationTask.tsx
@@ -47,6 +47,7 @@ export default function ClassificationTask({
   const frameOutput = useFrameOutput({
     pixelFormat: 'rgb',
     dropFramesWhileBusy: true,
+    enablePreviewSizedOutputBuffers: true,
     onFrame: useCallback(
       (frame: Frame) => {
         'worklet';
diff --git a/apps/computer-vision/components/vision_camera/tasks/ObjectDetectionTask.tsx b/apps/computer-vision/components/vision_camera/tasks/ObjectDetectionTask.tsx
@@ -1,5 +1,5 @@
 import React, { useCallback, useEffect, useRef, useState } from 'react';
-import { Platform, StyleSheet, Text, View } from 'react-native';
+import { StyleSheet, Text, View } from 'react-native';
 import { Frame, useFrameOutput } from 'react-native-vision-camera';
 import { scheduleOnRN } from 'react-native-worklets';
 import {
@@ -18,7 +18,7 @@ type Props = TaskProps & { activeModel: ObjModelId };
 export default function ObjectDetectionTask({
   activeModel,
   canvasSize,
-  cameraPosition,
+  cameraPositionSync,
   frameKillSwitch,
   onFrameOutputChange,
   onReadyChange,
@@ -70,6 +70,7 @@ export default function ObjectDetectionTask({
   const frameOutput = useFrameOutput({
     pixelFormat: 'rgb',
     dropFramesWhileBusy: true,
+    enablePreviewSizedOutputBuffers: true,
     onFrame: useCallback(
       (frame: Frame) => {
         'worklet';
@@ -80,10 +81,11 @@ export default function ObjectDetectionTask({
         try {
           if (!detRof) return;
           const orientation = frame.orientation;
-          const swapAxes = orientation === 'left' || orientation === 'right';
+          // "up"/"down" = landscape orientations where buffer axes are swapped vs screen
+          const swapAxes = orientation === 'up' || orientation === 'down';
           const screenW = swapAxes ? frame.height : frame.width;
           const screenH = swapAxes ? frame.width : frame.height;
-          const result = detRof(frame, 0.5);
+          const result = detRof(frame, cameraPositionSync.getDirty(), 0.5);
           if (result) {
             scheduleOnRN(updateDetections, {
               results: result,
@@ -97,7 +99,7 @@ export default function ObjectDetectionTask({
           frame.dispose();
         }
       },
-      [detRof, frameKillSwitch, updateDetections]
+      [detRof, frameKillSwitch, updateDetections, cameraPositionSync]
     ),
   });
 
@@ -113,16 +115,7 @@ export default function ObjectDetectionTask({
   const offsetY = (canvasSize.height - imageSize.height * scale) / 2;
 
   return (
-    <View
-      style={[
-        StyleSheet.absoluteFill,
-        // TODO: remove when VisionCamera fixes front camera orientation reporting on iOS
-        // https://github.com/mrousavy/react-native-vision-camera/issues/124
-        Platform.OS === 'ios' &&
-          cameraPosition === 'front' && { transform: [{ scaleX: -1 }] },
-      ]}
-      pointerEvents="none"
-    >
+    <View style={[StyleSheet.absoluteFill]} pointerEvents="none">
       {detections.map((det, i) => {
         const left = det.bbox.x1 * scale + offsetX;
         const top = det.bbox.y1 * scale + offsetY;
@@ -146,8 +139,6 @@ export default function ObjectDetectionTask({
               style={[
                 styles.bboxLabel,
                 { backgroundColor: labelColorBg(det.label) },
-                Platform.OS === 'ios' &&
-                  cameraPosition === 'front' && { transform: [{ scaleX: -1 }] },
               ]}
             >
               <Text style={styles.bboxLabelText}>
diff --git a/apps/computer-vision/components/vision_camera/tasks/SegmentationTask.tsx b/apps/computer-vision/components/vision_camera/tasks/SegmentationTask.tsx
@@ -1,5 +1,5 @@
 import React, { useCallback, useEffect, useRef, useState } from 'react';
-import { Platform, StyleSheet, View } from 'react-native';
+import { StyleSheet, View } from 'react-native';
 import { Frame, useFrameOutput } from 'react-native-vision-camera';
 import { scheduleOnRN } from 'react-native-worklets';
 import {
@@ -37,7 +37,7 @@ type Props = TaskProps & { activeModel: SegModelId };
 export default function SegmentationTask({
   activeModel,
   canvasSize,
-  cameraPosition,
+  cameraPositionSync,
   frameKillSwitch,
   onFrameOutputChange,
   onReadyChange,
@@ -139,6 +139,7 @@ export default function SegmentationTask({
   const frameOutput = useFrameOutput({
     pixelFormat: 'rgb',
     dropFramesWhileBusy: true,
+    enablePreviewSizedOutputBuffers: true,
     onFrame: useCallback(
       (frame: Frame) => {
         'worklet';
@@ -148,7 +149,7 @@ export default function SegmentationTask({
         }
         try {
           if (!segRof) return;
-          const result = segRof(frame, [], false);
+          const result = segRof(frame, cameraPositionSync.getDirty(), [], false);
           if (result?.ARGMAX) {
             const argmax: Int32Array = result.ARGMAX;
             const side = Math.round(Math.sqrt(argmax.length));
@@ -179,7 +180,7 @@ export default function SegmentationTask({
           frame.dispose();
         }
       },
-      [colors, frameKillSwitch, segRof, updateMask]
+      [cameraPositionSync, colors, frameKillSwitch, segRof, updateMask]
     ),
   });
 
@@ -190,16 +191,7 @@ export default function SegmentationTask({
   if (!maskImage) return null;
 
   return (
-    <View
-      style={[
-        StyleSheet.absoluteFill,
-        // TODO: remove when VisionCamera fixes front camera orientation reporting on iOS
-        // https://github.com/mrousavy/react-native-vision-camera/issues/124
-        Platform.OS === 'ios' &&
-          cameraPosition === 'front' && { transform: [{ scaleX: -1 }] },
-      ]}
-      pointerEvents="none"
-    >
+    <View style={StyleSheet.absoluteFill} pointerEvents="none">
       <Canvas style={StyleSheet.absoluteFill}>
         <SkiaImage
           image={maskImage}
diff --git a/apps/computer-vision/components/vision_camera/tasks/types.ts b/apps/computer-vision/components/vision_camera/tasks/types.ts
@@ -5,6 +5,7 @@ export type TaskProps = {
   activeModel: string;
   canvasSize: { width: number; height: number };
   cameraPosition: 'front' | 'back';
+  cameraPositionSync: ReturnType<typeof createSynchronizable<'front' | 'back'>>;
   frameKillSwitch: ReturnType<typeof createSynchronizable<boolean>>;
   onFrameOutputChange: (frameOutput: ReturnType<typeof useFrameOutput>) => void;
   onReadyChange: (isReady: boolean) => void;
diff --git a/apps/computer-vision/package.json b/apps/computer-vision/package.json
@@ -32,13 +32,13 @@
     "react-native-image-picker": "^7.2.2",
     "react-native-loading-spinner-overlay": "^3.0.1",
     "react-native-nitro-image": "0.13.0",
-    "react-native-nitro-modules": "0.35.0",
+    "react-native-nitro-modules": "0.35.2",
     "react-native-reanimated": "~4.2.2",
     "react-native-safe-area-context": "~5.6.0",
     "react-native-screens": "~4.16.0",
     "react-native-svg": "15.15.3",
     "react-native-svg-transformer": "^1.5.3",
-    "react-native-vision-camera": "5.0.0-beta.6",
+    "react-native-vision-camera": "5.0.0-beta.7",
     "react-native-worklets": "0.7.4"
   },
   "devDependencies": {
diff --git a/packages/react-native-executorch/common/rnexecutorch/utils/FrameProcessor.cpp b/packages/react-native-executorch/common/rnexecutorch/utils/FrameProcessor.cpp
@@ -59,7 +59,29 @@ FrameOrientation readFrameOrientation(jsi::Runtime &runtime,
     if (val.isNumber()) frameHeight = static_cast<int>(val.asNumber());
   }
 
-  return {orientation, isMirrored, frameWidth, frameHeight};
+  std::string cameraPosition = "back";
+  if (obj.hasProperty(runtime, "cameraPosition")) {
+    auto val = obj.getProperty(runtime, "cameraPosition");
+    if (val.isString()) cameraPosition = val.getString(runtime).utf8(runtime);
+  }
+
+  // VisionCamera does not set isMirrored=true for front camera (known bug).
+  bool rotate180 = false;
+  if (cameraPosition == "front") {
+    isMirrored = !isMirrored;
+#ifdef __ANDROID__
+    // Android front camera reports orientation shifted by 180° vs iOS.
+    if (orientation == "up") orientation = "down";
+    else if (orientation == "down") orientation = "up";
+    else if (orientation == "left") orientation = "right";
+    else if (orientation == "right") orientation = "left";
+#else
+    // iOS front camera needs an extra 180° rotation after the main transform.
+    rotate180 = true;
+#endif
+  }
+
+  return {orientation, isMirrored, frameWidth, frameHeight, rotate180};
 }
 
 cv::Mat pixelsToMat(const JSTensorViewIn &pixelData) {
diff --git a/packages/react-native-executorch/common/rnexecutorch/utils/FrameTransform.cpp b/packages/react-native-executorch/common/rnexecutorch/utils/FrameTransform.cpp
@@ -47,6 +47,14 @@ void transformBbox(float &x1, float &y1, float &x2, float &y2,
     x1 = nx1; y1 = ny1;
     x2 = nx2; y2 = ny2;
   }
+
+  // Extra 180° in post-rotation screen space (screen dims are h x w after CW).
+  if (orient.rotate180) {
+    float nx1 = h - x2, ny1 = w - y2;
+    float nx2 = h - x1, ny2 = w - y1;
+    x1 = nx1; y1 = ny1;
+    x2 = nx2; y2 = ny2;
+  }
 }
 
 cv::Mat transformMat(const cv::Mat &mat, const FrameOrientation &orient) {
@@ -69,6 +77,10 @@ cv::Mat transformMat(const cv::Mat &mat, const FrameOrientation &orient) {
     cv::rotate(result, result, cv::ROTATE_90_CLOCKWISE);
   }
 
+  if (orient.rotate180) {
+    cv::rotate(result, result, cv::ROTATE_180);
+  }
+
   return result;
 }
 
diff --git a/packages/react-native-executorch/common/rnexecutorch/utils/FrameTransform.h b/packages/react-native-executorch/common/rnexecutorch/utils/FrameTransform.h
@@ -13,6 +13,7 @@ struct FrameOrientation {
   bool isMirrored;
   int frameWidth;  // raw frame width (sensor native, before any rotation)
   int frameHeight; // raw frame height (sensor native, before any rotation)
+  bool rotate180 = false; // apply extra 180° after main rotation (front camera correction)
 };
 
 /**
@@ -80,6 +81,11 @@ void transformPoints(std::array<P, 4> &points,
     }
     // "up" = landscape-left: no-op
 
+    if (orient.rotate180) {
+      nx = h - nx;
+      ny = w - ny;
+    }
+
     p.x = nx;
     p.y = ny;
   }
diff --git a/packages/react-native-executorch/src/modules/computer_vision/VisionModule.ts b/packages/react-native-executorch/src/modules/computer_vision/VisionModule.ts
@@ -2,6 +2,7 @@ import { BaseModule } from '../BaseModule';
 import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
 import { RnExecutorchError } from '../../errors/errorUtils';
 import { Frame, PixelData, ScalarType } from '../../types/common';
+import { Platform } from 'react-native';
 
 export function isPixelData(input: unknown): input is PixelData {
   return (
@@ -70,18 +71,20 @@ export abstract class VisionModule<TOutput> extends BaseModule {
     const nativeGenerateFromFrame = this.nativeModule.generateFromFrame;
 
     // Return worklet that captures ONLY the JSI function
-    return (frame: any, ...args: any[]): TOutput => {
+    return (frame: any, cameraPosition: string, ...args: any[]): TOutput => {
       'worklet';
 
       let nativeBuffer: any = null;
       try {
         nativeBuffer = frame.getNativeBuffer();
+        console.log(frame.orientation);
         const frameData = {
           nativeBuffer: nativeBuffer.pointer,
           orientation: frame.orientation,
           isMirrored: frame.isMirrored,
           frameWidth: frame.width,
           frameHeight: frame.height,
+          cameraPosition: cameraPosition ?? 'back',
         };
         return nativeGenerateFromFrame(frameData, ...args);
       } finally {
diff --git a/packages/react-native-executorch/src/types/objectDetection.ts b/packages/react-native-executorch/src/types/objectDetection.ts
@@ -143,6 +143,6 @@ export interface ObjectDetectionType<L extends LabelEnum> {
    * @returns Array of Detection objects representing detected items in the frame.
    */
   runOnFrame:
-    | ((frame: Frame, detectionThreshold: number) => Detection<L>[])
+    | ((frame: Frame, cameraPosition: string, detectionThreshold: number) => Detection<L>[])
     | null;
 }
diff --git a/packages/react-native-executorch/src/types/semanticSegmentation.ts b/packages/react-native-executorch/src/types/semanticSegmentation.ts
@@ -184,6 +184,7 @@ export interface SemanticSegmentationType<L extends LabelEnum> {
   runOnFrame:
     | ((
         frame: Frame,
+        cameraPosition: string,
         classesOfInterest?: string[],
         resizeToInput?: boolean
       ) => Record<'ARGMAX', Int32Array> & Record<string, Float32Array>)
diff --git a/yarn.lock b/yarn.lock
@@ -7271,13 +7271,13 @@ __metadata:
     react-native-image-picker: "npm:^7.2.2"
     react-native-loading-spinner-overlay: "npm:^3.0.1"
     react-native-nitro-image: "npm:0.13.0"
-    react-native-nitro-modules: "npm:0.35.0"
+    react-native-nitro-modules: "npm:0.35.2"
     react-native-reanimated: "npm:~4.2.2"
     react-native-safe-area-context: "npm:~5.6.0"
     react-native-screens: "npm:~4.16.0"
     react-native-svg: "npm:15.15.3"
     react-native-svg-transformer: "npm:^1.5.3"
-    react-native-vision-camera: "npm:5.0.0-beta.6"
+    react-native-vision-camera: "npm:5.0.0-beta.7"
     react-native-worklets: "npm:0.7.4"
   languageName: unknown
   linkType: soft
@@ -14486,13 +14486,13 @@ __metadata:
   languageName: node
   linkType: hard
 
-"react-native-nitro-modules@npm:0.35.0":
-  version: 0.35.0
-  resolution: "react-native-nitro-modules@npm:0.35.0"
+"react-native-nitro-modules@npm:0.35.2":
+  version: 0.35.2
+  resolution: "react-native-nitro-modules@npm:0.35.2"
   peerDependencies:
     react: "*"
     react-native: "*"
-  checksum: 10/6c9166a115a03bfc26d3cb9a75761a1fdf33a06bdfb853779539cfe3d7dc2239e242a7fbd4cbb7e9dc0af90a373606fc607fa8a09ef4ff49f7ff29ccff736bbf
+  checksum: 10/22977e5e22c63fb91526c90fa59fcfe5c7f97978c583400f433a1d3ef8ee32fbceb0ad1b009b5bf0d1794e94a11716ad752fd7702c2d67fc808a0256c6e35488
   languageName: node
   linkType: hard
 
@@ -14588,16 +14588,16 @@ __metadata:
   languageName: node
   linkType: hard
 
-"react-native-vision-camera@npm:5.0.0-beta.6":
-  version: 5.0.0-beta.6
-  resolution: "react-native-vision-camera@npm:5.0.0-beta.6"
+"react-native-vision-camera@npm:5.0.0-beta.7":
+  version: 5.0.0-beta.7
+  resolution: "react-native-vision-camera@npm:5.0.0-beta.7"
   peerDependencies:
     react: "*"
     react-native: "*"
     react-native-nitro-image: "*"
     react-native-nitro-modules: "*"
     react-native-worklets: "*"
-  checksum: 10/5c1f3104869e51b173d2bba88a69397c08949314a47a14d70e9ea01b6531c8c74ea6e15f8de4bab6c55eb82c673f8231b6be223d8b4b44a8df433f1cd35c0376
+  checksum: 10/da627aa3c5f5be836ca36f12c98852efc07e82fccbec336da2cffef05fcad9e368c4f20e812fa55229d63f1f2b457f511e96b8debdc17fa2cea4b4205973e1ac
   languageName: node
   linkType: hard
 

Original file line number	Diff line number	Diff line change
`@@ -47,6 +47,14 @@ void transformBbox(float &x1, float &y1, float &x2, float &y2,`
`47`	`47`	`x1 = nx1; y1 = ny1;`
`48`	`48`	`x2 = nx2; y2 = ny2;`
`49`	`49`	`}`
	`50`	`+`
	`51`	`+ // Extra 180° in post-rotation screen space (screen dims are h x w after CW).`
	`52`	`+ if (orient.rotate180) {`
	`53`	`+ float nx1 = h - x2, ny1 = w - y2;`
	`54`	`+ float nx2 = h - x1, ny2 = w - y1;`
	`55`	`+ x1 = nx1; y1 = ny1;`
	`56`	`+ x2 = nx2; y2 = ny2;`
	`57`	`+ }`
`50`	`58`	`}`
`51`	`59`
`52`	`60`	`cv::Mat transformMat(const cv::Mat &mat, const FrameOrientation &orient) {`
`@@ -69,6 +77,10 @@ cv::Mat transformMat(const cv::Mat &mat, const FrameOrientation &orient) {`
`69`	`77`	`cv::rotate(result, result, cv::ROTATE_90_CLOCKWISE);`
`70`	`78`	`}`
`71`	`79`
	`80`	`+ if (orient.rotate180) {`
	`81`	`+ cv::rotate(result, result, cv::ROTATE_180);`
	`82`	`+ }`
	`83`	`+`
`72`	`84`	`return result;`
`73`	`85`	`}`
`74`	`86`