feat: this is the way

Author: NorbertKlockiewicz

apps/computer-vision/app/vision_camera/index.tsx

@@ -95,6 +95,7 @@ const TASKS: Task[] = [
 // Module-level consts so worklets in task components can always reference the same stable objects.
 // Never replaced — only mutated via setBlocking to avoid closure staleness.
 const frameKillSwitch = createSynchronizable(false);
+const cameraPositionSync = createSynchronizable<'front' | 'back'>('back');
 
 export default function VisionCameraScreen() {
   const insets = useSafeAreaInsets();
@@ -135,6 +136,10 @@ export default function VisionCameraScreen() {
     return () => clearTimeout(id);
   }, [activeModel]);
 
+  useEffect(() => {
+    cameraPositionSync.setBlocking(cameraPosition);
+  }, [cameraPosition]);
+
   const handleFpsChange = useCallback((newFps: number, newMs: number) => {
     setFps(newFps);
     setFrameMs(newMs);
@@ -177,6 +182,7 @@ export default function VisionCameraScreen() {
   const taskProps = {
     activeModel,
     canvasSize,
+    cameraPositionSync,
     frameKillSwitch,
     onFrameOutputChange: setFrameOutput,
     onReadyChange: setIsReady,
@@ -195,7 +201,7 @@ export default function VisionCameraScreen() {
         outputs={frameOutput ? [frameOutput] : []}
         isActive={isFocused}
         format={format}
-        orientationSource="interface"
+        orientationSource="device"
       />
 
       {/* Layout sentinel — measures the full-screen area for bbox/canvas sizing */}

apps/computer-vision/components/vision_camera/tasks/ObjectDetectionTask.tsx

@@ -18,6 +18,7 @@ type Props = TaskProps & { activeModel: ObjModelId };
 export default function ObjectDetectionTask({
   activeModel,
   canvasSize,
+  cameraPositionSync,
   frameKillSwitch,
   onFrameOutputChange,
   onReadyChange,
@@ -80,8 +81,8 @@ export default function ObjectDetectionTask({
         }
         try {
           if (!detRof) return;
-          const result = detRof(frame, 0.5);
-          // C++ maps coords to screen space (portrait: frameH × frameW)
+          const isMirrored = cameraPositionSync.getDirty() === 'front';
+          const result = detRof(frame, isMirrored, 0.5);
           const screenW = frame.height;
           const screenH = frame.width;
           if (result) {
@@ -97,7 +98,7 @@ export default function ObjectDetectionTask({
           frame.dispose();
         }
       },
-      [detRof, frameKillSwitch, updateDetections]
+      [cameraPositionSync, detRof, frameKillSwitch, updateDetections]
     ),
   });
 

apps/computer-vision/components/vision_camera/tasks/SegmentationTask.tsx

@@ -37,6 +37,7 @@ type Props = TaskProps & { activeModel: SegModelId };
 export default function SegmentationTask({
   activeModel,
   canvasSize,
+  cameraPositionSync,
   frameKillSwitch,
   onFrameOutputChange,
   onReadyChange,
@@ -148,7 +149,8 @@ export default function SegmentationTask({
         }
         try {
           if (!segRof) return;
-          const result = segRof(frame, [], false);
+          const isMirrored = cameraPositionSync.getDirty() === 'front';
+          const result = segRof(frame, isMirrored, [], false);
           if (result?.ARGMAX) {
             const argmax: Int32Array = result.ARGMAX;
             const side = Math.round(Math.sqrt(argmax.length));
@@ -179,7 +181,7 @@ export default function SegmentationTask({
           frame.dispose();
         }
       },
-      [colors, frameKillSwitch, segRof, updateMask]
+      [cameraPositionSync, colors, frameKillSwitch, segRof, updateMask]
     ),
   });
 

apps/computer-vision/components/vision_camera/tasks/types.ts

@@ -4,6 +4,7 @@ import { createSynchronizable } from 'react-native-worklets';
 export type TaskProps = {
   activeModel: string;
   canvasSize: { width: number; height: number };
+  cameraPositionSync: ReturnType<typeof createSynchronizable<'front' | 'back'>>;
   frameKillSwitch: ReturnType<typeof createSynchronizable<boolean>>;
   onFrameOutputChange: (frameOutput: ReturnType<typeof useFrameOutput>) => void;
   onReadyChange: (isReady: boolean) => void;

packages/react-native-executorch/common/rnexecutorch/utils/FrameTransform.cpp

@@ -91,7 +91,21 @@ void inverseRotateBbox(float &x1, float &y1, float &x2, float &y2,
     x2 = nx2;
     y2 = ny2;
   }
-  // "left": no-op
+  // "left": no-op (coords already in screen space)
+
+#if defined(__APPLE__)
+  if (orient.isMirrored) {
+    // After CW/CCW rotation ("up"/"down") screen dims are swapped: rH × rW.
+    // After no-op/180° ("left"/"right") screen dims are unchanged: rW × rH.
+    bool swapped = (orient.orientation == "up" || orient.orientation == "down");
+    float sw = swapped ? h : w;
+    float sh = swapped ? w : h;
+    float nx1 = sw - x2, ny1 = sh - y2;
+    float nx2 = sw - x1, ny2 = sh - y1;
+    x1 = nx1; y1 = ny1;
+    x2 = nx2; y2 = ny2;
+  }
+#endif
 }
 
 cv::Mat inverseRotateMat(const cv::Mat &mat, const FrameOrientation &orient) {
@@ -103,7 +117,13 @@ cv::Mat inverseRotateMat(const cv::Mat &mat, const FrameOrientation &orient) {
   } else if (orient.orientation == "down") {
     cv::rotate(result, result, cv::ROTATE_90_COUNTERCLOCKWISE);
   }
-  // "left": no-op
+  // "left": no-op (mask already in screen space)
+
+#if defined(__APPLE__)
+  if (orient.isMirrored) {
+    cv::rotate(result, result, cv::ROTATE_180);
+  }
+#endif
   return result;
 }
 

packages/react-native-executorch/src/modules/computer_vision/VisionModule.ts

@@ -70,7 +70,7 @@ export abstract class VisionModule<TOutput> extends BaseModule {
     const nativeGenerateFromFrame = this.nativeModule.generateFromFrame;
 
     // Return worklet that captures ONLY the JSI function
-    return (frame: any, ...args: any[]): TOutput => {
+    return (frame: any, isMirrored: boolean, ...args: any[]): TOutput => {
       'worklet';
 
       let nativeBuffer: any = null;
@@ -79,7 +79,7 @@ export abstract class VisionModule<TOutput> extends BaseModule {
         const frameData = {
           nativeBuffer: nativeBuffer.pointer,
           orientation: frame.orientation,
-          isMirrored: frame.isMirrored,
+          isMirrored,
           frameWidth: frame.width,
           frameHeight: frame.height,
         };

packages/react-native-executorch/src/types/objectDetection.ts

@@ -143,6 +143,10 @@ export interface ObjectDetectionType<L extends LabelEnum> {
    * @returns Array of Detection objects representing detected items in the frame.
    */
   runOnFrame:
-    | ((frame: Frame, detectionThreshold: number) => Detection<L>[])
+    | ((
+        frame: Frame,
+        isMirrored: boolean,
+        detectionThreshold: number
+      ) => Detection<L>[])
     | null;
 }

packages/react-native-executorch/src/types/semanticSegmentation.ts

@@ -184,6 +184,7 @@ export interface SemanticSegmentationType<L extends LabelEnum> {
   runOnFrame:
     | ((
         frame: Frame,
+        isMirrored: boolean,
         classesOfInterest?: string[],
         resizeToInput?: boolean
       ) => Record<'ARGMAX', Int32Array> & Record<string, Float32Array>)