From c26e30d330b72fab9127472889d927fdeddc49ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 25 Mar 2026 14:22:07 +0100 Subject: [PATCH 1/6] docs: Add fixes in docs section --- .../02-computer-vision/useInstanceSegmentation.md | 2 ++ .../src/hooks/computer_vision/useInstanceSegmentation.ts | 4 ++-- .../src/types/instanceSegmentation.ts | 4 +++- .../react-native-executorch/src/types/objectDetection.ts | 8 ++++---- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/docs/03-hooks/02-computer-vision/useInstanceSegmentation.md b/docs/docs/03-hooks/02-computer-vision/useInstanceSegmentation.md index 6f0d8de3b9..e12a932543 100644 --- a/docs/docs/03-hooks/02-computer-vision/useInstanceSegmentation.md +++ b/docs/docs/03-hooks/02-computer-vision/useInstanceSegmentation.md @@ -53,6 +53,8 @@ For more information on loading resources, take a look at [loading models](../.. - `error` - An error object if the model failed to load or encountered a runtime error. - `downloadProgress` - A value between 0 and 1 representing the download progress of the model binary. - `forward` - A function to run inference on an image. +- `getAvailableInputSizes` - Returns the available input sizes for the loaded model, or `undefined` if the model accepts only a single input size. Use this to populate UI controls for selecting the input resolution. +- `runOnFrame` - A synchronous worklet function for real-time VisionCamera frame processing. Returns `null` until the model is ready. See [VisionCamera Integration](./visioncamera-integration.md) for usage. ## Running the model diff --git a/packages/react-native-executorch/src/hooks/computer_vision/useInstanceSegmentation.ts b/packages/react-native-executorch/src/hooks/computer_vision/useInstanceSegmentation.ts index 75f9fee72d..36827828d5 100644 --- a/packages/react-native-executorch/src/hooks/computer_vision/useInstanceSegmentation.ts +++ b/packages/react-native-executorch/src/hooks/computer_vision/useInstanceSegmentation.ts @@ -13,10 +13,10 @@ import { useModuleFactory } from '../useModuleFactory'; * React hook for managing an Instance Segmentation model instance. * @typeParam C - A {@link InstanceSegmentationModelSources} config specifying which model to load. * @param props - Configuration object containing `model` config and optional `preventLoad` flag. - * @returns An object with model state (`error`, `isReady`, `isGenerating`, `downloadProgress`) and a typed `forward` function. + * @returns An object with model state (`error`, `isReady`, `isGenerating`, `downloadProgress`), a typed `forward` function, `getAvailableInputSizes` helper, and a `runOnFrame` worklet for VisionCamera integration. * @example * ```ts - * const { isReady, isGenerating, forward, error, downloadProgress } = + * const { isReady, isGenerating, forward, error, downloadProgress, getAvailableInputSizes, runOnFrame } = * useInstanceSegmentation({ * model: { * modelName: 'yolo26n-seg', diff --git a/packages/react-native-executorch/src/types/instanceSegmentation.ts b/packages/react-native-executorch/src/types/instanceSegmentation.ts index 55b52def3b..869f0cdcd7 100644 --- a/packages/react-native-executorch/src/types/instanceSegmentation.ts +++ b/packages/react-native-executorch/src/types/instanceSegmentation.ts @@ -196,7 +196,9 @@ export interface InstanceSegmentationType { * **Use this for VisionCamera frame processing in worklets.** * For async processing, use `forward()` instead. * - * Available after model is loaded (`isReady: true`). + * `null` until the model is ready (`isReady: true`). The property itself is + * `null` when the model has not loaded yet — the function always returns an + * array (never `null`) once called. * @param frame - VisionCamera Frame object * @param isFrontCamera - Whether the front camera is active (for mirroring correction). * @param options - Optional configuration for the segmentation process. diff --git a/packages/react-native-executorch/src/types/objectDetection.ts b/packages/react-native-executorch/src/types/objectDetection.ts index ca33697d93..271676c439 100644 --- a/packages/react-native-executorch/src/types/objectDetection.ts +++ b/packages/react-native-executorch/src/types/objectDetection.ts @@ -6,10 +6,10 @@ export { CocoLabel }; /** * Represents a bounding box for a detected object in an image. * @category Types - * @property {number} x1 - The x-coordinate of the bottom-left corner of the bounding box. - * @property {number} y1 - The y-coordinate of the bottom-left corner of the bounding box. - * @property {number} x2 - The x-coordinate of the top-right corner of the bounding box. - * @property {number} y2 - The y-coordinate of the top-right corner of the bounding box. + * @property {number} x1 - The x-coordinate of the top-left corner of the bounding box. + * @property {number} y1 - The y-coordinate of the top-left corner of the bounding box. + * @property {number} x2 - The x-coordinate of the bottom-right corner of the bounding box. + * @property {number} y2 - The y-coordinate of the bottom-right corner of the bounding box. */ export interface Bbox { x1: number; From 688ecb31fef6db4e583ab18f974701227fafe6d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 25 Mar 2026 15:02:16 +0100 Subject: [PATCH 2/6] chore: add another fixes to instance segmentation --- .../computer-vision/components/ImageWithMasks.tsx | 2 +- .../computer_vision/InstanceSegmentationModule.ts | 15 +++++++-------- .../computer_vision/ObjectDetectionModule.ts | 15 +++++++-------- .../src/modules/computer_vision/VisionModule.ts | 7 ++++--- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/apps/computer-vision/components/ImageWithMasks.tsx b/apps/computer-vision/components/ImageWithMasks.tsx index bd768909b2..0686964df3 100644 --- a/apps/computer-vision/components/ImageWithMasks.tsx +++ b/apps/computer-vision/components/ImageWithMasks.tsx @@ -106,7 +106,7 @@ function createMaskImage( { width: dstW, height: dstH, - alphaType: AlphaType.Premul, + alphaType: AlphaType.Unpremul, colorType: ColorType.RGBA_8888, }, data, diff --git a/packages/react-native-executorch/src/modules/computer_vision/InstanceSegmentationModule.ts b/packages/react-native-executorch/src/modules/computer_vision/InstanceSegmentationModule.ts index 4703a46d51..d469400c71 100644 --- a/packages/react-native-executorch/src/modules/computer_vision/InstanceSegmentationModule.ts +++ b/packages/react-native-executorch/src/modules/computer_vision/InstanceSegmentationModule.ts @@ -276,15 +276,14 @@ export class InstanceSegmentationModule< * Override runOnFrame to add label mapping for VisionCamera integration. * The parent's runOnFrame returns raw native results with class indices; * this override maps them to label strings and provides an options-based API. - * @returns A worklet function for VisionCamera frame processing, or null if the model is not loaded. + * @returns A worklet function for VisionCamera frame processing. + * @throws {RnExecutorchError} If the underlying native worklet is unavailable (should not occur on a loaded module). */ - override get runOnFrame(): - | (( - frame: Frame, - isFrontCamera: boolean, - options?: InstanceSegmentationOptions> - ) => SegmentedInstance>[]) - | null { + override get runOnFrame(): ( + frame: Frame, + isFrontCamera: boolean, + options?: InstanceSegmentationOptions> + ) => SegmentedInstance>[] { const baseRunOnFrame = super.runOnFrame; if (!baseRunOnFrame) { throw new RnExecutorchError( diff --git a/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts b/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts index 4d8bc3ad89..42f450f451 100644 --- a/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts +++ b/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts @@ -144,15 +144,14 @@ export class ObjectDetectionModule< /** * Override runOnFrame to provide an options-based API for VisionCamera integration. - * @returns A worklet function for frame processing, or null if the model is not loaded. + * @returns A worklet function for frame processing. + * @throws {RnExecutorchError} If the underlying native worklet is unavailable (should not occur on a loaded module). */ - override get runOnFrame(): - | (( - frame: any, - isFrontCamera: boolean, - options?: ObjectDetectionOptions> - ) => Detection>[]) - | null { + override get runOnFrame(): ( + frame: any, + isFrontCamera: boolean, + options?: ObjectDetectionOptions> + ) => Detection>[] { const baseRunOnFrame = super.runOnFrame; if (!baseRunOnFrame) { throw new RnExecutorchError( diff --git a/packages/react-native-executorch/src/modules/computer_vision/VisionModule.ts b/packages/react-native-executorch/src/modules/computer_vision/VisionModule.ts index 553a3ab112..f8724b139d 100644 --- a/packages/react-native-executorch/src/modules/computer_vision/VisionModule.ts +++ b/packages/react-native-executorch/src/modules/computer_vision/VisionModule.ts @@ -32,7 +32,7 @@ export abstract class VisionModule extends BaseModule { /** * Synchronous worklet function for real-time VisionCamera frame processing. * - * Only available after the model is loaded. Returns null if not loaded. + * Only available after the model is loaded. * * **Use this for VisionCamera frame processing in worklets.** * For async processing, use `forward()` instead. @@ -55,9 +55,10 @@ export abstract class VisionModule extends BaseModule { * } * }); * ``` - * @returns A worklet function for frame processing, or null if the model is not loaded. + * @returns A worklet function for frame processing. + * @throws {RnExecutorchError} If the model is not loaded. */ - get runOnFrame(): ((frame: Frame, ...args: any[]) => TOutput) | null { + get runOnFrame(): (frame: Frame, ...args: any[]) => TOutput { if (!this.nativeModule) { throw new RnExecutorchError( RnExecutorchErrorCode.ModuleNotLoaded, From 09150b329e930375873b79c7ccdbe7a3d268f149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 25 Mar 2026 15:06:36 +0100 Subject: [PATCH 3/6] chore: revert changes in masks --- apps/computer-vision/components/ImageWithMasks.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/computer-vision/components/ImageWithMasks.tsx b/apps/computer-vision/components/ImageWithMasks.tsx index 0686964df3..bd768909b2 100644 --- a/apps/computer-vision/components/ImageWithMasks.tsx +++ b/apps/computer-vision/components/ImageWithMasks.tsx @@ -106,7 +106,7 @@ function createMaskImage( { width: dstW, height: dstH, - alphaType: AlphaType.Unpremul, + alphaType: AlphaType.Premul, colorType: ColorType.RGBA_8888, }, data, From c58d407031ab3773fbfc6f0024ae18d5a3642796 Mon Sep 17 00:00:00 2001 From: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com> Date: Wed, 25 Mar 2026 15:10:57 +0100 Subject: [PATCH 4/6] Apply suggestion from @mkopcins Co-authored-by: Mateusz Kopcinski <120639731+mkopcins@users.noreply.github.com> --- .../src/modules/computer_vision/ObjectDetectionModule.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts b/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts index 42f450f451..bb166da0c6 100644 --- a/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts +++ b/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts @@ -148,7 +148,7 @@ export class ObjectDetectionModule< * @throws {RnExecutorchError} If the underlying native worklet is unavailable (should not occur on a loaded module). */ override get runOnFrame(): ( - frame: any, + frame: Frame, isFrontCamera: boolean, options?: ObjectDetectionOptions> ) => Detection>[] { From 95b45aaa1204172dd67f0d20197f6606ff3eacfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 25 Mar 2026 15:18:16 +0100 Subject: [PATCH 5/6] chore: fix lint --- .../src/modules/computer_vision/ObjectDetectionModule.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts b/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts index bb166da0c6..31e5751d15 100644 --- a/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts +++ b/packages/react-native-executorch/src/modules/computer_vision/ObjectDetectionModule.ts @@ -1,4 +1,9 @@ -import { LabelEnum, PixelData, ResourceSource } from '../../types/common'; +import { + Frame, + LabelEnum, + PixelData, + ResourceSource, +} from '../../types/common'; import { Detection, ObjectDetectionConfig, From f532f9ebfa98b155b19b765bba608c4f6e647851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 25 Mar 2026 15:23:07 +0100 Subject: [PATCH 6/6] docs: fix mention in runOnFrame --- .../useInstanceSegmentation.md | 2 +- .../02-computer-vision/useObjectDetection.md | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/docs/03-hooks/02-computer-vision/useInstanceSegmentation.md b/docs/docs/03-hooks/02-computer-vision/useInstanceSegmentation.md index e12a932543..a93041c2ae 100644 --- a/docs/docs/03-hooks/02-computer-vision/useInstanceSegmentation.md +++ b/docs/docs/03-hooks/02-computer-vision/useInstanceSegmentation.md @@ -54,7 +54,7 @@ For more information on loading resources, take a look at [loading models](../.. - `downloadProgress` - A value between 0 and 1 representing the download progress of the model binary. - `forward` - A function to run inference on an image. - `getAvailableInputSizes` - Returns the available input sizes for the loaded model, or `undefined` if the model accepts only a single input size. Use this to populate UI controls for selecting the input resolution. -- `runOnFrame` - A synchronous worklet function for real-time VisionCamera frame processing. Returns `null` until the model is ready. See [VisionCamera Integration](./visioncamera-integration.md) for usage. +- `runOnFrame` - A synchronous worklet function for real-time VisionCamera frame processing. See [VisionCamera Integration](./visioncamera-integration.md) for usage. ## Running the model diff --git a/docs/docs/03-hooks/02-computer-vision/useObjectDetection.md b/docs/docs/03-hooks/02-computer-vision/useObjectDetection.md index 56a4c7f023..b2bf9a9de5 100644 --- a/docs/docs/03-hooks/02-computer-vision/useObjectDetection.md +++ b/docs/docs/03-hooks/02-computer-vision/useObjectDetection.md @@ -62,6 +62,7 @@ You need more details? Check the following resources: - `downloadProgress` - A value between 0 and 1 representing the download progress of the model binary. - `forward` - A function to run inference on an image. - `getAvailableInputSizes` - A function that returns available input sizes for multi-method models (YOLO). Returns `undefined` for single-method models. +- `runOnFrame` - A synchronous worklet function for real-time VisionCamera frame processing. See [VisionCamera Integration](./visioncamera-integration.md) for usage. ## Running the model @@ -117,15 +118,15 @@ See the full guide: [VisionCamera Integration](./visioncamera-integration.md). ## Supported models -| Model | Number of classes | Class list | Multi-size Support | -| ----------------------------------------------------------------------------------------------------------------------------- | ----------------- | ------------------------------------------------------------ | ------------------ | -| [SSDLite320 MobileNetV3 Large](https://huggingface.co/software-mansion/react-native-executorch-ssdlite320-mobilenet-v3-large) | 91 | [COCO](../../06-api-reference/enumerations/CocoLabel.md) | No | -| [RF-DETR Nano](https://huggingface.co/software-mansion/react-native-executorch-rf-detr-nano) | 80 | [COCO](../../06-api-reference/enumerations/CocoLabel.md) | No | -| [YOLO26N](https://huggingface.co/software-mansion/react-native-executorch-yolo26) | 80 | [COCO YOLO](../../06-api-reference/enumerations/CocoLabel.md) | Yes (384/512/640) | -| [YOLO26S](https://huggingface.co/software-mansion/react-native-executorch-yolo26) | 80 | [COCO YOLO](../../06-api-reference/enumerations/CocoLabel.md) | Yes (384/512/640) | -| [YOLO26M](https://huggingface.co/software-mansion/react-native-executorch-yolo26) | 80 | [COCO YOLO](../../06-api-reference/enumerations/CocoLabel.md) | Yes (384/512/640) | -| [YOLO26L](https://huggingface.co/software-mansion/react-native-executorch-yolo26) | 80 | [COCO YOLO](../../06-api-reference/enumerations/CocoLabel.md) | Yes (384/512/640) | -| [YOLO26X](https://huggingface.co/software-mansion/react-native-executorch-yolo26) | 80 | [COCO YOLO](../../06-api-reference/enumerations/CocoLabel.md) | Yes (384/512/640) | +| Model | Number of classes | Class list | Multi-size Support | +| ----------------------------------------------------------------------------------------------------------------------------- | ----------------- | ------------------------------------------------------------- | ------------------ | +| [SSDLite320 MobileNetV3 Large](https://huggingface.co/software-mansion/react-native-executorch-ssdlite320-mobilenet-v3-large) | 91 | [COCO](../../06-api-reference/enumerations/CocoLabel.md) | No | +| [RF-DETR Nano](https://huggingface.co/software-mansion/react-native-executorch-rf-detr-nano) | 80 | [COCO](../../06-api-reference/enumerations/CocoLabel.md) | No | +| [YOLO26N](https://huggingface.co/software-mansion/react-native-executorch-yolo26) | 80 | [COCO YOLO](../../06-api-reference/enumerations/CocoLabel.md) | Yes (384/512/640) | +| [YOLO26S](https://huggingface.co/software-mansion/react-native-executorch-yolo26) | 80 | [COCO YOLO](../../06-api-reference/enumerations/CocoLabel.md) | Yes (384/512/640) | +| [YOLO26M](https://huggingface.co/software-mansion/react-native-executorch-yolo26) | 80 | [COCO YOLO](../../06-api-reference/enumerations/CocoLabel.md) | Yes (384/512/640) | +| [YOLO26L](https://huggingface.co/software-mansion/react-native-executorch-yolo26) | 80 | [COCO YOLO](../../06-api-reference/enumerations/CocoLabel.md) | Yes (384/512/640) | +| [YOLO26X](https://huggingface.co/software-mansion/react-native-executorch-yolo26) | 80 | [COCO YOLO](../../06-api-reference/enumerations/CocoLabel.md) | Yes (384/512/640) | :::tip YOLO models support multiple input sizes (384px, 512px, 640px). Smaller sizes are faster but less accurate, while larger sizes are more accurate but slower. Choose based on your speed/accuracy requirements.