Add docs and fix some typechecks

benITo47 · benITo47 · commit cc6484cc2f59 · 2026-03-02T14:43:07.000+01:00
diff --git a/apps/computer-vision/app/instance_segmentation/index.tsx b/apps/computer-vision/app/instance_segmentation/index.tsx
@@ -1,7 +1,14 @@
 import Spinner from '../../components/Spinner';
 import { BottomBar } from '../../components/BottomBar';
 import { getImage } from '../../utils';
-import { useInstanceSegmentation } from 'react-native-executorch';
+import {
+  useInstanceSegmentation,
+  YOLO26N_SEG,
+  YOLO26L_SEG,
+  YOLO26M_SEG,
+  YOLO26S_SEG,
+  YOLO26X_SEG,
+} from 'react-native-executorch';
 import {
   Canvas,
   Image as SkiaImage,
@@ -41,15 +48,54 @@ const instanceColors = [
 // Available input sizes for YOLO models
 const AVAILABLE_INPUT_SIZES = [384, 416, 512, 640, 1024];
 
+/**
+ * EXAMPLE: Using built-in YOLO models with COCO labels (80 classes)
+ *
+ * const { forward } = useInstanceSegmentation({
+ *   model: YOLO26N_SEG, // or YOLO26S_SEG, YOLO26M_SEG, etc.
+ * });
+ *
+ * // Filter by specific COCO classes:
+ * const results = await forward(imageUri, {
+ *   classesOfInterest: ['PERSON', 'CAR', 'DOG'],
+ *   confidenceThreshold: 0.5,
+ *   inputSize: 640,
+ * });
+ *
+ *
+ * EXAMPLE: Using a custom model with custom labels
+ *
+ * // 1. Define your custom label enum
+ * const MyLabels = { APPLE: 0, ORANGE: 1, BANANA: 2 } as const;
+ *
+ * // 2. Load using InstanceSegmentationModule directly (not the hook)
+ * const customModel = await InstanceSegmentationModule.fromCustomConfig(
+ *   'https://example.com/my-model.pte',
+ *   {
+ *     labelMap: MyLabels,
+ *     availableInputSizes: [640],
+ *     defaultInputSize: 640,
+ *     postprocessorConfig: {
+ *       type: 'yolo',
+ *       defaultConfidenceThreshold: 0.5,
+ *       defaultIouThreshold: 0.45,
+ *       applyNMS: true,
+ *     },
+ *   }
+ * );
+ *
+ * // 3. Run inference
+ * const results = await customModel.forward(imageUri, {
+ *   classesOfInterest: ['APPLE', 'BANANA'],
+ * });
+ */
+
 export default function InstanceSegmentationScreen() {
   const { setGlobalGenerating } = useContext(GeneratingContext);
 
   const { isReady, isGenerating, downloadProgress, forward, error } =
     useInstanceSegmentation({
-      model: {
-        modelName: 'yolo26n-seg',
-        modelSource: 'http://192.168.83.59:3000/yolo26n-seg.pte',
-      },
+      model: YOLO26X_SEG,
     });
 
   const [imageUri, setImageUri] = useState('');
@@ -81,7 +127,7 @@ export default function InstanceSegmentationScreen() {
     try {
       const output = await forward(imageUri, {
         confidenceThreshold: 0.5,
-        iouThreshold: 0.45,
+        iouThreshold: 0.55,
         maxInstances: 20,
         returnMaskAtOriginalResolution: true,
         inputSize: selectedInputSize,
diff --git a/docs/docs/02-benchmarks/inference-time.md b/docs/docs/02-benchmarks/inference-time.md
@@ -127,6 +127,23 @@ Times presented in the tables are measured as consecutive runs of the model. Ini
 | ----------------- | ---------------------------- | -------------------------------- | --------------------------------- |
 | DEELABV3_RESNET50 | 1000                         | 670                              | 700                               |
 
+## Instance Segmentation
+
+:::warning
+Times presented in the tables are measured as consecutive runs of the model. Initial run times may be up to 2x longer due to model loading and initialization.
+:::
+:::warning
+Times presented in the tables are measured for forward method with input size equal to 512. Other input sizes may yeild slower or faster inference times.
+:::
+
+| Model       | Samsung Galaxy S24 (XNNPACK) [ms] | Iphone 17 pro (XNNPACK) [ms] |
+| ----------- | --------------------------------- | ---------------------------- |
+| YOLO26n_SEG | 92                                | 90                           |
+| YOLO26s_SEG | 220                               | 188                          |
+| YOLO26m_SEG | 570                               | 550                          |
+| YOLO26l_SEG | 680                               | 608                          |
+| YOLO26x_SEG | 1410                              | 1338                         |
+
 ## Text to image
 
 | Model                 | iPhone 17 Pro (XNNPACK) [ms] | iPhone 16 Pro (XNNPACK) [ms] | iPhone SE 3 (XNNPACK) [ms] | Samsung Galaxy S24 (XNNPACK) [ms] | OnePlus 12 (XNNPACK) [ms] |
diff --git a/docs/docs/02-benchmarks/memory-usage.md b/docs/docs/02-benchmarks/memory-usage.md
@@ -93,6 +93,20 @@ Data presented in the following sections is based on inference with non-resized
 | ----------------- | ---------------------- | ------------------ |
 | DEELABV3_RESNET50 | 930                    | 660                |
 
+## Instance Segmentation
+
+:::warning
+Data presented in the following sections is based on inference with forward_1024 method.
+:::
+
+| Model       | Android (XNNPACK) [MB] | iOS (XNNPACK) [MB] |
+| ----------- | ---------------------- | ------------------ |
+| YOLO26n_SEG | 92                     | 668                |
+| YOLO26s_SEG | 220                    | 712                |
+| YOLO26m_SEG | 570                    | 815                |
+| YOLO26l_SEG | 680                    | 1024               |
+| YOLO26x_SEG | 1410                   | 1450               |
+
 ## Text to image
 
 | Model                 | Android (XNNPACK) [MB] | iOS (XNNPACK) [MB] |
diff --git a/packages/react-native-executorch/src/constants/modelUrls.ts b/packages/react-native-executorch/src/constants/modelUrls.ts
@@ -657,6 +657,55 @@ export const SELFIE_SEGMENTATION = {
   modelSource: SELFIE_SEGMENTATION_MODEL,
 } as const;
 
+/**
+ * @category Models - Instance Segmentation
+ */
+const YOLO26N_SEG_MODEL = `${URL_PREFIX}-yolo26-seg/${NEXT_VERSION_TAG}/yolo26n-seg/xnnpack/yolo26n-seg.pte`;
+const YOLO26S_SEG_MODEL = `${URL_PREFIX}-yolo26-seg/${NEXT_VERSION_TAG}/yolo26s-seg/xnnpack/yolo26s-seg.pte`;
+const YOLO26M_SEG_MODEL = `${URL_PREFIX}-yolo26-seg/${NEXT_VERSION_TAG}/yolo26m-seg/xnnpack/yolo26m-seg.pte`;
+const YOLO26L_SEG_MODEL = `${URL_PREFIX}-yolo26-seg/${NEXT_VERSION_TAG}/yolo26l-seg/xnnpack/yolo26l-seg.pte`;
+const YOLO26X_SEG_MODEL = `${URL_PREFIX}-yolo26-seg/${NEXT_VERSION_TAG}/yolo26x-seg/xnnpack/yolo26x-seg.pte`;
+
+/**
+ * @category Models - Instance Segmentation
+ */
+export const YOLO26N_SEG = {
+  modelName: 'yolo26n-seg',
+  modelSource: YOLO26N_SEG_MODEL,
+} as const;
+
+/**
+ * @category Models - Instance Segmentation
+ */
+export const YOLO26S_SEG = {
+  modelName: 'yolo26s-seg',
+  modelSource: YOLO26S_SEG_MODEL,
+} as const;
+
+/**
+ * @category Models - Instance Segmentation
+ */
+export const YOLO26M_SEG = {
+  modelName: 'yolo26m-seg',
+  modelSource: YOLO26M_SEG_MODEL,
+} as const;
+
+/**
+ * @category Models - Instance Segmentation
+ */
+export const YOLO26L_SEG = {
+  modelName: 'yolo26l-seg',
+  modelSource: YOLO26L_SEG_MODEL,
+} as const;
+
+/**
+ * @category Models - Instance Segmentation
+ */
+export const YOLO26X_SEG = {
+  modelName: 'yolo26x-seg',
+  modelSource: YOLO26X_SEG_MODEL,
+} as const;
+
 // Image Embeddings
 const CLIP_VIT_BASE_PATCH32_IMAGE_MODEL = `${URL_PREFIX}-clip-vit-base-patch32/${VERSION_TAG}/clip-vit-base-patch32-vision_xnnpack.pte`;
 
diff --git a/packages/react-native-executorch/src/hooks/computer_vision/useInstanceSegmentation.ts b/packages/react-native-executorch/src/hooks/computer_vision/useInstanceSegmentation.ts
@@ -47,9 +47,7 @@ export const useInstanceSegmentation = <
 >({
   model,
   preventLoad = false,
-}: InstanceSegmentationProps<C>): InstanceSegmentationType<
-  InstanceSegmentationLabels<InstanceModelNameOf<C>>
-> => {
+}: InstanceSegmentationProps<C>): InstanceSegmentationType<LabelEnum> => {
   const [error, setError] = useState<RnExecutorchError | null>(null);
   const [isReady, setIsReady] = useState(false);
   const [isGenerating, setIsGenerating] = useState(false);
@@ -96,9 +94,7 @@ export const useInstanceSegmentation = <
 
   const forward = async (
     imageSource: string,
-    options?: InstanceSegmentationOptions<
-      InstanceSegmentationLabels<InstanceModelNameOf<C>>
-    >
+    options?: InstanceSegmentationOptions<LabelEnum>
   ) => {
     if (!isReady || !instance) {
       throw new RnExecutorchError(
diff --git a/packages/react-native-executorch/src/modules/computer_vision/InstanceSegmentationModule.ts b/packages/react-native-executorch/src/modules/computer_vision/InstanceSegmentationModule.ts
@@ -31,10 +31,7 @@ const ModelConfigs = {
   'yolo26m-seg': YOLO_SEG_CONFIG,
   'yolo26l-seg': YOLO_SEG_CONFIG,
   'yolo26x-seg': YOLO_SEG_CONFIG,
-} as const satisfies Record<
-  InstanceSegmentationModelName,
-  InstanceSegmentationConfig<LabelEnum>
->;
+} as const;
 
 /** @internal */
 type ModelConfigsType = typeof ModelConfigs;
@@ -90,11 +87,11 @@ export class InstanceSegmentationModule<
   T extends InstanceSegmentationModelName | LabelEnum,
 > extends BaseModule {
   private labelMap: ResolveLabels<T>;
-  private modelConfig: InstanceSegmentationConfig<ResolveLabels<T>>;
+  private modelConfig: InstanceSegmentationConfig<LabelEnum>;
 
   private constructor(
     labelMap: ResolveLabels<T>,
-    modelConfig: InstanceSegmentationConfig<ResolveLabels<T>>,
+    modelConfig: InstanceSegmentationConfig<LabelEnum>,
     nativeModule: unknown
   ) {
     super();
@@ -127,7 +124,7 @@ export class InstanceSegmentationModule<
     onDownloadProgress: (progress: number) => void = () => {}
   ): Promise<InstanceSegmentationModule<InstanceModelNameOf<C>>> {
     const { modelName, modelSource } = config;
-    const modelConfig = ModelConfigs[modelName];
+    const modelConfig = ModelConfigs[modelName as keyof typeof ModelConfigs];
 
     const paths = await ResourceFetcher.fetch(onDownloadProgress, modelSource);
     if (!paths?.[0]) {
@@ -155,9 +152,7 @@ export class InstanceSegmentationModule<
 
     return new InstanceSegmentationModule<InstanceModelNameOf<C>>(
       modelConfig.labelMap as ResolveLabels<InstanceModelNameOf<C>>,
-      modelConfig as InstanceSegmentationConfig<
-        ResolveLabels<InstanceModelNameOf<C>>
-      >,
+      modelConfig,
       nativeModule
     );
   }
@@ -221,7 +216,7 @@ export class InstanceSegmentationModule<
 
     return new InstanceSegmentationModule<L>(
       config.labelMap as ResolveLabels<L>,
-      config as InstanceSegmentationConfig<ResolveLabels<L>>,
+      config,
       nativeModule
     );
   }
@@ -298,6 +293,8 @@ export class InstanceSegmentationModule<
         })
       : [];
 
+    // Measure inference time
+    const startTime = performance.now();
     const nativeResult = await this.nativeModule.generate(
       imageSource,
       confidenceThreshold,
@@ -307,10 +304,18 @@ export class InstanceSegmentationModule<
       returnMaskAtOriginalResolution,
       inputSize // Pass inputSize as number instead of methodName as string
     );
+    const endTime = performance.now();
+    const inferenceTime = endTime - startTime;
+
+    console.log(
+      `[Instance Segmentation] Inference completed in ${inferenceTime.toFixed(2)}ms | Input size: ${inputSize}x${inputSize} | Detected: ${nativeResult.length} instances`
+    );
 
     // Convert label indices back to label names
     // YOLO outputs 0-indexed class IDs, but COCO labels are 1-indexed, so add 1
-    const reverseLabelMap = Object.entries(this.labelMap).reduce(
+    const reverseLabelMap = Object.entries(
+      this.labelMap as Record<string, number>
+    ).reduce(
       (acc, [key, value]) => {
         acc[value as number] = key;
         return acc;
diff --git a/packages/react-native-executorch/src/modules/computer_vision/SemanticSegmentationModule.ts b/packages/react-native-executorch/src/modules/computer_vision/SemanticSegmentationModule.ts
@@ -185,13 +185,6 @@ export class SemanticSegmentationModule<
     );
   }
 
-  InstanceSegConfig = {
-    'yolo26n-seg': {
-      labelMap: LabelEnum,
-      prepocessorConfig: undefined,
-      availableInputSizes: [Number],
-    },
-  };
   /**
    * Executes the model's forward pass to perform semantic segmentation on the provided image.
    *
diff --git a/packages/react-native-executorch/src/types/instanceSegmentation.ts b/packages/react-native-executorch/src/types/instanceSegmentation.ts
@@ -1,6 +1,6 @@
 import { RnExecutorchError } from '../errors/errorUtils';
 import { LabelEnum, ResourceSource } from './common';
-import { Bbox, CocoLabel } from './objectDetection';
+import { Bbox } from './objectDetection';
 
 /**
  * Represents a single detected instance in instance segmentation output.
@@ -133,7 +133,6 @@ export type InstanceSegmentationModelSources =
   | { modelName: 'yolo26s-seg'; modelSource: ResourceSource }
   | { modelName: 'yolo26m-seg'; modelSource: ResourceSource }
   | { modelName: 'yolo26l-seg'; modelSource: ResourceSource }
-  | { modelName: 'rfdetr_seg'; modelSource: ResourceSource }
   | { modelName: 'yolo26x-seg'; modelSource: ResourceSource };
 
 /**
@@ -145,20 +144,12 @@ export type InstanceSegmentationModelName =
   InstanceSegmentationModelSources['modelName'];
 
 /**
- * Extracts the model name from a {@link InstanceSegmentationModelSources} config object.
- *
- * @category Types
- */
-export type ModelNameOf<C extends InstanceSegmentationModelSources> =
-  C['modelName'];
-
-/**
- * Alias for {@link ModelNameOf}. Extracts the instance segmentation model name from a config object.
+ * Extracts the instance segmentation model name from a {@link InstanceSegmentationModelSources} config object.
  *
  * @category Types
  */
 export type InstanceModelNameOf<C extends InstanceSegmentationModelSources> =
-  ModelNameOf<C>;
+  C['modelName'];
 
 /**
  * Props for the `useInstanceSegmentation` hook.

Original file line number	Diff line number	Diff line change
`@@ -185,13 +185,6 @@ export class SemanticSegmentationModule<`
`185`	`185`	`);`
`186`	`186`	`}`
`187`	`187`
`188`		`- InstanceSegConfig = {`
`189`		`- 'yolo26n-seg': {`
`190`		`- labelMap: LabelEnum,`
`191`		`- prepocessorConfig: undefined,`
`192`		`- availableInputSizes: [Number],`
`193`		`- },`
`194`		`- };`
`195`	`188`	`/**`
`196`	`189`	`* Executes the model's forward pass to perform semantic segmentation on the provided image.`
`197`	`190`	`*`