Merge branch 'main' into @ms/add-lfm2.5-instruct

Author: msluszniak

.cspell-wordlist.txt

@@ -21,6 +21,7 @@ microcontrollers
 notimestamps
 seqs
 smollm
+llms
 qwen
 XNNPACK
 EFFICIENTNET
@@ -111,4 +112,7 @@ logprob
 RNFS
 pogodin
 kesha
-antonov
+antonov
+rfdetr
+basemodule
+IMAGENET

.github/workflows/api-reference-up-to-date-check.yml

@@ -0,0 +1,52 @@
+name: Check if API reference is up-to-date
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+      - 'packages/react-native-executorch/src/**'
+      - '.github/workflows/api-reference-up-to-date-check.yml'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+      - 'packages/react-native-executorch/src/**'
+      - '.github/workflows/api-reference-up-to-date-check.yml'
+  workflow_dispatch:
+jobs:
+  check:
+    if: github.repository == 'software-mansion/react-native-executorch'
+    runs-on: ubuntu-latest
+    concurrency:
+      group: api-reference-up-to-date-${{ github.ref }}
+      cancel-in-progress: true
+    env:
+      WORKING_DIRECTORY: docs
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Use Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Install root dependencies
+        run: yarn install --immutable
+      - name: Install node docs dependencies
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+        run: yarn install --immutable
+      - name: Check TypeDoc is up-to-date
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+        run: |
+          yarn docusaurus generate-typedoc
+          yarn prettier-api-reference
+          git status
+          git diff
+          if ! git diff --quiet; then
+            echo "Differences found. Look at the 'git diff' output above."
+            exit 1
+          fi

RELEASE.md

@@ -10,7 +10,7 @@ The release process of new minor version consists of the following steps:
 4. Create a new release branch `release/{MAJOR}.{MINOR}`and push it to the remote.
 5. Stability tests are performed on the release branch and all fixes to the new-found issues are pushed into the main branch and cherry-picked into the release branch. This allows for further development on the main branch without interfering with the release process.
 6. Once all tests are passed, tag the release branch with proper version tag `v{MAJOR}.{MINOR}.0` and run `npm publish`.
-7. Create versioned docs by running from repo root `(cd docs && yarn docusaurus docs:version {MAJOR}.{MINOR}.x)` (the 'x' part is intentional and is not to be substituted). Also, make sure that all the links in `api-reference` are not broken.
+7. Create versioned docs by running from repo root `(cd docs && yarn docs:version {MAJOR}.{MINOR}.x)` (the 'x' part is intentional and is not to be substituted). Also, make sure that all the links in `api-reference` are not broken.
 8. Create a PR with the updated docs.
 9. Create the release notes on GitHub.
 10. Update README.md with release video, if available.

apps/computer-vision/app/image_segmentation/index.tsx

@@ -2,9 +2,8 @@ import Spinner from '../../components/Spinner';
 import { BottomBar } from '../../components/BottomBar';
 import { getImage } from '../../utils';
 import {
-  useImageSegmentation,
   DEEPLAB_V3_RESNET50,
-  DeeplabLabel,
+  useImageSegmentation,
 } from 'react-native-executorch';
 import {
   Canvas,
@@ -44,16 +43,20 @@ const numberToColor: number[][] = [
 ];
 
 export default function ImageSegmentationScreen() {
-  const model = useImageSegmentation({ model: DEEPLAB_V3_RESNET50 });
   const { setGlobalGenerating } = useContext(GeneratingContext);
-  useEffect(() => {
-    setGlobalGenerating(model.isGenerating);
-  }, [model.isGenerating, setGlobalGenerating]);
+  const { isReady, isGenerating, downloadProgress, forward } =
+    useImageSegmentation({
+      model: DEEPLAB_V3_RESNET50,
+    });
   const [imageUri, setImageUri] = useState('');
   const [imageSize, setImageSize] = useState({ width: 0, height: 0 });
   const [segImage, setSegImage] = useState<SkImage | null>(null);
   const [canvasSize, setCanvasSize] = useState({ width: 0, height: 0 });
 
+  useEffect(() => {
+    setGlobalGenerating(isGenerating);
+  }, [isGenerating, setGlobalGenerating]);
+
   const handleCameraPress = async (isCamera: boolean) => {
     const image = await getImage(isCamera);
     if (!image?.uri) return;
@@ -69,12 +72,8 @@ export default function ImageSegmentationScreen() {
     if (!imageUri || imageSize.width === 0 || imageSize.height === 0) return;
     try {
       const { width, height } = imageSize;
-      const output = await model.forward(imageUri, [DeeplabLabel.ARGMAX]);
-      const argmax = output[DeeplabLabel.ARGMAX] || [];
-      const uniqueValues = new Set<number>();
-      for (let i = 0; i < argmax.length; i++) {
-        uniqueValues.add(argmax[i]);
-      }
+      const output = await forward(imageUri, [], true);
+      const argmax = output.ARGMAX || [];
       const pixels = new Uint8Array(width * height * 4);
 
       for (let row = 0; row < height; row++) {
@@ -105,11 +104,11 @@ export default function ImageSegmentationScreen() {
     }
   };
 
-  if (!model.isReady) {
+  if (!isReady) {
     return (
       <Spinner
-        visible={!model.isReady}
-        textContent={`Loading the model ${(model.downloadProgress * 100).toFixed(0)} %`}
+        visible={!isReady}
+        textContent={`Loading the model ${(downloadProgress * 100).toFixed(0)} %`}
       />
     );
   }

docs/docs/03-hooks/01-natural-language-processing/useLLM.md

@@ -192,7 +192,7 @@ To configure model (i.e. change system prompt, load initial conversation history
 
   - [`initialMessageHistory`](../../06-api-reference/interfaces/ChatConfig.md#initialmessagehistory) - Object that represent the conversation history. This can be used to provide initial context to the model.
 
-  - [`contextWindowLength`](../../06-api-reference/interfaces/ChatConfig.md#contextwindowlength) - The number of messages from the current conversation that the model will use to generate a response. Keep in mind that using larger context windows will result in longer inference time and higher memory usage.
+  - [`contextStrategy`](../../06-api-reference/interfaces/ChatConfig.md#contextstrategy) - Object implementing [`ContextStrategy`](../../06-api-reference/interfaces/ContextStrategy.md) interface used to manage conversation context, including trimming history if necessary. Custom strategies can be implemented or one of the built-in options can be used (e.g. [`NoopContextStrategy`](../../06-api-reference/classes/NoopContextStrategy.md), [`MessageCountContextStrategy`](../../06-api-reference/classes/MessageCountContextStrategy.md) or the default [`SlidingWindowContextStrategy`](../../06-api-reference/classes/SlidingWindowContextStrategy.md)).
 
 - [`toolsConfig`](../../06-api-reference/interfaces/LLMConfig.md#toolsconfig) - Object configuring options for enabling and managing tool use. **It will only have effect if your model's chat template support it**. Contains following properties:
   - [`tools`](../../06-api-reference/interfaces/ToolsConfig.md#tools) - List of objects defining tools.

docs/docs/03-hooks/02-computer-vision/useImageSegmentation.md

@@ -21,12 +21,15 @@ import {
   DEEPLAB_V3_RESNET50,
 } from 'react-native-executorch';
 
-const model = useImageSegmentation({ model: DEEPLAB_V3_RESNET50 });
+const model = useImageSegmentation({
+  model: DEEPLAB_V3_RESNET50,
+});
 
 const imageUri = 'file::///Users/.../cute_cat.png';
 
 try {
-  const outputDict = await model.forward(imageUri);
+  const result = await model.forward(imageUri);
+  // result.ARGMAX is an Int32Array of per-pixel class indices
 } catch (error) {
   console.error(error);
 }
@@ -36,9 +39,13 @@ try {
 
 `useImageSegmentation` takes [`ImageSegmentationProps`](../../06-api-reference/interfaces/ImageSegmentationProps.md) that consists of:
 
-- `model` containing [`modelSource`](../../06-api-reference/interfaces/ImageSegmentationProps.md#modelsource).
+- `model` - An object containing:
+  - `modelName` - The name of a built-in model. See [`ModelSources`](../../06-api-reference/type-aliases/ModelSources.md) for the list of supported models.
+  - `modelSource` - The location of the model binary (a URL or a bundled resource).
 - An optional flag [`preventLoad`](../../06-api-reference/interfaces/ImageSegmentationProps.md#preventload) which prevents auto-loading of the model.
 
+The hook is generic over the model config — TypeScript automatically infers the correct label type based on the `modelName` you provide. No explicit generic parameter is needed.
+
 You need more details? Check the following resources:
 
 - For detailed information about `useImageSegmentation` arguments check this section: [`useImageSegmentation` arguments](../../06-api-reference/functions/useImageSegmentation.md#parameters).
@@ -47,45 +54,70 @@ You need more details? Check the following resources:
 
 ### Returns
 
-`useImageSegmentation` returns an object called `ImageSegmentationType` containing bunch of functions to interact with image segmentation models. To get more details please read: [`ImageSegmentationType` API Reference](../../06-api-reference/interfaces/ImageSegmentationType.md).
+`useImageSegmentation` returns an [`ImageSegmentationType`](../../06-api-reference/interfaces/ImageSegmentationType.md) object containing:
+
+- `isReady` - Whether the model is loaded and ready to process images.
+- `isGenerating` - Whether the model is currently processing an image.
+- `error` - An error object if the model failed to load or encountered a runtime error.
+- `downloadProgress` - A value between 0 and 1 representing the download progress of the model binary.
+- `forward` - A function to run inference on an image.
 
 ## Running the model
 
-To run the model, you can use the [`forward`](../../06-api-reference/interfaces/ImageSegmentationType.md#forward) method. It accepts three arguments: a required image - can be a remote URL, a local file URI, or a base64-encoded image (whole URI or only raw base64), an optional list of classes, and an optional flag whether to resize the output to the original dimensions.
+To run the model, use the [`forward`](../../06-api-reference/interfaces/ImageSegmentationType.md#forward) method. It accepts three arguments:
 
-- The image can be a remote URL, a local file URI, or a base64-encoded image.
-- The [`classesOfInterest`](../../06-api-reference/interfaces/ImageSegmentationType.md#classesofinterest) list contains classes for which to output the full results. By default the list is empty, and only the most probable classes are returned (essentially an arg max for each pixel). Look at [`DeeplabLabel`](../../06-api-reference/enumerations/DeeplabLabel.md) enum for possible classes.
-- The [`resizeToInput`](../../06-api-reference/interfaces/ImageSegmentationType.md#resizetoinput) flag specifies whether the output will be rescaled back to the size of the input image. The default is `true`. The model runs inference on a scaled (probably smaller) version of your image (224x224 for `DEEPLAB_V3_RESNET50`). If you choose to resize, the output will be `number[]` of size `width * height` of your original image.
+- [`imageSource`](../../06-api-reference/interfaces/ImageSegmentationType.md#forward) (required) - The image to segment. Can be a remote URL, a local file URI, or a base64-encoded image (whole URI or only raw base64).
+- [`classesOfInterest`](../../06-api-reference/interfaces/ImageSegmentationType.md#forward) (optional) - An array of label keys indicating which per-class probability masks to include in the output. Defaults to `[]` (no class masks). The `ARGMAX` map is always returned regardless of this parameter.
+- [`resizeToInput`](../../06-api-reference/interfaces/ImageSegmentationType.md#forward) (optional) - Whether to resize the output masks to the original input image dimensions. Defaults to `true`. If `false`, returns the raw model output dimensions (e.g. 224x224 for `DEEPLAB_V3_RESNET50`).
 
 :::warning
 Setting `resizeToInput` to `false` will make `forward` faster.
 :::
 
-[`forward`](../../06-api-reference/interfaces/ImageSegmentationType.md#forward) returns a promise which can resolve either to an error or a dictionary containing number arrays with size depending on [`resizeToInput`](../../06-api-reference/interfaces/ImageSegmentationType.md#resizetoinput):
+`forward` returns a promise resolving to an object containing:
+
+- `ARGMAX` - An `Int32Array` where each element is the class index with the highest probability for that pixel.
+- For each label included in `classesOfInterest`, a `Float32Array` of per-pixel probabilities for that class.
 
-- For the key [`DeeplabLabel.ARGMAX`](../../06-api-reference/enumerations/DeeplabLabel.md#argmax) the array contains for each pixel an integer corresponding to the class with the highest probability.
-- For every other key from [`DeeplabLabel`](../../06-api-reference/enumerations/DeeplabLabel.md), if the label was included in [`classesOfInterest`](../../06-api-reference/interfaces/ImageSegmentationType.md#classesofinterest) the dictionary will contain an array of floats corresponding to the probability of this class for every pixel.
+The return type is fully typed — TypeScript narrows it based on the labels you pass in `classesOfInterest`.
 
 ## Example
 
 ```typescript
+import {
+  useImageSegmentation,
+  DEEPLAB_V3_RESNET50,
+  DeeplabLabel,
+} from 'react-native-executorch';
+
 function App() {
-  const model = useImageSegmentation({ model: DEEPLAB_V3_RESNET50 });
+  const model = useImageSegmentation({
+    model: DEEPLAB_V3_RESNET50,
+  });
 
-  // ...
-  const imageUri = 'file::///Users/.../cute_cat.png';
+  const handleSegment = async () => {
+    if (!model.isReady) return;
+
+    const imageUri = 'file::///Users/.../cute_cat.png';
+
+    try {
+      const result = await model.forward(imageUri, ['CAT', 'PERSON'], true);
+
+      // result.ARGMAX — Int32Array of per-pixel class indices
+      // result.CAT — Float32Array of per-pixel probabilities for CAT
+      // result.PERSON — Float32Array of per-pixel probabilities for PERSON
+    } catch (error) {
+      console.error(error);
+    }
+  };
 
-  try {
-    const outputDict = await model.forward(imageUri, [DeeplabLabel.CAT], true);
-  } catch (error) {
-    console.error(error);
-  }
   // ...
 }
 ```
 
 ## Supported models
 
-| Model                                                                                            | Number of classes | Class list                                                          |
-| ------------------------------------------------------------------------------------------------ | ----------------- | ------------------------------------------------------------------- |
-| [deeplabv3_resnet50](https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3) | 21                | [DeeplabLabel](../../06-api-reference/enumerations/DeeplabLabel.md) |
+| Model                                                                                            | Number of classes | Class list                                                                                |
+| ------------------------------------------------------------------------------------------------ | ----------------- | ----------------------------------------------------------------------------------------- |
+| [deeplabv3_resnet50](https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3) | 21                | [DeeplabLabel](../../06-api-reference/enumerations/DeeplabLabel.md)                       |
+| selfie-segmentation                                                                              | 2                 | [SelfieSegmentationLabel](../../06-api-reference/enumerations/SelfieSegmentationLabel.md) |

docs/docs/04-typescript-api/01-natural-language-processing/LLMModule.md

@@ -96,7 +96,7 @@ To configure model (i.e. change system prompt, load initial conversation history
 
   - [`initialMessageHistory`](../../06-api-reference/interfaces/ChatConfig.md#initialmessagehistory) - Object that represent the conversation history. This can be used to provide initial context to the model.
 
-  - [`contextWindowLength`](../../06-api-reference/interfaces/ChatConfig.md#contextwindowlength) - The number of messages from the current conversation that the model will use to generate a response. Keep in mind that using larger context windows will result in longer inference time and higher memory usage.
+  - [`contextStrategy`](../../06-api-reference/interfaces/ChatConfig.md#contextstrategy) - Object implementing [`ContextStrategy`](../../06-api-reference/interfaces/ContextStrategy.md) interface used to manage conversation context, including trimming history if necessary. Custom strategies can be implemented or one of the built-in options can be used (e.g. [`NoopContextStrategy`](../../06-api-reference/classes/NoopContextStrategy.md), [`MessageCountContextStrategy`](../../06-api-reference/classes/MessageCountContextStrategy.md) or the default [`SlidingWindowContextStrategy`](../../06-api-reference/classes/SlidingWindowContextStrategy.md)).
 
 - [`toolsConfig`](../../06-api-reference/interfaces/ToolsConfig.md) - Object configuring options for enabling and managing tool use. **It will only have effect if your model's chat template support it**. Contains following properties:
   - [`tools`](../../06-api-reference/interfaces/ToolsConfig.md#tools) - List of objects defining tools.