Merge branch 'main' into exir-flatbuffer-serialize-fastpath_v2

Author: JacobSzwejbka

.ci/docker/ci_commit_pins/optimum-executorch.txt

@@ -1 +1 @@
-5bf1aeb587e9b1f3572b0bd60265c5dafd007b73
+a9592258daacad7423fd5f39aaa59c6e36471520

.ci/scripts/export_model_artifact.sh

@@ -34,7 +34,7 @@ Arguments:
 
   output_dir   Output directory for artifacts (optional, default: current directory)
 
-  mode         Export mode (optional, default: auto-detect based on model and device)
+  mode         Export mode (optional, default: vr-streaming)
                Supported modes:
                  - vr-streaming: Voxtral Realtime streaming mode
                  - vr-offline: Voxtral Realtime offline mode
@@ -141,6 +141,14 @@ case "$HF_MODEL" in
     PREPROCESSOR_FEATURE_SIZE=""
     PREPROCESSOR_OUTPUT=""
     ;;
+  Qwen/Qwen3-0.6B)
+    MODEL_NAME="qwen3"
+    TASK="text-generation"
+    MAX_SEQ_LEN="64"
+    EXTRA_PIP=""
+    PREPROCESSOR_FEATURE_SIZE=""
+    PREPROCESSOR_OUTPUT=""
+    ;;
   nvidia/parakeet-tdt)
     MODEL_NAME="parakeet"
     TASK=""
@@ -159,7 +167,7 @@ case "$HF_MODEL" in
     ;;
   *)
     echo "Error: Unsupported model '$HF_MODEL'"
-    echo "Supported models: mistralai/Voxtral-Mini-3B-2507, mistralai/Voxtral-Mini-4B-Realtime-2602, openai/whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo}, google/gemma-3-4b-it, nvidia/parakeet-tdt"
+    echo "Supported models: mistralai/Voxtral-Mini-3B-2507, mistralai/Voxtral-Mini-4B-Realtime-2602, openai/whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo}, google/gemma-3-4b-it, Qwen/Qwen3-0.6B, nvidia/parakeet-tdt"
     exit 1
     ;;
 esac
@@ -256,16 +264,9 @@ if [ "$MODEL_NAME" = "voxtral_realtime" ]; then
   fi
 
   # Determine streaming mode based on MODE parameter
-  USE_STREAMING="false"
-  if [ "$MODE" = "vr-streaming" ]; then
-    USE_STREAMING="true"
-  elif [ "$MODE" = "vr-offline" ]; then
+  USE_STREAMING="true"
+  if [ "$MODE" = "vr-offline" ]; then
     USE_STREAMING="false"
-  elif [ -z "$MODE" ]; then
-    # Auto-detect: XNNPACK uses streaming, others use offline
-    if [ "$DEVICE" = "xnnpack" ]; then
-      USE_STREAMING="true"
-    fi
   fi
 
   # Configure export and preprocessor based on streaming mode

.ci/scripts/test_huggingface_optimum_model.py

@@ -142,21 +142,50 @@ def test_text_generation(model_id, model_dir, recipe, *, quantize=True, run_only
                 "--qembedding",
                 "8w",
             ]
+    elif recipe == "cuda":
+        command += [
+            "--dtype",
+            "bfloat16",
+            "--device",
+            "cuda",
+        ]
+        if quantize:
+            command += [
+                "--qlinear",
+                "4w",
+                "--qlinear_packing_format",
+                "tile_packed_to_4d",
+                "--qembedding",
+                "8w",
+            ]
     else:
         assert (
             not quantize
-        ), "Quantization is only supported for XnnPack and CoreML recipes at the moment."
+        ), "Quantization is only supported for XnnPack, CoreML, and CUDA recipes at the moment."
 
     if not run_only:
         cli_export(command, model_dir)
 
+    if recipe == "cuda":
+        model_path = Path(model_dir) / "model.pte"
+        cuda_blob_path = Path(model_dir) / "aoti_cuda_blob.ptd"
+        assert model_path.exists(), f"Main model file not found: {model_path}"
+        assert cuda_blob_path.exists(), f"CUDA blob not found: {cuda_blob_path}"
+
     tokenizer = AutoTokenizer.from_pretrained(model_id)
     saved_files = tokenizer.save_pretrained(model_dir)
     tokenizer_path = get_tokenizer_path(model_dir, saved_files)
 
     from executorch.extension.llm.runner import GenerationConfig, TextLLMRunner
 
-    runner = TextLLMRunner(f"{model_dir}/model.pte", tokenizer_path)
+    if recipe == "cuda":
+        runner = TextLLMRunner(
+            f"{model_dir}/model.pte",
+            tokenizer_path,
+            f"{model_dir}/aoti_cuda_blob.ptd",
+        )
+    else:
+        runner = TextLLMRunner(f"{model_dir}/model.pte", tokenizer_path)
     tokens = []
     runner.generate(
         "Simply put, the theory of relativity states that",

.ci/scripts/test_lora.sh

@@ -12,7 +12,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 cmake_install_executorch_libraries() {
     echo "Installing libexecutorch.a, libextension_module.so, libportable_ops_lib.a"
     rm -rf cmake-out
-    cmake --workflow llm-release
+    cmake --preset llm-release -DEXECUTORCH_ENABLE_LOGGING=ON
+    cmake --build --preset llm-release-install
 }
 
 cmake_build_llama_runner() {

.ci/scripts/test_lora_multimethod.sh

@@ -12,7 +12,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 cmake_install_executorch_libraries() {
     echo "Installing libexecutorch.a, libextension_module.so, libportable_ops_lib.a"
     rm -rf cmake-out
-    cmake --workflow llm-release
+    cmake --preset llm-release -DEXECUTORCH_ENABLE_LOGGING=ON
+    cmake --build --preset llm-release-install
 }
 
 cmake_build_llama_runner() {

.ci/scripts/test_model_e2e.sh

@@ -21,6 +21,7 @@ Arguments:
                 - mistralai/Voxtral-Mini-3B-2507
                 - openai/whisper series (whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo})
                 - google/gemma-3-4b-it
+                - Qwen/Qwen3-0.6B
                 - nvidia/parakeet-tdt
                 - mistralai/Voxtral-Mini-4B-Realtime-2602
 
@@ -151,6 +152,18 @@ case "$HF_MODEL" in
     AUDIO_FILE=""
     IMAGE_PATH="docs/source/_static/img/et-logo.png"
     ;;
+  Qwen/Qwen3-0.6B)
+    MODEL_NAME="qwen3"
+    RUNNER_TARGET="llama_main"
+    RUNNER_PATH="llama"
+    EXPECTED_OUTPUT="Paris"
+    PREPROCESSOR=""
+    TOKENIZER_URL="https://huggingface.co/Qwen/Qwen3-0.6B/resolve/main" # @lint-ignore
+    TOKENIZER_FILE=""
+    AUDIO_URL=""
+    AUDIO_FILE=""
+    IMAGE_PATH=""
+    ;;
   nvidia/parakeet-tdt)
     MODEL_NAME="parakeet"
     RUNNER_TARGET="parakeet_runner"
@@ -177,7 +190,7 @@ case "$HF_MODEL" in
     ;;
   *)
     echo "Error: Unsupported model '$HF_MODEL'"
-    echo "Supported models: mistralai/Voxtral-Mini-3B-2507, mistralai/Voxtral-Mini-4B-Realtime-2602, openai/whisper series (whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo}), google/gemma-3-4b-it, nvidia/parakeet-tdt"
+    echo "Supported models: mistralai/Voxtral-Mini-3B-2507, mistralai/Voxtral-Mini-4B-Realtime-2602, openai/whisper series (whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo}), google/gemma-3-4b-it, Qwen/Qwen3-0.6B, nvidia/parakeet-tdt"
     exit 1
     ;;
 esac
@@ -246,9 +259,14 @@ if [ "$(uname -s)" = "Darwin" ] && [ -f "$RUNNER_BIN" ]; then
     install_name_tool -change /opt/llvm-openmp/lib/libomp.dylib @rpath/libomp.dylib "$RUNNER_BIN"
   fi
 fi
-# For CUDA, add data_path argument (Metal embeds data in .pte)
+# For CUDA, add named data argument (Metal embeds data in .pte).
+# Llama runner uses --data_paths, other runners use --data_path.
 if [ "$DEVICE" = "cuda" ]; then
-  RUNNER_ARGS="$RUNNER_ARGS --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd"
+  if [ "$RUNNER_PATH" = "llama" ]; then
+    RUNNER_ARGS="$RUNNER_ARGS --data_paths ${MODEL_DIR}/aoti_cuda_blob.ptd"
+  else
+    RUNNER_ARGS="$RUNNER_ARGS --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd"
+  fi
 fi
 
 # Add model-specific arguments
@@ -262,6 +280,15 @@ case "$MODEL_NAME" in
   gemma3)
     RUNNER_ARGS="$RUNNER_ARGS --tokenizer_path ${MODEL_DIR}/ --image_path $IMAGE_PATH"
     ;;
+  qwen3)
+    PROMPT_FILE="${MODEL_DIR}/qwen3_prompt.txt"
+    cat > "${PROMPT_FILE}" << 'EOF'
+<|im_start|>user
+What is the capital of France?<|im_end|>
+<|im_start|>assistant
+EOF
+    RUNNER_ARGS="$RUNNER_ARGS --tokenizer_path ${MODEL_DIR}/ --prompt_file ${PROMPT_FILE}"
+    ;;
   parakeet)
     RUNNER_ARGS="--model_path ${MODEL_DIR}/model.pte --audio_path ${MODEL_DIR}/$AUDIO_FILE --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE"
     # For CUDA, add data_path argument (Metal embeds data in .pte)
@@ -272,16 +299,9 @@ case "$MODEL_NAME" in
   voxtral_realtime)
     RUNNER_ARGS="--model_path ${MODEL_DIR}/model.pte --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE --preprocessor_path ${MODEL_DIR}/$PREPROCESSOR --audio_path ${MODEL_DIR}/$AUDIO_FILE --temperature 0"
     # Determine streaming mode based on MODE parameter
-    USE_STREAMING="false"
-    if [ "$MODE" = "vr-streaming" ]; then
-      USE_STREAMING="true"
-    elif [ "$MODE" = "vr-offline" ]; then
+    USE_STREAMING="true"
+    if [ "$MODE" = "vr-offline" ]; then
       USE_STREAMING="false"
-    elif [ -z "$MODE" ]; then
-      # Auto-detect: XNNPACK uses streaming, others use offline
-      if [ "$DEVICE" = "xnnpack" ]; then
-        USE_STREAMING="true"
-      fi
     fi
     # Add streaming flag if needed
     if [ "$USE_STREAMING" = "true" ]; then

.ci/scripts/test_wheel_package_qnn.sh

@@ -86,6 +86,14 @@ EOF
 # ----------------------------
 echo "=== Building Wheel Package ==="
 source .ci/scripts/utils.sh
+
+# Ensure QNN SDK is available so setup.py auto-detects it.
+source backends/qualcomm/scripts/install_qnn_sdk.sh
+install_qnn
+
+# Make QNN SDK libraries available for runtime loading (e.g. libQnnHtp.so)
+export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}"
+
 install_executorch
 EXECUTORCH_BUILDING_WHEEL=1 python setup.py bdist_wheel
 unset EXECUTORCH_BUILDING_WHEEL

.ci/scripts/wheel/pre_build_script.sh

@@ -44,3 +44,16 @@ fi
 # able to see the installed torch package.
 
 "${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh" --example
+
+# Download Qualcomm QNN SDK on Linux x86_64 so the wheel build can include the
+# QNN backend.  The SDK is large, so we download it here (outside CMake) rather
+# than during cmake configure.
+if [[ "$(uname -s)" == "Linux" && "$(uname -m)" == "x86_64" ]]; then
+  echo "Downloading Qualcomm QNN SDK..."
+  QNN_SDK_ROOT=$(python3 \
+    "${GITHUB_WORKSPACE}/${REPOSITORY}/backends/qualcomm/scripts/download_qnn_sdk.py" \
+    --print-sdk-path)
+  export QNN_SDK_ROOT
+  echo "QNN_SDK_ROOT=${QNN_SDK_ROOT}" >> "${GITHUB_ENV}"
+  echo "QNN SDK downloaded to ${QNN_SDK_ROOT}"
+fi

.ci/scripts/wheel/test_linux.py

@@ -5,10 +5,25 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import platform
+
 import test_base
 from examples.models import Backend, Model
 
 if __name__ == "__main__":
+    # On Linux x86_64 the wheel is built with the Qualcomm backend.
+    # Verify that it was registered correctly.
+    if platform.system() == "Linux" and platform.machine() in ("x86_64", "amd64"):
+        from executorch.extension.pybindings.portable_lib import (
+            _get_registered_backend_names,
+        )
+
+        registered = _get_registered_backend_names()
+        assert (
+            "QnnBackend" in registered
+        ), f"QnnBackend not found in registered backends: {registered}"
+        print("✓ QnnBackend is registered")
+
     test_base.run_tests(
         model_tests=[
             test_base.ModelTest(