Update base for Update on "Use caching allocator for runner (#15730)"

Summary:

We observed that on iOS it improves perf by 6% because SDPA op does temp allocations.

No significant difference on android though.
ghstack-source-id: 328001114
exported-using-ghexport

Reviewed By: navsud, derekdixu

Differential Revision: D86120038




[ghstack-poisoned]

Author: kimishpatel

.ci/scripts/cuda_benchmark.py

@@ -18,7 +18,9 @@ class RunMetrics:
     """Metrics from a single run."""
 
     generated_tokens: int
+    prompt_tokens: int
     tokens_per_sec: float
+    prefill_tokens_per_sec: float
     model_load_time_ms: float
     total_inference_time_ms: float
     encoder_time_ms: float
@@ -28,7 +30,8 @@ class RunMetrics:
     def __repr__(self):
         return (
             f"Tokens: {self.generated_tokens}, "
-            f"Throughput: {self.tokens_per_sec:.2f} t/s, "
+            f"Prefill: {self.prefill_tokens_per_sec:.2f} t/s ({self.prompt_tokens} tokens), "
+            f"Decode: {self.tokens_per_sec:.2f} t/s, "
             f"Model load: {self.model_load_time_ms:.0f}ms, "
             f"Total inference: {self.total_inference_time_ms:.0f}ms, "
             f"Encoder: {self.encoder_time_ms:.0f}ms, "
@@ -49,6 +52,7 @@ def parse_pytorch_observer_log(log_line: str) -> Optional[RunMetrics]:
 
         # Extract values
         generated_tokens = data.get("generated_tokens", 0)
+        prompt_tokens = data.get("prompt_tokens", 0)
         inference_start_ms = data.get("inference_start_ms", 0)
         inference_end_ms = data.get("inference_end_ms", 0)
         prompt_eval_end_ms = data.get("prompt_eval_end_ms", 0)
@@ -72,12 +76,20 @@ def parse_pytorch_observer_log(log_line: str) -> Optional[RunMetrics]:
             if generation_time_ms > 0
             else 0
         )
+
+        # Calculate prefill throughput
+        prefill_tokens_per_sec = (
+            (prompt_tokens / encoder_time_ms * 1000) if encoder_time_ms > 0 else 0
+        )
+
         model_load_time_ms = model_load_end_ms - model_load_start_ms
         first_token_latency_ms = first_token_ms - prompt_eval_end_ms
 
         return RunMetrics(
             generated_tokens=generated_tokens,
+            prompt_tokens=prompt_tokens,
             tokens_per_sec=tokens_per_sec,
+            prefill_tokens_per_sec=prefill_tokens_per_sec,
             model_load_time_ms=model_load_time_ms,
             total_inference_time_ms=total_inference_time_ms,
             encoder_time_ms=encoder_time_ms,
@@ -505,6 +517,7 @@ class BenchmarkResults:
 
     # Metrics
     throughput: MetricStats
+    prefill_throughput: MetricStats
     model_load_time: MetricStats
     total_inference_time: MetricStats
     encoder_time: MetricStats
@@ -529,6 +542,10 @@ def to_dict(self) -> dict:
             "throughput_min": self.throughput.min_val,
             "throughput_max": self.throughput.max_val,
             "throughput_stdev": self.throughput.stdev,
+            "prefill_throughput_mean": self.prefill_throughput.mean,
+            "prefill_throughput_min": self.prefill_throughput.min_val,
+            "prefill_throughput_max": self.prefill_throughput.max_val,
+            "prefill_throughput_stdev": self.prefill_throughput.stdev,
             "model_load_time_mean": self.model_load_time.mean,
             "model_load_time_min": self.model_load_time.min_val,
             "model_load_time_max": self.model_load_time.max_val,
@@ -601,6 +618,13 @@ def to_v3_format(
                 runner_type,
                 base_extra_info,
             ),
+            self.prefill_throughput.create_v3_record(
+                model_name_with_quant,
+                backend,
+                runner_name,
+                runner_type,
+                base_extra_info,
+            ),
             self.model_load_time.create_v3_record(
                 model_name_with_quant,
                 backend,
@@ -696,6 +720,11 @@ def create_metric_stats(
             "t/s",
             {"trimmed_runs": len(trimmed_throughput)},
         ),
+        prefill_throughput=create_metric_stats(
+            "prefill_encoder_throughput(tokens/sec)",
+            [r.prefill_tokens_per_sec for r in results],
+            "t/s",
+        ),
         model_load_time=create_metric_stats(
             "model_load_time(ms)",
             [r.model_load_time_ms for r in results],
@@ -740,6 +769,7 @@ def print_summary(summary: BenchmarkResults) -> None:
 
     # Print all metrics using their print_stats method
     summary.throughput.print_stats()
+    summary.prefill_throughput.print_stats()
     summary.model_load_time.print_stats()
     summary.total_inference_time.print_stats()
     summary.encoder_time.print_stats()

.ci/scripts/cuda_perf_prompts/qwen3_5_moe_long_prompt.txt

@@ -0,0 +1,25 @@
+Please analyze and summarize the following text in detail:
+
+The Transformer architecture, introduced by Vaswani et al. in the 2017 paper "Attention Is All You Need," fundamentally reshaped the landscape of natural language processing and, more broadly, machine learning. Prior to its introduction, sequence modeling had been dominated by recurrent neural networks such as the Long Short-Term Memory (LSTM) and the Gated Recurrent Unit (GRU), as well as by convolutional approaches that attempted to capture local context efficiently. While these architectures achieved respectable results on a variety of tasks, they suffered from inherent limitations: recurrent computations could not be easily parallelized along the time dimension, gradients tended to vanish or explode across very long sequences, and convolutional models had difficulty capturing global dependencies without resorting to deep stacks of layers or specialized dilated kernels.
+
+The Transformer addressed these limitations by replacing recurrence with a mechanism known as self-attention. In self-attention, every token in a sequence computes a weighted sum of representations from every other token, where the weights are determined by a compatibility function between learned queries and keys. This formulation has several appealing properties. First, the computation across positions is fully parallelizable, which dramatically accelerates training on modern accelerator hardware such as graphics processing units and tensor processing units. Second, the path length between any two tokens is constant, allowing the model to capture long-range dependencies far more directly than recurrent networks ever could. Third, the attention weights themselves carry interpretable information about which parts of the input the model is using to produce each output, offering a window into model behavior that earlier architectures rarely provided.
+
+A standard Transformer block stacks multi-head self-attention with a position-wise feed-forward network, surrounded by residual connections and layer normalization. The multi-head design allows the model to attend to information from multiple representation subspaces simultaneously: each head learns its own projection of the input into queries, keys, and values, computes its own attention pattern, and contributes a slice of the final output. Empirically, different heads often specialize in different linguistic phenomena, with some focusing on syntactic relationships such as subject-verb agreement, others on coreference, and still others on broader semantic context.
+
+Because self-attention is permutation invariant, the Transformer must inject information about the order of tokens explicitly. The original paper used fixed sinusoidal positional encodings added to the token embeddings, but subsequent work has explored a wide variety of alternatives. Learned positional embeddings, relative position representations, rotary positional embeddings, and attention-with-linear-biases schemes have all been proposed and adopted in different families of models. Each approach offers a different trade-off between expressiveness, generalization to longer contexts, and computational cost.
+
+The original Transformer was an encoder-decoder model designed for machine translation, but the architecture quickly proved general enough to power a remarkable range of subsequent systems. Encoder-only models such as BERT framed pretraining as a masked language modeling task and dramatically improved performance on classification, question answering, and information retrieval. Decoder-only models such as the GPT family treated language modeling as next-token prediction at very large scale, and demonstrated that with sufficient data and compute, a single architecture could exhibit emergent capabilities such as few-shot learning, code generation, and chain-of-thought reasoning. Encoder-decoder variants such as T5 and BART unified many tasks under a common text-to-text framing, simplifying the engineering of multitask systems.
+
+Scaling laws, established through systematic empirical study, have shown that Transformer performance improves predictably as a function of model parameters, dataset size, and compute budget. This insight motivated a wave of increasingly large models, culminating in dense networks with hundreds of billions of parameters and sparsely activated mixture-of-experts networks with trillions. Mixture-of-experts approaches in particular have become attractive because they decouple total parameter count from the per-token computation cost: a router network selects a small subset of experts for each token, allowing the model to grow in capacity without a proportional increase in inference latency. This trade-off makes mixture-of-experts especially appealing for deployment scenarios where memory bandwidth dominates compute as the bottleneck.
+
+Beyond text, the Transformer has been adapted to images, audio, video, proteins, source code, and combinations thereof. Vision Transformers split images into patches and treat each patch as a token; speech models tokenize raw audio or spectrogram features; multimodal systems share a common Transformer backbone across modalities by aligning their embedding spaces. The flexibility of the architecture, combined with the practical advantages of large-scale pretraining followed by task-specific fine-tuning or in-context prompting, has made the Transformer the de facto foundation of contemporary deep learning research and applications.
+
+Of course, the Transformer is not without its drawbacks. The quadratic memory and compute cost of standard self-attention with respect to sequence length remains a significant practical limitation. A vibrant subfield of research focuses on efficient attention variants, including sparse, low-rank, and kernelized formulations, as well as state-space models that recover linear complexity while preserving competitive accuracy. Long-context inference also stresses the key-value cache, motivating techniques such as paged attention, grouped-query attention, sliding-window attention, and aggressive quantization. Quantization in particular has become a critical tool for deploying large models on commodity hardware, with low-bit integer formats and weight-only quantization schemes enabling inference on consumer GPUs and edge devices that would otherwise be unable to host a model.
+
+Tooling has co-evolved with the architecture. Compilers and runtimes such as ExecuTorch, TensorRT, vLLM, and various ONNX-based stacks specialize in lowering Transformer graphs onto target accelerators while applying optimizations such as kernel fusion, operator scheduling, and memory planning. These systems make it feasible to take a research model trained in a high-level framework and deploy it efficiently on production hardware ranging from data center GPUs to mobile system-on-chips. The end-to-end pipeline of training, fine-tuning, quantization, export, and runtime execution has become a recognizable engineering discipline in its own right.
+
+Training a frontier-scale Transformer is itself a substantial systems undertaking. Modern pretraining runs combine data parallelism, tensor parallelism, pipeline parallelism, and sequence parallelism, often coordinated through libraries such as PyTorch FSDP, Megatron, and DeepSpeed ZeRO. Practitioners must balance compute and memory carefully, choosing micro-batch sizes that maximize accelerator utilization without exceeding device memory, designing checkpointing schemes that survive node failures over runs that can last for months, and overlapping communication with computation to hide network latency. Activation checkpointing trades extra computation for reduced memory pressure, while mixed precision training with bfloat16 or FP8 formats shrinks memory bandwidth requirements and unlocks newer hardware features.
+
+Inference brings its own set of challenges. The autoregressive nature of decoder-only Transformers means each generated token requires a full forward pass, and the dominant cost shifts from raw matrix multiplication during prefill to memory-bandwidth-bound key-value cache reads during decode. Techniques such as speculative decoding, continuous batching, and prefix caching attempt to claw back utilization. For latency-sensitive deployments, careful kernel fusion, paged attention, and ahead-of-time compilation can reduce per-token overhead substantially, and the rise of small distilled or sparsely activated models offers an alternative path to acceptable quality at a fraction of the cost.
+
+Looking ahead, the Transformer's dominance is being challenged by alternative architectures such as state-space models, linear recurrent networks, and hybrid designs that interleave attention with other mixing primitives. Whether any of these will displace the Transformer entirely remains to be seen, but it is already clear that the ideas the Transformer popularized — content-based mixing of tokens, parallelizable training, and large-scale pretraining followed by adaptation — will continue to shape the field for years to come.

.ci/scripts/test_yolo12.sh .ci/scripts/test_yolo26.sh.ci/scripts/test_yolo12.sh renamed to .ci/scripts/test_yolo26.sh

@@ -6,6 +6,7 @@
 # LICENSE file in the root directory of this source tree.
 
 set -ex
+
 # shellcheck source=/dev/null
 source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 
@@ -50,21 +51,21 @@ PT2E_QUANTIZE="${PT2E_QUANTIZE:-}"
 # Default CMake Build Type to release mode
 CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release}
 
-if [[ $# -lt 5 ]]; then # Assuming 4 mandatory args
-    echo "Expecting atleast 5 positional arguments"
-    echo "Usage: [...]"
-fi
 if [[ -z "${MODEL_NAME:-}" ]]; then
   echo "Missing model name, exiting..."
   exit 1
 fi
 
-
 if [[ -z "${MODE:-}" ]]; then
   echo "Missing mode, choose openvino or xnnpack, exiting..."
   exit 1
 fi
 
+if [[ -z "${VIDEO_PATH:-}" ]]; then
+  echo "Missing video path, exiting..."
+  exit 1
+fi
+
 if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
   PYTHON_EXECUTABLE=python3
 fi
@@ -75,21 +76,13 @@ if [[ "${MODE}" =~ .*openvino.* ]]; then
   OPENVINO=ON
   TARGET_LIBS="$TARGET_LIBS openvino_backend "
 
-  git clone https://github.com/openvinotoolkit/openvino.git
-  cd openvino && git b16b776ac119dafda51f69a80f1e6b7376d02c3b
-  git submodule update --init --recursive
-  sudo ./install_build_dependencies.sh
-  mkdir build && cd build
-  cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=ON
-  make -j$(nproc)
-
-  cd ..
-  cmake --install build --prefix dist
-
-  source dist/setupvars.sh
-  cd ../backends/openvino
-  pip install -r requirements.txt
-  cd ../../
+  # Install specific OpenVINO runtime from pip.
+  $PYTHON_EXECUTABLE -m pip install --pre openvino==2026.1.0.dev20260131 --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
+  $PYTHON_EXECUTABLE -m pip install -r backends/openvino/requirements.txt
+
+  # Set OPENVINO_LIB_PATH so the C++ demo runner can also find libopenvino_c.so.
+  OPENVINO_LIB_PATH=$($PYTHON_EXECUTABLE -c "import openvino, os, glob; print(sorted(glob.glob(os.path.join(os.path.dirname(openvino.__file__), 'libs', 'libopenvino_c.so*')))[-1])")
+  export OPENVINO_LIB_PATH
 else
   OPENVINO=OFF
 fi
@@ -103,9 +96,10 @@ fi
 
 which "${PYTHON_EXECUTABLE}"
 
+TORCH_URL=https://download.pytorch.org/whl/cpu
 
-DIR="examples/models/yolo12"
-$PYTHON_EXECUTABLE -m pip install -r ${DIR}/requirements.txt
+DIR="examples/models/yolo26"
+$PYTHON_EXECUTABLE -m pip install --upgrade-strategy only-if-needed --extra-index-url "$TORCH_URL" -r ${DIR}/requirements.txt
 
 cmake_install_executorch_libraries() {
     rm -rf cmake-out
@@ -142,11 +136,11 @@ cmake_install_executorch_libraries() {
 
     echo $TARGET_LIBS
     export CMAKE_BUILD_ARGS="--target $TARGET_LIBS"
-    pip install . --no-build-isolation
+    $PYTHON_EXECUTABLE -m pip install . --no-build-isolation
 }
 
 cmake_build_demo() {
-    echo "Building yolo12 runner"
+    echo "Building yolo26 runner"
     retry cmake \
         -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \
         -DUSE_OPENVINO_BACKEND="$OPENVINO" \
@@ -174,24 +168,29 @@ prepare_artifacts_upload() {
 
 
 # Export model.
-EXPORTED_MODEL_NAME="${MODEL_NAME}_fp32_${MODE}.pte"
-echo "Exporting ${EXPORTED_MODEL_NAME}"
 EXPORT_ARGS="--model_name=${MODEL_NAME} --backend=${MODE}"
+if [[ -n "${PT2E_QUANTIZE}" ]]; then
+  EXPORTED_MODEL_NAME="${MODEL_NAME}_int8_${MODE}.pte"
+  EXPORT_ARGS="${EXPORT_ARGS} --quantize --video_path=${VIDEO_PATH}"
+else
+  EXPORTED_MODEL_NAME="${MODEL_NAME}_fp32_${MODE}.pte"
+fi
+echo "Exporting ${EXPORTED_MODEL_NAME}"
 
 # Add dynamically linked library location
 cmake_install_executorch_libraries
 
-$PYTHON_EXECUTABLE -m examples.models.yolo12.export_and_validate ${EXPORT_ARGS}
+$PYTHON_EXECUTABLE -m examples.models.yolo26.export_and_validate ${EXPORT_ARGS}
 
 
 RUNTIME_ARGS="--model_path=${EXPORTED_MODEL_NAME} --input_path=${VIDEO_PATH}"
 # Check build tool.
 cmake_build_demo
-# Run yolo12 runner
+# Run yolo26 runner
 NOW=$(date +"%H:%M:%S")
-echo "Starting to run yolo12 runner at ${NOW}"
+echo "Starting to run yolo26 runner at ${NOW}"
 # shellcheck source=/dev/null
-cmake-out/examples/models/yolo12/Yolo12DetectionDemo ${RUNTIME_ARGS} > result.txt
+cmake-out/examples/models/yolo26/Yolo26DetectionDemo ${RUNTIME_ARGS} > result.txt
 NOW=$(date +"%H:%M:%S")
 echo "Finished at ${NOW}"
 

.claude/settings.json

@@ -0,0 +1,15 @@
+{
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Bash",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "if [ -x .wiki/fb/hooks/resync-guard.sh ]; then bash .wiki/fb/hooks/resync-guard.sh; fi"
+          }
+        ]
+      }
+    ]
+  }
+}