Add the EAGLE-3 speculative-decoding runner (CUDA)

A C++ runner that drives the speculator .pte with the shifted (vLLM-EAGLE)
scheme: the draft pairs the target hidden state at position t with token t+1, so
each round runs one target forward (target_verify) and reseeds the next draft
chain from the hidden states verify already produced -- no standalone target
decode. Greedy verification keeps output identical to greedy target decoding.
target_verify runs on stable input buffers and can be captured as a CUDA graph.

It requires the .pte metadata (fails loudly if absent) and enforces the exported
prefill range [get_min_prefill_chunk, get_max_prefill_chunk] (no chunking). The
prefill bonus token is always emitted; the speculative loop runs only when more
tokens are requested, the bonus was not EOS, and a K-token verify window fits
within get_max_seq_len (so a one-token or near-context request returns without
seeding the draft). The chat template and stop tokens are flags defaulting to
Gemma 4 IT (--chat_prefix/--chat_suffix/--stop_ids/--stop_token, --bos_id -1 to
skip) so other target/tokenizer pairs run without code changes. Device-to-host
reads are error-checked; the printed tau excludes the free prefill token.

Authored with assistance from Claude Code.


ghstack-source-id: 54fea57
ghstack-comment-id: 4661634339
Pull-Request: #20156

Author: digantdesai

examples/models/eagle3/CMakeLists.txt

@@ -0,0 +1,69 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+cmake_minimum_required(VERSION 3.24)
+project(eagle3_speculator)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
+
+include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake)
+
+set(_common_include_directories ${EXECUTORCH_ROOT}/..)
+
+# gflags
+set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/gflags)
+find_package(gflags REQUIRED)
+
+# executorch
+list(APPEND CMAKE_FIND_ROOT_PATH ${CMAKE_CURRENT_BINARY_DIR}/../../..)
+find_package(executorch CONFIG REQUIRED FIND_ROOT_PATH_BOTH)
+executorch_target_link_options_shared_lib(executorch)
+
+set(link_libraries executorch gflags)
+
+# CPU ops (host-side helpers not delegated to CUDA)
+list(APPEND link_libraries optimized_native_cpu_ops_lib cpublas eigen_blas)
+executorch_target_link_options_shared_lib(optimized_native_cpu_ops_lib)
+
+# Extensions
+list(
+  APPEND
+  link_libraries
+  extension_llm_runner
+  extension_module
+  extension_data_loader
+  extension_tensor
+  extension_flat_tensor
+)
+
+# Backend: CUDA (AOTI). The EAGLE-3 speculator export is CUDA-only.
+if(EXECUTORCH_BUILD_CUDA)
+  find_package(CUDAToolkit REQUIRED)
+  list(APPEND link_libraries aoti_cuda_backend)
+  executorch_target_link_options_shared_lib(aoti_cuda_backend)
+  add_compile_definitions(EXECUTORCH_BUILD_CUDA)
+else()
+  message(FATAL_ERROR "EAGLE-3 speculator runner requires EXECUTORCH_BUILD_CUDA=ON")
+endif()
+
+# Tokenizer (HuggingFace tokenizer.json)
+list(APPEND link_libraries tokenizers::tokenizers)
+
+add_executable(eagle3_speculator_runner main.cpp)
+target_include_directories(
+  eagle3_speculator_runner PUBLIC ${_common_include_directories}
+)
+target_link_libraries(eagle3_speculator_runner PUBLIC ${link_libraries})
+
+if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+  target_link_options_gc_sections(eagle3_speculator_runner)
+  if(NOT APPLE AND NOT MSVC)
+    target_link_options(eagle3_speculator_runner PRIVATE "LINKER:-s")
+  endif()
+endif()

examples/models/eagle3/CMakePresets.json

@@ -0,0 +1,29 @@
+{
+    "version": 6,
+    "configurePresets": [
+        {
+            "name": "eagle3-cuda",
+            "displayName": "EAGLE-3 speculator runner (CUDA)",
+            "binaryDir": "${sourceDir}/../../../cmake-out/examples/models/eagle3",
+            "cacheVariables": {
+                "CMAKE_BUILD_TYPE": "Release",
+                "CMAKE_FIND_ROOT_PATH": "${sourceDir}/../../../cmake-out",
+                "CMAKE_PREFIX_PATH": "${sourceDir}/../../../cmake-out",
+                "EXECUTORCH_BUILD_CUDA": "ON"
+            },
+            "condition": {
+                "type": "inList",
+                "string": "${hostSystemName}",
+                "list": ["Linux", "Windows"]
+            }
+        }
+    ],
+    "buildPresets": [
+        {
+            "name": "eagle3-cuda",
+            "displayName": "Build EAGLE-3 speculator runner (CUDA)",
+            "configurePreset": "eagle3-cuda",
+            "targets": ["eagle3_speculator_runner"]
+        }
+    ]
+}