From 43dfc81ffd4326345d51c979e8552be9281e2606 Mon Sep 17 00:00:00 2001 From: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:51:08 +0000 Subject: [PATCH 1/2] [None][infra] Check in most recent lock file from nightly pipeline Signed-off-by: TensorRT LLM <90828364+tensorrt-cicd@users.noreply.github.com> --- .../examples/auto_deploy/poetry.lock | 8 +++---- .../llm-eval/lm-eval-harness/poetry.lock | 8 +++---- .../examples/ray_orchestrator/poetry.lock | 24 +++++++++---------- security_scanning/examples/serve/poetry.lock | 8 +++---- .../examples/trtllm-eval/poetry.lock | 8 +++---- security_scanning/metadata.json | 4 ++-- security_scanning/poetry.lock | 8 +++---- 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/security_scanning/examples/auto_deploy/poetry.lock b/security_scanning/examples/auto_deploy/poetry.lock index 7e304e863bfb..e8a3dc2f2f0c 100644 --- a/security_scanning/examples/auto_deploy/poetry.lock +++ b/security_scanning/examples/auto_deploy/poetry.lock @@ -1880,22 +1880,22 @@ dill = ">=0.4.1" [[package]] name = "narwhals" -version = "2.23.0" +version = "2.24.0" description = "Extremely lightweight compatibility layer between dataframe libraries" optional = false python-versions = ">=3.10" groups = ["main"] markers = "python_version >= \"3.11\"" files = [ - {file = "narwhals-2.23.0-py3-none-any.whl", hash = "sha256:769e7b9ab102c93d8fa019f6b4cd1a657909b04a20bf6210e5a35aae06814ae9"}, - {file = "narwhals-2.23.0.tar.gz", hash = "sha256:13e7ff5b4bb4a2f77b907c2e4d8a76e273dfc1323a3c997440a2f9fd26aed408"}, + {file = "narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489"}, + {file = "narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d"}, ] [package.extras] cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] dask = ["dask[dataframe] (>=2024.8)"] duckdb = ["duckdb (>=1.1)"] -ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)", "rich (>=12.4.4)"] +ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)"] modin = ["modin (>=0.22.0)"] pandas = ["pandas (>=1.3.4)"] polars = ["polars (>=0.20.4)"] diff --git a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock index 632ee222e37d..b03fdf0b61c4 100644 --- a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock +++ b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock @@ -1650,22 +1650,22 @@ dill = ">=0.4.1" [[package]] name = "narwhals" -version = "2.23.0" +version = "2.24.0" description = "Extremely lightweight compatibility layer between dataframe libraries" optional = false python-versions = ">=3.10" groups = ["main"] markers = "python_version >= \"3.11\"" files = [ - {file = "narwhals-2.23.0-py3-none-any.whl", hash = "sha256:769e7b9ab102c93d8fa019f6b4cd1a657909b04a20bf6210e5a35aae06814ae9"}, - {file = "narwhals-2.23.0.tar.gz", hash = "sha256:13e7ff5b4bb4a2f77b907c2e4d8a76e273dfc1323a3c997440a2f9fd26aed408"}, + {file = "narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489"}, + {file = "narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d"}, ] [package.extras] cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] dask = ["dask[dataframe] (>=2024.8)"] duckdb = ["duckdb (>=1.1)"] -ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)", "rich (>=12.4.4)"] +ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)"] modin = ["modin (>=0.22.0)"] pandas = ["pandas (>=1.3.4)"] polars = ["polars (>=0.20.4)"] diff --git a/security_scanning/examples/ray_orchestrator/poetry.lock b/security_scanning/examples/ray_orchestrator/poetry.lock index 4b1e62530027..3442d2db7973 100644 --- a/security_scanning/examples/ray_orchestrator/poetry.lock +++ b/security_scanning/examples/ray_orchestrator/poetry.lock @@ -747,32 +747,32 @@ grpc = ["grpcio (>=1.41.0,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version > [[package]] name = "google-auth" -version = "2.55.2" +version = "2.56.0" description = "Google Authentication Library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "google_auth-2.55.2-py3-none-any.whl", hash = "sha256:d715f265f2cafc6a5f1bf0dc19870d20e3119f6f6682785a250bce3d03d38a3b"}, - {file = "google_auth-2.55.2.tar.gz", hash = "sha256:97ae7790ff740f2bc9db60eb864a7804f4ac19f5f02c38b3d942f2fea6e9b9ae"}, + {file = "google_auth-2.56.0-py3-none-any.whl", hash = "sha256:6e88c10217e07a92bfd01cac8ee99e32ccfb08414c3102e6c5b8d58f37a0d1e0"}, + {file = "google_auth-2.56.0.tar.gz", hash = "sha256:f90fa030b569a92654b9d690665a073841df33d57487be53db583a9a0867a553"}, ] [package.dependencies] -cryptography = ">=38.0.3" +cryptography = {version = ">=38.0.3", markers = "python_version < \"3.14\""} pyasn1-modules = ">=0.2.1" [package.extras] -aiohttp = ["aiohttp (>=3.8.0,<4.0.0)", "requests (>=2.20.0,<3.0.0)"] -cryptography = ["cryptography (>=38.0.3)"] -enterprise-cert = ["cryptography (>=38.0.3)"] +aiohttp = ["aiohttp (>=3.8.0,<4.0.0) ; python_version < \"3.14\"", "aiohttp (>=3.9.0,<4.0.0) ; python_version >= \"3.14\"", "requests (>=2.30.0,<3.0.0)"] +cryptography = ["cryptography (>=38.0.3) ; python_version < \"3.14\"", "cryptography (>=41.0.5) ; python_version >= \"3.14\""] +enterprise-cert = ["cryptography (>=38.0.3) ; python_version < \"3.14\"", "cryptography (>=41.0.5) ; python_version >= \"3.14\""] grpc = ["grpcio (>=1.59.0,<2.0.0) ; python_version < \"3.14\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\""] pyjwt = ["pyjwt (>=2.0)"] -pyopenssl = ["cryptography (>=38.0.3)"] +pyopenssl = ["cryptography (>=38.0.3) ; python_version < \"3.14\"", "cryptography (>=41.0.5) ; python_version >= \"3.14\""] reauth = ["pyu2f (>=0.1.5)"] -requests = ["requests (>=2.20.0,<3.0.0)"] -rsa = ["rsa (>=3.1.4,<5)"] -testing = ["aiohttp (>=3.8.0,<4.0.0)", "aioresponses", "flask", "freezegun", "grpcio (>=1.59.0,<2.0.0) ; python_version < \"3.14\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "packaging", "pyjwt (>=2.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] -urllib3 = ["packaging", "urllib3"] +requests = ["requests (>=2.30.0,<3.0.0)"] +rsa = ["rsa (>=4.0.0,<5)"] +testing = ["aiohttp (>=3.8.0,<4.0.0) ; python_version < \"3.14\"", "aiohttp (>=3.9.0,<4.0.0) ; python_version >= \"3.14\"", "aioresponses", "flask", "freezegun", "grpcio (>=1.59.0,<2.0.0) ; python_version < \"3.14\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "packaging (>=20.0)", "pyjwt (>=2.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.30.0,<3.0.0)", "responses", "urllib3 (>=1.26.15,<3.0.0)"] +urllib3 = ["packaging (>=20.0)", "urllib3 (>=1.26.15,<3.0.0)"] [[package]] name = "googleapis-common-protos" diff --git a/security_scanning/examples/serve/poetry.lock b/security_scanning/examples/serve/poetry.lock index e500b96552b3..9763fed10c4d 100644 --- a/security_scanning/examples/serve/poetry.lock +++ b/security_scanning/examples/serve/poetry.lock @@ -2671,21 +2671,21 @@ dill = ">=0.4.1" [[package]] name = "narwhals" -version = "2.23.0" +version = "2.24.0" description = "Extremely lightweight compatibility layer between dataframe libraries" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "narwhals-2.23.0-py3-none-any.whl", hash = "sha256:769e7b9ab102c93d8fa019f6b4cd1a657909b04a20bf6210e5a35aae06814ae9"}, - {file = "narwhals-2.23.0.tar.gz", hash = "sha256:13e7ff5b4bb4a2f77b907c2e4d8a76e273dfc1323a3c997440a2f9fd26aed408"}, + {file = "narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489"}, + {file = "narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d"}, ] [package.extras] cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] dask = ["dask[dataframe] (>=2024.8)"] duckdb = ["duckdb (>=1.1)"] -ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)", "rich (>=12.4.4)"] +ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)"] modin = ["modin (>=0.22.0)"] pandas = ["pandas (>=1.3.4)"] polars = ["polars (>=0.20.4)"] diff --git a/security_scanning/examples/trtllm-eval/poetry.lock b/security_scanning/examples/trtllm-eval/poetry.lock index 30ab9deab026..27cf9cafed09 100644 --- a/security_scanning/examples/trtllm-eval/poetry.lock +++ b/security_scanning/examples/trtllm-eval/poetry.lock @@ -1652,22 +1652,22 @@ dill = ">=0.4.1" [[package]] name = "narwhals" -version = "2.23.0" +version = "2.24.0" description = "Extremely lightweight compatibility layer between dataframe libraries" optional = false python-versions = ">=3.10" groups = ["main"] markers = "python_version >= \"3.11\"" files = [ - {file = "narwhals-2.23.0-py3-none-any.whl", hash = "sha256:769e7b9ab102c93d8fa019f6b4cd1a657909b04a20bf6210e5a35aae06814ae9"}, - {file = "narwhals-2.23.0.tar.gz", hash = "sha256:13e7ff5b4bb4a2f77b907c2e4d8a76e273dfc1323a3c997440a2f9fd26aed408"}, + {file = "narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489"}, + {file = "narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d"}, ] [package.extras] cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] dask = ["dask[dataframe] (>=2024.8)"] duckdb = ["duckdb (>=1.1)"] -ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)", "rich (>=12.4.4)"] +ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)"] modin = ["modin (>=0.22.0)"] pandas = ["pandas (>=1.3.4)"] polars = ["polars (>=0.20.4)"] diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index af546b51479f..117e591faada 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "770ebc5db5c9c1486b8792928fdebf35de73eff3", - "timestamp": "2026-07-13T02:48:18Z" + "commit_hash": "e8321d20974ec61976c037d4a9f3e57ffbd73b80", + "timestamp": "2026-07-13T19:34:01Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index f61067a96476..15a9bb849265 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -3050,21 +3050,21 @@ dill = ">=0.3.8" [[package]] name = "narwhals" -version = "2.23.0" +version = "2.24.0" description = "Extremely lightweight compatibility layer between dataframe libraries" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "narwhals-2.23.0-py3-none-any.whl", hash = "sha256:769e7b9ab102c93d8fa019f6b4cd1a657909b04a20bf6210e5a35aae06814ae9"}, - {file = "narwhals-2.23.0.tar.gz", hash = "sha256:13e7ff5b4bb4a2f77b907c2e4d8a76e273dfc1323a3c997440a2f9fd26aed408"}, + {file = "narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489"}, + {file = "narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d"}, ] [package.extras] cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] dask = ["dask[dataframe] (>=2024.8)"] duckdb = ["duckdb (>=1.1)"] -ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)", "rich (>=12.4.4)"] +ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)"] modin = ["modin (>=0.22.0)"] pandas = ["pandas (>=1.3.4)"] polars = ["polars (>=0.20.4)"] From 54d484fd3c7f07799f5518a3eed27ba16bee8fb1 Mon Sep 17 00:00:00 2001 From: Zheyu Fu Date: Mon, 13 Jul 2026 14:52:29 -0700 Subject: [PATCH 2/2] [TRTLLM-11558][feat] BREAKING: Acceptance rate based speculation off in one model path (#12905) Signed-off-by: Zheyu Fu --- docs/source/developer-guide/telemetry.md | 22 ++-- tensorrt_llm/_torch/pyexecutor/py_executor.py | 103 ++++++++++------ .../_torch/speculative/speculation_gate.py | 87 +++++++------- tensorrt_llm/llmapi/llm_args.py | 18 +-- .../usage/llm_args_golden_manifest.json | 12 +- .../speculative/hw_agnostic/test_spec_gate.py | 110 +++++++++--------- .../references/decoding_base_config.yaml | 35 ++++++ 7 files changed, 236 insertions(+), 151 deletions(-) create mode 100644 tests/unittest/api_stability/references/decoding_base_config.yaml diff --git a/docs/source/developer-guide/telemetry.md b/docs/source/developer-guide/telemetry.md index e5b763c19386..207c4604cce1 100644 --- a/docs/source/developer-guide/telemetry.md +++ b/docs/source/developer-guide/telemetry.md @@ -28,7 +28,7 @@ unset or when the safety sanitizer rejects the runtime value. ### `TorchLlmArgs` -258 captured fields. +261 captured fields. | Captured key | Annotation | Kind | Converter | Allowed values | |--------------|------------|------|-----------|----------------| @@ -50,6 +50,7 @@ unset or when the safety sanitizer rejects the runtime value. | `batch_wait_timeout_iters` | `` | `value` | | | | `batch_wait_timeout_ms` | `` | `value` | | | | `cache_transceiver_config.backend` | `Optional[Literal['DEFAULT', 'UCX', 'NIXL', 'MOONCAKE', 'MPI']]` | `categorical` | | `DEFAULT`, `UCX`, `NIXL`, `MOONCAKE`, `MPI` | +| `cache_transceiver_config.kv_cache_bounce_size_mb` | `` | `value` | | | | `cache_transceiver_config.kv_transfer_poll_interval_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | | `cache_transceiver_config.kv_transfer_sender_future_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | | `cache_transceiver_config.kv_transfer_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | @@ -87,6 +88,7 @@ unset or when the safety sanitizer rejects the runtime value. | `enable_layerwise_nvtx_marker` | `` | `value` | | | | `enable_lm_head_tp_in_adp` | `` | `value` | | | | `enable_lora` | `` | `value` | | | +| `enable_low_latency_host_dispatch` | `` | `value` | | | | `enable_min_latency` | `` | `value` | | | | `enable_resource_governor` | `` | `value` | | | | `enable_speculative_beam_history_d2h` | `` | `value` | | | @@ -154,6 +156,8 @@ unset or when the safety sanitizer rejects the runtime value. | `moe_config.use_low_precision_moe_combine` | `` | `value` | | | | `moe_expert_parallel_size` | `Optional[int]` | `value` | | | | `moe_tensor_parallel_size` | `Optional[int]` | `value` | | | +| `multimodal_config.encoder_side_stream_max_ahead` | `` | `value` | | | +| `multimodal_config.video_pruning_rate` | `Optional[float]` | `value` | | | | `mx_config.preshard_strategy` | `` | `categorical` | allowlist | `per_module` | | `mx_config.server_query_timeout_s` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | | `num_postprocess_workers` | `` | `value` | | | @@ -230,8 +234,8 @@ unset or when the safety sanitizer rejects the runtime value. | `sparse_attention_config.use_cute_dsl_paged_mqa_logits` | `` | `value` | | | | `sparse_attention_config.use_cute_dsl_topk` | `` | `value` | | | | `sparse_attention_config.window_size` | `` | `value` | | | -| `speculative_config.acceptance_length_threshold` | `Optional[Annotated[float, Ge(ge=0)]]` | `value` | | | -| `speculative_config.acceptance_window` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | +| `speculative_config.acceptance_rate_threshold` | `Optional[float]` | `value` | | | +| `speculative_config.acceptance_rate_window_size` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | | `speculative_config.allow_advanced_sampling` | `` | `value` | | | | `speculative_config.begin_thinking_phase_token` | `` | `value` | | | | `speculative_config.decoding_type` | `Literal['AUTO']` | `categorical` | | `AUTO`, `DFlash`, `Draft_Target`, `Eagle3`, `Eagle`, `Lookahead`, `MTP`, `Medusa`, `NGram`, `PARD`, `SA`, `SaveState`, `User_Provided` | @@ -288,11 +292,10 @@ unset or when the safety sanitizer rejects the runtime value. | `use_cute_dsl_bf16_gemm` | `` | `value` | | | | `use_cute_dsl_blockscaling_bmm` | `` | `value` | | | | `use_cute_dsl_blockscaling_mm` | `` | `value` | | | -| `video_pruning_rate` | `Optional[float]` | `value` | | | ### `TrtLlmArgs` -279 captured fields. +280 captured fields. | Captured key | Annotation | Kind | Converter | Allowed values | |--------------|------------|------|-----------|----------------| @@ -368,6 +371,7 @@ unset or when the safety sanitizer rejects the runtime value. | `build_config.weight_sparsity` | `` | `value` | | | | `build_config.weight_streaming` | `` | `value` | | | | `cache_transceiver_config.backend` | `Optional[Literal['DEFAULT', 'UCX', 'NIXL', 'MOONCAKE', 'MPI']]` | `categorical` | | `DEFAULT`, `UCX`, `NIXL`, `MOONCAKE`, `MPI` | +| `cache_transceiver_config.kv_cache_bounce_size_mb` | `` | `value` | | | | `cache_transceiver_config.kv_transfer_poll_interval_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | | `cache_transceiver_config.kv_transfer_sender_future_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | | `cache_transceiver_config.kv_transfer_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | @@ -480,11 +484,11 @@ unset or when the safety sanitizer rejects the runtime value. | `quant_config.clamp_val` | `Optional[List[float]]` | `value` | | | | `quant_config.group_size` | `Optional[int]` | `value` | | | | `quant_config.has_zero_point` | `` | `value` | | | -| `quant_config.kv_cache_quant_algo` | `Optional[tensorrt_llm.quantization.mode.QuantAlgo]` | `categorical` | | `W8A16`, `W4A16`, `W4A16_AWQ`, `W4A8_AWQ`, `W8A16_GPTQ`, `W4A16_GPTQ`, `W8A8_SQ_PER_CHANNEL`, `W8A8_SQ_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN`, `W4A8_QSERVE_PER_GROUP`, `W4A8_QSERVE_PER_CHANNEL`, `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `FP8_BLOCK_SCALES`, `INT8`, `MIXED_PRECISION`, `NVFP4`, `W4A8_NVFP4_FP8`, `W4A8_MXFP4_FP8`, `W4A8_MXFP4_MXFP8`, `W4A16_MXFP4`, `MXFP8`, `NVFP4_AWQ`, `NVFP4_ARC`, `NO_QUANT` | +| `quant_config.kv_cache_quant_algo` | `Optional[tensorrt_llm.quantization.mode.QuantAlgo]` | `categorical` | | `W8A16`, `W4A16`, `W4A16_AWQ`, `W4A8_AWQ`, `W8A16_GPTQ`, `W4A16_GPTQ`, `W8A8_SQ_PER_CHANNEL`, `W8A8_SQ_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN`, `W4A8_QSERVE_PER_GROUP`, `W4A8_QSERVE_PER_CHANNEL`, `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `FP8_BLOCK_SCALES`, `INT8`, `MIXED_PRECISION`, `NVFP4`, `W4A8_NVFP4_FP8`, `W4A8_MXFP4_FP8`, `W4A8_MXFP4_MXFP8`, `W4A16_MXFP4`, `MXFP8`, `W4A16_NVFP4`, `NVFP4_AWQ`, `NVFP4_ARC`, `NO_QUANT` | | `quant_config.mamba_ssm_philox_rounds` | `` | `value` | | | | `quant_config.mamba_ssm_stochastic_rounding` | `` | `value` | | | | `quant_config.pre_quant_scale` | `` | `value` | | | -| `quant_config.quant_algo` | `Optional[tensorrt_llm.quantization.mode.QuantAlgo]` | `categorical` | | `W8A16`, `W4A16`, `W4A16_AWQ`, `W4A8_AWQ`, `W8A16_GPTQ`, `W4A16_GPTQ`, `W8A8_SQ_PER_CHANNEL`, `W8A8_SQ_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN`, `W4A8_QSERVE_PER_GROUP`, `W4A8_QSERVE_PER_CHANNEL`, `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `FP8_BLOCK_SCALES`, `INT8`, `MIXED_PRECISION`, `NVFP4`, `W4A8_NVFP4_FP8`, `W4A8_MXFP4_FP8`, `W4A8_MXFP4_MXFP8`, `W4A16_MXFP4`, `MXFP8`, `NVFP4_AWQ`, `NVFP4_ARC`, `NO_QUANT` | +| `quant_config.quant_algo` | `Optional[tensorrt_llm.quantization.mode.QuantAlgo]` | `categorical` | | `W8A16`, `W4A16`, `W4A16_AWQ`, `W4A8_AWQ`, `W8A16_GPTQ`, `W4A16_GPTQ`, `W8A8_SQ_PER_CHANNEL`, `W8A8_SQ_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN`, `W4A8_QSERVE_PER_GROUP`, `W4A8_QSERVE_PER_CHANNEL`, `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `FP8_BLOCK_SCALES`, `INT8`, `MIXED_PRECISION`, `NVFP4`, `W4A8_NVFP4_FP8`, `W4A8_MXFP4_FP8`, `W4A8_MXFP4_MXFP8`, `W4A16_MXFP4`, `MXFP8`, `W4A16_NVFP4`, `NVFP4_AWQ`, `NVFP4_ARC`, `NO_QUANT` | | `quant_config.smoothquant_val` | `` | `value` | | | | `quant_config.use_meta_recipe` | `` | `value` | | | | `reasoning_parser` | `Optional[str]` | `categorical` | allowlist | `auto`, `deepseek-r1`, `laguna`, `qwen3`, `qwen3_5`, `minimax_m2`, `minimax_m2_append_think`, `nano-v3`, `gemma4`, `kimi_k2`, `kimi_k25` | @@ -528,8 +532,8 @@ unset or when the safety sanitizer rejects the runtime value. | `sparse_attention_config.use_cute_dsl_paged_mqa_logits` | `` | `value` | | | | `sparse_attention_config.use_cute_dsl_topk` | `` | `value` | | | | `sparse_attention_config.window_size` | `` | `value` | | | -| `speculative_config.acceptance_length_threshold` | `Optional[Annotated[float, Ge(ge=0)]]` | `value` | | | -| `speculative_config.acceptance_window` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | +| `speculative_config.acceptance_rate_threshold` | `Optional[float]` | `value` | | | +| `speculative_config.acceptance_rate_window_size` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | | `speculative_config.allow_advanced_sampling` | `` | `value` | | | | `speculative_config.begin_thinking_phase_token` | `` | `value` | | | | `speculative_config.decoding_type` | `Literal['AUTO']` | `categorical` | | `AUTO`, `DFlash`, `Draft_Target`, `Eagle3`, `Eagle`, `Lookahead`, `MTP`, `Medusa`, `NGram`, `PARD`, `SA`, `SaveState`, `User_Provided` | diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index f69041fa7ab8..7f2d1b4895cc 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -603,19 +603,16 @@ def __init__( self.num_fetch_requests = 0 self.shutdown_event = threading.Event() - # Rolling acceptance tracking for spec decode (disable speculation if rolling acceptance is below threshold) - spec_config = getattr(self.model_engine, 'spec_config', None) - self.acceptance_window = getattr( - spec_config, 'acceptance_window', - None) if spec_config is not None else None - self.acceptance_length_threshold = getattr( - spec_config, 'acceptance_length_threshold', - None) if spec_config is not None else None + # Rolling true acceptance-rate tracking for permanent speculation + # disable. self.speculation_permanently_disabled = False self.speculation_gate = None - if self.acceptance_window and self.acceptance_length_threshold is not None: - self.speculation_gate = SpeculationGate( - self.acceptance_window, self.acceptance_length_threshold) + spec_config = getattr(self.model_engine, 'spec_config', None) + if spec_config is not None: + window = getattr(spec_config, 'acceptance_rate_window_size', None) + threshold = getattr(spec_config, 'acceptance_rate_threshold', None) + if window and threshold is not None: + self.speculation_gate = SpeculationGate(window, threshold) # response used data self.response_lock = threading.Lock() @@ -2145,6 +2142,42 @@ def _update_iter_stats( return stats + def _update_batch_acceptance_rate( + self, + scheduled_batch: ScheduledRequests, + sample_state: SampleState, + iteration_id: Optional[int] = None) -> Tuple[bool, Optional[float]]: + if (self.speculation_gate is None + or self.speculation_permanently_disabled or self.is_warmup): + return False, None + + if (getattr(self.dist, 'has_pp', False) + and not self.dist.is_last_pp_rank): + return False, None + new_tokens_lens = getattr(sample_state.host, 'new_tokens_lens', None) + if new_tokens_lens is None: + return False, None + new_tokens_lens_list = (new_tokens_lens.tolist() if hasattr( + new_tokens_lens, 'tolist') else list(new_tokens_lens)) + total_draft_tokens = 0 + total_accepted_tokens = 0 + for request in scheduled_batch.generation_requests: + draft_len = request.num_draft_tokens + if draft_len <= 0 or request.is_dummy: + continue + total_draft_tokens += draft_len + total_accepted_tokens += request.py_num_accepted_draft_tokens + + if total_draft_tokens <= 0: + return False, None + + acceptance_rate = total_accepted_tokens / total_draft_tokens + disabled_now, avg = self.speculation_gate.record_acceptance_rate( + acceptance_rate, sample_id=iteration_id) + if disabled_now: + self.speculation_permanently_disabled = True + return disabled_now, avg + def _append_iter_stats(self, stats: IterationStats, req_stats: Optional[List[RequestStats]] = None, @@ -3033,6 +3066,11 @@ def _handle_executed_batch(self, executed_batch: Optional[BatchStatePP]): if executed_batch is not None: with torch.cuda.nvtx.range("_handle_executed_batch_pp"): self._update_requests(executed_batch.sample_state) + if self.speculation_gate is not None: + self._update_batch_acceptance_rate( + executed_batch.scheduled_requests, + executed_batch.sample_state, + iteration_id=self.iter_counter) scheduled_requests = executed_batch.scheduled_requests if self._is_kv_manager_v2: @@ -3115,6 +3153,12 @@ def _handle_dynamic_draft_len(self, if not hasattr(self.model_engine, 'max_draft_len'): return + if self.speculation_permanently_disabled: + for request in scheduled_batch.generation_requests: + request.py_draft_tokens = [] + self.model_engine.runtime_draft_len = 0 + return + if (self.model_engine.spec_config is not None and self.model_engine.spec_config.draft_len_schedule is not None and self.model_engine.spec_config.spec_dec_mode. @@ -3954,6 +3998,11 @@ def _executor_loop(self): self._update_request_states(scheduled_batch) self._update_requests(sample_state, self.resource_manager) + if self.speculation_gate is not None: + self._update_batch_acceptance_rate( + scheduled_batch, + sample_state, + iteration_id=self.iter_counter) if self._is_kv_manager_v2: # Finalize V2 context KV before disagg transfer/response @@ -4408,6 +4457,13 @@ def _executor_loop_overlap(self): if self.previous_batch is not None and should_process_previous_batch: self._update_requests(self.previous_batch.sample_state) + # Turning off speculative decoding when Acceptance Rate is low. + # In overlap scheduler path, it will do an extra iter with spec decode on. + if self.speculation_gate is not None: + self._update_batch_acceptance_rate( + self.previous_batch.scheduled_requests, + self.previous_batch.sample_state, + iteration_id=self.iter_counter) self._send_kv_async( self.previous_batch.scheduled_requests.all_requests()) @@ -6369,31 +6425,6 @@ def _handle_responses(self, emit_first_iter: bool = True): new_responses.append((req_id, response)) if request_done: - if (self.drafter is not None and getattr( - self.model_engine, 'enable_spec_decode', False) - and not self.speculation_permanently_disabled - and not request.is_dummy and not self.is_warmup): - if self.speculation_gate is not None: - # Response handling runs on multiple PP ranks. Only the last PP rank performs - # sampling; restrict rolling stat updates to it to avoid overcounting. - if (not getattr(self.dist, 'has_pp', - False)) or self.dist.is_last_pp_rank: - avg_decoded = getattr( - request, 'avg_decoded_tokens_per_iter', None) - if avg_decoded is not None: - disabled_now, _ = self.speculation_gate.record_avg_decoded( - avg_decoded, - request_id=getattr(request, 'py_request_id', - None)) - if disabled_now: - # disable speculation permanently - # starting from next iteration, _prepare_and_schedule_batch will set self.use_spec_decode to False - self.speculation_permanently_disabled = True - else: - logger.debug( - f"Request {request.py_request_id} has no avg_decoded_tokens_per_iter" - ) - # PP=1-only early termination; _end_transfer_and_maybe_terminate # gates on the same flag so the request terminates exactly once. force_terminate_for_partial_reuse = ( diff --git a/tensorrt_llm/_torch/speculative/speculation_gate.py b/tensorrt_llm/_torch/speculative/speculation_gate.py index 69b4fa22e99e..84d99985b2d2 100644 --- a/tensorrt_llm/_torch/speculative/speculation_gate.py +++ b/tensorrt_llm/_torch/speculative/speculation_gate.py @@ -1,77 +1,82 @@ from collections import deque -from typing import Optional, Tuple +from typing import Deque, Optional, Tuple from tensorrt_llm.logger import logger class SpeculationGate: """ - Tracks rolling average of accepted draft tokens per iteration over the last N completed requests. - Permanently disables speculation when average falls below a threshold. - """ + Tracks a rolling average of true acceptance-rate samples over the last N + speculation-enabled decoding iterations. + + Permanently disables speculation when the rolling average falls below the + configured threshold. + """ def __init__(self, window: int, threshold: float): self.window = window self.threshold = threshold - self.acceptance_history: Deque[float] = deque() - self.acceptance_sum: float = 0.0 - self.num_completed_for_acceptance = 0 + self.acceptance_rate_history: Deque[float] = deque() + self.acceptance_rate_sum: float = 0.0 + self.num_recorded_samples = 0 self.disabled = False logger.debug( f"[SpeculationGate] SpeculationGate initialized with window={self.window}, threshold={self.threshold}" ) def reset(self) -> None: - self.acceptance_history.clear() - self.acceptance_sum = 0.0 - self.num_completed_for_acceptance = 0 + self.acceptance_rate_history.clear() + self.acceptance_rate_sum = 0.0 + self.num_recorded_samples = 0 self.disabled = False - def record_avg_decoded( + def record_acceptance_rate( self, - avg_decoded_tokens_per_iter: float, - request_id: Optional[int] = None) -> Tuple[bool, Optional[float]]: + acceptance_rate: float, + sample_id: Optional[int] = None) -> Tuple[bool, Optional[float]]: """ - Record a completed request's avg_decoded_tokens_per_iter. - Returns (disabled_now, current_avg_accept) where disabled_now is True only when the call causes disable. + Record one speculation-enabled iteration's true acceptance rate. + + Returns (disabled_now, current_avg_acceptance_rate) where + disabled_now is True only when this call causes permanent disable. """ if self.disabled or self.window is None or self.window <= 0 or self.threshold is None: return False, None - # Extra Guard: if caller passed None, skip updating the rolling stats - if avg_decoded_tokens_per_iter is None: + if acceptance_rate is None: return False, None - accepted_len = 0.0 - accepted_len = max(0.0, float(avg_decoded_tokens_per_iter) - 1.0) + acceptance_rate = float(acceptance_rate) + if acceptance_rate < 0.0 or acceptance_rate > 1.0: + raise ValueError("acceptance_rate must be in the range [0.0, 1.0], " + f"got {acceptance_rate}") - # Log per-request completion for debug - if request_id is not None: - logger.debug( - f"[SpeculationGate] Request {request_id} completed: avg_decoded={avg_decoded_tokens_per_iter if avg_decoded_tokens_per_iter is not None else 'None'}, accepted_len={accepted_len:.3f}" - ) + if sample_id is not None: + logger.debug(f"[SpeculationGate] Iteration {sample_id} recorded " + f"acceptance_rate={acceptance_rate:.3f}") # O(1) rolling update - self.acceptance_history.append(accepted_len) - logger.debug( - f"[SpeculationGate] Acceptance history: {self.acceptance_history}") - self.acceptance_sum += accepted_len - if len(self.acceptance_history) > self.window: - removed = self.acceptance_history.popleft() - self.acceptance_sum -= removed + self.acceptance_rate_history.append(acceptance_rate) + logger.debug(f"[SpeculationGate] Acceptance-rate history: " + f"{self.acceptance_rate_history}") + self.acceptance_rate_sum += acceptance_rate + if len(self.acceptance_rate_history) > self.window: + removed = self.acceptance_rate_history.popleft() + self.acceptance_rate_sum -= removed - self.num_completed_for_acceptance += 1 + self.num_recorded_samples += 1 - if self.num_completed_for_acceptance >= self.window: - avg_accept = self.acceptance_sum / len(self.acceptance_history) - if avg_accept < self.threshold: + if self.num_recorded_samples >= self.window: + avg_acceptance_rate = (self.acceptance_rate_sum / + len(self.acceptance_rate_history)) + if avg_acceptance_rate < self.threshold: self.disabled = True logger.info( - f"[SpeculationGate] Speculative decoding disabled: rolling acceptance avg {avg_accept:.3f} < threshold {self.threshold} over last {self.window} requests" - ) - return True, avg_accept - else: - # speculation is still enabled - return False, avg_accept + "[SpeculationGate] Speculative decoding disabled: " + f"rolling acceptance rate avg " + f"{avg_acceptance_rate:.3f} < threshold " + f"{self.threshold} over last {self.window} iterations") + return True, avg_acceptance_rate + return False, avg_acceptance_rate return False, None diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 5c6c89e88b3c..faaeea64bc1c 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -1610,19 +1610,23 @@ class DecodingBaseConfig(StrictBaseModel): load_format: Optional[str] = Field( default=None, description="The load format of the speculative model.") - acceptance_window: Optional[NonNegativeInt] = Field( + acceptance_rate_window_size: Optional[NonNegativeInt] = Field( default=None, description= - "The rolling average window size (N) for acceptance length across completed requests. " + "The rolling average window size (N) for acceptance rate across " + "speculation-enabled decoding iterations. " "If not set or set to 0, the feature is disabled. PyTorch backend only." ) - acceptance_length_threshold: Optional[NonNegativeFloat] = Field( + acceptance_rate_threshold: Optional[float] = Field( default=None, - description= - "The threshold for average acceptance length; speculation will be disabled permanently once the " - "rolling average over the last N completed requests (N = acceptance_window) drops below this value. " - "PyTorch backend only.") + ge=0.0, + le=1.0, + description="The threshold for average true acceptance rate " + "(accepted_draft_tokens / drafted_tokens); speculation will be " + "disabled permanently once the rolling average over the last N " + "speculation-enabled decoding iterations " + "(N = acceptance_rate_window_size) drops below this value. ") use_rejection_sampling: bool = Field( default=False, diff --git a/tensorrt_llm/usage/llm_args_golden_manifest.json b/tensorrt_llm/usage/llm_args_golden_manifest.json index 1362f671ccfb..95e5bf391667 100644 --- a/tensorrt_llm/usage/llm_args_golden_manifest.json +++ b/tensorrt_llm/usage/llm_args_golden_manifest.json @@ -1566,17 +1566,17 @@ }, { "allowed_values": [], - "annotation": "Optional[Annotated[float, Ge(ge=0)]]", + "annotation": "Optional[float]", "converter": "", "kind": "value", - "path": "speculative_config.acceptance_length_threshold" + "path": "speculative_config.acceptance_rate_threshold" }, { "allowed_values": [], "annotation": "Optional[Annotated[int, Ge(ge=0)]]", "converter": "", "kind": "value", - "path": "speculative_config.acceptance_window" + "path": "speculative_config.acceptance_rate_window_size" }, { "allowed_values": [], @@ -3931,17 +3931,17 @@ }, { "allowed_values": [], - "annotation": "Optional[Annotated[float, Ge(ge=0)]]", + "annotation": "Optional[float]", "converter": "", "kind": "value", - "path": "speculative_config.acceptance_length_threshold" + "path": "speculative_config.acceptance_rate_threshold" }, { "allowed_values": [], "annotation": "Optional[Annotated[int, Ge(ge=0)]]", "converter": "", "kind": "value", - "path": "speculative_config.acceptance_window" + "path": "speculative_config.acceptance_rate_window_size" }, { "allowed_values": [], diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_spec_gate.py b/tests/unittest/_torch/speculative/hw_agnostic/test_spec_gate.py index 871bc8e96bbb..cb0001e0b718 100644 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_spec_gate.py +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_spec_gate.py @@ -35,29 +35,33 @@ def test_spec_gate_e2e(enforce_single_worker): eagle_model_dir = f"{models_path}/EAGLE3-LLaMA3.1-Instruct-8B" target_model_dir = f"{models_path}/llama-3.1-model/Llama-3.1-8B-Instruct" - max_batch_size = 2 + max_batch_size = 3 max_draft_len = 4 - acceptance_window = 3 - acceptance_threshold = 0.6 - kv_cache_config = KvCacheConfig(enable_block_reuse=True, max_tokens=8192) + acceptance_rate_window_size = 3 + acceptance_rate_threshold = 0.6 + kv_cache_config = KvCacheConfig( + enable_block_reuse=False, + free_gpu_memory_fraction=0.6, + ) cuda_graph_config = CudaGraphConfig(batch_sizes=[1]) llm_common_config = dict( model=target_model_dir, attn_backend="TRTLLM", - disable_overlap_scheduler=True, - cuda_graph_config=cuda_graph_config, + disable_overlap_scheduler=False, max_batch_size=max_batch_size, kv_cache_config=kv_cache_config, - max_seq_len=4096, + cuda_graph_config=cuda_graph_config, + enable_chunked_prefill=False, + max_num_tokens=8192, ) spec_config = Eagle3DecodingConfig( max_draft_len=max_draft_len, speculative_model=eagle_model_dir, - eagle3_one_model=False, - acceptance_window=acceptance_window, - acceptance_length_threshold=acceptance_threshold, + eagle3_one_model=True, + acceptance_rate_window_size=acceptance_rate_window_size, + acceptance_rate_threshold=acceptance_rate_threshold, ) prompts = [ @@ -67,28 +71,27 @@ def test_spec_gate_e2e(enforce_single_worker): ] sampling_params = SamplingParams(max_tokens=20, temperature=0) - # Track calls to record_avg_decoded and the disabled state + # Track calls to record_acceptance_rate and the disabled state. gate_state = {"record_calls": [], "gate_disabled": False} - original_record_avg_decoded = SpeculationGate.record_avg_decoded + original_record_acceptance_rate = SpeculationGate.record_acceptance_rate - def mock_record_avg_decoded(self, avg_decoded_tokens_per_iter, request_id=None): + def mock_record_acceptance_rate(self, acceptance_rate, sample_id=None): """ - Mock that simulates low acceptance rate (1.2 tokens/iter = 0.2 accepted). - This is below the threshold of 0.6, so the gate should trigger after the window fills. + Mock that simulates a low true acceptance rate. + This is below the threshold of 0.6, so the gate should trigger after + the window fills. """ - # Simulate low acceptance: avg_decoded = 1.2 means accepted_len = 0.2 - # This is below threshold (0.6), so gate should trigger - simulated_low_avg = 1.2 - disabled_now, avg = original_record_avg_decoded(self, simulated_low_avg, request_id) + simulated_low_rate = 0.2 + disabled_now, avg = original_record_acceptance_rate(self, simulated_low_rate, sample_id) gate_state["record_calls"].append( { - "original_avg": avg_decoded_tokens_per_iter, - "simulated_avg": simulated_low_avg, + "original_rate": acceptance_rate, + "simulated_rate": simulated_low_rate, "disabled_now": disabled_now, - "avg_accept": avg, - "request_id": request_id, + "avg_acceptance_rate": avg, + "sample_id": sample_id, } ) if disabled_now: @@ -99,40 +102,43 @@ def mock_record_avg_decoded(self, avg_decoded_tokens_per_iter, request_id=None): llm_spec = LLM(**llm_common_config, speculative_config=spec_config) try: - with patch.object(SpeculationGate, "record_avg_decoded", mock_record_avg_decoded): + with patch.object(SpeculationGate, "record_acceptance_rate", mock_record_acceptance_rate): llm_spec.generate(prompts, sampling_params) # Verify the mock was called (requests completed) - assert len(gate_state["record_calls"]) > 0, "record_avg_decoded should have been called" + assert len(gate_state["record_calls"]) > 0, "record_acceptance_rate should have been called" # Verify the gate was disabled after enough requests with low acceptance assert gate_state["gate_disabled"], ( f"Gate should have been disabled with simulated low acceptance. Calls: {gate_state['record_calls']}" ) - # Verify the gate triggered at the right time (after window is filled) - # The gate should trigger on the `acceptance_window`-th call (index = window - 1) + # Verify the gate triggered at the right time (after the window is filled). + # The gate should trigger on the `acceptance_rate_window_size`-th call + # (index = window - 1). disable_indices = [ i for i, call in enumerate(gate_state["record_calls"]) if call["disabled_now"] ] assert len(disable_indices) == 1, ( f"Gate should have triggered exactly once, but triggered at indices: {disable_indices}" ) - assert disable_indices[0] >= acceptance_window - 1, ( - f"Gate should trigger after window ({acceptance_window}) is filled, " - f"but triggered at index {disable_indices[0]}" + assert disable_indices[0] >= acceptance_rate_window_size - 1, ( + f"Gate should trigger after window ({acceptance_rate_window_size}) " + f"is filled, but triggered at index {disable_indices[0]}" ) - # Verify the average acceptance was below threshold when disabled + # Verify the average acceptance rate was below threshold when disabled. disable_call = gate_state["record_calls"][disable_indices[0]] - assert disable_call["avg_accept"] is not None - assert disable_call["avg_accept"] < acceptance_threshold, ( - f"Avg acceptance ({disable_call['avg_accept']}) should be below threshold ({acceptance_threshold})" + assert disable_call["avg_acceptance_rate"] is not None + assert disable_call["avg_acceptance_rate"] < acceptance_rate_threshold, ( + f"Avg acceptance rate ({disable_call['avg_acceptance_rate']}) " + f"should be below threshold ({acceptance_rate_threshold})" ) logger.debug(f"Gate correctly triggered after {disable_indices[0] + 1} requests") logger.debug( - f"Final avg acceptance: {disable_call['avg_accept']:.3f} < threshold {acceptance_threshold}" + f"Final avg acceptance rate: {disable_call['avg_acceptance_rate']:.3f} " + f"< threshold {acceptance_rate_threshold}" ) finally: llm_spec.shutdown() @@ -141,56 +147,56 @@ def mock_record_avg_decoded(self, avg_decoded_tokens_per_iter, request_id=None): def test_returns_none_until_window_and_enabled_when_above_threshold(): gate = SpeculationGate(window=3, threshold=0.5) - disabled, avg = gate.record_avg_decoded(2.0, request_id=1) + disabled, avg = gate.record_acceptance_rate(0.8, sample_id=1) assert disabled is False and avg is None assert gate.disabled is False - disabled, avg = gate.record_avg_decoded(2.0, request_id=2) + disabled, avg = gate.record_acceptance_rate(0.8, sample_id=2) assert disabled is False and avg is None assert gate.disabled is False - disabled, avg = gate.record_avg_decoded(2.0, request_id=3) + disabled, avg = gate.record_acceptance_rate(0.8, sample_id=3) assert disabled is False - assert avg == pytest.approx(1.0, rel=1e-6) + assert avg == pytest.approx(0.8, rel=1e-6) assert gate.disabled is False def test_disables_when_avg_below_threshold_and_stays_disabled(): - gate = SpeculationGate(window=3, threshold=0.7) + gate = SpeculationGate(window=3, threshold=0.3) - gate.record_avg_decoded(1.1) - gate.record_avg_decoded(1.2) + gate.record_acceptance_rate(0.1) + gate.record_acceptance_rate(0.2) - disabled, avg = gate.record_avg_decoded(1.3) + disabled, avg = gate.record_acceptance_rate(0.3) assert disabled is True assert avg == pytest.approx(0.2, rel=1e-6) assert gate.disabled is True # Once disabled, subsequent calls do nothing and return (False, None) - disabled, avg = gate.record_avg_decoded(100.0) + disabled, avg = gate.record_acceptance_rate(1.0) assert disabled is False and avg is None assert gate.disabled is True - disabled, avg = gate.record_avg_decoded(200.0) + disabled, avg = gate.record_acceptance_rate(1.0) assert disabled is False and avg is None assert gate.disabled is True def test_rolling_window_and_disable_on_drop(): - gate = SpeculationGate(window=3, threshold=0.8) + gate = SpeculationGate(window=3, threshold=0.7) # First three high-acceptance requests keep it enabled - gate.record_avg_decoded(2.0) - gate.record_avg_decoded(2.0) - disabled, avg = gate.record_avg_decoded(2.0) + gate.record_acceptance_rate(0.9) + gate.record_acceptance_rate(0.9) + disabled, avg = gate.record_acceptance_rate(0.9) assert disabled is False - assert avg == pytest.approx(1.0, rel=1e-6) + assert avg == pytest.approx(0.9, rel=1e-6) assert gate.disabled is False # Fourth lower value enters window -> average drops below threshold -> disable - disabled, avg = gate.record_avg_decoded(1.2) + disabled, avg = gate.record_acceptance_rate(0.2) assert disabled is True - assert avg == pytest.approx((1.0 + 1.0 + 0.2) / 3.0, rel=1e-6) + assert avg == pytest.approx((0.9 + 0.9 + 0.2) / 3.0, rel=1e-6) assert gate.disabled is True diff --git a/tests/unittest/api_stability/references/decoding_base_config.yaml b/tests/unittest/api_stability/references/decoding_base_config.yaml new file mode 100644 index 000000000000..cff72989e197 --- /dev/null +++ b/tests/unittest/api_stability/references/decoding_base_config.yaml @@ -0,0 +1,35 @@ +methods: + __init__: + parameters: + acceptance_rate_threshold: + annotation: Optional[float] + default: null + acceptance_rate_window_size: + annotation: Optional[Annotated[int, Ge(ge=0)]] + default: null + allow_advanced_sampling: + annotation: bool + default: false + draft_len_schedule: + annotation: Optional[dict[int, int]] + default: null + load_format: + annotation: Optional[str] + default: null + max_concurrency: + annotation: Optional[Annotated[int, Gt(gt=0)]] + default: null + max_draft_len: + annotation: Optional[Annotated[int, Ge(ge=0)]] + default: null + max_total_draft_tokens: + annotation: Optional[int] + default: null + speculative_model: + annotation: Union[str, pathlib.Path, NoneType] + default: null + use_rejection_sampling: + annotation: bool + default: false + return_annotation: None +properties: {}