Skip to content

Commit 54b2279

Browse files
[#13534][chore] AutoDeploy: Remove Two Model Speculative Decoding Support (#13532)
Signed-off-by: Govind Ramnarayan <105831528+govind-ramnarayan@users.noreply.github.com>
1 parent 579a10f commit 54b2279

10 files changed

Lines changed: 161 additions & 596 deletions

File tree

tensorrt_llm/_torch/auto_deploy/llm_args.py

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -109,35 +109,57 @@ def ensure_no_custom_parallel_config(cls, value: Any, info: ValidationInfo) -> A
109109
msg = "AutoDeploy only supports parallelization via the `world_size` argument."
110110
return _check_for_default_value_only(cls, value, info, msg)
111111

112+
@model_validator(mode="after")
113+
def validate_supported_speculative_config(self):
114+
spec_config = self.speculative_config
115+
if spec_config is None:
116+
return self
117+
118+
if isinstance(spec_config, MTPDecodingConfig):
119+
if not spec_config.mtp_eagle_one_model or spec_config.use_mtp_vanilla:
120+
raise ValueError(
121+
"AutoDeploy only supports MTP speculative decoding with "
122+
"mtp_eagle_one_model=True and use_mtp_vanilla=False "
123+
f"(got mtp_eagle_one_model={spec_config.mtp_eagle_one_model}, "
124+
f"use_mtp_vanilla={spec_config.use_mtp_vanilla})."
125+
)
126+
elif isinstance(spec_config, EagleDecodingConfig):
127+
if not spec_config.eagle3_one_model:
128+
raise ValueError(
129+
"AutoDeploy only supports Eagle speculative decoding with "
130+
f"eagle3_one_model=True (got eagle3_one_model={spec_config.eagle3_one_model})."
131+
)
132+
else:
133+
raise ValueError(
134+
"AutoDeploy only supports speculative decoding via "
135+
"MTPDecodingConfig(mtp_eagle_one_model=True) or "
136+
"EagleDecodingConfig(eagle3_one_model=True)."
137+
)
138+
139+
self.model_factory = "eagle_one_model"
140+
return self
141+
112142
@model_validator(mode="after")
113143
def setup_hidden_state_capture(self):
114144
spec_config = self.speculative_config
115145
if spec_config is None:
116146
return self
117147

118148
if isinstance(spec_config, MTPDecodingConfig):
119-
if not spec_config.mtp_eagle_one_model:
120-
return self
121-
if spec_config.use_mtp_vanilla:
122-
raise ValueError("mtp_eagle_one_model and use_mtp_vanilla cannot both be enabled")
123149
if spec_config.max_draft_len is None:
124150
raise ValueError(
125151
"MTPDecodingConfig.max_draft_len must not be None when mtp_eagle_one_model is "
126152
"enabled. Ensure num_nextn_predict_layers is set in the model config."
127153
)
128154
capture_layers = {-1}
129-
self.model_factory = "eagle_one_model"
130-
elif isinstance(spec_config, EagleDecodingConfig):
155+
else:
156+
assert isinstance(spec_config, EagleDecodingConfig)
131157
if spec_config.max_draft_len is None:
132158
raise ValueError(
133159
"EagleDecodingConfig.max_draft_len must not be None. "
134160
"Provide a positive integer for max_draft_len."
135161
)
136162
capture_layers = spec_config.eagle3_layers_to_capture
137-
if spec_config.eagle3_one_model:
138-
self.model_factory = "eagle_one_model"
139-
else:
140-
return self
141163

142164
self.transforms["detect_hidden_states_for_capture"]["enabled"] = True
143165
self.transforms["detect_hidden_states_for_capture"]["eagle3_layers_to_capture"] = (
@@ -223,11 +245,6 @@ def validate_and_init_tokenizer(self):
223245

224246
device: str = Field(default="cuda", description="The device to use for the model.", frozen=True)
225247

226-
draft_checkpoint_loader: Optional[object] = Field(
227-
default=None,
228-
description="The checkpoint loader to use for the draft model when using speculative decoding with two models.",
229-
)
230-
231248
### INFERENCE OPTIMIZER CONFIG #################################################################
232249
mode: Literal["graph", "transformers"] = Field(
233250
default="graph",

0 commit comments

Comments
 (0)