polish

Signed-off-by: h-guo18 <67671475+h-guo18@users.noreply.github.com>

Author: h-guo18

modelopt/torch/speculative/plugins/megatron_eagle.py

@@ -750,8 +750,6 @@ def modify(
             with FakeTensorMode():
                 d2t = torch.load(draft_vocab_cache, mmap=True)
             self.eagle_config.draft_vocab_size = d2t.shape[0]
-        else:
-            self.eagle_config.draft_vocab_size = self.vocab_size
 
         if self.eagle_config.draft_vocab_size != self.eagle_config.vocab_size:
             assert eagle_self_logit_distillation, (