support qwen2.5 eagle3 training (#220)

Author: liusong1222

angelslim/compressor/speculative/train/configs/qwen2.5-7b-eagle3.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "Eagle3LlamaForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "llama",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 1,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.1",
+  "use_cache": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064,
+  "draft_vocab_size": 32000
+}

angelslim/compressor/speculative/train/data/chat_templates.py

@@ -27,6 +27,7 @@ class ChatTemplateType(Enum):
     """Supported chat template types."""
 
     QWEN2_AUDIO = "qwen2_audio"
+    QWEN2_5 = "qwen2.5"
     QWEN3 = "qwen3"
     HUNYUAN = "hunyuan"
     QWEN3_VL = "qwen3_vl"
@@ -37,6 +38,7 @@ class ChatTemplateType(Enum):
 # String to ChatTemplateType mapping
 CHAT_TEMPLATE_TYPE_MAPPING = {
     "qwen2_audio": ChatTemplateType.QWEN2_AUDIO,
+    "qwen2.5": ChatTemplateType.QWEN2_5,
     "qwen3": ChatTemplateType.QWEN3,
     "hunyuan": ChatTemplateType.HUNYUAN,
     "hunyuan_7b": ChatTemplateType.HUNYUAN_7B,
@@ -86,6 +88,21 @@ def _initialize_templates(self) -> Dict[ChatTemplateType, ChatTemplate]:
                     "please don't share false information."
                 ),
             ),
+            ChatTemplateType.QWEN2_5: ChatTemplate(
+                user_header="<|im_start|>user\n",
+                assistant_header="<|im_start|>assistant\n",
+                system_prompt=(
+                    "You are a helpful, respectful and honest assistant. "
+                    "Always answer as helpfully as possible, while being safe. "
+                    "Your answers should not include any harmful, unethical, racist, "
+                    "sexist, toxic, dangerous, or illegal content. Please ensure that "
+                    "your responses are socially unbiased and positive in nature.\n\n"
+                    "If a question does not make any sense, or is not factually "
+                    "coherent, explain why instead of answering something not "
+                    "correct. If you don't know the answer to a question, "
+                    "please don't share false information."
+                ),
+            ),
             ChatTemplateType.HUNYUAN: ChatTemplate(
                 user_header="<｜hy_User｜>",
                 assistant_header="<｜hy_Assistant｜>",

docs/source/features/speculative_decoding/eagle/eagle.md

@@ -73,7 +73,7 @@ bash scripts/speculative/generate_vlm_hidden_for_draft_model.sh
 - `MODEL_NAME`: 目标模型的HF名称或本地路径
 - `TARGET_BACKEND`: 目标模型后端，目前仅支持HF
 - `MODEL_MAX_LENGTH`: 生成数据的上下文长度
-- `CHAT_TEMPLATE_TYPE`: 目标模型的目标类型，目前支持qwen3/hunyuan
+- `CHAT_TEMPLATE_TYPE`: 目标模型的目标类型，目前支持qwen3/qwen2.5/hunyuan/hunyuan-7b
 - `OUTPUT_DIR`: 生成的数据集输出路径