Skip to content

Commit f3a65b4

Browse files
yuanyuxuanclaude
yuanyuxuan
and
claude
committed
feat: VLM DFlash multi-model support (Qwen3-VL/Qwen3.5/Qwen3.6, HF backend)
Extend DFlash training to support 5 VLM model families with unified transformers >= 5.7.0 requirement and HF backend. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent e1f10ff commit f3a65b4

7 files changed

Lines changed: 1160 additions & 152 deletions

‎configs/qwen3-vl-30b-a3b-dflash-vlm-8layer.json‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"architectures": [
3+
"DFlashDraftModel"
4+
],
5+
"attention_bias": false,
6+
"attention_dropout": 0.0,
7+
"block_size": 8,
8+
"bos_token_id": 151643,
9+
"dflash_config": {
10+
"target_layer_ids": [3, 13, 24, 34, 44]
11+
},
12+
"dtype": "bfloat16",
13+
"eos_token_id": 151645,
14+
"head_dim": 128,
15+
"hidden_act": "silu",
16+
"hidden_size": 2048,
17+
"initializer_range": 0.02,
18+
"intermediate_size": 6144,
19+
"layer_types": [
20+
"full_attention",
21+
"full_attention",
22+
"full_attention",
23+
"full_attention",
24+
"full_attention"
25+
],
26+
"max_position_embeddings": 262144,
27+
"model_type": "qwen3_vl_text",
28+
"num_attention_heads": 32,
29+
"num_hidden_layers": 5,
30+
"num_key_value_heads": 4,
31+
"num_target_layers": 48,
32+
"rms_norm_eps": 1e-06,
33+
"rope_scaling": {
34+
"mrope_interleaved": true,
35+
"mrope_section": [
36+
24,
37+
20,
38+
20
39+
],
40+
"rope_type": "default"
41+
},
42+
"rope_theta": 5000000,
43+
"tie_word_embeddings": false,
44+
"use_cache": true,
45+
"vocab_size": 151936
46+
}

‎configs/qwen3-vl-8b-dflash-vlm-8layer.json‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"architectures": [
3+
"DFlashDraftModel"
4+
],
5+
"attention_bias": false,
6+
"attention_dropout": 0.0,
7+
"block_size": 8,
8+
"bos_token_id": 151643,
9+
"dflash_config": {
10+
"target_layer_ids": [3, 10, 18, 25, 32]
11+
},
12+
"dtype": "bfloat16",
13+
"eos_token_id": 151645,
14+
"head_dim": 128,
15+
"hidden_act": "silu",
16+
"hidden_size": 4096,
17+
"initializer_range": 0.02,
18+
"intermediate_size": 12288,
19+
"layer_types": [
20+
"full_attention",
21+
"full_attention",
22+
"full_attention",
23+
"full_attention",
24+
"full_attention"
25+
],
26+
"max_position_embeddings": 262144,
27+
"model_type": "qwen3_vl_text",
28+
"num_attention_heads": 32,
29+
"num_hidden_layers": 5,
30+
"num_key_value_heads": 8,
31+
"num_target_layers": 36,
32+
"rms_norm_eps": 1e-06,
33+
"rope_scaling": {
34+
"mrope_interleaved": true,
35+
"mrope_section": [
36+
24,
37+
20,
38+
20
39+
],
40+
"rope_type": "default"
41+
},
42+
"rope_theta": 5000000,
43+
"tie_word_embeddings": false,
44+
"use_cache": true,
45+
"vocab_size": 151936
46+
}

‎configs/qwen3.5-35b-a3b-dflash-vlm-8layer.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{
2+
"architectures": [
3+
"DFlashDraftModel"
4+
],
5+
"attention_bias": false,
6+
"attention_dropout": 0.0,
7+
"block_size": 8,
8+
"bos_token_id": 248044,
9+
"dflash_config": {
10+
"target_layer_ids": [1, 10, 19, 28, 37]
11+
},
12+
"dtype": "bfloat16",
13+
"eos_token_id": 248046,
14+
"head_dim": 256,
15+
"hidden_act": "silu",
16+
"hidden_size": 2048,
17+
"initializer_range": 0.02,
18+
"intermediate_size": 6144,
19+
"layer_types": [
20+
"full_attention",
21+
"full_attention",
22+
"full_attention",
23+
"full_attention",
24+
"full_attention"
25+
],
26+
"max_position_embeddings": 262144,
27+
"model_type": "qwen3_vl_text",
28+
"num_attention_heads": 16,
29+
"num_hidden_layers": 5,
30+
"num_key_value_heads": 2,
31+
"num_target_layers": 40,
32+
"partial_rotary_factor": 0.25,
33+
"rms_norm_eps": 1e-06,
34+
"rope_scaling": {
35+
"mrope_interleaved": true,
36+
"mrope_section": [
37+
11,
38+
11,
39+
10
40+
],
41+
"rope_type": "default"
42+
},
43+
"rope_theta": 10000000,
44+
"tie_word_embeddings": false,
45+
"use_cache": true,
46+
"vocab_size": 248320
47+
}

‎configs/qwen3.5-9b-dflash-vlm-8layer.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{
2+
"architectures": [
3+
"DFlashDraftModel"
4+
],
5+
"attention_bias": false,
6+
"attention_dropout": 0.0,
7+
"block_size": 8,
8+
"bos_token_id": 248044,
9+
"dflash_config": {
10+
"target_layer_ids": [1, 8, 15, 22, 29]
11+
},
12+
"dtype": "bfloat16",
13+
"eos_token_id": 248046,
14+
"head_dim": 256,
15+
"hidden_act": "silu",
16+
"hidden_size": 4096,
17+
"initializer_range": 0.02,
18+
"intermediate_size": 12288,
19+
"layer_types": [
20+
"full_attention",
21+
"full_attention",
22+
"full_attention",
23+
"full_attention",
24+
"full_attention"
25+
],
26+
"max_position_embeddings": 262144,
27+
"model_type": "qwen3_vl_text",
28+
"num_attention_heads": 16,
29+
"num_hidden_layers": 5,
30+
"num_key_value_heads": 4,
31+
"num_target_layers": 32,
32+
"partial_rotary_factor": 0.25,
33+
"rms_norm_eps": 1e-06,
34+
"rope_scaling": {
35+
"mrope_interleaved": true,
36+
"mrope_section": [
37+
11,
38+
11,
39+
10
40+
],
41+
"rope_type": "default"
42+
},
43+
"rope_theta": 10000000,
44+
"tie_word_embeddings": false,
45+
"use_cache": true,
46+
"vocab_size": 248320
47+
}

0 commit comments

Comments
 (0)