Skip to content

Commit d379cbf

Browse files
PotterluoPotterluo
committed
feat(kv_cache_calculator): Add sliding window and hybrid architecture support
- Update Model Config
1 parent 3fb9524 commit d379cbf

1 file changed

Lines changed: 36 additions & 91 deletions

File tree

‎docs/source/_static/kv_cache_calculator.html‎

Lines changed: 36 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -1127,18 +1127,8 @@ <h2 class="section-title">
11271127
"num_hidden_layers": 126,
11281128
"num_key_value_heads": 8
11291129
},
1130-
// GLM Models (Zhipu AI)
1131-
"zai-org/GLM-4.5":{
1132-
"hidden_size": 6144,
1133-
"num_attention_heads": 64,
1134-
"num_hidden_layers": 78,
1135-
"num_key_value_heads": 64,
1136-
"kv_lora_rank": 512,
1137-
"qk_rope_head_dim": 64
1138-
},
1139-
11401130
// =====================================================
1141-
// Additional 2025 Mainstream Models
1131+
// Additional 2026 Mainstream Models
11421132
// =====================================================
11431133

11441134
// DeepSeek V3.2 (MLA)
@@ -1152,33 +1142,40 @@ <h2 class="section-title">
11521142
},
11531143

11541144
// GLM Series
1155-
"zai-org/GLM-4.7": {
1145+
"zai-org/GLM-4.5":{
11561146
"hidden_size": 5120,
11571147
"num_attention_heads": 96,
11581148
"num_hidden_layers": 92,
11591149
"num_key_value_heads": 8,
11601150
"head_dim": 128
11611151
},
1162-
"zai-org/GLM-4.7-Flash": {
1163-
"hidden_size": 5120,
1152+
"zai-org/GLM-4.5-Air": {
1153+
"hidden_size": 4096,
11641154
"num_attention_heads": 96,
1165-
"num_hidden_layers": 47,
1155+
"num_hidden_layers": 46,
11661156
"num_key_value_heads": 8,
1167-
"kv_lora_rank": 512,
1168-
"qk_rope_head_dim": 64
1157+
"head_dim": 128
11691158
},
1170-
"zai-org/GLM-4.5-Air": {
1159+
"zai-org/GLM-4.7": {
11711160
"hidden_size": 5120,
1172-
"num_attention_heads": 40,
1173-
"num_hidden_layers": 46,
1161+
"num_attention_heads": 96,
1162+
"num_hidden_layers": 92,
11741163
"num_key_value_heads": 8,
11751164
"head_dim": 128
11761165
},
1166+
"zai-org/GLM-4.7-Flash": {
1167+
"hidden_size": 2048,
1168+
"num_attention_heads": 20,
1169+
"num_hidden_layers": 47,
1170+
"num_key_value_heads": 20,
1171+
"kv_lora_rank": 512,
1172+
"qk_rope_head_dim": 64
1173+
},
11771174
"zai-org/GLM-5": {
1178-
"hidden_size": 7168,
1179-
"num_attention_heads": 128,
1175+
"hidden_size": 6144,
1176+
"num_attention_heads": 64,
11801177
"num_hidden_layers": 78,
1181-
"num_key_value_heads": 128,
1178+
"num_key_value_heads": 64,
11821179
"kv_lora_rank": 512,
11831180
"qk_rope_head_dim": 64
11841181
},
@@ -1208,54 +1205,33 @@ <h2 class="section-title">
12081205

12091206
// Qwen3 Series (GQA with explicit head_dim, some Hybrid)
12101207
"Qwen/Qwen3.5-397B-A17B": {
1211-
"hidden_size": 5120,
1212-
"num_attention_heads": 80,
1213-
"num_hidden_layers": 94,
1214-
"num_key_value_heads": 2,
1215-
"head_dim": 128
1216-
},
1217-
"Qwen/Qwen3-235B-A22B": {
12181208
"hidden_size": 4096,
1219-
"num_attention_heads": 64,
1220-
"num_hidden_layers": 94,
1221-
"num_key_value_heads": 4,
1222-
"head_dim": 128
1209+
"num_attention_heads": 32,
1210+
"num_hidden_layers": 60,
1211+
"num_key_value_heads": 2,
1212+
"head_dim": 256
12231213
},
12241214
"Qwen/Qwen3.5-122B-A10B": {
1225-
"hidden_size": 5120,
1226-
"num_attention_heads": 80,
1227-
"num_hidden_layers": 94,
1215+
"hidden_size": 3072,
1216+
"num_attention_heads": 32,
1217+
"num_hidden_layers": 48,
12281218
"num_key_value_heads": 2,
1229-
"head_dim": 128
1219+
"head_dim": 256
1220+
},
1221+
"Qwen/Qwen3.5-35B-A3B": {
1222+
"hidden_size": 2048,
1223+
"num_attention_heads": 16,
1224+
"num_hidden_layers": 40,
1225+
"num_key_value_heads": 2,
1226+
"head_dim": 256
12301227
},
12311228
"Qwen/Qwen3.5-27B": {
12321229
"hidden_size": 5120,
12331230
"num_attention_heads": 40,
12341231
"num_hidden_layers": 64,
12351232
"num_key_value_heads": 4,
1236-
"head_dim": 128
1237-
},
1238-
1239-
// Gemma4 Series (Hybrid + Sliding Window + Multimodal)
1240-
"google/gemma-4-31B": {
1241-
"hidden_size": 2816,
1242-
"num_attention_heads": 16,
1243-
"num_hidden_layers": 30,
1244-
"num_key_value_heads": 8,
1245-
"head_dim": 256,
1246-
"sliding_window": 1024,
1247-
"attention_layer_count": 6
1248-
},
1249-
"google/gemma-4-26B-A4B-it": {
1250-
"hidden_size": 2816,
1251-
"num_attention_heads": 16,
1252-
"num_hidden_layers": 30,
1253-
"num_key_value_heads": 8,
1254-
"head_dim": 256,
1255-
"sliding_window": 1024,
1256-
"attention_layer_count": 6
1233+
"head_dim": 256
12571234
},
1258-
12591235
// Kimi Series (MLA + Multimodal)
12601236
"moonshot/Kimi-K2.5": {
12611237
"hidden_size": 7168,
@@ -1272,37 +1248,6 @@ <h2 class="section-title">
12721248
"num_key_value_heads": 64,
12731249
"kv_lora_rank": 512,
12741250
"qk_rope_head_dim": 64
1275-
},
1276-
1277-
// MiMo Series (Hybrid + Sliding Window)
1278-
"mimo/MiMo-V2-Flash": {
1279-
"hidden_size": 4096,
1280-
"num_attention_heads": 64,
1281-
"num_hidden_layers": 48,
1282-
"num_key_value_heads": 4,
1283-
"head_dim": 192,
1284-
"sliding_window": 128,
1285-
"attention_layer_count": 39
1286-
},
1287-
1288-
// GPT-OSS Series (Hybrid + Sliding)
1289-
"openai/gpt-oss-20b": {
1290-
"hidden_size": 5120,
1291-
"num_attention_heads": 40,
1292-
"num_hidden_layers": 48,
1293-
"num_key_value_heads": 8,
1294-
"head_dim": 128,
1295-
"sliding_window": 128,
1296-
"attention_layer_count": 24
1297-
},
1298-
"openai/gpt-oss-120b": {
1299-
"hidden_size": 6144,
1300-
"num_attention_heads": 64,
1301-
"num_hidden_layers": 72,
1302-
"num_key_value_heads": 8,
1303-
"head_dim": 96,
1304-
"sliding_window": 128,
1305-
"attention_layer_count": 36
13061251
}
13071252
};
13081253
}

0 commit comments

Comments
 (0)