Skip to content

Commit 4fe1230

Browse files
irisliu10irisliu10
authored
add int8_dynamic quant config of Qwen VL models (#266)
1 parent 676ae7d commit 4fe1230

10 files changed

Lines changed: 284 additions & 0 deletions

‎configs/qwen2_5_vl/int8_dynamic/qwen2_5_vl-32b_int8_dynamic.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Global configuration of pipeline
2+
global:
3+
save_path: ./output
4+
5+
# Simplified Configuration for LLM compression
6+
model:
7+
name: QwenVL
8+
model_path: Qwen/Qwen2.5-VL-32B-Instruct
9+
trust_remote_code: true
10+
low_cpu_mem_usage: true
11+
use_cache: false
12+
torch_dtype: auto
13+
device_map: auto
14+
15+
# Compression configuration
16+
compression:
17+
name: PTQ
18+
quantization:
19+
name: int8_dynamic
20+
bits: 8
21+
quant_method:
22+
weight: "per-channel"
23+
activation: "per-token"
24+
ignore_layers: # Skip quantization for these layers
25+
- "model.visual.patch_embed.proj"
26+
- "model.lm_head"
27+
- "model.language_model.embed_tokens"
28+
quant_vit: false
29+

‎configs/qwen2_5_vl/int8_dynamic/qwen2_5_vl-3b_int8_dynamic.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Global configuration of pipeline
2+
global:
3+
save_path: ./output
4+
5+
# Simplified Configuration for LLM compression
6+
model:
7+
name: QwenVL
8+
model_path: Qwen/Qwen2.5-VL-3B-Instruct
9+
trust_remote_code: true
10+
low_cpu_mem_usage: true
11+
use_cache: false
12+
torch_dtype: auto
13+
device_map: auto
14+
15+
# Compression configuration
16+
compression:
17+
name: PTQ
18+
quantization:
19+
name: int8_dynamic
20+
bits: 8
21+
quant_method:
22+
weight: "per-channel"
23+
activation: "per-token"
24+
ignore_layers: # Skip quantization for these layers
25+
- "model.visual.patch_embed.proj"
26+
- "model.lm_head"
27+
- "model.language_model.embed_tokens"
28+
quant_vit: false

‎configs/qwen2_5_vl/int8_dynamic/qwen2_5_vl-72b_int8_dynamic.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Global configuration of pipeline
2+
global:
3+
save_path: ./output
4+
5+
# Simplified Configuration for LLM compression
6+
model:
7+
name: QwenVL
8+
model_path: Qwen/Qwen2.5-VL-72B-Instruct
9+
trust_remote_code: true
10+
low_cpu_mem_usage: true
11+
use_cache: false
12+
torch_dtype: auto
13+
device_map: auto
14+
15+
# Compression configuration
16+
compression:
17+
name: PTQ
18+
quantization:
19+
name: int8_dynamic
20+
bits: 8
21+
quant_method:
22+
weight: "per-channel"
23+
activation: "per-token"
24+
ignore_layers: # Skip quantization for these layers
25+
- "model.visual.patch_embed.proj"
26+
- "model.lm_head"
27+
- "model.language_model.embed_tokens"
28+
quant_vit: false
29+

‎configs/qwen2_5_vl/int8_dynamic/qwen2_5_vl-7b_int8_dynamic.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Global configuration of pipeline
2+
global:
3+
save_path: ./output
4+
5+
# Simplified Configuration for LLM compression
6+
model:
7+
name: QwenVL
8+
model_path: Qwen/Qwen2.5-VL-7B-Instruct
9+
trust_remote_code: true
10+
low_cpu_mem_usage: true
11+
use_cache: false
12+
torch_dtype: auto
13+
device_map: auto
14+
15+
# Compression configuration
16+
compression:
17+
name: PTQ
18+
quantization:
19+
name: int8_dynamic
20+
bits: 8
21+
quant_method:
22+
weight: "per-channel"
23+
activation: "per-token"
24+
ignore_layers: # Skip quantization for these layers
25+
- "model.visual.patch_embed.proj"
26+
- "model.lm_head"
27+
- "model.language_model.embed_tokens"
28+
quant_vit: false

‎configs/qwen3_vl/int8_dynamic/qwen3_vl-235b_a22b_int8_dynamic.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Global configuration of pipeline
2+
global:
3+
save_path: ./output
4+
5+
# Simplified Configuration for LLM compression
6+
model:
7+
name: Qwen3VLMoE
8+
model_path: Qwen/Qwen3-VL-235B-A22B-Instruct
9+
trust_remote_code: true
10+
low_cpu_mem_usage: true
11+
use_cache: false
12+
torch_dtype: auto
13+
device_map: auto
14+
15+
# Compression configuration
16+
compression:
17+
name: PTQ
18+
quantization:
19+
name: int8_dynamic
20+
bits: 8
21+
quant_method:
22+
weight: "per-channel"
23+
activation: "per-token"
24+
ignore_layers: # Skip quantization for these layers
25+
- "model.visual.patch_embed.proj"
26+
- "model.lm_head"
27+
- "model.language_model.embed_tokens"
28+
quant_vit: false

‎configs/qwen3_vl/int8_dynamic/qwen3_vl-2b_int8_dynamic.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Global configuration of pipeline
2+
global:
3+
save_path: ./output
4+
5+
# Simplified Configuration for LLM compression
6+
model:
7+
name: Qwen3VL
8+
model_path: Qwen/Qwen3-VL-2B-Instruct
9+
trust_remote_code: true
10+
low_cpu_mem_usage: true
11+
use_cache: false
12+
torch_dtype: auto
13+
device_map: auto
14+
15+
# Compression configuration
16+
compression:
17+
name: PTQ
18+
quantization:
19+
name: int8_dynamic
20+
bits: 8
21+
quant_method:
22+
weight: "per-channel"
23+
activation: "per-token"
24+
ignore_layers: # Skip quantization for these layers
25+
- "model.visual.patch_embed.proj"
26+
- "model.lm_head"
27+
- "model.language_model.embed_tokens"
28+
quant_vit: false

‎configs/qwen3_vl/int8_dynamic/qwen3_vl-30b_a3b_int8_dynamic.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Global configuration of pipeline
2+
global:
3+
save_path: ./output
4+
5+
# Simplified Configuration for LLM compression
6+
model:
7+
name: Qwen3VLMoE
8+
model_path: Qwen/Qwen3-VL-30B-A3B-Instruct
9+
trust_remote_code: true
10+
low_cpu_mem_usage: true
11+
use_cache: false
12+
torch_dtype: auto
13+
device_map: auto
14+
15+
# Compression configuration
16+
compression:
17+
name: PTQ
18+
quantization:
19+
name: int8_dynamic
20+
bits: 8
21+
quant_method:
22+
weight: "per-channel"
23+
activation: "per-token"
24+
ignore_layers: # Skip quantization for these layers
25+
- "model.visual.patch_embed.proj"
26+
- "model.lm_head"
27+
- "model.language_model.embed_tokens"
28+
quant_vit: false

‎configs/qwen3_vl/int8_dynamic/qwen3_vl-32b_int8_dynamic.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Global configuration of pipeline
2+
global:
3+
save_path: ./output
4+
5+
# Simplified Configuration for LLM compression
6+
model:
7+
name: Qwen3VL
8+
model_path: Qwen/Qwen3-VL-32B-Instruct
9+
trust_remote_code: true
10+
low_cpu_mem_usage: true
11+
use_cache: false
12+
torch_dtype: auto
13+
device_map: auto
14+
15+
# Compression configuration
16+
compression:
17+
name: PTQ
18+
quantization:
19+
name: int8_dynamic
20+
bits: 8
21+
quant_method:
22+
weight: "per-channel"
23+
activation: "per-token"
24+
ignore_layers: # Skip quantization for these layers
25+
- "model.visual.patch_embed.proj"
26+
- "model.lm_head"
27+
- "model.language_model.embed_tokens"
28+
quant_vit: false
29+

‎configs/qwen3_vl/int8_dynamic/qwen3_vl-4b_int8_dynamic.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Global configuration of pipeline
2+
global:
3+
save_path: ./output
4+
5+
# Simplified Configuration for LLM compression
6+
model:
7+
name: Qwen3VL
8+
model_path: Qwen/Qwen3-VL-4B-Instruct
9+
trust_remote_code: true
10+
low_cpu_mem_usage: true
11+
use_cache: false
12+
torch_dtype: auto
13+
device_map: auto
14+
15+
# Compression configuration
16+
compression:
17+
name: PTQ
18+
quantization:
19+
name: int8_dynamic
20+
bits: 8
21+
quant_method:
22+
weight: "per-channel"
23+
activation: "per-token"
24+
ignore_layers: # Skip quantization for these layers
25+
- "model.visual.patch_embed.proj"
26+
- "model.lm_head"
27+
- "model.language_model.embed_tokens"
28+
quant_vit: false

‎configs/qwen3_vl/int8_dynamic/qwen3_vl-8b_int8_dynamic.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Global configuration of pipeline
2+
global:
3+
save_path: ./output
4+
5+
# Simplified Configuration for LLM compression
6+
model:
7+
name: Qwen3VL
8+
model_path: Qwen/Qwen3-VL-8B-Instruct
9+
trust_remote_code: true
10+
low_cpu_mem_usage: true
11+
use_cache: false
12+
torch_dtype: auto
13+
device_map: auto
14+
15+
# Compression configuration
16+
compression:
17+
name: PTQ
18+
quantization:
19+
name: int8_dynamic
20+
bits: 8
21+
quant_method:
22+
weight: "per-channel"
23+
activation: "per-token"
24+
ignore_layers: # Skip quantization for these layers
25+
- "model.visual.patch_embed.proj"
26+
- "model.lm_head"
27+
- "model.language_model.embed_tokens"
28+
quant_vit: false
29+

0 commit comments

Comments
 (0)