Add/modify A4 & A4X Qwen2.5-VL-7B, Qwen3-235B-A22B, Llama3.1-405B, TRTLLM launcher, TRTLLM values YAML files for A4 & A4X

hmhv1222 · hmhv1222 · commit dc0a1877661c · 2026-04-21T00:52:51.000Z
diff --git a/inference/a4/single-host-serving/tensorrt-llm/values.yaml b/inference/a4/single-host-serving/tensorrt-llm/values.yaml
@@ -55,9 +55,8 @@ workload:
       value: "/usr/local/nvidia/lib64:/usr/local/lib/"
   benchmarks:
     experiments:
-      - isl: 1024 # input sequence length
-        osl: 4096 # output sequence length
-        # psl: 7900 # prefix sequence length
+      - isl: 1024 # Input sequence length
+        osl: 4096 # Output sequence length
         num_requests: 1000
 
 network:
diff --git a/inference/a4x/single-host-serving/tensorrt-llm/values.yaml b/inference/a4x/single-host-serving/tensorrt-llm/values.yaml
@@ -51,8 +51,8 @@ workload:
       value: "/workload/configs/serving-args.yaml"
   benchmarks:
     experiments:
-      - isl: 2048
-        osl: 2048
+      - isl: 1024 # Input sequence length
+        osl: 4096 # Output sequence length
         num_requests: 1000
 
 network:
diff --git a/src/frameworks/a4/trtllm-configs/llama-3-1-405b.yaml b/src/frameworks/a4/trtllm-configs/llama-3-1-405b.yaml
@@ -1,4 +1,4 @@
-tp_size: 4
+tp_size: 8
 pp_size: 1
 backend: pytorch
 kv_cache_free_gpu_mem_fraction: 0.90
diff --git a/src/frameworks/a4/trtllm-configs/qwen2-5-vl-7b-fp8.yaml b/src/frameworks/a4/trtllm-configs/qwen2-5-vl-7b-fp8.yaml
@@ -1,4 +1,9 @@
 tp_size: 1
 pp_size: 1
 backend: pytorch
-kv_cache_free_gpu_mem_fraction: 0.90
+modality: image # For multimodal VL models
+streaming: true # For multimodal VL models
+max_input_len: 8192 # For multimodal VL models
+max_batch_size: 2 # For multimodal VL models
+kv_cache_free_gpu_mem_fraction: 0.95
+dataset: /gcs/datasets/Qwen2.5-VL-7B-Instruct/Qwen_Qwen2.5-VL-7B-Instruct/inlen1024_outlen110_prefixlen0_MM_imgnum5.5_imgsize736x736.jsonl
diff --git a/src/frameworks/a4/trtllm-configs/qwen2-5-vl-7b-nvfp4.yaml b/src/frameworks/a4/trtllm-configs/qwen2-5-vl-7b-nvfp4.yaml
@@ -1,4 +1,9 @@
 tp_size: 1
 pp_size: 1
 backend: pytorch
-kv_cache_free_gpu_mem_fraction: 0.90
+modality: image # For multimodal VL models
+streaming: true # For multimodal VL models
+max_input_len: 8192 # For multimodal VL models
+max_batch_size: 2 # For multimodal VL models
+kv_cache_free_gpu_mem_fraction: 0.95
+dataset: /gcs/datasets/Qwen2.5-VL-7B-Instruct/Qwen_Qwen2.5-VL-7B-Instruct/inlen1024_outlen110_prefixlen0_MM_imgnum5.5_imgsize736x736.jsonl
diff --git a/src/frameworks/a4/trtllm-configs/qwen3-235b-a22b-fp8.yaml b/src/frameworks/a4/trtllm-configs/qwen3-235b-a22b-fp8.yaml
@@ -1,35 +1,18 @@
 tp_size: 4
 pp_size: 1
-ep_size: 1
 backend: pytorch
-kv_cache_free_gpu_mem_fraction: 0.20
-# llm_api_args:
-#   cuda_graph_config:
-#     batch_sizes:
-#     - 1
-#     - 2
-#     - 4
-#     - 8
-#     - 16
-#     - 20
-#     - 24
-#     - 32
-#     - 64
-#     - 96
-#     - 128
-#     - 160
-#     - 192
-#     - 256
-#     - 320
-#     - 384
-#     - 512
-#     enable_padding: true
-#   enable_attention_dp: true
-#   enable_chunked_prefill: true
-#   kv_cache_config:
-#     dtype: auto
-#     enable_block_reuse: false
-#     free_gpu_memory_fraction: 0.40
-#   moe_config:
-#     backend: CUTLASS
-#   print_iter_log: true
+moe_backend: DEEPGEMM
+llm_api_args:
+  print_iter_log: true
+  cuda_graph_config:
+    max_batch_size: 1762
+    enable_padding: true
+  torch_compile_config:
+    capture_num_tokens: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840]
+    enable_userbuffers: false
+    max_num_streams: 2
+    enable_piecewise_cuda_graph: true
+  enable_chunked_prefill: true
+  kv_cache_config:
+    enable_block_reuse: false
+    free_gpu_memory_fraction: 0.90
diff --git a/src/frameworks/a4/trtllm-configs/qwen3-235b-a22b-nvfp4.yaml b/src/frameworks/a4/trtllm-configs/qwen3-235b-a22b-nvfp4.yaml
@@ -1,4 +1,18 @@
 tp_size: 4
 pp_size: 1
 backend: pytorch
-kv_cache_free_gpu_mem_fraction: 0.90
+moe_backend: DEEPGEMM
+llm_api_args:
+  print_iter_log: true
+  cuda_graph_config:
+    max_batch_size: 1762
+    enable_padding: true
+  torch_compile_config:
+    capture_num_tokens: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840]
+    enable_userbuffers: false
+    max_num_streams: 2
+    enable_piecewise_cuda_graph: true
+  enable_chunked_prefill: true
+  kv_cache_config:
+    enable_block_reuse: false
+    free_gpu_memory_fraction: 0.90
diff --git a/src/frameworks/a4x/trtllm-configs/qwen2-5-vl-7b-fp8.yaml b/src/frameworks/a4x/trtllm-configs/qwen2-5-vl-7b-fp8.yaml
@@ -1,4 +1,9 @@
 tp_size: 1
 pp_size: 1
 backend: pytorch
-kv_cache_free_gpu_mem_fraction: 0.90
+modality: image # For multimodal VL models
+streaming: true # For multimodal VL models
+max_input_len: 8192 # For multimodal VL models
+max_batch_size: 2 # For multimodal VL models
+kv_cache_free_gpu_mem_fraction: 0.95
+dataset: /gcs/datasets/Qwen2.5-VL-7B-Instruct/Qwen_Qwen2.5-VL-7B-Instruct/inlen1024_outlen110_prefixlen0_MM_imgnum5.5_imgsize736x480.jsonl
diff --git a/src/frameworks/a4x/trtllm-configs/qwen2-5-vl-7b-nvfp4.yaml b/src/frameworks/a4x/trtllm-configs/qwen2-5-vl-7b-nvfp4.yaml
@@ -1,4 +1,9 @@
 tp_size: 1
 pp_size: 1
 backend: pytorch
-kv_cache_free_gpu_mem_fraction: 0.90
+modality: image # For multimodal VL models
+streaming: true # For multimodal VL models
+max_input_len: 8192 # For multimodal VL models
+max_batch_size: 2 # For multimodal VL models
+kv_cache_free_gpu_mem_fraction: 0.95
+dataset: /gcs/datasets/Qwen2.5-VL-7B-Instruct/Qwen_Qwen2.5-VL-7B-Instruct/inlen1024_outlen110_prefixlen0_MM_imgnum5.5_imgsize736x480.jsonl
diff --git a/src/frameworks/a4x/trtllm-configs/qwen3-235b-a22b-fp8.yaml b/src/frameworks/a4x/trtllm-configs/qwen3-235b-a22b-fp8.yaml
@@ -1,35 +1,18 @@
 tp_size: 4
 pp_size: 1
-ep_size: 1
 backend: pytorch
-kv_cache_free_gpu_mem_fraction: 0.20
-# llm_api_args:
-#   cuda_graph_config:
-#     batch_sizes:
-#     - 1
-#     - 2
-#     - 4
-#     - 8
-#     - 16
-#     - 20
-#     - 24
-#     - 32
-#     - 64
-#     - 96
-#     - 128
-#     - 160
-#     - 192
-#     - 256
-#     - 320
-#     - 384
-#     - 512
-#     enable_padding: true
-#   enable_attention_dp: true
-#   enable_chunked_prefill: true
-#   kv_cache_config:
-#     dtype: auto
-#     enable_block_reuse: false
-#     free_gpu_memory_fraction: 0.40
-#   moe_config:
-#     backend: CUTLASS
-#   print_iter_log: true
+moe_backend: DEEPGEMM
+llm_api_args:
+  print_iter_log: true
+  cuda_graph_config:
+    max_batch_size: 1762
+    enable_padding: true
+  torch_compile_config:
+    capture_num_tokens: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840]
+    enable_userbuffers: false
+    max_num_streams: 2
+    enable_piecewise_cuda_graph: true
+  enable_chunked_prefill: true
+  kv_cache_config:
+    enable_block_reuse: false
+    free_gpu_memory_fraction: 0.90
diff --git a/src/frameworks/a4x/trtllm-configs/qwen3-235b-a22b-nvfp4.yaml b/src/frameworks/a4x/trtllm-configs/qwen3-235b-a22b-nvfp4.yaml
@@ -1,4 +1,18 @@
 tp_size: 4
 pp_size: 1
 backend: pytorch
-kv_cache_free_gpu_mem_fraction: 0.90
+moe_backend: DEEPGEMM
+llm_api_args:
+  print_iter_log: true
+  cuda_graph_config:
+    max_batch_size: 1762
+    enable_padding: true
+  torch_compile_config:
+    capture_num_tokens: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840]
+    enable_userbuffers: false
+    max_num_streams: 2
+    enable_piecewise_cuda_graph: true
+  enable_chunked_prefill: true
+  kv_cache_config:
+    enable_block_reuse: false
+    free_gpu_memory_fraction: 0.90
diff --git a/src/launchers/trtllm-launcher.sh b/src/launchers/trtllm-launcher.sh
@@ -141,6 +141,10 @@ print_configuration() {
     echo "pipeline parallel size:  $pp_size"
     echo "expert parallel size:    $ep_size"
     echo "backend:                 $backend"
+    echo "modality:                $modality"
+    echo "streaming:               $streaming"
+    echo "max input length:        $max_input_len"
+    echo "max batch size:          $max_batch_size"
     echo "kv_cache_free_gpu_mem_fraction: $kv_cache_free_gpu_mem_fraction"
     echo "--------------------------------"
 }
@@ -172,6 +176,7 @@ run_benchmark() {
     if [ -n "$max_batch_size" ]; then vl_args="$vl_args --max_batch_size $max_batch_size"; fi
 
     dataset_file=$custom_dataset
+    # If custom_dataset is not set, generate a textual dataset with tokens sampled in normal distribution
     if [ -z "$dataset_file" ]; then
         dataset_file="/ssd/token-norm-dist_${model_name##*/}_${isl}_${osl}_tp${tp_size}.json"
         echo "Preparing dataset"
@@ -191,7 +196,7 @@ run_benchmark() {
     if [ -f "$extra_args_file" ]; then
         extra_args="--extra_llm_api_options $extra_args_file"
     fi
-    
+
     export TOKENIZERS_PARALLELISM=false
     echo "enable_cuda_graph: false" > /tmp/extra_llm_api_args.yaml
 
@@ -202,6 +207,7 @@ run_benchmark() {
         --model $model_name \
         --model_path /ssd/${model_name} throughput \
         --dataset $dataset_file \
+        --num_requests $num_requests \
         --tp $tp_size \
         --pp $pp_size \
         --ep $ep_size \
