Qwen3 235B A22B, Qwen 2.5 VL 7B & Llama 3.1 405B config files and launcher modification

Author: hmhv1222

inference/a4/single-host-serving/tensorrt-llm/values.yaml

@@ -37,7 +37,7 @@ service:
 workload:
   model:
     name:
-  gpus: 8
+  gpus: 4
   image:
   framework:
   configFile: serving-args.yaml
@@ -55,8 +55,9 @@ workload:
       value: "/usr/local/nvidia/lib64:/usr/local/lib/"
   benchmarks:
     experiments:
-      - isl: 1024
-        osl: 4096
+      - isl: 1024 # input sequence length
+        osl: 4096 # output sequence length
+        # psl: 7900 # prefix sequence length
         num_requests: 1000
 
 network:

inference/a4x/single-host-serving/tensorrt-llm/values.yaml

@@ -51,8 +51,8 @@ workload:
       value: "/workload/configs/serving-args.yaml"
   benchmarks:
     experiments:
-      - isl: 1024
-        osl: 1024
+      - isl: 2048
+        osl: 2048
         num_requests: 1000
 
 network:

src/frameworks/a4/trtllm-configs/llama-3-1-405b.yaml

@@ -0,0 +1,4 @@
+tp_size: 4
+pp_size: 1
+backend: pytorch
+kv_cache_free_gpu_mem_fraction: 0.90

src/frameworks/a4/trtllm-configs/qwen2-5-vl-7b-fp8.yaml

@@ -0,0 +1,4 @@
+tp_size: 1
+pp_size: 1
+backend: pytorch
+kv_cache_free_gpu_mem_fraction: 0.90

src/frameworks/a4/trtllm-configs/qwen2-5-vl-7b-nvfp4.yaml

@@ -0,0 +1,4 @@
+tp_size: 1
+pp_size: 1
+backend: pytorch
+kv_cache_free_gpu_mem_fraction: 0.90

src/frameworks/a4/trtllm-configs/qwen3-235b-a22b-fp8.yaml

@@ -0,0 +1,35 @@
+tp_size: 4
+pp_size: 1
+ep_size: 1
+backend: pytorch
+kv_cache_free_gpu_mem_fraction: 0.20
+# llm_api_args:
+#   cuda_graph_config:
+#     batch_sizes:
+#     - 1
+#     - 2
+#     - 4
+#     - 8
+#     - 16
+#     - 20
+#     - 24
+#     - 32
+#     - 64
+#     - 96
+#     - 128
+#     - 160
+#     - 192
+#     - 256
+#     - 320
+#     - 384
+#     - 512
+#     enable_padding: true
+#   enable_attention_dp: true
+#   enable_chunked_prefill: true
+#   kv_cache_config:
+#     dtype: auto
+#     enable_block_reuse: false
+#     free_gpu_memory_fraction: 0.40
+#   moe_config:
+#     backend: CUTLASS
+#   print_iter_log: true

src/frameworks/a4/trtllm-configs/qwen3-235b-a22b-nvfp4.yaml

@@ -1,4 +1,4 @@
-tp_size: 1
+tp_size: 4
 pp_size: 1
 backend: pytorch
 kv_cache_free_gpu_mem_fraction: 0.90

src/frameworks/a4x/trtllm-configs/llama-3-1-405b.yaml

@@ -1,4 +1,4 @@
 tp_size: 4
 pp_size: 1
 backend: pytorch
-kv_cache_free_gpu_mem_fraction: 0.90
+kv_cache_free_gpu_mem_fraction: 0.90

src/frameworks/a4x/trtllm-configs/qwen2-5-vl-7b-fp8.yaml

@@ -0,0 +1,4 @@
+tp_size: 1
+pp_size: 1
+backend: pytorch
+kv_cache_free_gpu_mem_fraction: 0.90

src/frameworks/a4x/trtllm-configs/qwen2-5-vl-7b-nvfp4.yaml

@@ -0,0 +1,4 @@
+tp_size: 1
+pp_size: 1
+backend: pytorch
+kv_cache_free_gpu_mem_fraction: 0.90