From 4778a52c41ceaf27a738ece0222e4844443cd3fa Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Sat, 18 Jul 2026 12:01:11 -0700 Subject: [PATCH 1/2] feat: add DeepSeek V4 GB300 TensorRT-LLM agentic config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add eight disaggregated MTP topologies and the agentic routing, container-cache fallback, and GB300 launch support they require. 中文：添加八组分离式 MTP 拓扑，并补齐所需的 Agentic 会话路由、容器缓存回退及 GB300 启动支持。 --- benchmarks/benchmark_lib.sh | 22 ++- ...00-1p1d-dep8-dep32-c144-mtp-kvoffload.yaml | 164 ++++++++++++++++ ...gb300-1p3d-dep4-tep8-c3-mtp-kvoffload.yaml | 164 ++++++++++++++++ ...gb300-1p5d-dep4-tep4-c5-mtp-kvoffload.yaml | 162 ++++++++++++++++ ...00-2p1d-dep8-dep16-c480-mtp-kvoffload.yaml | 167 ++++++++++++++++ ...00-2p1d-dep8-dep32-c416-mtp-kvoffload.yaml | 164 ++++++++++++++++ ...00-3p1d-dep8-dep32-c544-mtp-kvoffload.yaml | 165 ++++++++++++++++ ...00-4p1d-dep8-dep16-c800-mtp-kvoffload.yaml | 171 +++++++++++++++++ ...0-5p1d-dep8-dep16-c1200-mtp-kvoffload.yaml | 179 ++++++++++++++++++ configs/nvidia-master.yaml | 145 ++++++++++++++ perf-changelog.yaml | 7 + runners/launch_gb300-nv.sh | 93 +++++++-- 12 files changed, 1581 insertions(+), 22 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c144-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tep8-c3-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p5d-dep4-tep4-c5-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c480-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c416-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep32-c544-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep8-dep16-c800-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-5p1d-dep8-dep16-c1200-mtp-kvoffload.yaml diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index b4c947932b..9cb3797c43 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1761,7 +1761,8 @@ build_replay_cmd() { REPLAY_CMD+=" --endpoint /v1/chat/completions" REPLAY_CMD+=" --endpoint-type chat" REPLAY_CMD+=" --streaming" - REPLAY_CMD+=" --model $MODEL" + local replay_model="${AIPERF_MODEL_NAME:-${SERVED_MODEL_NAME:-$MODEL}}" + REPLAY_CMD+=" --model $replay_model" REPLAY_CMD+=" --concurrency $CONC" REPLAY_CMD+=" --benchmark-duration $duration" REPLAY_CMD+=" --random-seed 42" @@ -1795,12 +1796,19 @@ build_replay_cmd() { # binding by itself. AIPerf emits nvext.session_control bind/close actions # keyed by the stable conversation correlation ID when this flag is set. if [[ "${FRAMEWORK:-}" == dynamo-* ]]; then - REPLAY_CMD+=" --use-dynamo-conv-aware-routing" - # The upstream 300s affinity TTL is shorter than an overloaded - # high-concurrency agentic request. Keep bindings alive across long - # prefills, generation, and capped inter-turn delay. This controls the - # router's inactivity lease; it does not relax HTTP/request failures. - REPLAY_CMD+=" --dynamo-session-timeout-seconds ${AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS:-3600}" + # Newer Dynamo versions accept session IDs through X-Session-ID. Opt in + # only when the recipe requests that path; older configurations retain + # AIPerf's conversation-aware routing flags. + if [[ -n "${AIPERF_NEW_DYNAMO_SESSION_CONTROL:-}" ]]; then + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + else + REPLAY_CMD+=" --use-dynamo-conv-aware-routing" + # The upstream 300s affinity TTL is shorter than an overloaded + # high-concurrency agentic request. Keep bindings alive across long + # prefills, generation, and capped inter-turn delay. This controls the + # router's inactivity lease; it does not relax HTTP/request failures. + REPLAY_CMD+=" --dynamo-session-timeout-seconds ${AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS:-3600}" + fi fi # Disable DCGM GPU telemetry collection. aiperf's GpuMetricTimeSeries # freezes its metric schema on the first DCGM scrape, then KeyErrors when diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c144-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c144-mtp-kvoffload.yaml new file mode 100644 index 0000000000..4c661b2208 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c144-mtp-kvoffload.yaml @@ -0,0 +1,164 @@ +name: disagg-gb300-1p1d-dep8-dep32-c144-mtp-kvoffload +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 + precision: fp4 +dynamo: + install: true + wheel: "1.3.0.dev2026071601" + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 1 + gpus_per_prefill: 8 + decode_nodes: 8 + decode_workers: 1 + gpus_per_decode: 32 +backend: + type: trtllm + publish_events_and_metrics: false + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + pool_ratio: + - 0.3 + - 0.4 + - 0.3 + tokens_per_block: 256 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + tokens_per_block: 256 + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Dynamo's 10-second default can expire during brief etcd stalls while + # many long-loading DEP ranks initialize. Slurm still detects hard exits. + ETCD_LEASE_TTL: "120" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TRACK_PREFILL_TOKENS: "1" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + # DYN_ROUTER_TEMPERATURE: "10000000" + # Keep session affinity alive across long-running agentic requests. + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" + args: + # Session-based routing is handled by Dynamo. + # no-kv-events for approximate routing (WAR some indexer issues, currently + # okay as conversation aware routing is the major routing here) + router-mode: "kv" + no-kv-events: true + router-reset-states: true + # orchestrator_placement: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + SERVED_MODEL_NAME: "DeepSeek-V4-Pro" + AIPERF_NEW_DYNAMO_SESSION_CONTROL: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tep8-c3-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tep8-c3-mtp-kvoffload.yaml new file mode 100644 index 0000000000..aee25f64b4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tep8-c3-mtp-kvoffload.yaml @@ -0,0 +1,164 @@ +name: disagg-gb300-1p3d-dep4-tep8-c3-mtp-kvoffload +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 + precision: fp4 +dynamo: + install: true + wheel: "1.3.0.dev2026071601" + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 6 + decode_workers: 3 + gpus_per_decode: 8 +backend: + type: trtllm + publish_events_and_metrics: false + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + pool_ratio: + - 0.3 + - 0.4 + - 0.3 + tokens_per_block: 256 + max_batch_size: 128 + max_num_tokens: 4096 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + host_cache_size: 137438953472 + tokens_per_block: 256 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 996595 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Dynamo's 10-second default can expire during brief etcd stalls while + # many long-loading DEP ranks initialize. Slurm still detects hard exits. + ETCD_LEASE_TTL: "120" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TRACK_PREFILL_TOKENS: "1" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + # DYN_ROUTER_TEMPERATURE: "10000000" + # Keep session affinity alive across long-running agentic requests. + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" + args: + # Session-based routing is handled by Dynamo. + # no-kv-events for approximate routing (WAR some indexer issues, currently + # okay as conversation aware routing is the major routing here) + router-mode: "kv" + no-kv-events: true + router-reset-states: true + # orchestrator_placement: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + SERVED_MODEL_NAME: "DeepSeek-V4-Pro" + AIPERF_NEW_DYNAMO_SESSION_CONTROL: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p5d-dep4-tep4-c5-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p5d-dep4-tep4-c5-mtp-kvoffload.yaml new file mode 100644 index 0000000000..334ba42eff --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p5d-dep4-tep4-c5-mtp-kvoffload.yaml @@ -0,0 +1,162 @@ +name: disagg-gb300-1p5d-dep4-tep4-c5-mtp-kvoffload +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 + precision: fp4 +dynamo: + install: true + wheel: "1.3.0.dev2026071601" + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 5 + decode_workers: 5 + gpus_per_decode: 4 +backend: + type: trtllm + publish_events_and_metrics: false + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + pool_ratio: + - 0.3 + - 0.4 + - 0.3 + tokens_per_block: 256 + max_batch_size: 128 + max_num_tokens: 4096 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + host_cache_size: 137438953472 + tokens_per_block: 256 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 996595 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Dynamo's 10-second default can expire during brief etcd stalls while + # many long-loading DEP ranks initialize. Slurm still detects hard exits. + ETCD_LEASE_TTL: "120" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TRACK_PREFILL_TOKENS: "1" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" + args: + # Session-based routing is handled by Dynamo. + # no-kv-events for approximate routing (WAR some indexer issues, currently + # okay as conversation aware routing is the major routing here) + router-mode: "kv" + no-kv-events: true + router-reset-states: true + # orchestrator_placement: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + SERVED_MODEL_NAME: "DeepSeek-V4-Pro" + AIPERF_NEW_DYNAMO_SESSION_CONTROL: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c480-mtp-kvoffload.yaml new file mode 100644 index 0000000000..de2bcc7597 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c480-mtp-kvoffload.yaml @@ -0,0 +1,167 @@ +name: disagg-gb300-2p1d-dep8-dep16-c480-mtp-kvoffload +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 + precision: fp4 +dynamo: + install: true + wheel: "1.3.0.dev2026071601" + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 2 + gpus_per_prefill: 8 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + publish_events_and_metrics: false + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + pool_ratio: + - 0.3 + - 0.4 + - 0.3 + tokens_per_block: 256 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + tokens_per_block: 256 + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 16 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Dynamo's 10-second default can expire during brief etcd stalls while + # many long-loading DEP ranks initialize. Slurm still detects hard exits. + ETCD_LEASE_TTL: "120" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TRACK_PREFILL_TOKENS: "1" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + # DYN_ROUTER_TEMPERATURE: "10000000" + # Keep session affinity alive across long-running agentic requests. + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" + args: + # Session-based routing is handled by Dynamo. + # no-kv-events for approximate routing (WAR some indexer issues, currently + # okay as conversation aware routing is the major routing here) + router-mode: "kv" + no-kv-events: true + router-reset-states: true + # orchestrator_placement: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + SERVED_MODEL_NAME: "DeepSeek-V4-Pro" + AIPERF_NEW_DYNAMO_SESSION_CONTROL: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c416-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c416-mtp-kvoffload.yaml new file mode 100644 index 0000000000..f02347f90e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c416-mtp-kvoffload.yaml @@ -0,0 +1,164 @@ +name: disagg-gb300-2p1d-dep8-dep32-c416-mtp-kvoffload +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 + precision: fp4 +dynamo: + install: true + wheel: "1.3.0.dev2026071601" + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 2 + gpus_per_prefill: 8 + decode_nodes: 8 + decode_workers: 1 + gpus_per_decode: 32 +backend: + type: trtllm + publish_events_and_metrics: false + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + pool_ratio: + - 0.3 + - 0.4 + - 0.3 + tokens_per_block: 256 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + tokens_per_block: 256 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Dynamo's 10-second default can expire during brief etcd stalls while + # many long-loading DEP ranks initialize. Slurm still detects hard exits. + ETCD_LEASE_TTL: "120" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TRACK_PREFILL_TOKENS: "1" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + # DYN_ROUTER_TEMPERATURE: "10000000" + # Keep session affinity alive across long-running agentic requests. + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" + args: + # Session-based routing is handled by Dynamo. + # no-kv-events for approximate routing (WAR some indexer issues, currently + # okay as conversation aware routing is the major routing here) + router-mode: "kv" + no-kv-events: true + router-reset-states: true + # orchestrator_placement: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + SERVED_MODEL_NAME: "DeepSeek-V4-Pro" + AIPERF_NEW_DYNAMO_SESSION_CONTROL: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep32-c544-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep32-c544-mtp-kvoffload.yaml new file mode 100644 index 0000000000..f71bccadc6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep32-c544-mtp-kvoffload.yaml @@ -0,0 +1,165 @@ +name: disagg-gb300-3p1d-dep8-dep32-c544-mtp-kvoffload +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 + precision: fp4 +dynamo: + install: true + wheel: "1.3.0.dev2026071601" + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 6 + prefill_workers: 3 + gpus_per_prefill: 8 + decode_nodes: 8 + decode_workers: 1 + gpus_per_decode: 32 +backend: + type: trtllm + publish_events_and_metrics: false + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + pool_ratio: + - 0.3 + - 0.4 + - 0.3 + tokens_per_block: 256 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + tokens_per_block: 256 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Dynamo's 10-second default can expire during brief etcd stalls while + # many long-loading DEP ranks initialize. Slurm still detects hard exits. + ETCD_LEASE_TTL: "120" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TRACK_PREFILL_TOKENS: "1" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + # DYN_ROUTER_TEMPERATURE: "10000000" + # Keep session affinity alive across long-running agentic requests. + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" + args: + # Session-based routing is handled by Dynamo. + # no-kv-events for approximate routing (WAR some indexer issues, currently + # okay as conversation aware routing is the major routing here) + router-mode: "kv" + no-kv-events: true + router-reset-states: true + # orchestrator_placement: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + SERVED_MODEL_NAME: "DeepSeek-V4-Pro" + AIPERF_NEW_DYNAMO_SESSION_CONTROL: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep8-dep16-c800-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep8-dep16-c800-mtp-kvoffload.yaml new file mode 100644 index 0000000000..250fef7f9e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep8-dep16-c800-mtp-kvoffload.yaml @@ -0,0 +1,171 @@ +name: disagg-gb300-4p1d-dep8-dep16-c800-mtp-kvoffload +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 + precision: fp4 +dynamo: + install: true + wheel: "1.3.0.dev2026071601" + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 8 + prefill_workers: 4 + gpus_per_prefill: 8 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + publish_events_and_metrics: false + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + pool_ratio: + - 0.3 + - 0.4 + - 0.3 + tokens_per_block: 256 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + tokens_per_block: 256 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 16 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Dynamo's 10-second default can expire during brief etcd stalls while + # many long-loading DEP ranks initialize. Slurm still detects hard exits. + ETCD_LEASE_TTL: "120" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TRACK_PREFILL_TOKENS: "1" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + # DYN_ROUTER_TEMPERATURE: "10000000" + # Keep session affinity alive across long-running agentic requests. + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" + args: + # Session-based routing is handled by Dynamo. + # no-kv-events for approximate routing (WAR some indexer issues, currently + # okay as conversation aware routing is the major routing here) + router-mode: "kv" + no-kv-events: true + router-reset-states: true + # orchestrator_placement: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + SERVED_MODEL_NAME: "DeepSeek-V4-Pro" + AIPERF_NEW_DYNAMO_SESSION_CONTROL: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-5p1d-dep8-dep16-c1200-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-5p1d-dep8-dep16-c1200-mtp-kvoffload.yaml new file mode 100644 index 0000000000..305723ad87 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic/disagg-gb300-5p1d-dep8-dep16-c1200-mtp-kvoffload.yaml @@ -0,0 +1,179 @@ +name: disagg-gb300-5p1d-dep8-dep16-c1200-mtp-kvoffload +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 + precision: fp4 +dynamo: + install: true + wheel: "1.3.0.dev2026071601" + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 10 + prefill_workers: 5 + gpus_per_prefill: 8 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + publish_events_and_metrics: false + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + DYN_ENGINE_CONV_AFFINITY: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.48' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + pool_ratio: + - 0.3 + - 0.4 + - 0.3 + tokens_per_block: 256 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + - 36 + - 40 + - 44 + - 48 + - 52 + - 56 + - 60 + - 64 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + tokens_per_block: 256 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 16 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Dynamo's 10-second default can expire during brief etcd stalls while + # many long-loading DEP ranks initialize. Slurm still detects hard exits. + ETCD_LEASE_TTL: "120" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TRACK_PREFILL_TOKENS: "1" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + # DYN_ROUTER_TEMPERATURE: "10000000" + # Keep session affinity alive across long-running agentic requests. + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" + args: + # Session-based routing is handled by Dynamo. + # no-kv-events for approximate routing (WAR some indexer issues, currently + # okay as conversation aware routing is the major routing here) + router-mode: "kv" + no-kv-events: true + router-reset-states: true + # orchestrator_placement: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + SERVED_MODEL_NAME: "DeepSeek-V4-Pro" + AIPERF_NEW_DYNAMO_SESSION_CONTROL: "true" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7d7af13236..518d61cad6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9891,3 +9891,148 @@ qwen3.5-fp8-gb300-dynamo-sglang: tp: 16 ep: 16 dp-attn: true + +dsv4-fp4-gb300-dynamo-trt-agentic: + image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc21 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.3.0.dev2026071601" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: + - 3 + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + # TRTLLM's KV offloading is controlled by 'host_cache_size' + additional-settings: + - CONFIG_FILE=recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tep8-c3-mtp-kvoffload.yaml + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: mtp + conc-list: + - 5 + - 10 + - 20 + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + # TRTLLM's KV offloading is controlled by 'host_cache_size' + additional-settings: + - CONFIG_FILE=recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p5d-dep4-tep4-c5-mtp-kvoffload.yaml + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: mtp + conc-list: + - 144 + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # TRTLLM's KV offloading is controlled by 'host_cache_size' + additional-settings: + - CONFIG_FILE=recipes/trtllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c144-mtp-kvoffload.yaml + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: mtp + conc-list: + - 416 + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + # TRTLLM's KV offloading is controlled by 'host_cache_size' + additional-settings: + - CONFIG_FILE=recipes/trtllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep32-c416-mtp-kvoffload.yaml + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: mtp + conc-list: + - 480 + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + # TRTLLM's KV offloading is controlled by 'host_cache_size' + additional-settings: + - CONFIG_FILE=recipes/trtllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c480-mtp-kvoffload.yaml + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: + - 544 + prefill: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + # TRTLLM's KV offloading is controlled by 'host_cache_size' + additional-settings: + - CONFIG_FILE=recipes/trtllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep32-c544-mtp-kvoffload.yaml + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: mtp + conc-list: + - 800 + prefill: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: true + # TRTLLM's KV offloading is controlled by 'host_cache_size' + additional-settings: + - CONFIG_FILE=recipes/trtllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep8-dep16-c800-mtp-kvoffload.yaml + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: + - 1200 + prefill: + num-worker: 5 + tp: 8 + ep: 8 + dp-attn: true + # TRTLLM's KV offloading is controlled by 'host_cache_size' + additional-settings: + - CONFIG_FILE=recipes/trtllm/deepseek-v4/agentic/disagg-gb300-5p1d-dep8-dep16-c1200-mtp-kvoffload.yaml + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b358f10bf1..a3cda30e27 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4933,3 +4933,10 @@ description: - "Add MiniMax M3 NVFP4 B300 Dynamo-vLLM disaggregated EAGLE3 recipes" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2182 + +- config-keys: + - dsv4-fp4-gb300-dynamo-trt-agentic + description: + - "Add DeepSeek V4 Pro FP4 GB300 disaggregated Dynamo TensorRT-LLM MTP agentic config with eight prefill/decode topologies" + - "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index d450cdb86d..b27e44f02c 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -103,29 +103,81 @@ NGINX_IMAGE="nginx:1.27.4" # symbolic links" bug from workflow worker NFS sessions on lockfiles # AND data files. /data/ has a separate NFS client cache that isn't # poisoned. See feedback_gb300_nfs_eloop_workaround for diagnosis. -SQUASH_FILE="/data/home/sa-shared/gharunners/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" -NGINX_SQUASH_FILE="/data/home/sa-shared/gharunners/squash/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" +SQUASH_DIR="${GB300_SQUASH_DIR:-/data/home/sa-shared/gharunners/squash}" +FALLBACK_SQUASH_DIR="${GITHUB_WORKSPACE:-/tmp}/.container-squash/gb300" +SQUASH_FILE="$SQUASH_DIR/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" +NGINX_SQUASH_FILE="$SQUASH_DIR/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" # Run the import on a compute node via srun, not on the login node: # the login node is x86_64 while the compute nodes are aarch64, so the # arm64 squash file has to be built on a compute node. -import_squash() { +run_squash_import() { local squash="$1" image="$2" - local lock="${squash}.lock" - srun --partition=$SLURM_PARTITION --exclusive --time=180 bash -c " - exec 9>\"$lock\" - flock -w 600 9 || { echo 'Failed to acquire lock for $squash' >&2; exit 1; } - if unsquashfs -l \"$squash\" > /dev/null 2>&1; then - echo 'Squash file already exists and is valid, skipping import: $squash' - else - rm -f \"$squash\" - enroot import -o \"$squash\" docker://$image + srun --partition="$SLURM_PARTITION" --exclusive --time=180 bash -s -- "$squash" "$image" <<'EOS' +set -euo pipefail + +squash="$1" +image="$2" +squash_dir="$(dirname "$squash")" + +if unsquashfs -l "$squash" > /dev/null 2>&1; then + echo "Squash file already exists and is valid, skipping import: $squash" + exit 0 +fi + +if ! mkdir -p "$squash_dir" 2>/dev/null; then + echo "Error: failed to create squash dir: $squash_dir" >&2 + exit 1 +fi + +probe="$squash_dir/.write-probe.${SLURM_JOB_ID:-$$}" +if ! touch "$probe" 2>/dev/null; then + echo "Error: squash dir is not writable: $squash_dir" >&2 + exit 1 +fi +rm -f "$probe" + +lock_dir="${GB300_SQUASH_LOCK_DIR:-${TMPDIR:-/tmp}/inferencex-gb300-squash-locks-${USER:-unknown}}" +mkdir -p "$lock_dir" +lock="$lock_dir/$(basename "$squash").lock" + +( + exec 9>"$lock" + flock -w 600 9 || { echo "Failed to acquire lock for $squash" >&2; exit 1; } + if unsquashfs -l "$squash" > /dev/null 2>&1; then + echo "Squash file already exists and is valid, skipping import: $squash" + else + tmp="${squash}.tmp.${SLURM_JOB_ID:-$$}.${RANDOM}" + rm -f "$tmp" + enroot import -o "$tmp" "docker://$image" + if ! unsquashfs -l "$tmp" > /dev/null 2>&1; then + echo "Error: enroot import did not produce a valid squash file: $tmp" >&2 + rm -f "$tmp" + exit 1 fi - " + mv -f "$tmp" "$squash" + chmod a+r "$squash" || true + fi +) +EOS +} + +import_squash() { + local var_name="$1" image="$2" + local squash="${!var_name}" + + if run_squash_import "$squash" "$image"; then + return 0 + fi + + local fallback="$FALLBACK_SQUASH_DIR/$(basename "$squash")" + echo "Warning: unable to use primary squash cache $squash; trying $fallback" >&2 + run_squash_import "$fallback" "$image" + printf -v "$var_name" '%s' "$fallback" } -import_squash "$SQUASH_FILE" "$IMAGE" -import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" +import_squash SQUASH_FILE "$IMAGE" +import_squash NGINX_SQUASH_FILE "$NGINX_IMAGE" export EVAL_ONLY="${EVAL_ONLY:-false}" @@ -149,6 +201,16 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == mkdir -p recipes/sglang/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then + # This release includes the custom benchmark hook, Dynamo wheel support, + # and --no-preflight required by the node-local DeepSeek V4 model path. + SRT_SLURM_MODEL_PREFIX="deepseek-ai/DeepSeek-V4-Pro" + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.30 + mkdir -p recipes/trtllm/deepseek-v4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentic" \ + recipes/trtllm/deepseek-v4/agentic elif [[ "$IS_AGENTIC" == "1" ]]; then # Agentic multi-node uses cquil11/srt-slurm-nv@cam/no-preflight-flag, # a thin branch off NVIDIA/srt-slurm@127597c that adds one CLI flag @@ -317,6 +379,7 @@ containers: v0.5.11: ${SQUASH_FILE} v0.5.13.post1: ${SQUASH_FILE} "${IMAGE}": ${SQUASH_FILE} + "${IMAGE//#//}": ${SQUASH_FILE} nginx-sqsh: ${NGINX_SQUASH_FILE} use_segment_sbatch_directive: false EOF From 7f4d54a6f8303638d01a3b45238934e3846bd03c Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Sat, 18 Jul 2026 12:02:24 -0700 Subject: [PATCH 2/2] chore: link perf changelog to upstream PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the placeholder with the published upstream PR URL. 中文：将性能变更日志中的占位链接更新为已发布的上游 PR URL。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a3cda30e27..df9b454b24 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4939,4 +4939,4 @@ description: - "Add DeepSeek V4 Pro FP4 GB300 disaggregated Dynamo TensorRT-LLM MTP agentic config with eight prefill/decode topologies" - "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2274