models.yaml

# Model-specific SGLang server configurations for disaggregated inference.
#
# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR).
#
# To add a new model: add a new top-level entry following the same schema.
# No script changes are required.
#
# Schema:
#   <model-name>:
#     base_flags: str          # Common flags for both prefill and decode
#     mtp_flags: str           # Appended to decode when DECODE_MTP_SIZE > 0
#     dp_flags: str            # Appended when DP attention is enabled (prefill or decode)
#     ep_flags: str            # Appended when EP is enabled. EP-specific MoE knobs only
#                              # (a2a backend, deepep mode, ep-dispatch algorithm). With
#                              # ep=1 these are dropped so the MoE runs tensor-parallel (TP).
#     prefill:
#       mem_fraction_static: float
#       disable_radix_cache: bool
#       dp:                              # Config when data-parallel attention is enabled
#         max_running_requests: int
#         chunked_prefill_size: str      # Can be integer or bash arithmetic expression
#         cuda_graph_bs: str             # Space-separated values
#       no_dp:                           # Config when data-parallel attention is disabled
#         max_running_requests: int
#         chunked_prefill_size: int
#         cuda_graph_bs_range: str       # "start-end" expanded via seq
#     decode:
#       mem_fraction_static: float
#       prefill_round_robin_balance: bool
#       dp:
#         max_running_requests: int
#         chunked_prefill_size: str
#         cuda_graph_bs_range: str
#       ep_only:                         # Config when EP is enabled but DP is disabled
#         max_running_requests: int
#         chunked_prefill_size: int
#         cuda_graph_bs_range: str
#       no_dp:
#         max_running_requests: int
#         chunked_prefill_size: int
#         cuda_graph_bs_range: str

DeepSeek-V3:
  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori"
  mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1"
  dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
  ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    dp:
      max_running_requests: 24
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"
      cuda_graph_bs: "1 2 3"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
      cuda_graph_bs_range: "1-160"
    ep_only:
      max_running_requests: 256
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-256"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"

DeepSeek-V3-0324:
  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori"
  mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1"
  dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
  ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    dp:
      max_running_requests: 24
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"
      cuda_graph_bs: "1 2 3"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
      cuda_graph_bs_range: "1-160"
    ep_only:
      max_running_requests: 256
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-256"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"

DeepSeek-R1:
  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori"
  mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1"
  dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
  ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    dp:
      max_running_requests: 24
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"
      cuda_graph_bs: "1 2 3"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
      cuda_graph_bs_range: "1-160"
    ep_only:
      max_running_requests: 256
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-256"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"

DeepSeek-R1-0528:
  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori"
  mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1"
  dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
  ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    dp:
      max_running_requests: 24
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"
      cuda_graph_bs: "1 2 3"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
      cuda_graph_bs_range: "1-160"
    ep_only:
      max_running_requests: 256
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-256"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"

Qwen3.5-397B-A17B-MXFP4:
  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1"
  mtp_flags: ""
  dp_flags: "--enable-dp-attention --enable-dp-lm-head"
  ep_flags: "--moe-a2a-backend mori"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    dp:
      max_running_requests: 24
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"
      cuda_graph_bs: "1 2 3"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
      cuda_graph_bs_range: "1-160"
    ep_only:
      max_running_requests: 256
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-256"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"

Qwen3.5-397B-A17B-FP8:
  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1"
  mtp_flags: ""
  dp_flags: "--enable-dp-attention --enable-dp-lm-head"
  ep_flags: "--moe-a2a-backend mori"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    dp:
      max_running_requests: 24
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"
      cuda_graph_bs: "1 2 3"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
      cuda_graph_bs_range: "1-160"
    ep_only:
      max_running_requests: 256
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-256"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"

GLM-5-FP8:
  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'"
  mtp_flags: ""
  dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
  ep_flags: "--moe-a2a-backend mori"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    dp:
      max_running_requests: 24
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"
      cuda_graph_bs: "1 2 3"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
      cuda_graph_bs_range: "1-160"
    ep_only:
      max_running_requests: 256
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-256"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"

DeepSeek-R1-0528-MXFP4-Preview:
  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori"
  mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1"
  dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
  ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    dp:
      max_running_requests: 24
      chunked_prefill_size: 16384
      cuda_graph_bs: "1 2 3"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 16384
      cuda_graph_bs_range: "1-128"
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
      cuda_graph_bs_range: "1-160"
    ep_only:
      max_running_requests: 256
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-256"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"

DeepSeek-R1-0528-MXFP4:
  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori"
  mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1"
  dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
  ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    dp:
      max_running_requests: 24
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"
      cuda_graph_bs: "1 2 3"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 16384
      cuda_graph_bs_range: "1-128"
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
      cuda_graph_bs_range: "1-160"
    ep_only:
      max_running_requests: 256
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-256"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"

DeepSeek-R1-0528-MXFP4-v2:
  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori"
  mtp_flags: "--speculative-draft-model-path SGLang/DeepSeek-R1-NextN --speculative-algorithm NEXTN --speculative-eagle-topk 1 --speculative-attention-mode decode "
  dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head --stream-interval 100 --tokenizer-worker-num 32 "
  ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"
      cuda_graph_bs: "1 2 3"
      context_length: 9217
      max_total_tokens: 131072
      enable_two_batch_overlap: true
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 16384
      cuda_graph_bs_range: "1-128"
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 4096
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
      cuda_graph_bs_range: "1-512"
    ep_only:
      max_running_requests: 256
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-256"
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 262144
      cuda_graph_bs_range: "1-128"

DeepSeek-V4-Pro-AgentX:
  base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error"
  dp_flags: "--enable-dp-attention --enable-prefill-delayer --enable-two-batch-overlap"
  ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: false
    disable_cuda_graph: false
    dp:
      max_running_requests: 1024
      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"  # dsv4 compressor kernel uint16 token cap (255*256)
      context_length: 1048576
      # max_total_tokens: 1048576
    no_dp:
      max_running_requests: 128
      # Small prefill chunks interleave long-context agentic prefills across
      # requests instead of letting one ~100K-token prefill monopolize the
      # engine (the conc>=16 queue-saturation / decode-stall failure mode).
      # Mirrors the single-node DSv4 agentic recipe (dsv4_fp4_mi355x.sh=8192).
      # Was 65280 (255*256, the dsv4 compressor kernel uint16 token cap); 8192
      # (32*256) stays a page-size multiple well under that cap.
      chunked_prefill_size: 8192
      context_length: 1048576
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    disagg_decode_enable_radix_cache: false
    dp:
      max_running_requests: 1024
      cuda_graph_bs_range: "1-128"
    no_dp:
      max_running_requests: 128
      cuda_graph_bs_range: "1-128"

DeepSeek-V4-Pro-DI:
  base_flags: "--decode-log-interval 100 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori"
  dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
  ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
  prefill:
    mem_fraction_static: 0.8
    disable_radix_cache: true
    disable_cuda_graph: true
    dp:
      max_running_requests: 1024
      chunked_prefill_size: 131072
      context_length: 9217
      max_total_tokens: 2097152
    no_dp:
      max_running_requests: 128
      chunked_prefill_size: 131072
      context_length: 9217
      max_total_tokens: 2097152
  decode:
    mem_fraction_static: 0.85
    prefill_round_robin_balance: true
    dp:
      max_running_requests: 1024
      cuda_graph_bs_range: "1-128"
    no_dp:
      max_running_requests: 128
      cuda_graph_bs_range: "1-128"