diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml index 13456c2dcb..6d2cfb6deb 100644 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ b/benchmarks/multi_node/amd_utils/models_vllm.yaml @@ -59,3 +59,27 @@ MiniMax-M3-MXFP8: decode_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" hf_dir: "models--MiniMaxAI--MiniMax-M3-MXFP8" + +DeepSeek-V4-Pro: + # DeepSeek-V4-Pro is mixed-precision FP4+FP8 (FP4 MoE expert weights dominate + # the ~960 GB footprint; FP8 on attention/norm/router; FP8 KV cache at runtime). + # InferenceX classifies this as the fp4 variant. + # + # Serving flags reuse the validated single-node MI355X recipe + # (benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm.sh, from + # vllm-project/recipes#433) so the per-node engine config is identical to the + # known-good aggregated run; disaggregation only adds the MoRIIO kv-transfer + # role (injected by server_vllm.sh). Each P/D worker is a full TP=8 node, EP=1 + # — matching the aggregated recipe, which runs DSv4 on TP=8 without expert + # parallelism. DEP decode is a follow-up. + # + # --moe-backend triton_unfused is REQUIRED for the FP4 MoE expert weight format; + # the auto backend doesn't register the FP4 scale params and safetensors load + # raises KeyError. --enforce-eager (no CUDA graphs) keeps the first disagg recipe + # robust against cudagraph/MoRIIO-hook interactions; FULL/PIECEWISE capture is a + # follow-up. --async-scheduling is intentionally omitted (not used by the kimi / + # minimax vllm-disagg recipes). + prefill_flags: "--tensor-parallel-size 8 --distributed-executor-backend mp --kv-cache-dtype fp8 --moe-backend triton_unfused --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-enable-prefix-caching --gpu-memory-utilization 0.9 --max-num-batched-tokens 8192 --enforce-eager" + decode_flags: "--tensor-parallel-size 8 --distributed-executor-backend mp --kv-cache-dtype fp8 --moe-backend triton_unfused --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-enable-prefix-caching --gpu-memory-utilization 0.9 --max-num-batched-tokens 8192 --enforce-eager" + env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ENGINE_READY_TIMEOUT_S=3600" + hf_dir: "models--deepseek-ai--DeepSeek-V4-Pro" diff --git a/benchmarks/multi_node/dsv4_fp4_mi355x_vllm-disagg.sh b/benchmarks/multi_node/dsv4_fp4_mi355x_vllm-disagg.sh new file mode 100755 index 0000000000..e47cc153a1 --- /dev/null +++ b/benchmarks/multi_node/dsv4_fp4_mi355x_vllm-disagg.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# DeepSeek-V4-Pro disaggregated prefill/decode on MI355X via vLLM + MoRI-IO. +# Thin, model-agnostic launcher: all serving flags / env live in the +# DeepSeek-V4-Pro entry of amd_utils/models_vllm.yaml, and topology (P/D node +# counts, TP/EP) comes from amd-master.yaml. Identical in shape to the kimi / +# minimax vllm-disagg wrappers; only the model differs. + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + ISL \ + OSL \ + IMAGE \ + SPEC_DECODING \ + MODEL_PATH \ + PREFILL_NUM_WORKERS \ + PREFILL_TP \ + PREFILL_EP \ + PREFILL_DP_ATTN \ + DECODE_NUM_WORKERS \ + DECODE_TP \ + DECODE_EP \ + DECODE_DP_ATTN \ + PREFILL_NODES \ + DECODE_NODES \ + RANDOM_RANGE_RATIO \ + FRAMEWORK + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +set -x + +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 + +export TIME_LIMIT="08:00:00" +export MODEL_PATH=$MODEL_PATH +export MODEL_NAME=$MODEL_NAME +export CONTAINER_IMAGE=$IMAGE + +# Same EP/DP booleans as kimik2.5_fp4_mi355x_vllm-disagg.sh → amd_utils/submit.sh +if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then + export PREFILL_ENABLE_EP=false +else + export PREFILL_ENABLE_EP=true +fi + +if [[ "$PREFILL_DP_ATTN" == "true" ]]; then + export PREFILL_ENABLE_DP=true +else + export PREFILL_ENABLE_DP=false +fi + +if [[ "${DECODE_EP:-1}" -eq 1 ]]; then + export DECODE_ENABLE_EP=false +else + export DECODE_ENABLE_EP=true +fi + +if [[ "$DECODE_DP_ATTN" == "true" ]]; then + export DECODE_ENABLE_DP=true +else + export DECODE_ENABLE_DP=false +fi + +# Parameter order matches SGLang disagg submit.sh; arg 16 is optional NODELIST. +JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ + $PREFILL_NUM_WORKERS \ + $DECODE_NODES \ + $DECODE_NUM_WORKERS \ + $ISL $OSL "${CONC_LIST// /x}" inf \ + ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ + ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ + ${PREFILL_TP} ${DECODE_TP} \ + ${RANDOM_RANGE_RATIO} \ + "${NODELIST:-}") + +if [[ $? -ne 0 ]]; then + echo "Failed to submit job" >&2 + exit 1 +fi + +echo "$JOB_ID" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d3f1c70e6c..df65bf9410 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1326,6 +1326,60 @@ kimik2.5-fp4-mi355x-vllm-disagg: additional-settings: - "DECODE_NODES=2" +# DeepSeek-V4-Pro disaggregated prefill/decode on MI355X via vLLM + MoRI-IO. +# Combines the validated single-node DSv4 vLLM serving recipe +# (dsv4-fp4-mi355x-vllm, vllm-project/recipes#433) with the vLLM-disagg +# framework (the kimi / minimax vllm-disagg recipes). Routes to +# benchmarks/multi_node/dsv4_fp4_mi355x_vllm-disagg.sh; per-node serving flags +# + env live in the DeepSeek-V4-Pro entry of amd_utils/models_vllm.yaml. +# +# DeepSeek-V4-Pro is FP4+FP8 mixed (FP4 MoE expert weights, FP8 for the rest); +# InferenceX classifies this as fp4 — same as dsv4-fp4-mi355x-{sglang,vllm,atom}. +# +# Image: latest vLLM ROCm nightly (2026-07-04). The patch-free MoRI-IO path +# (vllm#40344) and the DSv4 model class (vllm#40871) are both long since in +# nightlies; the release tags still trail on #40344, hence nightly. +# +# Topology 1P1D: each prefill/decode worker is a full TP=8 node (EP=1), +# matching the aggregated recipe which runs DSv4 on TP=8 without expert +# parallelism (no all2all backend needed). DEP decode + 1P2D are follow-ups +# once the base disagg path validates. +dsv4-fp4-mi355x-vllm-disagg: + image: vllm/vllm-openai-rocm:nightly-f329ce405b12623fb8b1cf1830f12e5a712523be + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: mi355x-disagg + precision: fp4 + framework: vllm-disagg + multinode: true + disagg: true + scenarios: + fixed-seq-len: + # Smoke test: a single ISL/OSL (8k/1k) at a single concurrency (conc=32) to + # validate the DSv4 vLLM-disagg path (image, MoRIIO transport, serving + # flags, model staging) end-to-end before expanding to the full 1k1k + 8k1k + # conc sweep that the kimi / minimax vllm-disagg recipes run. + - isl: 8192 + osl: 1024 + search-space: + # 1P1D: 1 prefill node (co-located with proxy) + 1 decode node = 2 nodes + - spec-decoding: "none" + conc-list: [ 32 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + dsr1-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 model: amd/DeepSeek-R1-0528-MXFP4-v2 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 320289d243..db695aea45 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4455,3 +4455,12 @@ description: - "Update SGLang image from qwen3.5-fp8-h200-sglang: lmsysorg/sglang:v0.5.12-cu130 / qwen3.5-fp8-h200-sglang-mtp: lmsysorg/sglang:v0.5.12 to lmsysorg/sglang:v0.5.14-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2061 + +- config-keys: + - dsv4-fp4-mi355x-vllm-disagg + description: + - "New recipe: DeepSeek-V4-Pro disaggregated prefill/decode on MI355X via vLLM + MoRI-IO, combining the validated single-node DSv4 vLLM serving recipe (dsv4-fp4-mi355x-vllm, vllm-project/recipes#433) with the existing vllm-disagg framework (MoRI-IO P/D, standalone router)." + - "New benchmarks/multi_node/dsv4_fp4_mi355x_vllm-disagg.sh launcher + DeepSeek-V4-Pro entry in amd_utils/models_vllm.yaml. Per-node flags reuse the aggregated recipe verbatim (--moe-backend triton_unfused for the FP4 expert format, deepseek_v4 tokenizer/reasoning parser, fp8 KV cache, --enforce-eager); the framework adds the MoRIIO kv-transfer role with the patch-free read_mode path already on main." + - "Image: latest vLLM ROCm nightly vllm/vllm-openai-rocm:nightly-f329ce405b12623fb8b1cf1830f12e5a712523be (2026-07-04)." + - "1P1D topology, TP8/EP1 prefill+decode. Smoke test: single ISL/OSL (8k/1k) at conc=32 to validate the path end-to-end before expanding to the full sweep." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2081