From f13ebcc136c720f3daeb43c74684942ac7855080 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 15:56:26 -0500 Subject: [PATCH 01/16] feat: add DeepSeek-V4-Pro-0813 llm-d AgentX on GB200 through srt-slurm Port #2719's llm-d submission to srt-slurm's llm-d frontend: disagg 1P1D DEP8/DEP8 and 2P1D DEP8/DEP16 with Mooncake on prefill, and agg TP8, DEP8, and DEP8 with Mooncake. Drop the MiniMax-M3 smoke keys. Co-authored-by: Ilya Markov Signed-off-by: Cam Quilici --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 239 +++++++++++++++ .../vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 274 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 102 +++++++ .../infx/launch/drivers/srt/lanes.py | 2 +- inferencex-e2e/perf-changelog.yaml | 10 + 5 files changed, 626 insertions(+), 1 deletion(-) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml new file mode 100644 index 0000000000..a998dc8771 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -0,0 +1,239 @@ +# DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router +# (srt-slurm frontend.type: llm-d): one aggregated worker over two trays, as TP8, +# DEP8, or DEP8 with an embedded Mooncake prefix-cache store. Image, vLLM flags, +# environment, and EPP schedulers follow #2719's llm-d recipes. srt-slurm launches a +# DEP8 worker as one `vllm serve` that balances its eight DP ranks itself, so the EPP +# routes to the worker rather than to one rank, and VLLM_USE_RUST_FRONTEND is left +# unset (the Python API server). +# Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-v4-pro-0813 + container: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + precision: fp4 + identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro-0813} + container: {image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95"} + dynamo: + install: false + slurm: {time_limit: "8:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: gb200 + gpus_per_node: 4 + observability: + enabled: false + tachometer: + enabled: false + frontend: + type: llm-d + enable_multiple_frontends: false + # #2719's scheduler, without its file-discovery plugin (srtctl adds its own). + epp_config: + plugins: + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 5858 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 20000 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 1499703 + - type: active-request-scorer + - type: queue-scorer + - type: max-score-picker + schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-affinity-filter + - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer + weight: 3 + - pluginRef: max-score-picker + engine: + type: vllm + connector: + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONHASHSEED: "0" + HF_HUB_CACHE: /hf_hub_cache + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_IB_ENABLE_IBGDA: "false" + NVSHMEM_CUMEM_HANDLE_TYPE: FABRIC + NVSHMEM_DISABLE_CUDA_VMM: "0" + UCX_TLS: cuda_copy,cuda_ipc,rc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: NVL + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.88 + max-num-batched-tokens: 8192 + block-size: 256 + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + moe-backend: deep_gemm_mega_moe + enable-ep-weight-filter: true + enable-cumem-allocator: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"}' + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +# TP8 with five DSpark tokens, no EP, and a load-only scheduler. +override_tp8: + name: dsv4-gb200-vllm-agentic-dspark-agg-tp8-llm-d + frontend: + epp_config: + plugins: + - type: active-request-scorer + - type: queue-scorer + - type: weighted-random-picker + schedulingProfiles: + - name: default + plugins: + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: weighted-random-picker + roles: + agg: + env: + PYTHONHASHSEED: + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: + VLLM_USE_DEEP_GEMM: + NVSHMEM_REMOTE_TRANSPORT: + NVSHMEM_IB_ENABLE_IBGDA: + NVSHMEM_CUMEM_HANDLE_TYPE: + NVSHMEM_DISABLE_CUDA_VMM: + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: auto + UCX_TLS: cuda_copy,cuda_ipc,tcp + UCX_MEMTYPE_REG_WHOLE: "n" + args: + tensor-parallel-size: 8 + data-parallel-size: + enable-expert-parallel: + moe-backend: + enable-ep-weight-filter: + numa-bind: + gpu-memory-utilization: 0.85 + disable-custom-all-reduce: true + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"}' + +override_dep8: + name: dsv4-gb200-vllm-agentic-dspark-agg-dep8-llm-d + +# DEP8 with an embedded Mooncake store at 45% of each tray's DRAM (~900 GB over four +# GPUs); NIXL, SimpleCPUOffloadConnector, and MooncakeStoreConnector in a MultiConnector. +override_dep8_mooncake: + name: dsv4-gb200-vllm-agentic-dspark-agg-dep8-mooncake-llm-d + services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "101GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + frontend: + epp_config: + plugins: + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 40810 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 20000 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 1499703 + - type: active-request-scorer + - type: queue-scorer + - type: max-score-picker + roles: + agg: + env: + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + gpu-memory-utilization: 0.87 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":42949672960}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml new file mode 100644 index 0000000000..ba5558e892 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -0,0 +1,274 @@ +# DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router +# (srt-slurm frontend.type: llm-d): DEP8 prefill workers over two trays each and one +# DEP8 or DEP16 decode worker. KV moves over NIXL and the prefill workers share prefix +# cache through an embedded Mooncake store. Image, vLLM flags, environment, and EPP +# scheduler follow #2719's disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml, with these +# differences: +# - srt-slurm launches each worker as one `vllm serve` over its trays that balances its +# DP ranks itself, so the EPP routes to the worker rather than to one rank. +# - Prefill drafts three DSpark tokens like decode (#2719: one): the srt launch planner +# gives both roles the decode role's golden acceptance length. +# - VLLM_USE_RUST_FRONTEND is left unset (the Python API server). +# Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-v4-pro-0813 + container: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + precision: fp4 + identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro-0813} + container: {image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95"} + dynamo: + install: false + slurm: {time_limit: "8:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: gb200 + gpus_per_node: 4 + observability: + enabled: false + tachometer: + enabled: false + services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + # Each prefill rank's embedded store: 2/3 of a tray's ~900 GB DRAM over 4 GPUs. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + frontend: + type: llm-d + enable_multiple_frontends: false + # #2719's scheduler, without its file-discovery plugin (srtctl adds its own). + epp_config: + plugins: + - type: prefill-filter + - type: decode-filter + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 43949 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 4783 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 3000000 + - type: active-request-scorer + - type: queue-scorer + - type: always-disagg-pd-decider + - type: disagg-profile-handler + parameters: + deciders: + prefill: always-disagg-pd-decider + - type: max-score-picker + name: prefill-picker + - type: max-score-picker + name: decode-picker + schedulingProfiles: + - name: prefill + plugins: + - pluginRef: prefill-filter + - pluginRef: prefix-cache-affinity-filter + - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: queue-scorer + weight: 3 + - pluginRef: prefill-picker + - name: decode + plugins: + - pluginRef: decode-filter + - pluginRef: active-request-scorer + - pluginRef: decode-picker + engine: + type: vllm + connector: + roles: + prefill: + gpus: 8 + env: + PYTHONHASHSEED: "0" + HF_HUB_CACHE: /hf_hub_cache + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED: "1" + VLLM_NO_USAGE_STATS: "1" + VLLM_LOGGING_LEVEL: INFO + TQDM_DISABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_IB_ENABLE_IBGDA: "false" + NVSHMEM_CUMEM_HANDLE_TYPE: FABRIC + NVSHMEM_DISABLE_CUDA_VMM: "0" + UCX_TLS: cuda_copy,cuda_ipc,rc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: NVL + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_RPC_CLIENT_IO_THREADS: "32" + MC_TE_RPC_CLIENT_IO_THREADS: "32" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + # NIXL hands KV to decode; the Mooncake store shares prefix cache across prefill ranks. + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"kv_lease_duration":1800}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + max-num-seqs: 16 + max-model-len: 1048576 + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}' + enable-cumem-allocator: true + block-size: 256 + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + moe-backend: deep_gemm_mega_moe + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONHASHSEED: "0" + HF_HUB_CACHE: /hf_hub_cache + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_NO_USAGE_STATS: "1" + VLLM_LOGGING_LEVEL: INFO + TQDM_DISABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_IB_ENABLE_IBGDA: "false" + NVSHMEM_CUMEM_HANDLE_TYPE: FABRIC + NVSHMEM_DISABLE_CUDA_VMM: "0" + UCX_TLS: cuda_copy,cuda_ipc,rc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: NVL + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"kv_lease_duration":1800}}' + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-num-batched-tokens: 256 + max-num-seqs: 32 + max-cudagraph-capture-size: 256 + max-model-len: 1048576 + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}' + enable-cumem-allocator: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + moe-backend: deep_gemm_mega_moe + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +override_1p1d: + name: dsv4-gb200-vllm-agentic-dspark-disagg-1p1d-dep8-dep8-llm-d + roles: + prefill: + nodes: 2 + workers: 1 + +override_2p1d: + name: dsv4-gb200-vllm-agentic-dspark-disagg-2p1d-dep8-dep16-llm-d + roles: + prefill: + nodes: 4 + workers: 2 + decode: + nodes: 4 + gpus: 16 + args: + data-parallel-size: 16 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 12c4cc160e..f288080aee 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6076,6 +6076,108 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: additional-settings: - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c40_b40" +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200 behind the llm-d router (srt-slurm +# frontend.type: llm-d). The image bundles epp and pd-sidecar from llm-d-router v0.10.0. +dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: vllm + router: { name: llm-d-router, version: "0.10.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.666667 + search-space: + # 1P DEP8 + 1D DEP8 (4 trays), Mooncake prefix cache on prefill. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [64, 128, 160, 256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_1p1d" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 2P DEP8 + 1D DEP16 (8 trays), Mooncake prefix cache on prefill. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [256, 384, 576] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_2p1d" + decode: + num-worker: 1 + tp: 8 + ep: 16 + dp-attn: true + +# Aggregated TP8 and DEP8 over two trays; only the Mooncake arm offloads KV to DRAM. +dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-agg: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: vllm + router: { name: llm-d-router, version: "0.10.0" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.45 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 2, 4, 8, 12] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_tp8" + - spec-decoding: mtp + kv-offloading: none + conc-list: [12, 16, 32] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_dep8" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [52, 72] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_dep8_mooncake" + dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index 57d31ce99d..58cd13eadc 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -66,7 +66,7 @@ class SrtLane: ), ("b300-dsxe", LaunchPath.SRT_MULTI): SrtLane(frameworks=_DYNAMO), ("gb200-nv", LaunchPath.SRT_MULTI): SrtLane( - frameworks=_DYNAMO, + frameworks=any_of("dynamo-sglang", "dynamo-trt", "dynamo-vllm", "vllm"), setup_scripts={"dynamo-sglang": "install-torchao.sh"}, mounts=( *_AGENTIC_CACHES, diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 4892c67d61..42c8e8aef5 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9528,3 +9528,13 @@ - "TP4 c64: 8K prefill chunks with a 16-step prefill-decode interval instead of 16K chunks with none. Local runs with the fold: 82,586 -> 85,093 tok/s/GPU at 86.8 -> 136.8 tok/s/user median (46.3 -> 72.1 P90); P90 TTFT 5.7 -> 12.5 s." - "TP2 c128: admit all 128 sessions (max-running-requests and cuda-graph-max-bs-decode 64 -> 128) with a 4-step prefill-decode interval and mem-fraction-static 0.85. The PR #3421 sweep measured 18,631 tok/s/GPU at 130.9 tok/s/user median at this point; locally the new settings gave 147,763 tok/s/GPU at 37.9 tok/s/user median with a 51 s P90 TTFT (measured before the fold was added)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3782 + +- config-keys: + - dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg + - dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4-Pro-0813 DSpark FP4/FP8-KV AgentX on GB200 behind the llm-d router (EPP and pd-sidecar from llm-d-router v0.10.0, Envoy) through srt-slurm's llm-d frontend, porting #2719's llm-d submission: disagg 1P1D DEP8/DEP8 (conc 64/128/160/256) and 2P1D DEP8/DEP16 (conc 256/384/576) with NIXL P/D KV and a Mooncake prefix-cache store on prefill at 66.7% node DRAM; agg TP8 (conc 1-12), DEP8 (conc 12-32), and DEP8 with Mooncake at 45% node DRAM (conc 52/72)." + - "Same image, vLLM flags, environment, and EPP schedulers as #2719. Each DEP worker is one vllm serve that balances its DP ranks itself, so the EPP routes to workers. Throughput uses committed golden synthetic acceptance (TP8 K=5, DEP K=3 on both prefill and decode); evals use real verification." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741 From 046a1ea0aa3276d0941eb480bbac6653c1f24b79 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 15:52:52 -0500 Subject: [PATCH 02/16] fix: declare the 2P1D decode worker as TP16 so results count 32 GPUs Signed-off-by: Cam Quilici --- inferencex-e2e/configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index f288080aee..f7cd612d17 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6124,7 +6124,7 @@ dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg: - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_2p1d" decode: num-worker: 1 - tp: 8 + tp: 16 ep: 16 dp-attn: true From 8840a82cf5f800f6c045f5ef2154f5b1ae0c7317 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 15:55:45 -0500 Subject: [PATCH 03/16] fix: stage gb200-nv dsv4 vllm runs on the shared run root Signed-off-by: Cam Quilici --- inferencex-e2e/infx/launch/drivers/srt/lanes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index 58cd13eadc..d0bd95d6ef 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -79,7 +79,7 @@ class SrtLane: ), shared_run_root=( Match(any_of("minimaxm3", "kimik3", "qwen3.5", "glm5.2")), - Match(any_of("dsv4"), frameworks=any_of("dynamo-vllm")), + Match(any_of("dsv4"), frameworks=any_of("dynamo-vllm", "vllm")), Match(any_of("dsv4"), frameworks=any_of("dynamo-sglang"), agentic=True), ), ), From 67c1918970d1a052f7a3b661e612291b24710a52 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 15:56:10 -0500 Subject: [PATCH 04/16] chore: drop PR references from the llm-d recipes Signed-off-by: Cam Quilici --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 5 ++--- .../dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 15 ++++++--------- inferencex-e2e/perf-changelog.yaml | 4 ++-- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml index a998dc8771..da2fb35ff4 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -1,7 +1,6 @@ # DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router # (srt-slurm frontend.type: llm-d): one aggregated worker over two trays, as TP8, -# DEP8, or DEP8 with an embedded Mooncake prefix-cache store. Image, vLLM flags, -# environment, and EPP schedulers follow #2719's llm-d recipes. srt-slurm launches a +# DEP8, or DEP8 with an embedded Mooncake prefix-cache store. srt-slurm launches a # DEP8 worker as one `vllm serve` that balances its eight DP ranks itself, so the EPP # routes to the worker rather than to one rank, and VLLM_USE_RUST_FRONTEND is left # unset (the Python API server). @@ -31,7 +30,7 @@ base: frontend: type: llm-d enable_multiple_frontends: false - # #2719's scheduler, without its file-discovery plugin (srtctl adds its own). + # srtctl adds the file-discovery plugin. epp_config: plugins: - type: inflight-load-producer diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml index ba5558e892..f4c419cc81 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -1,14 +1,11 @@ # DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router # (srt-slurm frontend.type: llm-d): DEP8 prefill workers over two trays each and one # DEP8 or DEP16 decode worker. KV moves over NIXL and the prefill workers share prefix -# cache through an embedded Mooncake store. Image, vLLM flags, environment, and EPP -# scheduler follow #2719's disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml, with these -# differences: -# - srt-slurm launches each worker as one `vllm serve` over its trays that balances its -# DP ranks itself, so the EPP routes to the worker rather than to one rank. -# - Prefill drafts three DSpark tokens like decode (#2719: one): the srt launch planner -# gives both roles the decode role's golden acceptance length. -# - VLLM_USE_RUST_FRONTEND is left unset (the Python API server). +# cache through an embedded Mooncake store. srt-slurm launches each worker as one +# `vllm serve` over its trays that balances its DP ranks itself, so the EPP routes to the +# worker rather than to one rank. Prefill drafts three DSpark tokens like decode: the srt +# launch planner gives both roles the decode role's golden acceptance length. +# VLLM_USE_RUST_FRONTEND is left unset (the Python API server). # Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_. schema: 2 @@ -48,7 +45,7 @@ base: frontend: type: llm-d enable_multiple_frontends: false - # #2719's scheduler, without its file-discovery plugin (srtctl adds its own). + # srtctl adds the file-discovery plugin. epp_config: plugins: - type: prefill-filter diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 42c8e8aef5..e1dce5326f 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9535,6 +9535,6 @@ scenario-type: - agentic-coding description: - - "Add DeepSeek-V4-Pro-0813 DSpark FP4/FP8-KV AgentX on GB200 behind the llm-d router (EPP and pd-sidecar from llm-d-router v0.10.0, Envoy) through srt-slurm's llm-d frontend, porting #2719's llm-d submission: disagg 1P1D DEP8/DEP8 (conc 64/128/160/256) and 2P1D DEP8/DEP16 (conc 256/384/576) with NIXL P/D KV and a Mooncake prefix-cache store on prefill at 66.7% node DRAM; agg TP8 (conc 1-12), DEP8 (conc 12-32), and DEP8 with Mooncake at 45% node DRAM (conc 52/72)." - - "Same image, vLLM flags, environment, and EPP schedulers as #2719. Each DEP worker is one vllm serve that balances its DP ranks itself, so the EPP routes to workers. Throughput uses committed golden synthetic acceptance (TP8 K=5, DEP K=3 on both prefill and decode); evals use real verification." + - "Add DeepSeek-V4-Pro-0813 DSpark FP4/FP8-KV AgentX on GB200 behind the llm-d router (EPP and pd-sidecar from llm-d-router v0.10.0, Envoy) through srt-slurm's llm-d frontend: disagg 1P1D DEP8/DEP8 (conc 64/128/160/256) and 2P1D DEP8/DEP16 (conc 256/384/576) with NIXL P/D KV and a Mooncake prefix-cache store on prefill at 66.7% node DRAM; agg TP8 (conc 1-12), DEP8 (conc 12-32), and DEP8 with Mooncake at 45% node DRAM (conc 52/72)." + - "Each DEP worker is one vllm serve that balances its DP ranks itself, so the EPP routes to workers. Throughput uses committed golden synthetic acceptance (TP8 K=5, DEP K=3 on both prefill and decode); evals use real verification." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741 From 25adb730ab13d2c7c3129d0214d852f5505c185a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 16:00:36 -0500 Subject: [PATCH 05/16] fix: label the llm-d keys llmd-vllm Signed-off-by: Cam Quilici --- inferencex-e2e/configs/nvidia-master.yaml | 8 ++++---- inferencex-e2e/infx/launch/drivers/srt/lanes.py | 4 ++-- inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py | 1 + inferencex-e2e/perf-changelog.yaml | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index f7cd612d17..560d2d03ae 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6078,13 +6078,13 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: # DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200 behind the llm-d router (srt-slurm # frontend.type: llm-d). The image bundles epp and pd-sidecar from llm-d-router v0.10.0. -dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg: +dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg: image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:gb200-nv precision: fp4 - framework: vllm + framework: llmd-vllm router: { name: llm-d-router, version: "0.10.0" } kv-p2p-transfer: nixl multinode: true @@ -6129,13 +6129,13 @@ dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg: dp-attn: true # Aggregated TP8 and DEP8 over two trays; only the Mooncake arm offloads KV to DRAM. -dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-agg: +dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg: image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:gb200-nv precision: fp4 - framework: vllm + framework: llmd-vllm router: { name: llm-d-router, version: "0.10.0" } multinode: true disagg: false diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index d0bd95d6ef..49611d7d8b 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -66,7 +66,7 @@ class SrtLane: ), ("b300-dsxe", LaunchPath.SRT_MULTI): SrtLane(frameworks=_DYNAMO), ("gb200-nv", LaunchPath.SRT_MULTI): SrtLane( - frameworks=any_of("dynamo-sglang", "dynamo-trt", "dynamo-vllm", "vllm"), + frameworks=any_of("dynamo-sglang", "dynamo-trt", "dynamo-vllm", "llmd-vllm"), setup_scripts={"dynamo-sglang": "install-torchao.sh"}, mounts=( *_AGENTIC_CACHES, @@ -79,7 +79,7 @@ class SrtLane: ), shared_run_root=( Match(any_of("minimaxm3", "kimik3", "qwen3.5", "glm5.2")), - Match(any_of("dsv4"), frameworks=any_of("dynamo-vllm", "vllm")), + Match(any_of("dsv4"), frameworks=any_of("dynamo-vllm", "llmd-vllm")), Match(any_of("dsv4"), frameworks=any_of("dynamo-sglang"), agentic=True), ), ), diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 19ba42a60d..c30a78942c 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -24,6 +24,7 @@ "vllm-disagg": "vllm", "vllm": "vllm", "dynamo-vllm": "vllm", + "llmd-vllm": "vllm", "dynamo-sglang": "sglang", "trt": "trtllm", "dynamo-trt": "trtllm", diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e1dce5326f..6bdd41b7f2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9530,8 +9530,8 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3782 - config-keys: - - dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg - - dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-agg + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg scenario-type: - agentic-coding description: From 47dfac9f3984f4acf30129017e18fbadf5b9884e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 16:34:17 -0500 Subject: [PATCH 06/16] fix: pin huggingface-hub below 2 when installing lm-eval Signed-off-by: Cam Quilici --- inferencex-e2e/infx/bench/eval/lm_eval.py | 4 +++- inferencex-e2e/infx/tests/bench/test_eval_command.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/bench/eval/lm_eval.py b/inferencex-e2e/infx/bench/eval/lm_eval.py index cbc8d23b38..419f0820e2 100644 --- a/inferencex-e2e/infx/bench/eval/lm_eval.py +++ b/inferencex-e2e/infx/bench/eval/lm_eval.py @@ -33,7 +33,9 @@ def install(environ: Mapping[str, str]) -> None: # torchvision causes circular imports in ATOM; TRT-LLM/SGLang need it at module level. if "atom" in environ.get("IMAGE", ""): _pip(environ, "uninstall", "torchvision") - _pip(environ, "install", "lm-eval[api]") + # lm-eval does not depend on transformers, so without this range its resolution + # upgrades huggingface-hub past what transformers 5.x accepts. + _pip(environ, "install", "lm-eval[api]", "huggingface-hub>=1.5,<2") pinned = ("install", "--no-deps", "--reinstall") git = shutil.which("git", path=environ.get("PATH")) if git and _pip(environ, *pinned, f"git+{REPOSITORY}.git@{REF}"): diff --git a/inferencex-e2e/infx/tests/bench/test_eval_command.py b/inferencex-e2e/infx/tests/bench/test_eval_command.py index 59d679410e..898f64fcbb 100644 --- a/inferencex-e2e/infx/tests/bench/test_eval_command.py +++ b/inferencex-e2e/infx/tests/bench/test_eval_command.py @@ -404,7 +404,7 @@ def uv_step(argv: list[str]) -> str: """Name one recorded ``uv`` call of the lm-eval install into this interpreter.""" assert argv[0] == "pip" and argv[-2:] == ["--python", sys.executable] assert "--break-system-packages" in argv - spec = argv[-3] + spec = next(arg for arg in argv[2:] if not arg.startswith("-")) if argv[1] == "uninstall": return f"uninstall {spec}" if {"--no-deps", "--reinstall"} <= set(argv): From 995a5acf04911bbe9bb5975353e007e1f5a356c8 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 10 Oct 2026 00:10:52 -0500 Subject: [PATCH 07/16] fix: keep decode HTTP connections alive past the llm-d sidecar's idle timeout Signed-off-by: Cam Quilici --- .../dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml index f4c419cc81..f007fcc066 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -187,6 +187,9 @@ base: VLLM_LOG_STATS_INTERVAL: "1" VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + # Outlast the decode sidecar's 90 s idle-connection timeout, so it never reuses a + # connection the server has already closed (connection reset, 502). + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_USE_DEEP_GEMM: "1" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" From 05ddf73d64dff5dc6aa82a2a372fb0fbc4b38769 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 10 Oct 2026 00:56:46 -0500 Subject: [PATCH 08/16] fix: keep agg HTTP connections alive past Envoy's upstream idle timeout Signed-off-by: Cam Quilici --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml index da2fb35ff4..150cb3f00e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -82,6 +82,9 @@ base: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + # Outlast Envoy's default 1 h upstream idle timeout, so it never reuses a connection + # the server has already closed (503 UC). + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "3900" VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" From c801c9f2dbbd4d2aa8b32efbb83fbaa2a70a16f0 Mon Sep 17 00:00:00 2001 From: Alec Ibarra <93070681+adibarra@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:05:15 -0500 Subject: [PATCH 09/16] fix(lint): alias datetime import for ruff 0.17 ICN001 (#3896) --- inferencex-e2e/infx/bench/agentic/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/bench/agentic/run.py b/inferencex-e2e/infx/bench/agentic/run.py index 25b3c93790..5b90068a4f 100644 --- a/inferencex-e2e/infx/bench/agentic/run.py +++ b/inferencex-e2e/infx/bench/agentic/run.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse -import datetime +import datetime as dt import os import shlex import sys @@ -168,7 +168,7 @@ def _open_power_window(plan: Plan, python: str, env: Mapping[str, str]) -> int: return 0 # AIPerf exports naive local datetimes; the adapter needs the offset to convert the # profiling window to Unix time. - now = datetime.datetime.now(datetime.timezone.utc).astimezone() + now = dt.datetime.now(dt.timezone.utc).astimezone() (plan.result_dir / "agentic_power_timezone_offset.txt").write_text(f"{now:%z}\n") rc = _power_adapter(plan, python, env, *_window(plan, "running")) if rc: From 3020f4af0722af8aca3b510cef3cfd8b22b5d46a Mon Sep 17 00:00:00 2001 From: honglie Date: Mon, 12 Oct 2026 10:51:20 +0800 Subject: [PATCH 10/16] =?UTF-8?q?perf(amd):=20DSv4-Pro=20ATOM=20AgentX=20o?= =?UTF-8?q?n=20MegaMoE=20DEP=20with=20LMCache=20MP=20on=20the=201007=20nig?= =?UTF-8?q?htly=20/=20DSv4-Pro=20ATOM=20AgentX=20=E6=94=B9=E7=94=A8=20Mega?= =?UTF-8?q?MoE=20DEP=20=E5=B9=B6=E5=9C=A8=201007=20nightly=20=E5=90=AF?= =?UTF-8?q?=E7=94=A8=20LMCache=20MP=20(#3819)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(amd): DSv4-Pro ATOM AgentX on MegaMoE DEP with an LMCache MP tier on the 1007 nightly Move dsv4-fp4-mi355x-atom-agentic-mtp to rocm/atom-dev:nightly_202610071700 (ATOM 0.1.7.dev65+ga526f0d55). C64 and C128 switch from RCCL expert parallelism to MegaMoE with DP pad-row masking and a fixed-slot decode instance; C128 adds prefill-only EPLB that stops after its warmup rebalances. C256 and the new C384 add ATOM's lmcache_mp CPU KV tier, served by one LMCache server per node that the new dsv4-atom-lmcache-mp-server.sh setup script starts from the LMCache build in the image. C48 and C96 keep RCCL. 将 dsv4-fp4-mi355x-atom-agentic-mtp 更新到 rocm/atom-dev:nightly_202610071700 （ATOM 0.1.7.dev65+ga526f0d55）。C64、C128 从 RCCL 专家并行改为 MegaMoE， 启用 DP pad 行掩码和 fixed-slot decode 实例；C128 增加只按 prefill 统计的 EPLB，warmup 内重平衡后停止。C256 和新增的 C384 增加 ATOM 的 lmcache_mp CPU KV 层，由新的 dsv4-atom-lmcache-mp-server.sh setup 脚本用镜像自带的 LMCache 在每个节点启动一个 LMCache server。C48、C96 保持 RCCL。 Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie * chore: link the dsv4 ATOM AgentX changelog entry to #3819 将 dsv4 ATOM AgentX 的 perf-changelog 条目链接到 #3819。 Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie * fix(dsv4-atom): escape braces in the lmcache_mp extra-config env srtctl expands {node}/{node_id} in worker env values with str.format_map, which parsed the JSON in ATOM_KV_OFFLOAD_EXTRA_CONFIG as a format field and failed at worker start ("'str' object has no attribute 'mp'"). Double the braces so the expanded value is the original JSON. srtctl 用 str.format_map 展开 worker env 里的 {node}/{node_id}， ATOM_KV_OFFLOAD_EXTRA_CONFIG 中的 JSON 被当成格式字段解析，worker 启动即报错。 花括号加倍后展开结果即原 JSON（已用 srtctl load_config + 同样的 format_map 验证 c256/c384）。 Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie * perf(dsv4-atom): run C48 and C96 on MegaMoE with EPLB C48 and C96 move from RCCL expert parallelism to the MegaMoE backend with EPLB, like the other DEP points; their admission, prefill-decode interval and cudagraph sizes stay as before. C48、C96 从 RCCL 专家并行改为 MegaMoE 后端并开启 EPLB，与其他 DEP 点一致； max-num-seqs、prefill 间隔和 cudagraph 档位不变。 Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie * chore(dsv4-atom): describe the final DEP and LMCache config in the changelog The entry still described the first revision of the PR. It now lists the MegaMoE move for C48 to C128 (EPLB everywhere except C64), the max-num-seqs cap of 128 from C128 up, the lmcache_mp tier at C256 and the new C384 with its server size and LMCache version, and that C1 to C16 are unchanged. changelog 条目仍描述的是 PR 的第一版。现改为最终配置：C48 至 C128 从 RCCL 专家并行改用 MegaMoE（除 C64 外均开启 EPLB）；C128 起 max-num-seqs 上限为 128；C256 与新增的 C384 使用 lmcache_mp CPU KV 层，并注明 server 容量和 LMCache 版本；C1 至 C16 不变。 Co-Authored-By: Claude Opus 5.5 Signed-off-by: seungrokj * perf(dsv4-atom): start the C256/C384 LMCache server with srtctl's lmcache-server service Replace the custom setup script with srtctl's lmcache-server service, which runs `lmcache server` from the LMCache build in the ATOM image on port 8750 with the same server arguments; ATOM's lmcache_mp tier dials that port. C256/C384 的 LMCache server 改由 srtctl 自带的 lmcache-server 服务启动（使用镜像 自带的 LMCache，端口 8750，server 参数不变），删除自定义 setup 脚本。 Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie * Revise performance metrics in perf-changelog.yaml Updated performance changelog with new metrics and configurations for various TP settings. * perf(amd): DSv4-Pro ATOM AgentX on MegaMoE DEP with LMCache MP on the 1007 nightly - Update the image from rocm/atom-dev:nightly_202609291501 to rocm/atom-dev:nightly_202610071700 (digest sha256:bb3912c372dda0c82fc6831afdd76e366a372d5e9fe223373499db63b06cce65). - Move the C48, C64, C96 and C128 DEP points from RCCL expert parallelism to ATOM's MegaMoE backend (all2all-backend high-throughput, moe-backend mega), with EPLB at every point except C64. - Cap max-num-seqs at 128 from C128 up (C128 was 256, C256 was 512); C48, C64 and C96 keep admission at 2x CONC. - Run C256 and the new C384 on the same MegaMoE DEP setup with ATOM's lmcache_mp CPU KV tier: one LMCache server per node (1000 GB L1, LMCache 0.5.6.dev139+gf22dec28 as shipped in the image), started by srtctl's lmcache-server service, within a 1019 GB budget (dram-utilization 0.34). - C1 to C16 are unchanged. - No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4-Pro-0813 runs as shipped. Co-Authored-By: Claude Opus 5.5 --------- Signed-off-by: yihonglie Signed-off-by: seungrokj Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: seungrokj <144636725+seungrokj@users.noreply.github.com> Co-authored-by: seungrokj --- .../dsv4/atom/mi355x-fp4-mtp/agentic.yaml | 172 +++++++++++++++--- inferencex-e2e/configs/amd-master.yaml | 10 +- inferencex-e2e/perf-changelog.yaml | 13 ++ 3 files changed, 168 insertions(+), 27 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml index 592a91827b..f6dd120075 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml @@ -1,19 +1,20 @@ # DeepSeek-V4-Pro-0813 AgentX on MI355X with ATOM DSpark K6 (six draft tokens, -# a verify window of 7). Low concurrency is TP-only; the wide band runs DP -# attention with native RCCL expert parallelism. The KV cache is GPU-resident. +# a verify window of 7). Low concurrency is TP-only. The DP-attention band runs +# MegaMoE with EPLB from C48 (C64 without EPLB). C256 and C384 add an LMCache +# CPU KV tier (lmcache_mp); the other points keep the KV cache GPU-resident. base: schema: 2 name: dsv4-fp4-mi355x-atom-agentic model: path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: rocm/atom-dev:nightly_202609291501 + container: rocm/atom-dev:nightly_202610071700 precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + container_image: rocm/atom-dev@sha256:bb3912c372dda0c82fc6831afdd76e366a372d5e9fe223373499db63b06cce65 enable_multiple_frontends: false observability: enabled: false @@ -58,9 +59,9 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point. Admission is 2x CONC. KV is BF16 through concurrency 16 -# and FP8 in the DEP band. DEP pins sessions to one DP-attention rank so later -# turns reach their prefix KV, and captures every small per-rank decode batch. +# One variant per point. KV is BF16 through concurrency 16 and FP8 in the DEP +# band. DEP pins sessions to one DP-attention rank so later turns reach their +# prefix KV, and captures every small per-rank decode batch. override_tp8_c1: roles: agg: @@ -119,15 +120,22 @@ override_tp8_c48_dep: kv-cache-dtype: fp8 enable-expert-parallel: true enable-dp-attention: true - all2all-backend: rccl + all2all-backend: high-throughput dp-load-balance: least_tokens - moe-backend: standard + moe-backend: mega cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' env: ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' ATOM_ENABLE_PREFILL_DELAYER: '1' ATOM_PREFILL_DECODE_INTERVAL: '10' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '256' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '511' + ATOM_EPLB_MAX_REBALANCES: '1' benchmark: env: CONC: '48' @@ -142,15 +150,19 @@ override_tp8_c64_dep: kv-cache-dtype: fp8 enable-expert-parallel: true enable-dp-attention: true - all2all-backend: rccl + all2all-backend: high-throughput dp-load-balance: least_tokens - moe-backend: standard + moe-backend: mega cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' env: ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' ATOM_ENABLE_PREFILL_DELAYER: '1' ATOM_PREFILL_DECODE_INTERVAL: '10' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '256' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '511' benchmark: env: CONC: '64' @@ -165,15 +177,22 @@ override_tp8_c96_dep: kv-cache-dtype: fp8 enable-expert-parallel: true enable-dp-attention: true - all2all-backend: rccl + all2all-backend: high-throughput dp-load-balance: least_tokens - moe-backend: standard + moe-backend: mega cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' env: ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' ATOM_ENABLE_PREFILL_DELAYER: '1' ATOM_PREFILL_DECODE_INTERVAL: '10' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '512' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '1023' + ATOM_EPLB_MAX_REBALANCES: '1' benchmark: env: CONC: '96' @@ -184,44 +203,149 @@ override_tp8_c128_dep: roles: agg: args: - max-num-seqs: 256 + max-num-seqs: 128 kv-cache-dtype: fp8 enable-expert-parallel: true enable-dp-attention: true - all2all-backend: rccl + all2all-backend: high-throughput dp-load-balance: least_tokens - moe-backend: standard - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + moe-backend: mega + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' env: ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' ATOM_ENABLE_PREFILL_DELAYER: '1' - ATOM_PREFILL_DECODE_INTERVAL: '10' + ATOM_PREFILL_DECODE_INTERVAL: '20' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '512' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '1023' + ATOM_EPLB_MAX_REBALANCES: '2' benchmark: env: CONC: '128' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' AGENTIC_WARMUP_GRACE_PERIOD: '3600' -override_tp8_c256_dep: +# DRAM variants: ATOM's lmcache_mp tier backed by srtctl's lmcache-server +# service (one LMCache server per node on port 8750, run from the LMCache build in +# the ATOM image) within the 1019 GB budget (dram-utilization 0.34). +override_tp8_c256_dep_lmcache: + services: + - name: lmcache + type: lmcache-server + args: + - --chunk-size + - '256' + - --null-block-id + - '-1' + - --separate-object-groups + - --supported-transfer-mode + - lmcache_driven + - --l1-size-gb + - '1000' + - --l1-use-lazy + - --l1-read-ttl-seconds + - '900' + - --eviction-policy + - LRU + - --eviction-trigger-watermark + - '0.98' roles: agg: args: - max-num-seqs: 512 + max-num-seqs: 128 kv-cache-dtype: fp8 enable-expert-parallel: true enable-dp-attention: true - all2all-backend: rccl + all2all-backend: high-throughput dp-load-balance: least_tokens - moe-backend: standard - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + moe-backend: mega + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' env: ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' ATOM_ENABLE_PREFILL_DELAYER: '1' - ATOM_PREFILL_DECODE_INTERVAL: '10' + ATOM_PREFILL_DECODE_INTERVAL: '40' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '512' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '1023' + ATOM_EPLB_MAX_REBALANCES: '4' + ATOM_KV_OFFLOAD: lmcache_mp + # Braces doubled: srtctl expands {node}/{node_id} in env values with str.format_map. + ATOM_KV_OFFLOAD_EXTRA_CONFIG: '{{"lmcache.mp.host": "tcp://127.0.0.1", "lmcache.mp.port": 8750, "lmcache.mp.tp_rank_collapse": true, "lmcache.mp.max_pinned_state_bytes": 1000000000000}}' + LMCACHE_CHUNK_SIZE: '256' + OFFLOAD_MAX_PENDING_SAVES: '8' + OFFLOAD_MIN_LOAD_TOKENS: '8192' + OFFLOAD_MIN_SAVE_TOKENS: '8192' benchmark: env: CONC: '256' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1019' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c384_dep_lmcache: + services: + - name: lmcache + type: lmcache-server + args: + - --chunk-size + - '256' + - --null-block-id + - '-1' + - --separate-object-groups + - --supported-transfer-mode + - lmcache_driven + - --l1-size-gb + - '1000' + - --l1-use-lazy + - --l1-read-ttl-seconds + - '900' + - --eviction-policy + - LRU + - --eviction-trigger-watermark + - '0.98' + roles: + agg: + args: + max-num-seqs: 128 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: high-throughput + dp-load-balance: least_tokens + moe-backend: mega + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,72,80,88,96,104,112,120,128]' + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '20' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '1024' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '2047' + ATOM_EPLB_MAX_REBALANCES: '5' + ATOM_KV_OFFLOAD: lmcache_mp + # Braces doubled: srtctl expands {node}/{node_id} in env values with str.format_map. + ATOM_KV_OFFLOAD_EXTRA_CONFIG: '{{"lmcache.mp.host": "tcp://127.0.0.1", "lmcache.mp.port": 8750, "lmcache.mp.tp_rank_collapse": true, "lmcache.mp.max_pinned_state_bytes": 1000000000000}}' + LMCACHE_CHUNK_SIZE: '256' + OFFLOAD_MAX_PENDING_SAVES: '8' + OFFLOAD_MIN_LOAD_TOKENS: '8192' + OFFLOAD_MIN_SAVE_TOKENS: '8192' + benchmark: + env: + CONC: '384' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1019' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index e0ed3d6b51..3e3960355a 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -946,8 +946,10 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: # DeepSeek-V4-Pro-0813 FP4 AgentX on one MI355X node using ATOM DSpark K6. # Both throughput bands use thinking_on golden AL 3.77; eval uses real # acceptance. Keep the historical config key for changelog compatibility. +# C256 and C384 offload KV to an LMCache MP server whose 1000 GB L1 fits the +# 1019 GB budget of dram-utilization 0.34. dsv4-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202609291501 + image: rocm/atom-dev:nightly_202610071700 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:mi355x-amds @@ -956,9 +958,11 @@ dsv4-fp4-mi355x-atom-agentic-mtp: multinode: false scenarios: agentic-coding: - - search-space: + - dram-utilization: 0.34 + search-space: - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128, 256], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, spec-decoding: draft_model, conc-list: [256, 384], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 2334965697..9833118d71 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9538,3 +9538,16 @@ - "Run prefill context parallelism on the TP-only arms at conc 16, 32 and 48; conc 1 and 4 keep BCG." - "Enable SGLANG_OPT_FP8_WO_A_FUSED_INVROPE." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3778 + +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202609291501 to rocm/atom-dev:nightly_202610071700 (digest sha256:bb3912c372dda0c82fc6831afdd76e366a372d5e9fe223373499db63b06cce65)." + - "Move the C48, C64, C96 and C128 DEP points from RCCL expert parallelism to ATOM's MegaMoE backend (all2all-backend high-throughput, moe-backend mega), with EPLB at every point except C64." + - "Cap max-num-seqs at 128 from C128 up (C128 was 256, C256 was 512); C48, C64 and C96 keep admission at 2x CONC." + - "Run C256 and the new C384 on the same MegaMoE DEP setup with ATOM's lmcache_mp CPU KV tier: one LMCache server per node (1000 GB L1, LMCache 0.5.6.dev139+gf22dec28 as shipped in the image), started by srtctl's lmcache-server service, within a 1019 GB budget (dram-utilization 0.34)." + - "C1 to C16 are unchanged." + - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4-Pro-0813 runs as shipped." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3819 From 0f2778481f0848423bed401d3ceafafa89fb2250 Mon Sep 17 00:00:00 2001 From: honglie Date: Mon, 12 Oct 2026 11:14:40 +0800 Subject: [PATCH 11/16] =?UTF-8?q?feat(amd):=20bump=20DSv4.1-Flash=20ATOM?= =?UTF-8?q?=20AgentX=20to=201010=20nightly=20/=20=E5=B0=86=20DSv4.1-Flash?= =?UTF-8?q?=20ATOM=20AgentX=20=E6=9B=B4=E6=96=B0=E8=87=B3=201010=20nightly?= =?UTF-8?q?=20(#3901)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(amd): bump DSv4.1-Flash ATOM AgentX to 1010 nightly Update dsv41flash-fp4-mi355x-atom-agentic-dspark from rocm/atom-dev:nightly_202610070333 to nightly_202610100929 (ATOM 0.1.8.dev14+g73defd9b8). Recipe, points and server settings are unchanged. Co-Authored-By: Claude Opus 5.5 * chore: set perf-changelog pr-link to #3901 Co-Authored-By: Claude Opus 5.5 * feat(amd): bump DSv4.1-Flash ATOM AgentX to 1010 nightly - Update the image from rocm/atom-dev:nightly_202610070333 to nightly_202610100929 (digest sha256:cc60ba06b28050a7fdec71e21f1853c782a9ec8850f031e84ebf0e49f3180974; ATOM 0.1.8.dev14+g73defd9b8, ROCm 7.2.4) and update model.container to match. The recipe, points and server settings are unchanged; upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged between 7fc0926c6 and 73defd9b8. - ROCm/ATOM#2507 turns decoder SWA bounded replay on by default for DeepSeek-V4.1: a prefill runs layers 21..39 on each request's last 133 rows only, the same approximation as SGLang's --enable-decoder-swa-bounded-replay. Upstream reports +10.7% / +22.7% / +38.1% P90 interactivity at TP2 c16 / c32 / c64 on this AgentX workload, with no GSM8K or long-context retrieval change beyond noise. - ROCm/ATOM#2512 samples the DSpark Markov block from each TP rank's vocab shard instead of all-gathering the full logits (bit-identical ids; draft head 657 -> 302 us per decode step at TP2 c32). ROCm/ATOM#2513 packs the FP4 index logits by row visibility and moves BF16 decode to the two-source paged kernel (needs ROCm/aiter#6334). ROCm/ATOM#2495 fixes the FP4 sparse indexer output contract. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: seungrokj Co-authored-by: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com> --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 4 ++-- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 10 ++++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 56b6a87647..3ae0b754ec 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -1,13 +1,13 @@ # DeepSeek-V4.1-Flash AgentX on MI355X with ATOM DSpark (five draft tokens), # following ROCm/ATOM recipes/DeepSeek-V4.1-Flash-Agentic.md at the image's -# revision (7fc0926c698e3f19df9dca23091e399851d3d0ec). TP2 and TP4, no expert +# revision (73defd9b868663be51985fedff06115ef9bcab3d). TP2 and TP4, no expert # parallelism; the KV cache is GPU-resident. base: schema: 2 name: dsv41flash-fp4-mi355x-atom-agentic model: path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: rocm/atom-dev:nightly_202610070333 + container: rocm/atom-dev:nightly_202610100929 precision: fp4 resources: gpu_type: mi355x diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 3e3960355a..b35bc95eed 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1522,7 +1522,7 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: # DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610070333 + image: rocm/atom-dev:nightly_202610100929 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 9833118d71..c861540fcc 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9539,6 +9539,16 @@ - "Enable SGLANG_OPT_FP8_WO_A_FUSED_INVROPE." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3778 +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202610070333 to nightly_202610100929 (digest sha256:cc60ba06b28050a7fdec71e21f1853c782a9ec8850f031e84ebf0e49f3180974; ATOM 0.1.8.dev14+g73defd9b8, ROCm 7.2.4) and update model.container to match. The recipe, points and server settings are unchanged; upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged between 7fc0926c6 and 73defd9b8." + - "ROCm/ATOM#2507 turns decoder SWA bounded replay on by default for DeepSeek-V4.1: a prefill runs layers 21..39 on each request's last 133 rows only, the same approximation as SGLang's --enable-decoder-swa-bounded-replay. Upstream reports +10.7% / +22.7% / +38.1% P90 interactivity at TP2 c16 / c32 / c64 on this AgentX workload, with no GSM8K or long-context retrieval change beyond noise." + - "ROCm/ATOM#2512 samples the DSpark Markov block from each TP rank's vocab shard instead of all-gathering the full logits (bit-identical ids; draft head 657 -> 302 us per decode step at TP2 c32). ROCm/ATOM#2513 packs the FP4 index logits by row visibility and moves BF16 decode to the two-source paged kernel (needs ROCm/aiter#6334). ROCm/ATOM#2495 fixes the FP4 sparse indexer output contract." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3901 + - config-keys: - dsv4-fp4-mi355x-atom-agentic-mtp scenario-type: From 1f76fc1d9e8d4ab1aa3c26ab86656a9ed793a298 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:03:12 -0400 Subject: [PATCH 12/16] =?UTF-8?q?[AMD][MI355X]=20DSv4.1=20Flash:=20enable?= =?UTF-8?q?=20the=20vLLM=20mono=20decode=20kernels=20/=20[AMD][MI355X]=20D?= =?UTF-8?q?Sv4.1=20Flash=EF=BC=9A=E5=90=AF=E7=94=A8=20vLLM=20mono=20decode?= =?UTF-8?q?=20=E5=86=85=E6=A0=B8=EF=BC=88=E9=95=9C=E5=83=8F=E5=BE=85?= =?UTF-8?q?=E5=AE=9A=EF=BC=89=20(#3690)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(dsv41flash): repin MI355X to the 2026-10-06 ROCm 10.0 nightly Repin the MI355X DeepSeek-V4.1-Flash DSpark recipe to nightly-rocm100-21d93d0d and retune the ladder around it. - Leave VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4 unset. The a4w4 opt-in faults during graph capture at TP=2; all nine TP=2 points failed and all nine TP=4 points passed in run 37424787697. vllm-project/vllm#60273 tracks the fix, so the experts stay on the generic AITER CK a8w4 selector. - Enable VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 so tensor-parallel all-reduces above the quick-reduce size threshold use the INT4 codec. - Hold both TP rows at concurrency 64. Run 37510565585 measured TP=2 c128 at 150,683 tok/s/chip against 151,896 at c64 with prompt tokens within 0.5%, so the extra concurrency bought real prefill rather than scored tokens. - Set VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD=1024. Upstream caps the shared-expert overlap at 256 tokens, which predates speculative decoding: five DSpark drafts make a decode step CONC x 6 tokens, so c64 submits 384 and the overlap never engages above c42. 1024 covers the ladder and matches the ATOM default, and stays below the prefill chunk so memory profiling never takes the overlap. - Restore the upstream 16384 prefill chunk at TP2 c64, making the chunk uniform across all 14 points. That point had capped it at 8192 to buy KV room, but the room was taken by a profiling bug rather than by the chunk: the ROCm paged MXFP4 indexer sizes its decode logits workspace as (rows, max-model-len), and with no ambient vLLM config in scope the row count fell back to the batched-token count, so at this recipe's 1M context the reservation scaled linearly with the chunk. vllm-project/vllm#58014 bounds rows by max-num-seqs x (1 + 5 drafts), and the repinned image carries that fix. Rebased onto main: the configuration-procedure guides this branch had edited were removed by #3789, so those edits are dropped. Co-authored-by: Cursor Agent * perf(dsv41flash): enable mono decode on MI355X; image TBD - Set VLLM_ROCM_MONO_DECODE=1 (vllm-project/vllm#60397): two persistent launches a layer for decode steps of up to 48 rows; larger steps take the regular path. - Image TBD until the first ROCm 10.0 nightly with #60397 is published. - Drop the a4w4 notes and shorten the comments and the changelog entry. Co-Authored-By: Claude Opus 5.5 * perf(dsv41flash): drop the shared-expert stream threshold Keep the upstream default for simplicity. Co-Authored-By: Claude Opus 5.5 * perf(dsv41flash): pin the ROCm 10.0 nightly that carries mono decode Replace the TBD placeholder with nightly-rocm100-7d0b4e57, the first ROCm 10.0 nightly built after vllm-project/vllm#60397 merged, so the VLLM_ROCM_MONO_DECODE=1 setting on this recipe has kernels to run. 将 TBD 占位符替换为 nightly-rocm100-7d0b4e57，这是 vllm-project/vllm#60397 合并后构建的首个 ROCm 10.0 nightly，使本 recipe 中的 VLLM_ROCM_MONO_DECODE=1 设置有对应的 kernel 可用。 Co-authored-by: Cursor * docs(dsv41flash): align MI355X vLLM changelog entry with the #3690 diff - Update the image from nightly-rocm100-ac9126e58aa7bbab1856ba6593ba4d5003fea516 to nightly-rocm100-7d0b4e57aac4c4323225b0eee9ae960ce75ffffe, the first ROCm 10.0 nightly with the DeepSeek-V4.1 mono decode kernels of vllm-project/vllm#60397, and update model.container to match. The image also brings vllm-project/vllm#59685, #56638, #60159 and #60389. - Enable the mono decode kernels (VLLM_ROCM_MONO_DECODE=1) at TP2 and TP4: a decode step of up to 48 rows, i.e. concurrency 1-8 with five DSpark drafts, runs each backbone layer as two persistent launches (30 of 40 layers whole, the other 10 the FFN launch only). Larger steps take the regular path, so C16-C64 are expected unchanged. - Enable VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 (vllm-project/recipes#1061) for tensor-parallel all-reduces above the quick-reduce size threshold. - Restore --max-num-batched-tokens 16384 at TP2 c64: vllm-project/vllm#58014 removed the profiling over-reservation the 8192 cap worked around. The prefill chunk is now 16384 at every point and the ladder stays at concurrency 1-64. - No data-type or precision change to the DSpark draft: vllm-project/vllm#60397 skips the draft layers (layer id >= num_hidden_layers), so the bundled DSpark draft of deepseek-ai/DeepSeek-V4.1-Flash runs on vLLM's regular path as shipped. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Cursor Agent Co-authored-by: Claude Opus 5.5 Co-authored-by: seungrokj <144636725+seungrokj@users.noreply.github.com> Co-authored-by: seungrokj --- .../vllm/mi355x-fp4-mtp/agentic.yaml | 29 ++++++++++++------- inferencex-e2e/configs/amd-master.yaml | 14 ++++----- inferencex-e2e/perf-changelog.yaml | 12 ++++++++ 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml index 83a196ec81..12c4d1f43d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml @@ -6,10 +6,13 @@ base: name: dsv41flash-fp4-mi355x-vllm-agentic model: path: hf:deepseek-ai/DeepSeek-V4.1-Flash - # Repinned to a nightly build that includes vllm-project/vllm#58671 - # (paged MXFP4 sparse indexer), #58655 (mHC fused Triton seams) and - # #53492 (sparse MLA Gluon kernel). - container: vllm/vllm-openai-rocm:nightly-rocm100-ac9126e58aa7bbab1856ba6593ba4d5003fea516 + # The first ROCm 10.0 nightly carrying vllm-project/vllm#60397 (DSv4.1 + # mono decode kernels), which the environment below enables. It also + # brings #59685 (AITER MegaMoEV2 for DeepSeek V4), #56638 (shared + # combine_topk_swa_indices kernel in place of the Torch prefill combine), + # #60159 (native merge_attn_states on gfx950) and #60389 (MLA derived + # weights kept in place on weight reload). + container: vllm/vllm-openai-rocm:nightly-rocm100-7d0b4e57aac4c4323225b0eee9ae960ce75ffffe precision: fp4 resources: gpu_type: mi355x @@ -62,6 +65,13 @@ base: env: VLLM_ROCM_USE_AITER: '1' VLLM_ROCM_USE_AITER_MOE: '1' + # Mono decode kernels (vllm-project/vllm#60397): two persistent + # launches a layer for decode steps of up to 48 rows (c8 with five + # drafts); larger steps take the regular path. + VLLM_ROCM_MONO_DECODE: '1' + # INT4 quick all-reduce above its size threshold + # (vllm-project/recipes#1061). + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 AITER_TRITON_LOG_LEVEL: ERROR VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: 'True' # DeepseekV41ForCausalLM is not torch-compiled upstream; breakable @@ -80,11 +90,10 @@ base: WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -# One variant per point (c128 excluded for both TP2 and TP4). Engram tables -# move to pinned host memory at TP2, where resident tables leave too little -# KV. Graph capture covers 2x CONC x (1 + 5 drafts). The prefill chunk shrinks -# at TP2 from c64 to buy KV room, and where it drops below 1024 x 6 slots the -# scheduler is capped at the captured shape. +# One variant per point, concurrency 1 through 64 on both TP arms (c128 and +# c192 excluded). Engram tables move to pinned host memory at TP2, where +# resident tables leave too little KV. Graph capture covers 2x CONC x +# (1 + 5 drafts). The prefill chunk is the upstream 16384 everywhere. override_tp4_c1: roles: agg: @@ -262,7 +271,7 @@ override_tp2_c64: tensor-parallel-size: 2 engram-config: '{"cpu_offload":true}' max-cudagraph-capture-size: 1024 - max-num-batched-tokens: 8192 + max-num-batched-tokens: 16384 benchmark: env: CONC: '64' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index b35bc95eed..47eefeef43 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1498,9 +1498,9 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: # Two GPUs per server doubles the servers per node and is the layout that # decides whether DSv4.1-Flash is throughput- or interactivity-bound here. dsv41flash-fp4-mi355x-vllm-agentic-dspark: - # Repinned to a nightly build that includes vllm-project/vllm#58671, - # #58655 and #53492 (see agentic.yaml). - image: vllm/vllm-openai-rocm:nightly-rocm100-ac9126e58aa7bbab1856ba6593ba4d5003fea516 + # The first ROCm 10.0 nightly carrying vllm-project/vllm#60397 (see + # agentic.yaml). + image: vllm/vllm-openai-rocm:nightly-rocm100-7d0b4e57aac4c4323225b0eee9ae960ce75ffffe model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds @@ -1511,11 +1511,9 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: agentic-coding: - dram-utilization: 0.60 search-space: - # The recipe only changes the memory split where a point ran short of KV: - # the prefill chunk shrinks at TP=2 c64. c128 is excluded for both TP=2 - # and TP=4. Every point is measured here rather than combined from the - # earlier run: the image move leaves concurrency 1-32 measured only on - # the superseded nightly-eed1f3d0 pin. + # Concurrency 1-64 on both TP arms, every point re-measured on the new + # image. c128 and c192 are excluded: c128 did not raise the scored + # metric at TP=2 (150,683 against 151,896 tok/s/chip at c64). - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c861540fcc..c6ab19cc8c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9561,3 +9561,15 @@ - "C1 to C16 are unchanged." - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4-Pro-0813 runs as shipped." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3819 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from nightly-rocm100-ac9126e58aa7bbab1856ba6593ba4d5003fea516 to nightly-rocm100-7d0b4e57aac4c4323225b0eee9ae960ce75ffffe, the first ROCm 10.0 nightly with the DeepSeek-V4.1 mono decode kernels of vllm-project/vllm#60397, and update model.container to match. The image also brings vllm-project/vllm#59685, #56638, #60159 and #60389." + - "Enable the mono decode kernels (VLLM_ROCM_MONO_DECODE=1) at TP2 and TP4: a decode step of up to 48 rows, i.e. concurrency 1-8 with five DSpark drafts, runs each backbone layer as two persistent launches (30 of 40 layers whole, the other 10 the FFN launch only). Larger steps take the regular path, so C16-C64 are expected unchanged." + - "Enable VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 (vllm-project/recipes#1061) for tensor-parallel all-reduces above the quick-reduce size threshold." + - "Restore --max-num-batched-tokens 16384 at TP2 c64: vllm-project/vllm#58014 removed the profiling over-reservation the 8192 cap worked around. The prefill chunk is now 16384 at every point and the ladder stays at concurrency 1-64." + - "No data-type or precision change to the DSpark draft: vllm-project/vllm#60397 skips the draft layers (layer id >= num_hidden_layers), so the bundled DSpark draft of deepseek-ai/DeepSeek-V4.1-Flash runs on vLLM's regular path as shipped." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3690 From 4f00011e81e6d9eb9bda5c0f6870b58d2254f40b Mon Sep 17 00:00:00 2001 From: seungrokj <144636725+seungrokj@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:08:30 -0700 Subject: [PATCH 13/16] =?UTF-8?q?[AMD][MI355X]=20Kimi-K3=20vLLM=20AgentX?= =?UTF-8?q?=20nightly=2081198e97=20with=20dcp8=20c1-c48=20sweep=20/=20[AMD?= =?UTF-8?q?][MI355X]=20Kimi-K3=20vLLM=20AgentX=20=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E8=87=B3=20nightly=2081198e97=20=E5=B9=B6?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=20dcp8=20c1-c48=20=E6=89=AB=E6=8F=8F=20(#383?= =?UTF-8?q?3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(amd): retune Kimi-K3 MI355X vLLM AgentX sweep to dcp8 c10-c16 and c44-c50 Move the mid band to dcp8 (c10/c12/c14/c16) and the throughput band to c44/c48/c50 with a 4-token DSpark draft, dropping the unused c8 and c70 variants. CUDA graphs capture every even size up to 2x CONC x (1 + drafts). Add stream-interval 10 and VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm to the base recipe, and FP8 per-channel online quantization of linears to c1. 将中段改为 dcp8（c10/c12/c14/c16），吞吐段改为 c44/c48/c50 并使用 4 token DSpark 草稿，移除不再使用的 c8 与 c70 变体。CUDA graph 捕获不超过 2x CONC x (1 + drafts) 的所有偶数尺寸。基础配方新增 stream-interval 10 与 VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm，c1 新增线性层 FP8 per-channel 在线量化。 Co-Authored-By: Claude Opus 5.5 * chore(amd): narrow Kimi-K3 MI355X vLLM AgentX sweep to c1, dcp8 c10 and dcp8 c48 Comment out the full band lists in amd-master.yaml and run one point per band: c1 (dcp1, GPU-resident), c10 and c48 (dcp8, DRAM offload). 在 amd-master.yaml 中注释掉完整的分段列表，每段只运行一个点：c1（dcp1， GPU 常驻）、c10 与 c48（dcp8，DRAM offload）。 Co-Authored-By: Claude Opus 5.5 * chore(amd): run only Kimi-K3 MI355X vLLM c1 with dcp8 Add override_tp8_c1_dcp8, the c1 variant with decode-context-parallel-size 8 over a2a, and narrow the amd-master.yaml sweep to that single c1 dcp8 point. 新增 override_tp8_c1_dcp8：在 c1 变体基础上使用 a2a 的 decode-context-parallel-size 8，并将 amd-master.yaml 扫描缩小为该 c1 dcp8 单点。 Co-Authored-By: Claude Opus 5.5 * chore(amd): run only Kimi-K3 MI355X vLLM dcp8 c48 Re-enable the dcp8 DRAM-offload c48 point and disable c1 dcp8 to reproduce the c48 NCCL watchdog hang. 仅运行 Kimi-K3 MI355X vLLM dcp8 c48：重新启用 dcp8 DRAM 卸载 c48 配置点， 禁用 c1 dcp8，以复现 c48 的 NCCL watchdog 挂起问题。 Co-Authored-By: Claude Opus 5.5 * perf(amd): Kimi-K3 MI355X vLLM AgentX on nightly 81198e97 with dcp8 c1-c48 Move kimik3-fp4-mi355x-vllm-agentic-mtp and its recipe to the nightly-rocm100-81198e97 image and sweep dcp1 c1, dcp8 c1 (GPU-resident) and dcp8 DRAM-offload c4, c10-c16, c44, c48. Add override_tp8_c4_dcp8_simple and raise the c14/c16 DSpark draft from 3 to 4 tokens, with graph capture sized to 2 x CONC x (1 + drafts). Co-Authored-By: Claude Opus 5.5 * perf(amd): disable breakable CUDA graphs for Kimi-K3 MI355X vLLM c1/c4 Set VLLM_USE_BREAKABLE_CUDAGRAPH to 0 in override_tp8_c1, override_tp8_c1_dcp8, override_tp8_c4 and override_tp8_c4_dcp8_simple. Co-Authored-By: Claude Opus 5.5 * perf(amd): use FULL_AND_PIECEWISE on every Kimi-K3 MI355X vLLM point Switch c10/c12/c14/c16 from FULL to FULL_AND_PIECEWISE so all overrides request the same cudagraph mode, refresh the recipe comment, and add the perf-changelog entry for #3833. Co-Authored-By: Claude Opus 5.5 * docs(amd): align Kimi-K3 MI355X vLLM changelog entry with the #3833 diff Correct the kimik3-fp4-mi355x-vllm-agentic-mtp perf-changelog entry: - Every point requests FULL_AND_PIECEWISE (the mid band was FULL) with breakable CUDA graphs off; without a torch.compile submodule vLLM overrides it to FULL_DECODE_ONLY, as the c1 and c14 server logs show. - Golden (synthetic) acceptance applies at every draft point in agentic runs, not only the throughput band; note the c14 draft length 3 -> 4. Co-Authored-By: Claude Opus 5.5 * perf(amd): Kimi-K3 MI355X vLLM AgentX on nightly 81198e97 with dcp8 c1-c48 - Update the image from vllm/vllm-openai-rocm:nightly-rocm100-36768d1bfd39094681cdbc8cb37d4b31c0729c89 to vllm/vllm-openai-rocm:nightly-rocm100-81198e97ba7eee2a22540caaa756b7fdddcb4d93. - Sweep dcp1 concurrency 1 and dcp8 concurrency 1 GPU-resident, and dcp8 concurrency 4, 10, 12, 14, 16, 44 and 48 with the SimpleCPUOffloadConnector DRAM tier (224.875 GB/rank). This replaces dcp1 concurrency 1/4, dcp1 DRAM-offload concurrency 8-14 and dcp8 concurrency 44/48/70. The dcp8 points use the a2a DCP backend. - DSpark draft length per point: 7 at concurrency 1, 5 at 4, 4 at 10, 3 at 12, and 4 at 14, 16, 44 and 48 (concurrency 14 was 3). The throughput band (44, 48) previously had no speculative-config and now runs the 4-token DSpark draft. As at every draft point, agentic runs replace block rejection with the golden acceptance length for that draft length. - Admission is 2x concurrency. CUDA graphs capture every even size up to 2x concurrency x (1 + draft tokens), replacing the previous every-size lists (up to 8192 in the throughput band). - Every point requests cudagraph_mode FULL_AND_PIECEWISE (the mid band was FULL) with breakable CUDA graphs disabled (VLLM_USE_BREAKABLE_CUDAGRAPH=0, previously 1 at concurrency 1 and 4). The AMD Kimi-K3 model has no torch.compile submodule, so vLLM overrides the mode to FULL_DECODE_ONLY at load, as the concurrency 1 and 14 server logs show. - Add stream-interval 10 and VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm, which routes DCP multi-token verify through the AITER ASM round-robin decode, to every point. - Concurrency 1 (dcp1 and dcp8) adds quantization-config linear fp8_per_channel: vLLM quantizes the target's linear layers to FP8 W8A8 at load, except kv_b_proj, in_proj_qkvgfab, f_b_proj, fused_qkv_a_proj, self_attn.g_proj and layer 0 mlp.down_proj, which stay BF16. The routed experts stay MXFP4. - No change to the Inferact/Kimi-K3-DSpark draft's precision: its model config takes the speculative config's own quantization (unset) and never receives quantization-config, and get_draft_quant_config() returns None for K3DSparkModel, so its weights stay BF16 and its KV cache stays fp8. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../kimik3/vllm/mi355x-fp4-mtp/agentic.yaml | 138 +++++++++++++----- inferencex-e2e/configs/amd-master.yaml | 15 +- inferencex-e2e/perf-changelog.yaml | 15 ++ 3 files changed, 123 insertions(+), 45 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml index a1690473a7..e7d2c47351 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml @@ -1,17 +1,17 @@ # Kimi-K3 MXFP4 AgentX on MI355X with vLLM DSpark # (https://recipes.vllm.ai/moonshotai/Kimi-K3). TP8 only: the 1.56 TB checkpoint -# is ~195 GB per GPU. The KV cache is GPU-resident through c4 and backed by -# vLLM's SimpleCPUOffloadConnector (dram) from c8. Three bands, driven by -# kimik3-fp4-mi355x-vllm-agentic-mtp: interactive dcp1 c1/c4 (GPU-resident), -# mid dcp1 c8-c14 (DSpark draft + DRAM offload), throughput dcp8 c44/c48/c70 -# (KV read sharded across all 8 GPUs, DRAM offload, mtp/no draft with synthetic -# acceptance). The 1799 GB DRAM budget splits evenly across the eight ranks. +# is ~195 GB per GPU. The KV cache is GPU-resident at c1 and backed by +# vLLM's SimpleCPUOffloadConnector (dram) from dcp8 c4. Three bands, driven by +# kimik3-fp4-mi355x-vllm-agentic-mtp: interactive dcp1/dcp8 c1 (GPU-resident), +# mid dcp8 c4-c16 and throughput dcp8 c44/c48/c50 (KV read sharded across all +# 8 GPUs, DRAM offload, DSpark draft with synthetic acceptance). The 1799 GB +# DRAM budget splits evenly across the eight ranks. base: schema: 2 name: kimik3-fp4-mi355x-vllm-agentic model: path: hf:moonshotai/Kimi-K3 - container: vllm/vllm-openai-rocm:nightly-rocm100-36768d1bfd39094681cdbc8cb37d4b31c0729c89 + container: vllm/vllm-openai-rocm:nightly-rocm100-81198e97ba7eee2a22540caaa756b7fdddcb4d93 precision: fp4 resources: gpu_type: mi355x @@ -50,6 +50,7 @@ base: enable-prefix-caching: true kv-cache-dtype: fp8 attention-config: '{"mla_prefill_backend":"ROCM_AITER_FA"}' + stream-interval: 10 env: # Upstream AMD recipe environment. VLLM_ROCM_AITER_MLA_ASM_PADDING: asm @@ -68,6 +69,8 @@ base: VLLM_USE_DIRECT_DCP_A2A: '0' VLLM_USE_DIRECT_DCP_Q_GATHER: '0' VLLM_USE_DIRECT_DCP_KV_GATHER: '0' + # Route DCP multi-token verify through the AITER ASM round-robin decode. + VLLM_ROCM_AITER_MLA_DCP_VERIFY: asm benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh @@ -78,11 +81,13 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. The DSpark draft length descends with concurrency -# (7, 5, 4, 4, 3, 3); throughput runs replace block rejection with the golden -# acceptance length for that draft length. Admission is 2x CONC and graph -# capture covers every size up to 2x CONC x (1 + drafts). Breakable -# FULL_AND_PIECEWISE graphs cost KV pool, so only c1 and c4 use them. The -# offload points split the 1799 GB DRAM budget evenly across the eight ranks; +# (7, 5, 4, 3, 4, 4) and stays at 4 for the throughput band; agentic runs +# replace block rejection with the golden acceptance length for that draft +# length. Admission is 2x CONC and graph +# capture covers every even size up to 2x CONC x (1 + drafts). Every point +# requests FULL_AND_PIECEWISE; the AMD K3 model has no torch.compile +# submodule, so without breakable graphs vLLM falls back to FULL_DECODE_ONLY. +# The offload points split the 1799 GB DRAM budget evenly across the eight ranks; # identical prefixes must hash to identical block keys on every rank. override_tp8_c1: roles: @@ -90,10 +95,30 @@ override_tp8_c1: args: max-num-seqs: 2 max-num-batched-tokens: 16384 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16]}' speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + quantization-config: '{"linear":"fp8_per_channel","ignore":["*kv_b_proj","*vision_tower*","*mm_projector*","*in_proj_qkvgfab","*f_b_proj","*fused_qkv_a_proj","*self_attn.g_proj","*.layers.0.mlp.down_proj"]}' env: - VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +# c1 with dcp8: the KV read sharded across all 8 GPUs, still GPU-resident. +override_tp8_c1_dcp8: + roles: + agg: + args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-seqs: 2 + max-num-batched-tokens: 16384 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + quantization-config: '{"linear":"fp8_per_channel","ignore":["*kv_b_proj","*vision_tower*","*mm_projector*","*in_proj_qkvgfab","*f_b_proj","*fused_qkv_a_proj","*self_attn.g_proj","*.layers.0.mlp.down_proj"]}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' benchmark: env: CONC: '1' @@ -105,40 +130,48 @@ override_tp8_c4: args: max-num-seqs: 8 max-num-batched-tokens: 8192 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":48,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48]}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":48,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48]}' speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":5,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' env: - VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' benchmark: env: CONC: '4' KV_OFFLOADING: 'none' -override_tp8_c8_simple: +# c4 with dcp8: the KV read sharded across all 8 GPUs, backed by the DRAM tier. +override_tp8_c4_dcp8_simple: roles: agg: args: - max-num-seqs: 16 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-seqs: 8 max-num-batched-tokens: 8192 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":80,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80]}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":48,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":5,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' PYTHONHASHSEED: '42' benchmark: env: - CONC: '8' + CONC: '4' KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' -override_tp8_c10_simple: +# Mid band: dcp-size 8 shards the KV read across all 8 GPUs, backed by +# vLLM's SimpleCPUOffloadConnector DRAM tier, with the DSpark draft. +override_tp8_c10_dcp8_simple: roles: agg: args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a max-num-seqs: 20 max-num-batched-tokens: 8192 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":100,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100]}' + gpu-memory-utilization: 0.9 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":100,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100]}' speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: @@ -150,13 +183,16 @@ override_tp8_c10_simple: KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' -override_tp8_c12_simple: +override_tp8_c12_dcp8_simple: roles: agg: args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a max-num-seqs: 24 max-num-batched-tokens: 8192 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":96,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96]}' + gpu-memory-utilization: 0.9 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":96,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96]}' speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: @@ -168,14 +204,17 @@ override_tp8_c12_simple: KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' -override_tp8_c14_simple: +override_tp8_c14_dcp8_simple: roles: agg: args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a max-num-seqs: 28 max-num-batched-tokens: 8192 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":112,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112]}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + gpu-memory-utilization: 0.9 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":140,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' @@ -186,10 +225,30 @@ override_tp8_c14_simple: KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' -# Throughput band: dcp-size 8 shards the KV read across all 8 GPUs for the -# 100k+ token contexts, backed by the same SimpleCPUOffloadConnector DRAM tier. -# No draft model here (the a2a DCP path); spec-decoding stays mtp so the golden -# acceptance length drives throughput. Admission is 2x CONC. +override_tp8_c16_dcp8_simple: + roles: + agg: + args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-seqs: 32 + max-num-batched-tokens: 8192 + gpu-memory-utilization: 0.9 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":160,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +# Throughput band: same dcp8 DRAM-offload layout with a 4-token DSpark draft. +# Synthetic acceptance replaces block rejection with the golden acceptance +# length. FULL_AND_PIECEWISE graphs cover every even size up to 2x CONC x 5. override_tp8_c44_dcp8_simple: roles: agg: @@ -199,7 +258,8 @@ override_tp8_c44_dcp8_simple: max-num-seqs: 88 max-num-batched-tokens: 8192 gpu-memory-utilization: 0.9 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,128,256,512,1024,2048,4096,8192]}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":440,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164,166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200,202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236,238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272,274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308,310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344,346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380,382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416,418,420,422,424,426,428,430,432,434,436,438,440]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' @@ -220,7 +280,8 @@ override_tp8_c48_dcp8_simple: max-num-seqs: 96 max-num-batched-tokens: 8192 gpu-memory-utilization: 0.9 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,128,256,512,1024,2048,4096,8192]}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":480,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164,166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200,202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236,238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272,274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308,310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344,346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380,382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416,418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452,454,456,458,460,462,464,466,468,470,472,474,476,478,480]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' @@ -232,23 +293,24 @@ override_tp8_c48_dcp8_simple: TOTAL_CPU_DRAM_GB: '1799' SPEC_DECODING: 'mtp' -override_tp8_c70_dcp8_simple: +override_tp8_c50_dcp8_simple: roles: agg: args: decode-context-parallel-size: 8 dcp-comm-backend: a2a - max-num-seqs: 140 + max-num-seqs: 100 max-num-batched-tokens: 8192 gpu-memory-utilization: 0.9 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,256,512,1024,2048,4096,8192]}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":500,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164,166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200,202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236,238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272,274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308,310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344,346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380,382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416,418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452,454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488,490,492,494,496,498,500]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' PYTHONHASHSEED: '42' benchmark: env: - CONC: '70' + CONC: '50' KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' SPEC_DECODING: 'mtp' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 47eefeef43..39f858edf1 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -716,13 +716,13 @@ kimik3-fp4-mi355x-atom-agentic-mtp: # decoding. Companion to kimik3-fp4-mi355x-atom-agentic-mtp: same checkpoint and # runner. TP8 only -- the 1.56 TB MXFP4 checkpoint is ~195 GB/GPU. Three bands: # interactive (1, 4) GPU-resident, dcp 1, DSpark draft. -# mid (8-14) dcp 1, DSpark draft, SimpleCPUOffload DRAM tier. -# throughput (44-70) dcp 8 shards the KV read across all 8 GPUs, DRAM -# offload, mtp with synthetic acceptance (no draft). +# mid (10-16) dcp 8, DSpark draft, SimpleCPUOffload DRAM tier. +# throughput (44-50) dcp 8 shards the KV read across all 8 GPUs, DRAM +# offload, 4-token DSpark draft with synthetic acceptance. # dram-utilization 0.6 -> TOTAL_CPU_DRAM_GB 1799 at TP8 (node cap 2861022 MiB), # split evenly across the eight ranks by the launcher. kimik3-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-rocm100-36768d1bfd39094681cdbc8cb37d4b31c0729c89 + image: vllm/vllm-openai-rocm:nightly-rocm100-81198e97ba7eee2a22540caaa756b7fdddcb4d93 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds @@ -733,9 +733,10 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.6 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 10, 12, 14], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48, 70], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: none, conc-list: [1], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 10, 12, 14, 16], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } minimaxm3-fp4-mi355x-atom-agentic-mtp: image: rocm/atom-dev:nightly_202610070333 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c6ab19cc8c..24666cb3c2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9573,3 +9573,18 @@ - "Restore --max-num-batched-tokens 16384 at TP2 c64: vllm-project/vllm#58014 removed the profiling over-reservation the 8192 cap worked around. The prefill chunk is now 16384 at every point and the ladder stays at concurrency 1-64." - "No data-type or precision change to the DSpark draft: vllm-project/vllm#60397 skips the draft layers (layer id >= num_hidden_layers), so the bundled DSpark draft of deepseek-ai/DeepSeek-V4.1-Flash runs on vLLM's regular path as shipped." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3690 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image from vllm/vllm-openai-rocm:nightly-rocm100-36768d1bfd39094681cdbc8cb37d4b31c0729c89 to vllm/vllm-openai-rocm:nightly-rocm100-81198e97ba7eee2a22540caaa756b7fdddcb4d93." + - "Sweep dcp1 concurrency 1 and dcp8 concurrency 1 GPU-resident, and dcp8 concurrency 4, 10, 12, 14, 16, 44 and 48 with the SimpleCPUOffloadConnector DRAM tier (224.875 GB/rank). This replaces dcp1 concurrency 1/4, dcp1 DRAM-offload concurrency 8-14 and dcp8 concurrency 44/48/70. The dcp8 points use the a2a DCP backend." + - "DSpark draft length per point: 7 at concurrency 1, 5 at 4, 4 at 10, 3 at 12, and 4 at 14, 16, 44 and 48 (concurrency 14 was 3). The throughput band (44, 48) previously had no speculative-config and now runs the 4-token DSpark draft. As at every draft point, agentic runs replace block rejection with the golden acceptance length for that draft length." + - "Admission is 2x concurrency. CUDA graphs capture every even size up to 2x concurrency x (1 + draft tokens), replacing the previous every-size lists (up to 8192 in the throughput band)." + - "Every point requests cudagraph_mode FULL_AND_PIECEWISE (the mid band was FULL) with breakable CUDA graphs disabled (VLLM_USE_BREAKABLE_CUDAGRAPH=0, previously 1 at concurrency 1 and 4). The AMD Kimi-K3 model has no torch.compile submodule, so vLLM overrides the mode to FULL_DECODE_ONLY at load, as the concurrency 1 and 14 server logs show." + - "Add stream-interval 10 and VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm, which routes DCP multi-token verify through the AITER ASM round-robin decode, to every point." + - "Concurrency 1 (dcp1 and dcp8) adds quantization-config linear fp8_per_channel: vLLM quantizes the target's linear layers to FP8 W8A8 at load, except kv_b_proj, in_proj_qkvgfab, f_b_proj, fused_qkv_a_proj, self_attn.g_proj and layer 0 mlp.down_proj, which stay BF16. The routed experts stay MXFP4." + - "No change to the Inferact/Kimi-K3-DSpark draft's precision: its model config takes the speculative config's own quantization (unset) and never receives quantization-config, and get_draft_quant_config() returns None for K3DSparkModel, so its weights stay BF16 and its KV cache stays fp8." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3833 From 32c378de9ac7fe780a132023dcb119f472fd7874 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Mon, 12 Oct 2026 02:22:10 -0700 Subject: [PATCH 14/16] =?UTF-8?q?[NVIDIA]=20GLM-5.2=20NVFP4=20B300=20SGLan?= =?UTF-8?q?g=20AgentX:=20tune=20EAGLE=20speculative=20steps=20/=20[NVIDIA]?= =?UTF-8?q?=20GLM-5.2=20NVFP4=20B300=20SGLang=20AgentX=EF=BC=9A=E8=B0=83?= =?UTF-8?q?=E4=BC=98=20EAGLE=20=E6=8E=A8=E6=B5=8B=E6=AD=A5=E6=95=B0=20(#38?= =?UTF-8?q?90)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [NVIDIA] GLM-5.2 NVFP4 B300 SGLang AgentX: tune EAGLE speculative steps / [NVIDIA] GLM-5.2 NVFP4 B300 SGLang AgentX：调优 EAGLE 推测步数 * Add PR link to perf changelog * Restore unrelated tilert pr-link placeholder in perf changelog Co-Authored-By: Claude Sonnet 5.5 --------- Co-authored-by: Claude Sonnet 5.5 Co-authored-by: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com> --- .../glm5.2/sglang/b300-fp4-mtp/agentic.yaml | 11 ++++++++++- inferencex-e2e/configs/nvidia-master.yaml | 8 ++++---- inferencex-e2e/perf-changelog.yaml | 9 +++++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml index e32b72696a..621dd5438a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml @@ -69,13 +69,16 @@ base: TOTAL_CPU_DRAM_GB: '2399' # One variant per point: admission is 2x CONC and the graph batch matches it, -# capped at 64. +# capped at 64. Use K=5 at c1/c4/c8/c20; retain K=3 elsewhere. The connector +# selects the committed golden acceptance length for each variant. override_c1: roles: agg: args: max-running-requests: 2 cuda-graph-max-bs: 2 + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 benchmark: env: CONC: '1' @@ -86,6 +89,8 @@ override_c4: args: max-running-requests: 8 cuda-graph-max-bs: 8 + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 benchmark: env: CONC: '4' @@ -96,6 +101,8 @@ override_c8: args: max-running-requests: 16 cuda-graph-max-bs: 16 + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 benchmark: env: CONC: '8' @@ -126,6 +133,8 @@ override_c20: args: max-running-requests: 40 cuda-graph-max-bs: 40 + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 benchmark: env: CONC: '20' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 12c4cc160e..1cb191a29a 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -7332,10 +7332,10 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg: # GLM-5.2 B300 NVFP4 AgentX with EAGLE/MTP speculative decoding, following the # AgentX speculative-decoding policy in docs/MODELS.md. SGLang EAGLE runs off -# GLM-5.2's built-in nextn head (num-steps 3, -# eagle-topk 1, 4 draft tokens = 3 speculative tokens), with acceptance pinned -# to the golden AL 2.99 (infx/golden_al_distribution/glm5.2_mtp.yaml, thinking_on, -# K=3) through SGLANG_SIMULATE_ACC_*. The pinned nightly includes FlashInfer +# GLM-5.2's built-in nextn head (eagle-topk 1): K=5 / 6 draft tokens at +# c1/c4/c8/c20, K=3 / 4 draft tokens elsewhere. The connector selects golden +# AL 3.61 or 2.99 respectively from infx/golden_al_distribution/glm5.2_mtp.yaml +# for thinking_on; evals retain real verification. The pinned nightly includes FlashInfer # 0.6.18's BF16 TRTLLM MoE allocation fix for small-batch Blackwell execution # in GLM-5.2's unquantized EAGLE draft head. # diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 24666cb3c2..602fc4a4e7 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9529,6 +9529,15 @@ - "TP2 c128: admit all 128 sessions (max-running-requests and cuda-graph-max-bs-decode 64 -> 128) with a 4-step prefill-decode interval and mem-fraction-static 0.85. The PR #3421 sweep measured 18,631 tok/s/GPU at 130.9 tok/s/user median at this point; locally the new settings gave 147,763 tok/s/GPU at 37.9 tok/s/user median with a 51 s P90 TTFT (measured before the fold was added)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3782 +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use five EAGLE speculative steps and six draft tokens at concurrency 1, 4, 8, and 20 on GLM-5.2 NVFP4 B300 SGLang." + - "Keep three speculative steps and four draft tokens at the other concurrencies." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3890 + - config-keys: - dsv4-fp4-mi355x-sglang-agentic-mtp scenario-type: From 20c8fba9eabd41f7e0b49aada0d1c147a180e7fc Mon Sep 17 00:00:00 2001 From: weireweire Date: Mon, 12 Oct 2026 19:48:30 +0800 Subject: [PATCH 15/16] =?UTF-8?q?[NV]=20Move=20GB200=20DeepSeek-V4-Pro=20A?= =?UTF-8?q?gentX=20disagg=20to=20the=20Mooncake=20external=20linker=20/=20?= =?UTF-8?q?=E5=B0=86=20GB200=20DeepSeek-V4-Pro=20AgentX=20=E5=88=86?= =?UTF-8?q?=E7=A6=BB=E5=BC=8F=E9=85=8D=E7=BD=AE=E5=88=87=E6=8D=A2=E5=88=B0?= =?UTF-8?q?=20Mooncake=20external=20linker=20(#3701)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(agentx): GB200 DSV4 SGLang disagg KV offload via Mooncake unified-cache external linker * config(agentx): record Mooncake as the GB200 DSV4 SGLang disagg KV offload backend * docs(changelog): GB200 DSV4 SGLang AgentX Mooncake KV offload * docs(changelog): link #3701 * perf(agentx): 8k-token prefill chunks per rank on the GB200 DSV4 DEP16 prefill recipes Raise chunked-prefill-size from 65536 to 131072 on the c256/c768/c1024/c1280 DEP16 prefill workers (4k to 8k tokens per DP rank) and lower prefill mem-fraction-static from 0.85 to 0.80; 8k chunks run out of GPU memory at 0.85. * docs(changelog): GB200 DSV4 agg key, DEP16 prefill chunks; English-only entry List dsv4-fp4-gb200-dynamo-sglang-agentic-agg so the sweep reruns the low-latency TP8 c1/c4 points, describe the 8k-token DEP16 prefill chunks, and drop the Chinese bullets now that new changelog entries must be English-only. * Revert "perf(agentx): 8k-token prefill chunks per rank on the GB200 DSV4 DEP16 prefill recipes" This reverts commit 28b5f47ee758e4b316690dafe7c8b449289d8a7c. * docs(changelog): drop the reverted DEP16 prefill chunk change from the GB200 DSV4 entry --------- Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com> Co-authored-by: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com> --- ...0-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml | 79 ++++++++++++++++-- ...00-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml | 81 +++++++++++++++++-- ...200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml | 81 +++++++++++++++++-- ...-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml | 79 ++++++++++++++++-- ...-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml | 79 ++++++++++++++++-- ...0-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml | 79 ++++++++++++++++-- inferencex-e2e/configs/nvidia-master.yaml | 12 +-- inferencex-e2e/perf-changelog.yaml | 14 ++++ 8 files changed, 466 insertions(+), 38 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml index 9a2b0265a1..541934f0af 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml index 9461990907..a44d935fda 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml @@ -2,7 +2,7 @@ schema: 2 name: "disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload" # GB200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + HiCache KV offload), tuned for +# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + Mooncake unified-cache KV offload), tuned for # concurrency 128. # The DEP8 prefill worker spans two nodes; the DEP16 decode worker spans four. @@ -28,6 +28,11 @@ health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -42,6 +47,54 @@ services: node: dedicated options: max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 + # Decode ranks use a chunk cache, so their host DRAM only joins the store + # through standalone stores: four per decode node, so segments stay uniform + # with prefill's per-rank ones instead of one oversized segment per node. + - &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -95,6 +148,18 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -130,11 +195,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' decode: nodes: 4 @@ -169,6 +232,8 @@ roles: SGLANG_LOG_FORWARD_ITERS: '1' SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -215,6 +280,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml index d3bcb51688..f0181c16ad 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml @@ -2,7 +2,7 @@ schema: 2 name: "disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload" # GB200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + HiCache KV offload), tuned for +# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + Mooncake unified-cache KV offload), tuned for # concurrency 64. # The DEP8 prefill worker spans two nodes; the DEP16 decode worker spans four. @@ -28,6 +28,11 @@ health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -42,6 +47,54 @@ services: node: dedicated options: max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 + # Decode ranks use a chunk cache, so their host DRAM only joins the store + # through standalone stores: four per decode node, so segments stay uniform + # with prefill's per-rank ones instead of one oversized segment per node. + - &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -95,6 +148,18 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -130,11 +195,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' decode: nodes: 4 @@ -169,6 +232,8 @@ roles: SGLANG_LOG_FORWARD_ITERS: '1' SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -215,6 +280,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml index 5a21654eb0..f1672f9e54 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml index c597addbe6..ac2ed56b08 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml index 4b7b7ff7b0..ff4cd0a15f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 1cb191a29a..c32f89b47b 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6858,7 +6858,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [64] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 1 tp: 8 @@ -6874,7 +6874,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [128] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 1 tp: 8 @@ -6890,7 +6890,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [256] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 1 tp: 16 @@ -6906,7 +6906,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [768] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 2 tp: 16 @@ -6922,7 +6922,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [1024] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 2 tp: 16 @@ -6938,7 +6938,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [1280] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 2 tp: 16 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 602fc4a4e7..058145a6d1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9392,6 +9392,20 @@ - "Run TP4 at concurrency 1, 4, 8 and 32 and replace TP2 with DEP2 (TP1 x DP2 + EP2) at concurrency 8-32 and DEP4 (TP1 x DP4 + EP4) at concurrency 64-128, both with DeepGEMM MegaMoE and adaptive verification behind a consistent-hash vLLM Router. All points set gpu-memory-utilization 0.97." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3692 +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang-agentic-agg + - dsv4-fp4-gb200-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Move the GB200 DeepSeek-V4-Pro AgentX disaggregated recipes from hierarchical cache to the Mooncake unified-cache external linker with group semantics, as #3187 did for GB300. MoE settings are unchanged: the recipes keep MegaMoE without W4A4 MXFP4." + - "Decode nodes lend host DRAM to the store through srt-slurm mooncake-store services: four 180GB standalone stores per decode node, gated on their readiness ports with a 600-second timeout. Prefill keeps a 140GB segment per rank. GB200 nodes have the same host memory as GB300 nodes, so the sizes carry over." + - "The prefill role and the decode stores name their RDMA devices (MOONCAKE_DEVICE=mlx5_0..3, as the GB200 vLLM Mooncake recipes do) and set MC_TCP_BIND_ADDRESS to their node, so each store transfer engine advertises the node address instead of the first active interface." + - "The benchmark client runs on the dedicated etcd/nats node, because prefill nodes now also hold Mooncake segments, and Tachometer's per-node exporters are off for this throughput ladder." + - "The aggregate TP8 c1/c4 recipes are unchanged and stay on hierarchical cache; their config key is listed so this sweep covers the whole GB200 Pareto ladder, including the low-latency points." + - "Measured at concurrency 128 (1P DEP8 / 1D DEP16) against the current recipe with the same image, srt-slurm and golden acceptance length: 26,566 vs 23,763 total tok/s/GPU (+11.8%) at the same P90 TPOT (7.31 vs 7.22 ms), with TTFT p50 3.27 vs 4.95 s." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3701 + - config-keys: - "*" description: From 05dbdc61666236d19089070f4648b21c72e5e90d Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 12 Oct 2026 17:43:11 +0200 Subject: [PATCH 16/16] fix(srt-slurm): restore adaptive DSpark verification for llm-d AgentX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match Sep 28 speculative policy on GB200 llm-d recipes (adaptive disagg/DEP8, golden AL only on agg TP8) and skip golden injection when adaptive is enabled. 在 GB200 llm-d AgentX 配方中恢复 9 月 28 日的投机策略（disagg/DEP8 自适应验证、 仅 agg TP8 使用黄金 AL），并在启用自适应时跳过黄金接受长度注入。 --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 2 +- .../vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 8 +-- .../infx/srt_slurm/synthetic_acceptance.py | 11 +++- .../srt_slurm/test_synthetic_acceptance.py | 55 +++++++++++++++++++ inferencex-e2e/perf-changelog.yaml | 9 +++ 5 files changed, 79 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml index 150cb3f00e..c74530894e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -124,7 +124,7 @@ base: no-enable-flashinfer-autotune: true numa-bind: true compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - speculative-config: '{"method":"dspark","num_speculative_tokens":3,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"}' + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"enable_adaptive_verification":true,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"}' attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml index f007fcc066..43fed8d374 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -3,8 +3,8 @@ # DEP8 or DEP16 decode worker. KV moves over NIXL and the prefill workers share prefix # cache through an embedded Mooncake store. srt-slurm launches each worker as one # `vllm serve` over its trays that balances its DP ranks itself, so the EPP routes to the -# worker rather than to one rank. Prefill drafts three DSpark tokens like decode: the srt -# launch planner gives both roles the decode role's golden acceptance length. +# worker rather than to one rank. Prefill drafts one DSpark token and decode three, both with +# adaptive verification on throughput (golden synthetic acceptance applies only to agg TP8). # VLLM_USE_RUST_FRONTEND is left unset (the Python API server). # Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_. @@ -158,7 +158,7 @@ base: long-prefill-token-threshold: 1024 max-num-seqs: 16 max-model-len: 1048576 - speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}' + speculative-config: '{"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true}' enable-cumem-allocator: true block-size: 256 tokenizer-mode: deepseek_v4 @@ -224,7 +224,7 @@ base: max-num-seqs: 32 max-cudagraph-capture-size: 256 max-model-len: 1048576 - speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}' + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true}' enable-cumem-allocator: true block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index c30a78942c..d21efbff86 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -156,7 +156,16 @@ def build_overrides( if worker_engine == "vllm": if not worker_spec: continue - if worker_al is not None: + if worker_spec.get("enable_adaptive_verification") is True: + if ( + worker_spec.get("rejection_sample_method") == "synthetic" + or "synthetic_acceptance_length" in worker_spec + ): + worker_spec["rejection_sample_method"] = "block" + worker_spec.pop("synthetic_acceptance_length", None) + else: + continue + elif worker_al is not None: worker_spec.update( rejection_sample_method="synthetic", synthetic_acceptance_length=worker_al ) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index 6b43d9ac49..28a125c88d 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -146,6 +146,61 @@ def test_thinking_mode_and_decode_priority(golden_dir: Path) -> None: assert result["roles"]["prefill"] == {"args": {"tensor-parallel-size": 8}} +def test_adaptive_dspark_skips_golden_acceptance_on_throughput(golden_dir: Path) -> None: + recipe = { + "roles": { + "prefill": { + "args": { + "speculative-config": json.dumps( + { + "method": "dspark", + "num_speculative_tokens": 1, + "enable_adaptive_verification": True, + } + ) + } + }, + "decode": { + "args": { + "speculative-config": json.dumps( + { + "method": "dspark", + "num_speculative_tokens": 3, + "enable_adaptive_verification": True, + } + ) + } + }, + } + } + original = copy.deepcopy(recipe) + result = apply_native( + recipe, + build_overrides( + recipe, + "llmd-vllm", + {**ENV, "SPEC_DECODING": "dspark"}, + golden_dir=golden_dir, + ), + ) + assert result == original + tp8_recipe = vllm_recipe( + "dspark", num_speculative_tokens=3, enable_adaptive_verification=False + ) + tp8 = apply_native( + tp8_recipe, + build_overrides( + tp8_recipe, + "vllm", + {**ENV, "SPEC_DECODING": "dspark"}, + golden_dir=golden_dir, + ), + ) + spec = json.loads(tp8["roles"]["agg"]["args"]["speculative-config"]) + assert spec["synthetic_acceptance_length"] == 2.7 + assert spec["rejection_sample_method"] == "synthetic" + + @pytest.mark.parametrize("sampling", [None, "unknown"]) def test_kimi_curve_requires_an_explicit_supported_sampler( golden_dir: Path, sampling: str | None diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 66a6261e72..1a9e50e734 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9548,3 +9548,12 @@ - "Add DeepSeek-V4-Pro-0813 DSpark FP4/FP8-KV AgentX on GB200 behind the llm-d router (EPP and pd-sidecar from llm-d-router v0.10.0, Envoy) through srt-slurm's llm-d frontend: disagg 1P1D DEP8/DEP8 (conc 64/128/160/256) and 2P1D DEP8/DEP16 (conc 256/384/576) with NIXL P/D KV and a Mooncake prefix-cache store on prefill at 66.7% node DRAM; agg TP8 (conc 1-12), DEP8 (conc 12-32), and DEP8 with Mooncake at 45% node DRAM (conc 52/72)." - "Each DEP worker is one vllm serve that balances its DP ranks itself, so the EPP routes to workers. Throughput uses committed golden synthetic acceptance (TP8 K=5, DEP K=3 on both prefill and decode); evals use real verification." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Match the Sep 28 standalone llm-d AgentX speculative policy: disagg and agg DEP8 (including Mooncake) use adaptive verification with prefill K=1 and decode/agg K=3; only agg TP8 keeps golden synthetic acceptance at K=5 (3.61). The srt-slurm synthetic-acceptance connector skips golden AL injection when enable_adaptive_verification is true." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741