diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml index 9a2b0265a1..541934f0af 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml index 9461990907..a44d935fda 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml @@ -2,7 +2,7 @@ schema: 2 name: "disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload" # GB200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + HiCache KV offload), tuned for +# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + Mooncake unified-cache KV offload), tuned for # concurrency 128. # The DEP8 prefill worker spans two nodes; the DEP16 decode worker spans four. @@ -28,6 +28,11 @@ health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -42,6 +47,54 @@ services: node: dedicated options: max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 + # Decode ranks use a chunk cache, so their host DRAM only joins the store + # through standalone stores: four per decode node, so segments stay uniform + # with prefill's per-rank ones instead of one oversized segment per node. + - &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -95,6 +148,18 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -130,11 +195,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' decode: nodes: 4 @@ -169,6 +232,8 @@ roles: SGLANG_LOG_FORWARD_ITERS: '1' SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -215,6 +280,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml index d3bcb51688..f0181c16ad 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml @@ -2,7 +2,7 @@ schema: 2 name: "disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload" # GB200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + HiCache KV offload), tuned for +# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + Mooncake unified-cache KV offload), tuned for # concurrency 64. # The DEP8 prefill worker spans two nodes; the DEP16 decode worker spans four. @@ -28,6 +28,11 @@ health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -42,6 +47,54 @@ services: node: dedicated options: max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 + # Decode ranks use a chunk cache, so their host DRAM only joins the store + # through standalone stores: four per decode node, so segments stay uniform + # with prefill's per-rank ones instead of one oversized segment per node. + - &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -95,6 +148,18 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -130,11 +195,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' decode: nodes: 4 @@ -169,6 +232,8 @@ roles: SGLANG_LOG_FORWARD_ITERS: '1' SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -215,6 +280,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml index 5a21654eb0..f1672f9e54 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml index c597addbe6..ac2ed56b08 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml index 4b7b7ff7b0..ff4cd0a15f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml new file mode 100644 index 0000000000..c74530894e --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -0,0 +1,241 @@ +# DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router +# (srt-slurm frontend.type: llm-d): one aggregated worker over two trays, as TP8, +# DEP8, or DEP8 with an embedded Mooncake prefix-cache store. srt-slurm launches a +# DEP8 worker as one `vllm serve` that balances its eight DP ranks itself, so the EPP +# routes to the worker rather than to one rank, and VLLM_USE_RUST_FRONTEND is left +# unset (the Python API server). +# Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-v4-pro-0813 + container: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + precision: fp4 + identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro-0813} + container: {image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95"} + dynamo: + install: false + slurm: {time_limit: "8:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: gb200 + gpus_per_node: 4 + observability: + enabled: false + tachometer: + enabled: false + frontend: + type: llm-d + enable_multiple_frontends: false + # srtctl adds the file-discovery plugin. + epp_config: + plugins: + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 5858 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 20000 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 1499703 + - type: active-request-scorer + - type: queue-scorer + - type: max-score-picker + schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-affinity-filter + - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer + weight: 3 + - pluginRef: max-score-picker + engine: + type: vllm + connector: + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONHASHSEED: "0" + HF_HUB_CACHE: /hf_hub_cache + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + # Outlast Envoy's default 1 h upstream idle timeout, so it never reuses a connection + # the server has already closed (503 UC). + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "3900" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_IB_ENABLE_IBGDA: "false" + NVSHMEM_CUMEM_HANDLE_TYPE: FABRIC + NVSHMEM_DISABLE_CUDA_VMM: "0" + UCX_TLS: cuda_copy,cuda_ipc,rc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: NVL + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.88 + max-num-batched-tokens: 8192 + block-size: 256 + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + moe-backend: deep_gemm_mega_moe + enable-ep-weight-filter: true + enable-cumem-allocator: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"enable_adaptive_verification":true,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"}' + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +# TP8 with five DSpark tokens, no EP, and a load-only scheduler. +override_tp8: + name: dsv4-gb200-vllm-agentic-dspark-agg-tp8-llm-d + frontend: + epp_config: + plugins: + - type: active-request-scorer + - type: queue-scorer + - type: weighted-random-picker + schedulingProfiles: + - name: default + plugins: + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: weighted-random-picker + roles: + agg: + env: + PYTHONHASHSEED: + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: + VLLM_USE_DEEP_GEMM: + NVSHMEM_REMOTE_TRANSPORT: + NVSHMEM_IB_ENABLE_IBGDA: + NVSHMEM_CUMEM_HANDLE_TYPE: + NVSHMEM_DISABLE_CUDA_VMM: + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: auto + UCX_TLS: cuda_copy,cuda_ipc,tcp + UCX_MEMTYPE_REG_WHOLE: "n" + args: + tensor-parallel-size: 8 + data-parallel-size: + enable-expert-parallel: + moe-backend: + enable-ep-weight-filter: + numa-bind: + gpu-memory-utilization: 0.85 + disable-custom-all-reduce: true + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"}' + +override_dep8: + name: dsv4-gb200-vllm-agentic-dspark-agg-dep8-llm-d + +# DEP8 with an embedded Mooncake store at 45% of each tray's DRAM (~900 GB over four +# GPUs); NIXL, SimpleCPUOffloadConnector, and MooncakeStoreConnector in a MultiConnector. +override_dep8_mooncake: + name: dsv4-gb200-vllm-agentic-dspark-agg-dep8-mooncake-llm-d + services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "101GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + frontend: + epp_config: + plugins: + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 40810 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 20000 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 1499703 + - type: active-request-scorer + - type: queue-scorer + - type: max-score-picker + roles: + agg: + env: + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + gpu-memory-utilization: 0.87 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":42949672960}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml new file mode 100644 index 0000000000..43fed8d374 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -0,0 +1,274 @@ +# DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router +# (srt-slurm frontend.type: llm-d): DEP8 prefill workers over two trays each and one +# DEP8 or DEP16 decode worker. KV moves over NIXL and the prefill workers share prefix +# cache through an embedded Mooncake store. srt-slurm launches each worker as one +# `vllm serve` over its trays that balances its DP ranks itself, so the EPP routes to the +# worker rather than to one rank. Prefill drafts one DSpark token and decode three, both with +# adaptive verification on throughput (golden synthetic acceptance applies only to agg TP8). +# VLLM_USE_RUST_FRONTEND is left unset (the Python API server). +# Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-v4-pro-0813 + container: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + precision: fp4 + identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro-0813} + container: {image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95"} + dynamo: + install: false + slurm: {time_limit: "8:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: gb200 + gpus_per_node: 4 + observability: + enabled: false + tachometer: + enabled: false + services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + # Each prefill rank's embedded store: 2/3 of a tray's ~900 GB DRAM over 4 GPUs. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + frontend: + type: llm-d + enable_multiple_frontends: false + # srtctl adds the file-discovery plugin. + epp_config: + plugins: + - type: prefill-filter + - type: decode-filter + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 43949 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 4783 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 3000000 + - type: active-request-scorer + - type: queue-scorer + - type: always-disagg-pd-decider + - type: disagg-profile-handler + parameters: + deciders: + prefill: always-disagg-pd-decider + - type: max-score-picker + name: prefill-picker + - type: max-score-picker + name: decode-picker + schedulingProfiles: + - name: prefill + plugins: + - pluginRef: prefill-filter + - pluginRef: prefix-cache-affinity-filter + - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: queue-scorer + weight: 3 + - pluginRef: prefill-picker + - name: decode + plugins: + - pluginRef: decode-filter + - pluginRef: active-request-scorer + - pluginRef: decode-picker + engine: + type: vllm + connector: + roles: + prefill: + gpus: 8 + env: + PYTHONHASHSEED: "0" + HF_HUB_CACHE: /hf_hub_cache + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED: "1" + VLLM_NO_USAGE_STATS: "1" + VLLM_LOGGING_LEVEL: INFO + TQDM_DISABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_IB_ENABLE_IBGDA: "false" + NVSHMEM_CUMEM_HANDLE_TYPE: FABRIC + NVSHMEM_DISABLE_CUDA_VMM: "0" + UCX_TLS: cuda_copy,cuda_ipc,rc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: NVL + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_RPC_CLIENT_IO_THREADS: "32" + MC_TE_RPC_CLIENT_IO_THREADS: "32" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + # NIXL hands KV to decode; the Mooncake store shares prefix cache across prefill ranks. + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"kv_lease_duration":1800}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + max-num-seqs: 16 + max-model-len: 1048576 + speculative-config: '{"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true}' + enable-cumem-allocator: true + block-size: 256 + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + moe-backend: deep_gemm_mega_moe + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONHASHSEED: "0" + HF_HUB_CACHE: /hf_hub_cache + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + # Outlast the decode sidecar's 90 s idle-connection timeout, so it never reuses a + # connection the server has already closed (connection reset, 502). + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_NO_USAGE_STATS: "1" + VLLM_LOGGING_LEVEL: INFO + TQDM_DISABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_IB_ENABLE_IBGDA: "false" + NVSHMEM_CUMEM_HANDLE_TYPE: FABRIC + NVSHMEM_DISABLE_CUDA_VMM: "0" + UCX_TLS: cuda_copy,cuda_ipc,rc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: NVL + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"kv_lease_duration":1800}}' + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-num-batched-tokens: 256 + max-num-seqs: 32 + max-cudagraph-capture-size: 256 + max-model-len: 1048576 + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true}' + enable-cumem-allocator: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + moe-backend: deep_gemm_mega_moe + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +override_1p1d: + name: dsv4-gb200-vllm-agentic-dspark-disagg-1p1d-dep8-dep8-llm-d + roles: + prefill: + nodes: 2 + workers: 1 + +override_2p1d: + name: dsv4-gb200-vllm-agentic-dspark-disagg-2p1d-dep8-dep16-llm-d + roles: + prefill: + nodes: 4 + workers: 2 + decode: + nodes: 4 + gpus: 16 + args: + data-parallel-size: 16 diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml index 592a91827b..f6dd120075 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml @@ -1,19 +1,20 @@ # DeepSeek-V4-Pro-0813 AgentX on MI355X with ATOM DSpark K6 (six draft tokens, -# a verify window of 7). Low concurrency is TP-only; the wide band runs DP -# attention with native RCCL expert parallelism. The KV cache is GPU-resident. +# a verify window of 7). Low concurrency is TP-only. The DP-attention band runs +# MegaMoE with EPLB from C48 (C64 without EPLB). C256 and C384 add an LMCache +# CPU KV tier (lmcache_mp); the other points keep the KV cache GPU-resident. base: schema: 2 name: dsv4-fp4-mi355x-atom-agentic model: path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: rocm/atom-dev:nightly_202609291501 + container: rocm/atom-dev:nightly_202610071700 precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + container_image: rocm/atom-dev@sha256:bb3912c372dda0c82fc6831afdd76e366a372d5e9fe223373499db63b06cce65 enable_multiple_frontends: false observability: enabled: false @@ -58,9 +59,9 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point. Admission is 2x CONC. KV is BF16 through concurrency 16 -# and FP8 in the DEP band. DEP pins sessions to one DP-attention rank so later -# turns reach their prefix KV, and captures every small per-rank decode batch. +# One variant per point. KV is BF16 through concurrency 16 and FP8 in the DEP +# band. DEP pins sessions to one DP-attention rank so later turns reach their +# prefix KV, and captures every small per-rank decode batch. override_tp8_c1: roles: agg: @@ -119,15 +120,22 @@ override_tp8_c48_dep: kv-cache-dtype: fp8 enable-expert-parallel: true enable-dp-attention: true - all2all-backend: rccl + all2all-backend: high-throughput dp-load-balance: least_tokens - moe-backend: standard + moe-backend: mega cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' env: ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' ATOM_ENABLE_PREFILL_DELAYER: '1' ATOM_PREFILL_DECODE_INTERVAL: '10' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '256' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '511' + ATOM_EPLB_MAX_REBALANCES: '1' benchmark: env: CONC: '48' @@ -142,15 +150,19 @@ override_tp8_c64_dep: kv-cache-dtype: fp8 enable-expert-parallel: true enable-dp-attention: true - all2all-backend: rccl + all2all-backend: high-throughput dp-load-balance: least_tokens - moe-backend: standard + moe-backend: mega cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' env: ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' ATOM_ENABLE_PREFILL_DELAYER: '1' ATOM_PREFILL_DECODE_INTERVAL: '10' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '256' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '511' benchmark: env: CONC: '64' @@ -165,15 +177,22 @@ override_tp8_c96_dep: kv-cache-dtype: fp8 enable-expert-parallel: true enable-dp-attention: true - all2all-backend: rccl + all2all-backend: high-throughput dp-load-balance: least_tokens - moe-backend: standard + moe-backend: mega cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' env: ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' ATOM_ENABLE_PREFILL_DELAYER: '1' ATOM_PREFILL_DECODE_INTERVAL: '10' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '512' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '1023' + ATOM_EPLB_MAX_REBALANCES: '1' benchmark: env: CONC: '96' @@ -184,44 +203,149 @@ override_tp8_c128_dep: roles: agg: args: - max-num-seqs: 256 + max-num-seqs: 128 kv-cache-dtype: fp8 enable-expert-parallel: true enable-dp-attention: true - all2all-backend: rccl + all2all-backend: high-throughput dp-load-balance: least_tokens - moe-backend: standard - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + moe-backend: mega + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' env: ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' ATOM_ENABLE_PREFILL_DELAYER: '1' - ATOM_PREFILL_DECODE_INTERVAL: '10' + ATOM_PREFILL_DECODE_INTERVAL: '20' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '512' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '1023' + ATOM_EPLB_MAX_REBALANCES: '2' benchmark: env: CONC: '128' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' AGENTIC_WARMUP_GRACE_PERIOD: '3600' -override_tp8_c256_dep: +# DRAM variants: ATOM's lmcache_mp tier backed by srtctl's lmcache-server +# service (one LMCache server per node on port 8750, run from the LMCache build in +# the ATOM image) within the 1019 GB budget (dram-utilization 0.34). +override_tp8_c256_dep_lmcache: + services: + - name: lmcache + type: lmcache-server + args: + - --chunk-size + - '256' + - --null-block-id + - '-1' + - --separate-object-groups + - --supported-transfer-mode + - lmcache_driven + - --l1-size-gb + - '1000' + - --l1-use-lazy + - --l1-read-ttl-seconds + - '900' + - --eviction-policy + - LRU + - --eviction-trigger-watermark + - '0.98' roles: agg: args: - max-num-seqs: 512 + max-num-seqs: 128 kv-cache-dtype: fp8 enable-expert-parallel: true enable-dp-attention: true - all2all-backend: rccl + all2all-backend: high-throughput dp-load-balance: least_tokens - moe-backend: standard - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + moe-backend: mega + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' env: ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' ATOM_ENABLE_PREFILL_DELAYER: '1' - ATOM_PREFILL_DECODE_INTERVAL: '10' + ATOM_PREFILL_DECODE_INTERVAL: '40' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '512' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '1023' + ATOM_EPLB_MAX_REBALANCES: '4' + ATOM_KV_OFFLOAD: lmcache_mp + # Braces doubled: srtctl expands {node}/{node_id} in env values with str.format_map. + ATOM_KV_OFFLOAD_EXTRA_CONFIG: '{{"lmcache.mp.host": "tcp://127.0.0.1", "lmcache.mp.port": 8750, "lmcache.mp.tp_rank_collapse": true, "lmcache.mp.max_pinned_state_bytes": 1000000000000}}' + LMCACHE_CHUNK_SIZE: '256' + OFFLOAD_MAX_PENDING_SAVES: '8' + OFFLOAD_MIN_LOAD_TOKENS: '8192' + OFFLOAD_MIN_SAVE_TOKENS: '8192' benchmark: env: CONC: '256' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1019' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c384_dep_lmcache: + services: + - name: lmcache + type: lmcache-server + args: + - --chunk-size + - '256' + - --null-block-id + - '-1' + - --separate-object-groups + - --supported-transfer-mode + - lmcache_driven + - --l1-size-gb + - '1000' + - --l1-use-lazy + - --l1-read-ttl-seconds + - '900' + - --eviction-policy + - LRU + - --eviction-trigger-watermark + - '0.98' + roles: + agg: + args: + max-num-seqs: 128 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: high-throughput + dp-load-balance: least_tokens + moe-backend: mega + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,72,80,88,96,104,112,120,128]' + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '20' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '1024' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '2047' + ATOM_EPLB_MAX_REBALANCES: '5' + ATOM_KV_OFFLOAD: lmcache_mp + # Braces doubled: srtctl expands {node}/{node_id} in env values with str.format_map. + ATOM_KV_OFFLOAD_EXTRA_CONFIG: '{{"lmcache.mp.host": "tcp://127.0.0.1", "lmcache.mp.port": 8750, "lmcache.mp.tp_rank_collapse": true, "lmcache.mp.max_pinned_state_bytes": 1000000000000}}' + LMCACHE_CHUNK_SIZE: '256' + OFFLOAD_MAX_PENDING_SAVES: '8' + OFFLOAD_MIN_LOAD_TOKENS: '8192' + OFFLOAD_MIN_SAVE_TOKENS: '8192' + benchmark: + env: + CONC: '384' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1019' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 56b6a87647..3ae0b754ec 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -1,13 +1,13 @@ # DeepSeek-V4.1-Flash AgentX on MI355X with ATOM DSpark (five draft tokens), # following ROCm/ATOM recipes/DeepSeek-V4.1-Flash-Agentic.md at the image's -# revision (7fc0926c698e3f19df9dca23091e399851d3d0ec). TP2 and TP4, no expert +# revision (73defd9b868663be51985fedff06115ef9bcab3d). TP2 and TP4, no expert # parallelism; the KV cache is GPU-resident. base: schema: 2 name: dsv41flash-fp4-mi355x-atom-agentic model: path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: rocm/atom-dev:nightly_202610070333 + container: rocm/atom-dev:nightly_202610100929 precision: fp4 resources: gpu_type: mi355x diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml index 83a196ec81..12c4d1f43d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml @@ -6,10 +6,13 @@ base: name: dsv41flash-fp4-mi355x-vllm-agentic model: path: hf:deepseek-ai/DeepSeek-V4.1-Flash - # Repinned to a nightly build that includes vllm-project/vllm#58671 - # (paged MXFP4 sparse indexer), #58655 (mHC fused Triton seams) and - # #53492 (sparse MLA Gluon kernel). - container: vllm/vllm-openai-rocm:nightly-rocm100-ac9126e58aa7bbab1856ba6593ba4d5003fea516 + # The first ROCm 10.0 nightly carrying vllm-project/vllm#60397 (DSv4.1 + # mono decode kernels), which the environment below enables. It also + # brings #59685 (AITER MegaMoEV2 for DeepSeek V4), #56638 (shared + # combine_topk_swa_indices kernel in place of the Torch prefill combine), + # #60159 (native merge_attn_states on gfx950) and #60389 (MLA derived + # weights kept in place on weight reload). + container: vllm/vllm-openai-rocm:nightly-rocm100-7d0b4e57aac4c4323225b0eee9ae960ce75ffffe precision: fp4 resources: gpu_type: mi355x @@ -62,6 +65,13 @@ base: env: VLLM_ROCM_USE_AITER: '1' VLLM_ROCM_USE_AITER_MOE: '1' + # Mono decode kernels (vllm-project/vllm#60397): two persistent + # launches a layer for decode steps of up to 48 rows (c8 with five + # drafts); larger steps take the regular path. + VLLM_ROCM_MONO_DECODE: '1' + # INT4 quick all-reduce above its size threshold + # (vllm-project/recipes#1061). + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 AITER_TRITON_LOG_LEVEL: ERROR VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: 'True' # DeepseekV41ForCausalLM is not torch-compiled upstream; breakable @@ -80,11 +90,10 @@ base: WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -# One variant per point (c128 excluded for both TP2 and TP4). Engram tables -# move to pinned host memory at TP2, where resident tables leave too little -# KV. Graph capture covers 2x CONC x (1 + 5 drafts). The prefill chunk shrinks -# at TP2 from c64 to buy KV room, and where it drops below 1024 x 6 slots the -# scheduler is capped at the captured shape. +# One variant per point, concurrency 1 through 64 on both TP arms (c128 and +# c192 excluded). Engram tables move to pinned host memory at TP2, where +# resident tables leave too little KV. Graph capture covers 2x CONC x +# (1 + 5 drafts). The prefill chunk is the upstream 16384 everywhere. override_tp4_c1: roles: agg: @@ -262,7 +271,7 @@ override_tp2_c64: tensor-parallel-size: 2 engram-config: '{"cpu_offload":true}' max-cudagraph-capture-size: 1024 - max-num-batched-tokens: 8192 + max-num-batched-tokens: 16384 benchmark: env: CONC: '64' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml index e32b72696a..621dd5438a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml @@ -69,13 +69,16 @@ base: TOTAL_CPU_DRAM_GB: '2399' # One variant per point: admission is 2x CONC and the graph batch matches it, -# capped at 64. +# capped at 64. Use K=5 at c1/c4/c8/c20; retain K=3 elsewhere. The connector +# selects the committed golden acceptance length for each variant. override_c1: roles: agg: args: max-running-requests: 2 cuda-graph-max-bs: 2 + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 benchmark: env: CONC: '1' @@ -86,6 +89,8 @@ override_c4: args: max-running-requests: 8 cuda-graph-max-bs: 8 + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 benchmark: env: CONC: '4' @@ -96,6 +101,8 @@ override_c8: args: max-running-requests: 16 cuda-graph-max-bs: 16 + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 benchmark: env: CONC: '8' @@ -126,6 +133,8 @@ override_c20: args: max-running-requests: 40 cuda-graph-max-bs: 40 + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 benchmark: env: CONC: '20' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml index a1690473a7..e7d2c47351 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml @@ -1,17 +1,17 @@ # Kimi-K3 MXFP4 AgentX on MI355X with vLLM DSpark # (https://recipes.vllm.ai/moonshotai/Kimi-K3). TP8 only: the 1.56 TB checkpoint -# is ~195 GB per GPU. The KV cache is GPU-resident through c4 and backed by -# vLLM's SimpleCPUOffloadConnector (dram) from c8. Three bands, driven by -# kimik3-fp4-mi355x-vllm-agentic-mtp: interactive dcp1 c1/c4 (GPU-resident), -# mid dcp1 c8-c14 (DSpark draft + DRAM offload), throughput dcp8 c44/c48/c70 -# (KV read sharded across all 8 GPUs, DRAM offload, mtp/no draft with synthetic -# acceptance). The 1799 GB DRAM budget splits evenly across the eight ranks. +# is ~195 GB per GPU. The KV cache is GPU-resident at c1 and backed by +# vLLM's SimpleCPUOffloadConnector (dram) from dcp8 c4. Three bands, driven by +# kimik3-fp4-mi355x-vllm-agentic-mtp: interactive dcp1/dcp8 c1 (GPU-resident), +# mid dcp8 c4-c16 and throughput dcp8 c44/c48/c50 (KV read sharded across all +# 8 GPUs, DRAM offload, DSpark draft with synthetic acceptance). The 1799 GB +# DRAM budget splits evenly across the eight ranks. base: schema: 2 name: kimik3-fp4-mi355x-vllm-agentic model: path: hf:moonshotai/Kimi-K3 - container: vllm/vllm-openai-rocm:nightly-rocm100-36768d1bfd39094681cdbc8cb37d4b31c0729c89 + container: vllm/vllm-openai-rocm:nightly-rocm100-81198e97ba7eee2a22540caaa756b7fdddcb4d93 precision: fp4 resources: gpu_type: mi355x @@ -50,6 +50,7 @@ base: enable-prefix-caching: true kv-cache-dtype: fp8 attention-config: '{"mla_prefill_backend":"ROCM_AITER_FA"}' + stream-interval: 10 env: # Upstream AMD recipe environment. VLLM_ROCM_AITER_MLA_ASM_PADDING: asm @@ -68,6 +69,8 @@ base: VLLM_USE_DIRECT_DCP_A2A: '0' VLLM_USE_DIRECT_DCP_Q_GATHER: '0' VLLM_USE_DIRECT_DCP_KV_GATHER: '0' + # Route DCP multi-token verify through the AITER ASM round-robin decode. + VLLM_ROCM_AITER_MLA_DCP_VERIFY: asm benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh @@ -78,11 +81,13 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. The DSpark draft length descends with concurrency -# (7, 5, 4, 4, 3, 3); throughput runs replace block rejection with the golden -# acceptance length for that draft length. Admission is 2x CONC and graph -# capture covers every size up to 2x CONC x (1 + drafts). Breakable -# FULL_AND_PIECEWISE graphs cost KV pool, so only c1 and c4 use them. The -# offload points split the 1799 GB DRAM budget evenly across the eight ranks; +# (7, 5, 4, 3, 4, 4) and stays at 4 for the throughput band; agentic runs +# replace block rejection with the golden acceptance length for that draft +# length. Admission is 2x CONC and graph +# capture covers every even size up to 2x CONC x (1 + drafts). Every point +# requests FULL_AND_PIECEWISE; the AMD K3 model has no torch.compile +# submodule, so without breakable graphs vLLM falls back to FULL_DECODE_ONLY. +# The offload points split the 1799 GB DRAM budget evenly across the eight ranks; # identical prefixes must hash to identical block keys on every rank. override_tp8_c1: roles: @@ -90,10 +95,30 @@ override_tp8_c1: args: max-num-seqs: 2 max-num-batched-tokens: 16384 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16]}' speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + quantization-config: '{"linear":"fp8_per_channel","ignore":["*kv_b_proj","*vision_tower*","*mm_projector*","*in_proj_qkvgfab","*f_b_proj","*fused_qkv_a_proj","*self_attn.g_proj","*.layers.0.mlp.down_proj"]}' env: - VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +# c1 with dcp8: the KV read sharded across all 8 GPUs, still GPU-resident. +override_tp8_c1_dcp8: + roles: + agg: + args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-seqs: 2 + max-num-batched-tokens: 16384 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + quantization-config: '{"linear":"fp8_per_channel","ignore":["*kv_b_proj","*vision_tower*","*mm_projector*","*in_proj_qkvgfab","*f_b_proj","*fused_qkv_a_proj","*self_attn.g_proj","*.layers.0.mlp.down_proj"]}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' benchmark: env: CONC: '1' @@ -105,40 +130,48 @@ override_tp8_c4: args: max-num-seqs: 8 max-num-batched-tokens: 8192 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":48,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48]}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":48,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48]}' speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":5,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' env: - VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' benchmark: env: CONC: '4' KV_OFFLOADING: 'none' -override_tp8_c8_simple: +# c4 with dcp8: the KV read sharded across all 8 GPUs, backed by the DRAM tier. +override_tp8_c4_dcp8_simple: roles: agg: args: - max-num-seqs: 16 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-seqs: 8 max-num-batched-tokens: 8192 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":80,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80]}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":48,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":5,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' PYTHONHASHSEED: '42' benchmark: env: - CONC: '8' + CONC: '4' KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' -override_tp8_c10_simple: +# Mid band: dcp-size 8 shards the KV read across all 8 GPUs, backed by +# vLLM's SimpleCPUOffloadConnector DRAM tier, with the DSpark draft. +override_tp8_c10_dcp8_simple: roles: agg: args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a max-num-seqs: 20 max-num-batched-tokens: 8192 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":100,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100]}' + gpu-memory-utilization: 0.9 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":100,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100]}' speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: @@ -150,13 +183,16 @@ override_tp8_c10_simple: KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' -override_tp8_c12_simple: +override_tp8_c12_dcp8_simple: roles: agg: args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a max-num-seqs: 24 max-num-batched-tokens: 8192 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":96,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96]}' + gpu-memory-utilization: 0.9 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":96,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96]}' speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: @@ -168,14 +204,17 @@ override_tp8_c12_simple: KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' -override_tp8_c14_simple: +override_tp8_c14_dcp8_simple: roles: agg: args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a max-num-seqs: 28 max-num-batched-tokens: 8192 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":112,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112]}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + gpu-memory-utilization: 0.9 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":140,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' @@ -186,10 +225,30 @@ override_tp8_c14_simple: KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' -# Throughput band: dcp-size 8 shards the KV read across all 8 GPUs for the -# 100k+ token contexts, backed by the same SimpleCPUOffloadConnector DRAM tier. -# No draft model here (the a2a DCP path); spec-decoding stays mtp so the golden -# acceptance length drives throughput. Admission is 2x CONC. +override_tp8_c16_dcp8_simple: + roles: + agg: + args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-seqs: 32 + max-num-batched-tokens: 8192 + gpu-memory-utilization: 0.9 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":160,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +# Throughput band: same dcp8 DRAM-offload layout with a 4-token DSpark draft. +# Synthetic acceptance replaces block rejection with the golden acceptance +# length. FULL_AND_PIECEWISE graphs cover every even size up to 2x CONC x 5. override_tp8_c44_dcp8_simple: roles: agg: @@ -199,7 +258,8 @@ override_tp8_c44_dcp8_simple: max-num-seqs: 88 max-num-batched-tokens: 8192 gpu-memory-utilization: 0.9 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,128,256,512,1024,2048,4096,8192]}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":440,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164,166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200,202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236,238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272,274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308,310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344,346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380,382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416,418,420,422,424,426,428,430,432,434,436,438,440]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' @@ -220,7 +280,8 @@ override_tp8_c48_dcp8_simple: max-num-seqs: 96 max-num-batched-tokens: 8192 gpu-memory-utilization: 0.9 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,128,256,512,1024,2048,4096,8192]}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":480,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164,166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200,202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236,238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272,274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308,310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344,346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380,382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416,418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452,454,456,458,460,462,464,466,468,470,472,474,476,478,480]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' @@ -232,23 +293,24 @@ override_tp8_c48_dcp8_simple: TOTAL_CPU_DRAM_GB: '1799' SPEC_DECODING: 'mtp' -override_tp8_c70_dcp8_simple: +override_tp8_c50_dcp8_simple: roles: agg: args: decode-context-parallel-size: 8 dcp-comm-backend: a2a - max-num-seqs: 140 + max-num-seqs: 100 max-num-batched-tokens: 8192 gpu-memory-utilization: 0.9 - compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,256,512,1024,2048,4096,8192]}' + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":500,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164,166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200,202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236,238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272,274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308,310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344,346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380,382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416,418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452,454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488,490,492,494,496,498,500]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' env: VLLM_USE_BREAKABLE_CUDAGRAPH: '0' PYTHONHASHSEED: '42' benchmark: env: - CONC: '70' + CONC: '50' KV_OFFLOADING: 'dram' TOTAL_CPU_DRAM_GB: '1799' SPEC_DECODING: 'mtp' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index e0ed3d6b51..39f858edf1 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -716,13 +716,13 @@ kimik3-fp4-mi355x-atom-agentic-mtp: # decoding. Companion to kimik3-fp4-mi355x-atom-agentic-mtp: same checkpoint and # runner. TP8 only -- the 1.56 TB MXFP4 checkpoint is ~195 GB/GPU. Three bands: # interactive (1, 4) GPU-resident, dcp 1, DSpark draft. -# mid (8-14) dcp 1, DSpark draft, SimpleCPUOffload DRAM tier. -# throughput (44-70) dcp 8 shards the KV read across all 8 GPUs, DRAM -# offload, mtp with synthetic acceptance (no draft). +# mid (10-16) dcp 8, DSpark draft, SimpleCPUOffload DRAM tier. +# throughput (44-50) dcp 8 shards the KV read across all 8 GPUs, DRAM +# offload, 4-token DSpark draft with synthetic acceptance. # dram-utilization 0.6 -> TOTAL_CPU_DRAM_GB 1799 at TP8 (node cap 2861022 MiB), # split evenly across the eight ranks by the launcher. kimik3-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-rocm100-36768d1bfd39094681cdbc8cb37d4b31c0729c89 + image: vllm/vllm-openai-rocm:nightly-rocm100-81198e97ba7eee2a22540caaa756b7fdddcb4d93 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds @@ -733,9 +733,10 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.6 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 10, 12, 14], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48, 70], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: none, conc-list: [1], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 10, 12, 14, 16], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } minimaxm3-fp4-mi355x-atom-agentic-mtp: image: rocm/atom-dev:nightly_202610070333 @@ -946,8 +947,10 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: # DeepSeek-V4-Pro-0813 FP4 AgentX on one MI355X node using ATOM DSpark K6. # Both throughput bands use thinking_on golden AL 3.77; eval uses real # acceptance. Keep the historical config key for changelog compatibility. +# C256 and C384 offload KV to an LMCache MP server whose 1000 GB L1 fits the +# 1019 GB budget of dram-utilization 0.34. dsv4-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202609291501 + image: rocm/atom-dev:nightly_202610071700 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:mi355x-amds @@ -956,9 +959,11 @@ dsv4-fp4-mi355x-atom-agentic-mtp: multinode: false scenarios: agentic-coding: - - search-space: + - dram-utilization: 0.34 + search-space: - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128, 256], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, spec-decoding: draft_model, conc-list: [256, 384], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 @@ -1494,9 +1499,9 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: # Two GPUs per server doubles the servers per node and is the layout that # decides whether DSv4.1-Flash is throughput- or interactivity-bound here. dsv41flash-fp4-mi355x-vllm-agentic-dspark: - # Repinned to a nightly build that includes vllm-project/vllm#58671, - # #58655 and #53492 (see agentic.yaml). - image: vllm/vllm-openai-rocm:nightly-rocm100-ac9126e58aa7bbab1856ba6593ba4d5003fea516 + # The first ROCm 10.0 nightly carrying vllm-project/vllm#60397 (see + # agentic.yaml). + image: vllm/vllm-openai-rocm:nightly-rocm100-7d0b4e57aac4c4323225b0eee9ae960ce75ffffe model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds @@ -1507,18 +1512,16 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: agentic-coding: - dram-utilization: 0.60 search-space: - # The recipe only changes the memory split where a point ran short of KV: - # the prefill chunk shrinks at TP=2 c64. c128 is excluded for both TP=2 - # and TP=4. Every point is measured here rather than combined from the - # earlier run: the image move leaves concurrency 1-32 measured only on - # the superseded nightly-eed1f3d0 pin. + # Concurrency 1-64 on both TP arms, every point re-measured on the new + # image. c128 and c192 are excluded: c128 did not raise the scored + # metric at TP=2 (150,683 against 151,896 tok/s/chip at c64). - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } # DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610070333 + image: rocm/atom-dev:nightly_202610100929 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 12c4cc160e..f3366e67f4 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6076,6 +6076,108 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: additional-settings: - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c40_b40" +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200 behind the llm-d router (srt-slurm +# frontend.type: llm-d). The image bundles epp and pd-sidecar from llm-d-router v0.10.0. +dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: llmd-vllm + router: { name: llm-d-router, version: "0.10.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.666667 + search-space: + # 1P DEP8 + 1D DEP8 (4 trays), Mooncake prefix cache on prefill. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [64, 128, 160, 256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_1p1d" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 2P DEP8 + 1D DEP16 (8 trays), Mooncake prefix cache on prefill. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [256, 384, 576] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_2p1d" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + +# Aggregated TP8 and DEP8 over two trays; only the Mooncake arm offloads KV to DRAM. +dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: llmd-vllm + router: { name: llm-d-router, version: "0.10.0" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.45 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 2, 4, 8, 12] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_tp8" + - spec-decoding: mtp + kv-offloading: none + conc-list: [12, 16, 32] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_dep8" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [52, 72] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_dep8_mooncake" + dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 model: deepseek-ai/DeepSeek-V4-Pro @@ -6858,7 +6960,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [64] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 1 tp: 8 @@ -6874,7 +6976,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [128] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 1 tp: 8 @@ -6890,7 +6992,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [256] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 1 tp: 16 @@ -6906,7 +7008,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [768] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 2 tp: 16 @@ -6922,7 +7024,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [1024] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 2 tp: 16 @@ -6938,7 +7040,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [1280] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 2 tp: 16 @@ -7332,10 +7434,10 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg: # GLM-5.2 B300 NVFP4 AgentX with EAGLE/MTP speculative decoding, following the # AgentX speculative-decoding policy in docs/MODELS.md. SGLang EAGLE runs off -# GLM-5.2's built-in nextn head (num-steps 3, -# eagle-topk 1, 4 draft tokens = 3 speculative tokens), with acceptance pinned -# to the golden AL 2.99 (infx/golden_al_distribution/glm5.2_mtp.yaml, thinking_on, -# K=3) through SGLANG_SIMULATE_ACC_*. The pinned nightly includes FlashInfer +# GLM-5.2's built-in nextn head (eagle-topk 1): K=5 / 6 draft tokens at +# c1/c4/c8/c20, K=3 / 4 draft tokens elsewhere. The connector selects golden +# AL 3.61 or 2.99 respectively from infx/golden_al_distribution/glm5.2_mtp.yaml +# for thinking_on; evals retain real verification. The pinned nightly includes FlashInfer # 0.6.18's BF16 TRTLLM MoE allocation fix for small-batch Blackwell execution # in GLM-5.2's unquantized EAGLE draft head. # diff --git a/inferencex-e2e/infx/bench/agentic/run.py b/inferencex-e2e/infx/bench/agentic/run.py index 25b3c93790..5b90068a4f 100644 --- a/inferencex-e2e/infx/bench/agentic/run.py +++ b/inferencex-e2e/infx/bench/agentic/run.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse -import datetime +import datetime as dt import os import shlex import sys @@ -168,7 +168,7 @@ def _open_power_window(plan: Plan, python: str, env: Mapping[str, str]) -> int: return 0 # AIPerf exports naive local datetimes; the adapter needs the offset to convert the # profiling window to Unix time. - now = datetime.datetime.now(datetime.timezone.utc).astimezone() + now = dt.datetime.now(dt.timezone.utc).astimezone() (plan.result_dir / "agentic_power_timezone_offset.txt").write_text(f"{now:%z}\n") rc = _power_adapter(plan, python, env, *_window(plan, "running")) if rc: diff --git a/inferencex-e2e/infx/bench/eval/lm_eval.py b/inferencex-e2e/infx/bench/eval/lm_eval.py index cbc8d23b38..419f0820e2 100644 --- a/inferencex-e2e/infx/bench/eval/lm_eval.py +++ b/inferencex-e2e/infx/bench/eval/lm_eval.py @@ -33,7 +33,9 @@ def install(environ: Mapping[str, str]) -> None: # torchvision causes circular imports in ATOM; TRT-LLM/SGLang need it at module level. if "atom" in environ.get("IMAGE", ""): _pip(environ, "uninstall", "torchvision") - _pip(environ, "install", "lm-eval[api]") + # lm-eval does not depend on transformers, so without this range its resolution + # upgrades huggingface-hub past what transformers 5.x accepts. + _pip(environ, "install", "lm-eval[api]", "huggingface-hub>=1.5,<2") pinned = ("install", "--no-deps", "--reinstall") git = shutil.which("git", path=environ.get("PATH")) if git and _pip(environ, *pinned, f"git+{REPOSITORY}.git@{REF}"): diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index 57d31ce99d..49611d7d8b 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -66,7 +66,7 @@ class SrtLane: ), ("b300-dsxe", LaunchPath.SRT_MULTI): SrtLane(frameworks=_DYNAMO), ("gb200-nv", LaunchPath.SRT_MULTI): SrtLane( - frameworks=_DYNAMO, + frameworks=any_of("dynamo-sglang", "dynamo-trt", "dynamo-vllm", "llmd-vllm"), setup_scripts={"dynamo-sglang": "install-torchao.sh"}, mounts=( *_AGENTIC_CACHES, @@ -79,7 +79,7 @@ class SrtLane: ), shared_run_root=( Match(any_of("minimaxm3", "kimik3", "qwen3.5", "glm5.2")), - Match(any_of("dsv4"), frameworks=any_of("dynamo-vllm")), + Match(any_of("dsv4"), frameworks=any_of("dynamo-vllm", "llmd-vllm")), Match(any_of("dsv4"), frameworks=any_of("dynamo-sglang"), agentic=True), ), ), diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 19ba42a60d..d21efbff86 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -24,6 +24,7 @@ "vllm-disagg": "vllm", "vllm": "vllm", "dynamo-vllm": "vllm", + "llmd-vllm": "vllm", "dynamo-sglang": "sglang", "trt": "trtllm", "dynamo-trt": "trtllm", @@ -155,7 +156,16 @@ def build_overrides( if worker_engine == "vllm": if not worker_spec: continue - if worker_al is not None: + if worker_spec.get("enable_adaptive_verification") is True: + if ( + worker_spec.get("rejection_sample_method") == "synthetic" + or "synthetic_acceptance_length" in worker_spec + ): + worker_spec["rejection_sample_method"] = "block" + worker_spec.pop("synthetic_acceptance_length", None) + else: + continue + elif worker_al is not None: worker_spec.update( rejection_sample_method="synthetic", synthetic_acceptance_length=worker_al ) diff --git a/inferencex-e2e/infx/tests/bench/test_eval_command.py b/inferencex-e2e/infx/tests/bench/test_eval_command.py index 59d679410e..898f64fcbb 100644 --- a/inferencex-e2e/infx/tests/bench/test_eval_command.py +++ b/inferencex-e2e/infx/tests/bench/test_eval_command.py @@ -404,7 +404,7 @@ def uv_step(argv: list[str]) -> str: """Name one recorded ``uv`` call of the lm-eval install into this interpreter.""" assert argv[0] == "pip" and argv[-2:] == ["--python", sys.executable] assert "--break-system-packages" in argv - spec = argv[-3] + spec = next(arg for arg in argv[2:] if not arg.startswith("-")) if argv[1] == "uninstall": return f"uninstall {spec}" if {"--no-deps", "--reinstall"} <= set(argv): diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index 6b43d9ac49..28a125c88d 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -146,6 +146,61 @@ def test_thinking_mode_and_decode_priority(golden_dir: Path) -> None: assert result["roles"]["prefill"] == {"args": {"tensor-parallel-size": 8}} +def test_adaptive_dspark_skips_golden_acceptance_on_throughput(golden_dir: Path) -> None: + recipe = { + "roles": { + "prefill": { + "args": { + "speculative-config": json.dumps( + { + "method": "dspark", + "num_speculative_tokens": 1, + "enable_adaptive_verification": True, + } + ) + } + }, + "decode": { + "args": { + "speculative-config": json.dumps( + { + "method": "dspark", + "num_speculative_tokens": 3, + "enable_adaptive_verification": True, + } + ) + } + }, + } + } + original = copy.deepcopy(recipe) + result = apply_native( + recipe, + build_overrides( + recipe, + "llmd-vllm", + {**ENV, "SPEC_DECODING": "dspark"}, + golden_dir=golden_dir, + ), + ) + assert result == original + tp8_recipe = vllm_recipe( + "dspark", num_speculative_tokens=3, enable_adaptive_verification=False + ) + tp8 = apply_native( + tp8_recipe, + build_overrides( + tp8_recipe, + "vllm", + {**ENV, "SPEC_DECODING": "dspark"}, + golden_dir=golden_dir, + ), + ) + spec = json.loads(tp8["roles"]["agg"]["args"]["speculative-config"]) + assert spec["synthetic_acceptance_length"] == 2.7 + assert spec["rejection_sample_method"] == "synthetic" + + @pytest.mark.parametrize("sampling", [None, "unknown"]) def test_kimi_curve_requires_an_explicit_supported_sampler( golden_dir: Path, sampling: str | None diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 2334965697..ad9225c89c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9392,6 +9392,20 @@ - "Run TP4 at concurrency 1, 4, 8 and 32 and replace TP2 with DEP2 (TP1 x DP2 + EP2) at concurrency 8-32 and DEP4 (TP1 x DP4 + EP4) at concurrency 64-128, both with DeepGEMM MegaMoE and adaptive verification behind a consistent-hash vLLM Router. All points set gpu-memory-utilization 0.97." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3692 +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang-agentic-agg + - dsv4-fp4-gb200-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Move the GB200 DeepSeek-V4-Pro AgentX disaggregated recipes from hierarchical cache to the Mooncake unified-cache external linker with group semantics, as #3187 did for GB300. MoE settings are unchanged: the recipes keep MegaMoE without W4A4 MXFP4." + - "Decode nodes lend host DRAM to the store through srt-slurm mooncake-store services: four 180GB standalone stores per decode node, gated on their readiness ports with a 600-second timeout. Prefill keeps a 140GB segment per rank. GB200 nodes have the same host memory as GB300 nodes, so the sizes carry over." + - "The prefill role and the decode stores name their RDMA devices (MOONCAKE_DEVICE=mlx5_0..3, as the GB200 vLLM Mooncake recipes do) and set MC_TCP_BIND_ADDRESS to their node, so each store transfer engine advertises the node address instead of the first active interface." + - "The benchmark client runs on the dedicated etcd/nats node, because prefill nodes now also hold Mooncake segments, and Tachometer's per-node exporters are off for this throughput ladder." + - "The aggregate TP8 c1/c4 recipes are unchanged and stay on hierarchical cache; their config key is listed so this sweep covers the whole GB200 Pareto ladder, including the low-latency points." + - "Measured at concurrency 128 (1P DEP8 / 1D DEP16) against the current recipe with the same image, srt-slurm and golden acceptance length: 26,566 vs 23,763 total tok/s/GPU (+11.8%) at the same P90 TPOT (7.31 vs 7.22 ms), with TTFT p50 3.27 vs 4.95 s." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3701 + - config-keys: - "*" description: @@ -9529,6 +9543,15 @@ - "TP2 c128: admit all 128 sessions (max-running-requests and cuda-graph-max-bs-decode 64 -> 128) with a 4-step prefill-decode interval and mem-fraction-static 0.85. The PR #3421 sweep measured 18,631 tok/s/GPU at 130.9 tok/s/user median at this point; locally the new settings gave 147,763 tok/s/GPU at 37.9 tok/s/user median with a 51 s P90 TTFT (measured before the fold was added)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3782 +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use five EAGLE speculative steps and six draft tokens at concurrency 1, 4, 8, and 20 on GLM-5.2 NVFP4 B300 SGLang." + - "Keep three speculative steps and four draft tokens at the other concurrencies." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3890 + - config-keys: - dsv4-fp4-mi355x-sglang-agentic-mtp scenario-type: @@ -9538,3 +9561,62 @@ - "Run prefill context parallelism on the TP-only arms at conc 16, 32 and 48; conc 1 and 4 keep BCG." - "Enable SGLANG_OPT_FP8_WO_A_FUSED_INVROPE." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3778 + +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202610070333 to nightly_202610100929 (digest sha256:cc60ba06b28050a7fdec71e21f1853c782a9ec8850f031e84ebf0e49f3180974; ATOM 0.1.8.dev14+g73defd9b8, ROCm 7.2.4) and update model.container to match. The recipe, points and server settings are unchanged; upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged between 7fc0926c6 and 73defd9b8." + - "ROCm/ATOM#2507 turns decoder SWA bounded replay on by default for DeepSeek-V4.1: a prefill runs layers 21..39 on each request's last 133 rows only, the same approximation as SGLang's --enable-decoder-swa-bounded-replay. Upstream reports +10.7% / +22.7% / +38.1% P90 interactivity at TP2 c16 / c32 / c64 on this AgentX workload, with no GSM8K or long-context retrieval change beyond noise." + - "ROCm/ATOM#2512 samples the DSpark Markov block from each TP rank's vocab shard instead of all-gathering the full logits (bit-identical ids; draft head 657 -> 302 us per decode step at TP2 c32). ROCm/ATOM#2513 packs the FP4 index logits by row visibility and moves BF16 decode to the two-source paged kernel (needs ROCm/aiter#6334). ROCm/ATOM#2495 fixes the FP4 sparse indexer output contract." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3901 + +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202609291501 to rocm/atom-dev:nightly_202610071700 (digest sha256:bb3912c372dda0c82fc6831afdd76e366a372d5e9fe223373499db63b06cce65)." + - "Move the C48, C64, C96 and C128 DEP points from RCCL expert parallelism to ATOM's MegaMoE backend (all2all-backend high-throughput, moe-backend mega), with EPLB at every point except C64." + - "Cap max-num-seqs at 128 from C128 up (C128 was 256, C256 was 512); C48, C64 and C96 keep admission at 2x CONC." + - "Run C256 and the new C384 on the same MegaMoE DEP setup with ATOM's lmcache_mp CPU KV tier: one LMCache server per node (1000 GB L1, LMCache 0.5.6.dev139+gf22dec28 as shipped in the image), started by srtctl's lmcache-server service, within a 1019 GB budget (dram-utilization 0.34)." + - "C1 to C16 are unchanged." + - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4-Pro-0813 runs as shipped." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3819 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from nightly-rocm100-ac9126e58aa7bbab1856ba6593ba4d5003fea516 to nightly-rocm100-7d0b4e57aac4c4323225b0eee9ae960ce75ffffe, the first ROCm 10.0 nightly with the DeepSeek-V4.1 mono decode kernels of vllm-project/vllm#60397, and update model.container to match. The image also brings vllm-project/vllm#59685, #56638, #60159 and #60389." + - "Enable the mono decode kernels (VLLM_ROCM_MONO_DECODE=1) at TP2 and TP4: a decode step of up to 48 rows, i.e. concurrency 1-8 with five DSpark drafts, runs each backbone layer as two persistent launches (30 of 40 layers whole, the other 10 the FFN launch only). Larger steps take the regular path, so C16-C64 are expected unchanged." + - "Enable VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 (vllm-project/recipes#1061) for tensor-parallel all-reduces above the quick-reduce size threshold." + - "Restore --max-num-batched-tokens 16384 at TP2 c64: vllm-project/vllm#58014 removed the profiling over-reservation the 8192 cap worked around. The prefill chunk is now 16384 at every point and the ladder stays at concurrency 1-64." + - "No data-type or precision change to the DSpark draft: vllm-project/vllm#60397 skips the draft layers (layer id >= num_hidden_layers), so the bundled DSpark draft of deepseek-ai/DeepSeek-V4.1-Flash runs on vLLM's regular path as shipped." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3690 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image from vllm/vllm-openai-rocm:nightly-rocm100-36768d1bfd39094681cdbc8cb37d4b31c0729c89 to vllm/vllm-openai-rocm:nightly-rocm100-81198e97ba7eee2a22540caaa756b7fdddcb4d93." + - "Sweep dcp1 concurrency 1 and dcp8 concurrency 1 GPU-resident, and dcp8 concurrency 4, 10, 12, 14, 16, 44 and 48 with the SimpleCPUOffloadConnector DRAM tier (224.875 GB/rank). This replaces dcp1 concurrency 1/4, dcp1 DRAM-offload concurrency 8-14 and dcp8 concurrency 44/48/70. The dcp8 points use the a2a DCP backend." + - "DSpark draft length per point: 7 at concurrency 1, 5 at 4, 4 at 10, 3 at 12, and 4 at 14, 16, 44 and 48 (concurrency 14 was 3). The throughput band (44, 48) previously had no speculative-config and now runs the 4-token DSpark draft. As at every draft point, agentic runs replace block rejection with the golden acceptance length for that draft length." + - "Admission is 2x concurrency. CUDA graphs capture every even size up to 2x concurrency x (1 + draft tokens), replacing the previous every-size lists (up to 8192 in the throughput band)." + - "Every point requests cudagraph_mode FULL_AND_PIECEWISE (the mid band was FULL) with breakable CUDA graphs disabled (VLLM_USE_BREAKABLE_CUDAGRAPH=0, previously 1 at concurrency 1 and 4). The AMD Kimi-K3 model has no torch.compile submodule, so vLLM overrides the mode to FULL_DECODE_ONLY at load, as the concurrency 1 and 14 server logs show." + - "Add stream-interval 10 and VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm, which routes DCP multi-token verify through the AITER ASM round-robin decode, to every point." + - "Concurrency 1 (dcp1 and dcp8) adds quantization-config linear fp8_per_channel: vLLM quantizes the target's linear layers to FP8 W8A8 at load, except kv_b_proj, in_proj_qkvgfab, f_b_proj, fused_qkv_a_proj, self_attn.g_proj and layer 0 mlp.down_proj, which stay BF16. The routed experts stay MXFP4." + - "No change to the Inferact/Kimi-K3-DSpark draft's precision: its model config takes the speculative config's own quantization (unset) and never receives quantization-config, and get_draft_quant_config() returns None for K3DSparkModel, so its weights stay BF16 and its KV cache stays fp8." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3833 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Match the Sep 28 standalone llm-d AgentX speculative policy: disagg and agg DEP8 (including Mooncake) use adaptive verification with prefill K=1 and decode/agg K=3; only agg TP8 keeps golden synthetic acceptance at K=5 (3.61). The srt-slurm synthetic-acceptance connector skips golden AL injection when enable_adaptive_verification is true." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741