diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml index 9a2b0265a1..541934f0af 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml index 9461990907..a44d935fda 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml @@ -2,7 +2,7 @@ schema: 2 name: "disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload" # GB200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + HiCache KV offload), tuned for +# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + Mooncake unified-cache KV offload), tuned for # concurrency 128. # The DEP8 prefill worker spans two nodes; the DEP16 decode worker spans four. @@ -28,6 +28,11 @@ health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -42,6 +47,54 @@ services: node: dedicated options: max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 + # Decode ranks use a chunk cache, so their host DRAM only joins the store + # through standalone stores: four per decode node, so segments stay uniform + # with prefill's per-rank ones instead of one oversized segment per node. + - &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -95,6 +148,18 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -130,11 +195,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' decode: nodes: 4 @@ -169,6 +232,8 @@ roles: SGLANG_LOG_FORWARD_ITERS: '1' SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -215,6 +280,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml index d3bcb51688..f0181c16ad 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml @@ -2,7 +2,7 @@ schema: 2 name: "disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload" # GB200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + HiCache KV offload), tuned for +# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + Mooncake unified-cache KV offload), tuned for # concurrency 64. # The DEP8 prefill worker spans two nodes; the DEP16 decode worker spans four. @@ -28,6 +28,11 @@ health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -42,6 +47,54 @@ services: node: dedicated options: max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 + # Decode ranks use a chunk cache, so their host DRAM only joins the store + # through standalone stores: four per decode node, so segments stay uniform + # with prefill's per-rank ones instead of one oversized segment per node. + - &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 + - <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -95,6 +148,18 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -130,11 +195,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' decode: nodes: 4 @@ -169,6 +232,8 @@ roles: SGLANG_LOG_FORWARD_ITERS: '1' SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 @@ -215,6 +280,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml index 5a21654eb0..f1672f9e54 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml index c597addbe6..ac2ed56b08 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml index 4b7b7ff7b0..ff4cd0a15f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml @@ -18,6 +18,11 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 +# Tachometer's per-node exporters slow decode steps by a few percent and add +# host memory on the head node; this ladder is a throughput measurement. +observability: + tachometer: + enabled: false resources: gpu_type: gb200 gpus_per_node: 4 @@ -32,6 +37,54 @@ services: node: dedicated options: max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 +# Decode ranks use a chunk cache, so their host DRAM only joins the store +# through standalone stores: four per decode node, so segments stay uniform +# with prefill's per-rank ones instead of one oversized segment per node. +- &decode-store + name: store-decode-0 + type: mooncake-store + placement: + node: decode + args: ["--port", "8800"] + env: + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node_ip}" + preamble: | + ulimit -n 1048576 + ulimit -l unlimited + # Cold container starts on a decode node can exceed srtctl's 120 s default; + # the <<: merge is shallow, so every store repeats the timeout. + readiness: + port: 8800 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-1 + args: ["--port", "8801"] + readiness: + port: 8801 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-2 + args: ["--port", "8802"] + readiness: + port: 8802 + timeout_seconds: 600 +- <<: *decode-store + name: store-decode-3 + args: ["--port", "8803"] + readiness: + port: 8803 + timeout_seconds: 600 frontend: type: dynamo nginx_session_affinity: true @@ -84,6 +137,18 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + # The store only registers host buffers on RDMA; without an explicit list it + # falls back to NVLink for same-domain peers and aborts at the warmup put. + MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + # Advertise the store transfer engine on the node address instead of the first active + # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node). + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -117,11 +182,9 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' cuda-graph-max-bs-decode: 256 decode: nodes: 8 @@ -157,6 +220,8 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,6 +264,10 @@ srun_options: benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh + # The replay client holds ~300 GB of host memory; on the default head node it + # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. + placement: + node: dedicated env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 1cb191a29a..c32f89b47b 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6858,7 +6858,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [64] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 1 tp: 8 @@ -6874,7 +6874,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [128] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 1 tp: 8 @@ -6890,7 +6890,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [256] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 1 tp: 16 @@ -6906,7 +6906,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [768] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 2 tp: 16 @@ -6922,7 +6922,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [1024] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 2 tp: 16 @@ -6938,7 +6938,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [1280] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13" } prefill: num-worker: 2 tp: 16 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 602fc4a4e7..058145a6d1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9392,6 +9392,20 @@ - "Run TP4 at concurrency 1, 4, 8 and 32 and replace TP2 with DEP2 (TP1 x DP2 + EP2) at concurrency 8-32 and DEP4 (TP1 x DP4 + EP4) at concurrency 64-128, both with DeepGEMM MegaMoE and adaptive verification behind a consistent-hash vLLM Router. All points set gpu-memory-utilization 0.97." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3692 +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang-agentic-agg + - dsv4-fp4-gb200-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Move the GB200 DeepSeek-V4-Pro AgentX disaggregated recipes from hierarchical cache to the Mooncake unified-cache external linker with group semantics, as #3187 did for GB300. MoE settings are unchanged: the recipes keep MegaMoE without W4A4 MXFP4." + - "Decode nodes lend host DRAM to the store through srt-slurm mooncake-store services: four 180GB standalone stores per decode node, gated on their readiness ports with a 600-second timeout. Prefill keeps a 140GB segment per rank. GB200 nodes have the same host memory as GB300 nodes, so the sizes carry over." + - "The prefill role and the decode stores name their RDMA devices (MOONCAKE_DEVICE=mlx5_0..3, as the GB200 vLLM Mooncake recipes do) and set MC_TCP_BIND_ADDRESS to their node, so each store transfer engine advertises the node address instead of the first active interface." + - "The benchmark client runs on the dedicated etcd/nats node, because prefill nodes now also hold Mooncake segments, and Tachometer's per-node exporters are off for this throughput ladder." + - "The aggregate TP8 c1/c4 recipes are unchanged and stay on hierarchical cache; their config key is listed so this sweep covers the whole GB200 Pareto ladder, including the low-latency points." + - "Measured at concurrency 128 (1P DEP8 / 1D DEP16) against the current recipe with the same image, srt-slurm and golden acceptance length: 26,566 vs 23,763 total tok/s/GPU (+11.8%) at the same P90 TPOT (7.31 vs 7.22 ms), with TTFT p50 3.27 vs 4.95 s." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3701 + - config-keys: - "*" description: