[AgentX] DeepSeek-V4-Pro-0813 llm-d on GB200

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml

@@ -18,6 +18,11 @@ slurm:
 health_check:
   max_attempts: 1440
   interval_seconds: 10
+# Tachometer's per-node exporters slow decode steps by a few percent and add
+# host memory on the head node; this ladder is a throughput measurement.
+observability:
+  tachometer:
+    enabled: false
 resources:
   gpu_type: gb200
   gpus_per_node: 4
@@ -32,6 +37,54 @@ services:
     node: dedicated
   options:
     max_payload_mb: 32
+- name: mooncake-master
+  type: mooncake-master
+  args:
+    - --eviction_high_watermark_ratio=0.90
+# Decode ranks use a chunk cache, so their host DRAM only joins the store
+# through standalone stores: four per decode node, so segments stay uniform
+# with prefill's per-rank ones instead of one oversized segment per node.
+- &decode-store
+  name: store-decode-0
+  type: mooncake-store
+  placement:
+    node: decode
+  args: ["--port", "8800"]
+  env:
+    MOONCAKE_PROTOCOL: rdma
+    MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+    MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb
+    MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+    MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+    # Advertise the store transfer engine on the node address instead of the first active
+    # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node).
+    MC_TCP_BIND_ADDRESS: "{node_ip}"
+  preamble: |
+    ulimit -n 1048576
+    ulimit -l unlimited
+  # Cold container starts on a decode node can exceed srtctl's 120 s default;
+  # the <<: merge is shallow, so every store repeats the timeout.
+  readiness:
+    port: 8800
+    timeout_seconds: 600
+- <<: *decode-store
+  name: store-decode-1
+  args: ["--port", "8801"]
+  readiness:
+    port: 8801
+    timeout_seconds: 600
+- <<: *decode-store
+  name: store-decode-2
+  args: ["--port", "8802"]
+  readiness:
+    port: 8802
+    timeout_seconds: 600
+- <<: *decode-store
+  name: store-decode-3
+  args: ["--port", "8803"]
+  readiness:
+    port: 8803
+    timeout_seconds: 600
 frontend:
   type: dynamo
   nginx_session_affinity: true
@@ -84,6 +137,18 @@ roles:
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
       SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1'
       SGLANG_DSV4_MHC_PREWARM: '1'
+      MOONCAKE_PROTOCOL: rdma
+      # The store only registers host buffers on RDMA; without an explicit list it
+      # falls back to NVLink for same-domain peers and aborts at the warmup put.
+      MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      # Advertise the store transfer engine on the node address instead of the first active
+      # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node).
+      MC_TCP_BIND_ADDRESS: "{node}"
+      WITH_NVIDIA_PEERMEM: '0'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
@@ -117,11 +182,9 @@ roles:
       speculative-num-steps: 1
       speculative-eagle-topk: 1
       speculative-num-draft-tokens: 7
-      enable-hierarchical-cache: true
-      hicache-write-policy: write_back
-      hicache-ratio: 2
-      hicache-io-backend: direct
-      hicache-mem-layout: page_first_direct
+      enable-unified-cache-external-linker: true
+      unified-cache-external-linker-backend: mooncake
+      hicache-storage-backend-extra-config: '{"enable_group_semantics":true}'
       cuda-graph-max-bs-decode: 256
   decode:
     nodes: 8
@@ -157,6 +220,8 @@ roles:
       SGLANG_LOG_MS: '1'
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
       SGLANG_DSV4_MHC_PREWARM: '1'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
@@ -199,6 +264,10 @@ srun_options:
 benchmark:
   type: custom
   command: bash /infmax-workspace/benchmarks/srt_agentic.sh
+  # The replay client holds ~300 GB of host memory; on the default head node it
+  # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead.
+  placement:
+    node: dedicated
   env:
     INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
     RESULT_DIR: /logs/agentic

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml

@@ -2,7 +2,7 @@ schema: 2
 name: "disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload"
 
 # GB200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813
-# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + HiCache KV offload), tuned for
+# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + Mooncake unified-cache KV offload), tuned for
 # concurrency 128.
 # The DEP8 prefill worker spans two nodes; the DEP16 decode worker spans four.
 
@@ -28,6 +28,11 @@ health_check:
   max_attempts: 1440
   interval_seconds: 10
 
+# Tachometer's per-node exporters slow decode steps by a few percent and add
+# host memory on the head node; this ladder is a throughput measurement.
+observability:
+  tachometer:
+    enabled: false
 resources:
   gpu_type: gb200
   gpus_per_node: 4
@@ -42,6 +47,54 @@ services:
       node: dedicated
     options:
       max_payload_mb: 32
+  - name: mooncake-master
+    type: mooncake-master
+    args:
+      - --eviction_high_watermark_ratio=0.90
+  # Decode ranks use a chunk cache, so their host DRAM only joins the store
+  # through standalone stores: four per decode node, so segments stay uniform
+  # with prefill's per-rank ones instead of one oversized segment per node.
+  - &decode-store
+    name: store-decode-0
+    type: mooncake-store
+    placement:
+      node: decode
+    args: ["--port", "8800"]
+    env:
+      MOONCAKE_PROTOCOL: rdma
+      MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      # Advertise the store transfer engine on the node address instead of the first active
+      # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node).
+      MC_TCP_BIND_ADDRESS: "{node_ip}"
+    preamble: |
+      ulimit -n 1048576
+      ulimit -l unlimited
+    # Cold container starts on a decode node can exceed srtctl's 120 s default;
+    # the <<: merge is shallow, so every store repeats the timeout.
+    readiness:
+      port: 8800
+      timeout_seconds: 600
+  - <<: *decode-store
+    name: store-decode-1
+    args: ["--port", "8801"]
+    readiness:
+      port: 8801
+      timeout_seconds: 600
+  - <<: *decode-store
+    name: store-decode-2
+    args: ["--port", "8802"]
+    readiness:
+      port: 8802
+      timeout_seconds: 600
+  - <<: *decode-store
+    name: store-decode-3
+    args: ["--port", "8803"]
+    readiness:
+      port: 8803
+      timeout_seconds: 600
 frontend:
   type: dynamo
   nginx_session_affinity: true
@@ -95,6 +148,18 @@ roles:
       SGLANG_LOG_MS: '1'
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
       SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1'
+      MOONCAKE_PROTOCOL: rdma
+      # The store only registers host buffers on RDMA; without an explicit list it
+      # falls back to NVLink for same-domain peers and aborts at the warmup put.
+      MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      # Advertise the store transfer engine on the node address instead of the first active
+      # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node).
+      MC_TCP_BIND_ADDRESS: "{node}"
+      WITH_NVIDIA_PEERMEM: '0'
 
     args:
       host: 0.0.0.0
@@ -130,11 +195,9 @@ roles:
       speculative-num-steps: 1
       speculative-eagle-topk: 1
       speculative-num-draft-tokens: 7
-      enable-hierarchical-cache: true
-      hicache-write-policy: write_back
-      hicache-ratio: 8
-      hicache-io-backend: direct
-      hicache-mem-layout: page_first_direct
+      enable-unified-cache-external-linker: true
+      unified-cache-external-linker-backend: mooncake
+      hicache-storage-backend-extra-config: '{"enable_group_semantics":true}'
 
   decode:
     nodes: 4
@@ -169,6 +232,8 @@ roles:
       SGLANG_LOG_FORWARD_ITERS: '1'
       SGLANG_LOG_MS: '1'
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
 
     args:
       host: 0.0.0.0
@@ -215,6 +280,10 @@ srun_options:
 benchmark:
   type: custom
   command: bash /infmax-workspace/benchmarks/srt_agentic.sh
+  # The replay client holds ~300 GB of host memory; on the default head node it
+  # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead.
+  placement:
+    node: dedicated
   env:
     INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
     RESULT_DIR: /logs/agentic

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml

@@ -2,7 +2,7 @@ schema: 2
 name: "disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload"
 
 # GB200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813
-# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + HiCache KV offload), tuned for
+# (1P x DEP8 / 1D x DEP16, bundled DSpark K=6 + Mooncake unified-cache KV offload), tuned for
 # concurrency 64.
 # The DEP8 prefill worker spans two nodes; the DEP16 decode worker spans four.
 
@@ -28,6 +28,11 @@ health_check:
   max_attempts: 1440
   interval_seconds: 10
 
+# Tachometer's per-node exporters slow decode steps by a few percent and add
+# host memory on the head node; this ladder is a throughput measurement.
+observability:
+  tachometer:
+    enabled: false
 resources:
   gpu_type: gb200
   gpus_per_node: 4
@@ -42,6 +47,54 @@ services:
       node: dedicated
     options:
       max_payload_mb: 32
+  - name: mooncake-master
+    type: mooncake-master
+    args:
+      - --eviction_high_watermark_ratio=0.90
+  # Decode ranks use a chunk cache, so their host DRAM only joins the store
+  # through standalone stores: four per decode node, so segments stay uniform
+  # with prefill's per-rank ones instead of one oversized segment per node.
+  - &decode-store
+    name: store-decode-0
+    type: mooncake-store
+    placement:
+      node: decode
+    args: ["--port", "8800"]
+    env:
+      MOONCAKE_PROTOCOL: rdma
+      MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 180gb
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      # Advertise the store transfer engine on the node address instead of the first active
+      # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node).
+      MC_TCP_BIND_ADDRESS: "{node_ip}"
+    preamble: |
+      ulimit -n 1048576
+      ulimit -l unlimited
+    # Cold container starts on a decode node can exceed srtctl's 120 s default;
+    # the <<: merge is shallow, so every store repeats the timeout.
+    readiness:
+      port: 8800
+      timeout_seconds: 600
+  - <<: *decode-store
+    name: store-decode-1
+    args: ["--port", "8801"]
+    readiness:
+      port: 8801
+      timeout_seconds: 600
+  - <<: *decode-store
+    name: store-decode-2
+    args: ["--port", "8802"]
+    readiness:
+      port: 8802
+      timeout_seconds: 600
+  - <<: *decode-store
+    name: store-decode-3
+    args: ["--port", "8803"]
+    readiness:
+      port: 8803
+      timeout_seconds: 600
 frontend:
   type: dynamo
   nginx_session_affinity: true
@@ -95,6 +148,18 @@ roles:
       SGLANG_LOG_MS: '1'
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
       SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1'
+      MOONCAKE_PROTOCOL: rdma
+      # The store only registers host buffers on RDMA; without an explicit list it
+      # falls back to NVLink for same-domain peers and aborts at the warmup put.
+      MOONCAKE_DEVICE: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      # Advertise the store transfer engine on the node address instead of the first active
+      # interface, which can be a BMC Redfish host interface (the same 10.0.1.2 on every node).
+      MC_TCP_BIND_ADDRESS: "{node}"
+      WITH_NVIDIA_PEERMEM: '0'
 
     args:
       host: 0.0.0.0
@@ -130,11 +195,9 @@ roles:
       speculative-num-steps: 1
       speculative-eagle-topk: 1
       speculative-num-draft-tokens: 7
-      enable-hierarchical-cache: true
-      hicache-write-policy: write_back
-      hicache-ratio: 8
-      hicache-io-backend: direct
-      hicache-mem-layout: page_first_direct
+      enable-unified-cache-external-linker: true
+      unified-cache-external-linker-backend: mooncake
+      hicache-storage-backend-extra-config: '{"enable_group_semantics":true}'
 
   decode:
     nodes: 4
@@ -169,6 +232,8 @@ roles:
       SGLANG_LOG_FORWARD_ITERS: '1'
       SGLANG_LOG_MS: '1'
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
 
     args:
       host: 0.0.0.0
@@ -215,6 +280,10 @@ srun_options:
 benchmark:
   type: custom
   command: bash /infmax-workspace/benchmarks/srt_agentic.sh
+  # The replay client holds ~300 GB of host memory; on the default head node it
+  # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead.
+  placement:
+    node: dedicated
   env:
     INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
     RESULT_DIR: /logs/agentic