diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx index 5470bc51f009..bfc01de3e2c6 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx @@ -232,6 +232,21 @@ This section provides deployment configurations optimized for different hardware - **Xeon CPU service configuration**: Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. + + +### 3.3 Agentic Long-Context Deployment + +Agentic workloads, such as coding agents and other multi-turn, tool-using assistants, send long prompts whose prefixes grow and repeat from turn to turn. Serving them well depends on reusing the KV cache across turns, offloading it beyond GPU memory, and keeping each session on the workers that already hold its prefix. This section gives complete launch commands for such deployments of Qwen3.5-397B-A17B, each benchmarked in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) and linked to the run that measured it. Pick a hardware platform, checkpoint, deployment shape, concurrency, KV cache offloading mechanism and router to get every process of the deployment as a raw command, one tab per process in launch order. + +- **Router.** Each configuration was submitted behind one front door: Dynamo (`dynamo.frontend` with `dynamo.sglang` workers) or SGLang (`sglang.launch_server`, behind `sglang_router` when there are several workers or DP-attention ranks). +- **KV offload.** Choose among these KV cache offloading mechanisms: none (GPU KV only), HiCache (host DRAM), or a Mooncake store attached through the external linker. The Mooncake option needs an image whose SGLang build includes `--enable-unified-cache-external-linker`. +- **Container image.** Run the commands inside the image listed for each configuration. Several are SGLang nightly images, so flag names follow that image. + +import { AgentX } from "/src/snippets/_agentx.jsx"; +import { agentx } from "/src/snippets/agentx/Qwen/qwen3.5.jsx"; + + + ## 4. Model Invocation **NVIDIA:** @@ -1023,44 +1038,6 @@ Max ITL (ms): 1220.68 ================================================== ``` -#### 5.2.3 Agentic Long-Context with HiCache DRAM Offload (H200 FP8, MTP) - -For agentic workloads, here is how to enable HiCache and MTP. - -Container image (pinned for reproducibility): -`lmsysorg/sglang:nightly-dev-cu13-20260815-a5ba081f`. - -Server Launch Command: -```bash Command -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3.5-397B-A17B-FP8 \ - --served-model-name Qwen/Qwen3.5-397B-A17B-FP8 \ - --trust-remote-code \ - --tensor-parallel-size 8 \ - --data-parallel-size 1 \ - --expert-parallel-size 1 \ - --quantization fp8 \ - --kv-cache-dtype fp8_e4m3 \ - --mamba-ssm-dtype bfloat16 \ - --attention-backend flashinfer \ - --enable-flashinfer-allreduce-fusion \ - --mem-fraction-static 0.8 \ - --stream-interval 50 \ - --scheduler-recv-interval 10 \ - --tokenizer-worker-num 6 \ - --enable-metrics \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --page-size 64 \ - --enable-hierarchical-cache \ - --hicache-size 77 \ - --hicache-io-backend kernel \ - --hicache-mem-layout page_first \ - --hicache-write-policy write_through_selective -``` - ### 5.3 Vision Speed Benchmark We use SGLang's built-in benchmarking tool to conduct performance evaluation with random images. Each request has 128 input tokens, two 720p images, and 1024 output tokens. diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx index 9fd95259df19..dd5654c22e23 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx @@ -26,7 +26,7 @@ import { Deployment } from "/src/snippets/_deployment.jsx"; import { config } from "/src/snippets/configs/Qwen/qwen3.8-flash-next.jsx"; import { benchmarks } from "/src/snippets/configs/Qwen/qwen3.8-flash-next-benchmarks.jsx"; - + @@ -69,6 +69,19 @@ import { Playground } from "/src/snippets/_playground.jsx"; +## Agentic Long-Context Deployment + +Agentic workloads, such as coding agents and other multi-turn, tool-using assistants, send long prompts whose prefixes grow and repeat from turn to turn. Serving them well depends on reusing the KV cache across turns, offloading it beyond GPU memory, and keeping each session on the workers that already hold its prefix. This section gives complete launch commands for such deployments, each benchmarked in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) and linked to the run that measured it. Pick a hardware platform, checkpoint, deployment shape, concurrency, KV cache offloading mechanism and router to get every process of the deployment as a raw command, one tab per process in launch order. + +- **Router.** Each configuration was submitted behind one front door: Dynamo (`dynamo.frontend` with `dynamo.sglang` workers) or SGLang (`sglang.launch_server`, behind `sglang_router` when there are several workers or DP-attention ranks). +- **KV offload.** Choose among these KV cache offloading mechanisms: none (GPU KV only), HiCache (host DRAM), or a Mooncake store attached through the external linker. The Mooncake option needs an image whose SGLang build includes `--enable-unified-cache-external-linker`. +- **Container image.** Run the commands inside the image listed for each configuration. Several are SGLang nightly images, so flag names follow that image. + +import { AgentX } from "/src/snippets/_agentx.jsx"; +import { agentx } from "/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx"; + + + ## 1. Model Introduction **Qwen3.8-Flash-Next** is a multimodal Mixture-of-Experts model released as an early preview of the architecture Qwen4 is being built on — the same role Qwen3-Next played for Qwen3.5, whose hybrid Gated DeltaNet + Gated Attention design then carried through the Qwen3.5, Qwen3.6, Qwen3.7 and Qwen3.8 series. Qwen is publishing the architectural changes ahead of the full Qwen4 family so the community can evaluate them independently. @@ -218,45 +231,3 @@ Pending update — a sample transcript will be added here once the weights are p ``` - -### 2.3 GB300 long-context serving with HiCache and MTP - -Use this single-GPU, aggregated TP1/EP1 configuration for long-context, multi-turn workloads on GB300. It combines the NVIDIA NVFP4 checkpoint's embedded MTP head, CPU DRAM KV offload through HiCache, and pinned-memory PLE embedding offload. - -This serving configuration comes from [InferenceX #3816](https://github.com/SemiAnalysisAI/InferenceX/pull/3816). [Run 37714835620, attempt 1](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37714835620) passed concurrency 1, 2, 4, 8, 16, 32, 48 and 64 on commit `e67ca6fc9ecf4cd57f27654bda8a3e9059923820`. Its [matching GSM8K eval](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37714835620/job/113109083376) used real verification and scored 97.35% on 1,319 questions. - -The throughput sweep used InferenceX's simulated acceptance; the generated command uses normal MTP verification and does not claim those throughput numbers for production traffic. - -#### Prerequisites and memory budget - -- Use a Linux ARM64 GB300 host with Docker, NVIDIA Container Toolkit, and a driver compatible with the image's CUDA 13.0 runtime. See [Docker installation](/docs/get-started/install#method-3-using-docker). -- Reserve one GB300 GPU. The recipe keeps TP, PP, DP and EP at 1. -- Budget 224 GiB of CPU memory for the combined target/draft KV and Mamba HiCache host pool, plus approximately 48 GiB for the offloaded FP8 PLE table and additional headroom for checkpoint loading and host processes. `--hicache-size` sets the combined cache budget; it is separate from PLE weight offload. Keep sufficient free host memory for both. -- The Docker command mounts your Hugging Face cache to retain downloaded model files across restarts. The image digest and both target/draft checkpoint revisions are pinned to the validated configuration. - -#### Generate the launch command - -In the [Deployment panel](#deployment), select **GB300 → NVFP4 (NVDA) → Low Latency → Single Node**, and leave **PLE Offload** at **Auto** or **On**. The matched cell includes all five serving environment variables and the full argument set, including ReplaySSM, HiCache and pinned PLE offload. Choose **Docker** to use the pinned image, or **Python** for a matching local installation. Changing PLE Offload to **Off** produces an unverified configuration. - -The Docker wrapper comes from the shared command builder and is adapted from the validated Slurm/Enroot launch; it has not been run separately on a GB300 host. Use the command generated by the panel rather than a separate static copy. - - - -The validated run loaded the target and embedded draft through the stock image's mixed-precision loader. This excerpt is from the real-verification eval; timings and memory availability vary by host. - -```text Output -[2026-10-08 02:22:36] Load weight end. elapsed=1011.56 s, type=Qwen4ExpForConditionalGeneration, quant=modelopt_mixed, quant_algo=MIXED_PRECISION, avail mem=202.03 GB, mem usage=73.64 GB. -[2026-10-08 02:23:03] Load weight end. elapsed=25.97 s, type=Qwen4ExpForCausalLMMTP, quant=modelopt_mixed, quant_algo=MIXED_PRECISION, avail mem=197.09 GB, mem usage=4.89 GB. -``` - - - -#### Serving choices - -- **Draft precision:** revision `fc694b54` stores the MTP routed experts in block-scaled FP8 and the rest of the head in BF16. The pinned image, built from SGLang `e00930c5`, preserves this per-layer `modelopt_mixed` configuration. The draft uses the same checkpoint revision as the target. `SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE=0` disables the optional draft precision conversion path; target and draft KV caches stay in BF16. -- **Hybrid attention:** the recipe selects FlashInfer for linear-attention prefill/decode, Triton for linear verification, and `trtllm_mha` as the attention backend. ReplaySSM speculation, `extra_buffer_lazy`, a tracking interval of 256 and 480 Mamba cache slots retain the qualified state-cache configuration. -- **Admission and graphs:** the server admits up to 96 requests, uses 16,384-token prefill chunks, and captures full decode graphs at the listed batch sizes up to 96. Prefill graphs remain disabled. The qualification varied client concurrency while retaining these server settings. -- **CPU offload:** HiCache uses the `kernel` I/O backend, `page_first` layout and `write_back` policy. PLE uses a separate pinned-memory embedding table. See [HiCache design](/docs/advanced_features/hicache_design) for the cache tiers and policies. -- **API and telemetry:** send requests through the chat-completions API at `http://localhost:30000/v1`, using model name `nvidia/Qwen3.8-Flash-Next-NVFP4`. `--reasoning-parser auto` separates reasoning from answer content. Engine metrics and cache reporting remain enabled. - -The generated command omits arguments that match the pinned image's defaults, including TP/PP/DP/EP=1, the inherited served-model name and draft MoE backend, full decode graphs, and default Mamba tracking, HiCache layout, PLE backend, tokenizer and scheduler settings. It retains the qualified lane's non-default serving settings. InferenceX-only trace loading, simulated-acceptance controls, client concurrency overrides, result collection and Slurm allocation settings are excluded. diff --git a/docs/src/snippets/_agentx.jsx b/docs/src/snippets/_agentx.jsx index ded17304fe6f..75085e61bbc6 100644 --- a/docs/src/snippets/_agentx.jsx +++ b/docs/src/snippets/_agentx.jsx @@ -191,8 +191,10 @@ export const AgentX = ({ data }) => { }; const noteText = (S) => ({ "rename-dp-attention": <>Flags are mirrored from the submission image: --enable-dp-attention with --dp-size / --data-parallel-size. Newer SGLang deprecates them in favor of --attn-dp-size (#41818)., + "rename-mamba-strategy": <>Flags are mirrored from the submission image: --mamba-scheduler-strategy is the older spelling of --mamba-radix-cache-strategy (#28151). Current SGLang accepts only the new name., "hicache-size": <>--hicache-size is host DRAM per rank in GB, sized to SemiAnalysis's nodes. Scale it to your host memory., efa: <>MOONCAKE_PROTOCOL=efa reflects SemiAnalysis's AWS-EFA B300 cluster. Use rdma on InfiniBand., + "nixl-libfabric": <>SGLANG_DISAGGREGATION_NIXL_BACKEND=LIBFABRIC reflects SemiAnalysis's AWS-EFA B300 cluster. On InfiniBand, leave it unset to use NIXL's default UCX backend., "ib-devices": <>Set IB_DEVICES to your node's RDMA NICs (comma-separated, e.g. mlx5_0,mlx5_1)., "mooncake-segment": <>MOONCAKE_GLOBAL_SEGMENT_SIZE is the host DRAM each worker process contributes to the Mooncake store. Size it to your host memory., }); @@ -202,8 +204,10 @@ export const AgentX = ({ data }) => { const text = JSON.stringify(blocks); return [ names.has("--enable-dp-attention") && "rename-dp-attention", + names.has("--mamba-scheduler-strategy") && "rename-mamba-strategy", names.has("--hicache-size") && "hicache-size", env.MOONCAKE_PROTOCOL === "efa" && "efa", + env.SGLANG_DISAGGREGATION_NIXL_BACKEND === "LIBFABRIC" && "nixl-libfabric", env.MOONCAKE_GLOBAL_SEGMENT_SIZE !== undefined && "mooncake-segment", text.includes("$IB_DEVICES") && "ib-devices", ].filter(Boolean); @@ -582,7 +586,7 @@ export const AgentX = ({ data }) => { )} {notesFor(blocks).map((n) => noteText(S)[n] &&
  • {noteText(S)[n]}
    • )} {routerData.install.map((cmd) => ( -
  • {cmd.startsWith("#") ? <>Dynamo: {cmd.replace(/^# /, "")} : <>Dynamo install used by SA: {cmd}}
    • +
  • {cmd.startsWith("#") ? <>Dynamo: {cmd.replace(/^# /, "")} : router === cell.submitted ? <>Dynamo install used by SA: {cmd} : <>This image does not bundle ai-dynamo. Install the release built for its SGLang version first: {cmd}}
    • ))} {router === "dynamo" && !routerData.install.length &&
  • Dynamo: the ai-dynamo release bundled in the image.
    • } {cell.setup.map((s, i) => ( diff --git a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx new file mode 100644 index 000000000000..665a7824e3ad --- /dev/null +++ b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx @@ -0,0 +1,12051 @@ +// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. +// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions +// (SemiAnalysisAI/InferenceX main@ceb50ba21208 plus open PRs #2978, #3606, #3701, #3744, #3782, #3816, #3817, #3836, #3840), with SA-specific settings removed. +export const agentx = { + "model": "Qwen3.5", + "generatedOn": "2026-10-08", + "source": { + "repo": "https://github.com/SemiAnalysisAI/InferenceX", + "main": "ceb50ba212080174e0563d2be1db9b0d654ab2ef", + "prs": { + "3816": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37714835620", + "sha": "e67ca6fc9ecf4cd57f27654bda8a3e9059923820", + "status": "verified-pr" + }, + "3840": { + "run": null, + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "status": "in-progress" + }, + "2978": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34548026742", + "sha": "6b7dcd07ab0c8c0805ac7fbec2f2ded38d450c20", + "status": "verified-pr" + }, + "3817": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", + "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", + "status": "verified-pr" + }, + "3606": { + "run": null, + "sha": "5b394d395eefcaacd5168e04aca95020adac5312", + "status": "in-progress" + }, + "3701": { + "run": null, + "sha": "d9185455e166dfd034dcdfaf9608bd8335618b54", + "status": "in-progress" + }, + "3782": { + "run": null, + "sha": "f00c81ab40297f98a68d2726d4301b44a2892419", + "status": "in-progress" + }, + "3744": { + "run": null, + "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "status": "in-progress" + }, + "3836": { + "run": null, + "sha": "03ffb35db157a0fecc78206bea748449daa1385a", + "status": "in-progress" + } + } + }, + "hardware": [ + { + "id": "gb300", + "label": "GB300" + }, + { + "id": "gb200", + "label": "GB200" + }, + { + "id": "b300", + "label": "B300" + }, + { + "id": "b200", + "label": "B200" + }, + { + "id": "h200", + "label": "H200" + }, + { + "id": "h100", + "label": "H100" + } + ], + "checkpoints": [ + { + "id": "Qwen/Qwen3.5-397B-A17B-FP8" + }, + { + "id": "nvidia/Qwen3.5-397B-A17B-NVFP4" + }, + { + "id": "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + } + ], + "envs": [ + { + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "MC_FORCE_MNNVL": "1", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1", + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", + "SGLANG_ENABLE_JIT_DEEPGEMM": "true", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", + "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", + "SGLANG_MOE_NVFP4_DISPATCH": "1", + "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", + "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "1", + "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", + "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" + }, + { + "NCCL_NVLS_ENABLE": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_CUDA_ARCH_LIST": "10.0" + }, + { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "MC_INTRANODE_NVLINK": "1", + "NCCL_NVLS_ENABLE": "1", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", + "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "INTRA_NODE_NVLINK", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_CUDA_ARCH_LIST": "10.0" + }, + { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "MC_INTRANODE_NVLINK": "1", + "NCCL_NVLS_ENABLE": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "INTRA_NODE_NVLINK", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" + } + ], + "kvMooncake": { + "donor": "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300", + "master": { + "dynamo": { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": { + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + }, + "sglang": { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": {}, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + } + }, + "args": [ + ["--enable-unified-cache-external-linker"], + ["--hicache-storage-backend-extra-config", "{\"enable_group_semantics\":true}"], + ["--unified-cache-external-linker-backend", "mooncake"] + ], + "env": { + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", + "MC_TCP_BIND_ADDRESS": "$NODE_IP", + "MOONCAKE_DEVICE": "$IB_DEVICES", + "MOONCAKE_GLOBAL_SEGMENT_SIZE": "140gb", + "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", + "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", + "MOONCAKE_PROTOCOL": "rdma", + "MOONCAKE_STANDALONE_STORAGE": "0", + "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata" + }, + "decode_env": { + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", + "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", + "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", + "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata" + } + }, + "files": { + "nats.conf#48a4de48": { + "name": "nats.conf", + "content": "max_payload: 8388608\njetstream { store_dir: \"/tmp/nats\" }\n" + }, + "nginx.conf#033bddfb": { + "name": "nginx.conf", + "content": "# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n# SPDX-License-Identifier: Apache-2.0\n# Defines the group of servers to which NGINX will proxy requests.\n# NGINX will cycle through these servers in a round-robin fashion by default.\nworker_processes auto;\n\n\nhttp {\n # Raise the request-body cap (nginx default is 1 MiB) so the load balancer\n # does not reject large client requests with 413 before they reach the\n # frontend. Agentic replay traces (e.g. multi-turn cc-traces warmup bodies)\n # exceed 1 MiB; this mirrors the dynamo frontend's own body limit so nginx\n # is never the bottleneck and is never more permissive than the backend.\n client_max_body_size 45m;\n access_log off;\n\n # Keep a Dynamo session on one frontend. Anonymous requests use NGINX's\n # generated request ID so an empty session header cannot pin all traffic.\n\n map $http_x_dynamo_session_id $frontend_hash_key {\n default $http_x_dynamo_session_id;\n \"\" $request_id;\n }\n\n\n # Agentic replay reproduces recorded think-time, so a session's connection can\n # sit idle for minutes between turns. nginx's default keepalive_timeout of 75s\n # closes it underneath a pooled client connection, and the client's next write\n # on that socket surfaces as \"broken pipe\" / \"server disconnected\" with nothing\n # logged server-side.\n keepalive_timeout 600s;\n\n upstream backend_servers {\n\n hash $frontend_hash_key consistent;\n\n \n server $FRONTEND_0_IP:8180;\n \n server $FRONTEND_1_IP:8180;\n \n # Reuse upstream connections instead of a fresh TCP handshake per request.\n keepalive 512;\n keepalive_timeout 600s;\n }\n\n # The main server block that listens for incoming traffic.\n server {\n listen 8000; # Public-facing port for incoming HTTP requests.\n\n location / {\n # Pass the request to the upstream group defined above.\n proxy_pass http://backend_servers;\n\n proxy_buffering off;\n proxy_read_timeout 24h;\n proxy_send_timeout 24h;\n\n # Required for the upstream keepalive pool above: without these nginx\n # talks HTTP/1.0 to the backend and opens a new connection per request.\n proxy_http_version 1.1;\n proxy_set_header Connection \"\";\n }\n }\n}\n\nevents {\n\n #\n # Determines how many clients will be served by each worker process.\n # (Max clients = worker_connections * worker_processes)\n # Should be equal to `ulimit -n / worker_processes`\n #\n worker_connections 65535;\n\n #\n # Let each process accept multiple connections.\n # Accept as many connections as possible, after nginx gets notification\n # about a new connection.\n # May flood worker_connections, if that option is set too low.\n #\n multi_accept on;\n\n #\n # Preferred connection method for newer linux versions.\n # Essential for linux, optimized to serve many clients with each thread.\n #\n use epoll;\n}" + } + }, + "cells": [ + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve#0", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP4", + "detail": "MTP 6-1-7", + "gpus": 4, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "pr": "3840", + "status": "in-progress" + }, + "points": [ + { + "concs": [1], + "item": 1, + "goldenAL": "4.8", + "kv": "hicache", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml" + }, + { + "concs": [44], + "item": 0, + "goldenAL": "4.8", + "kv": "hicache", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "MC_FORCE_MNNVL": "1", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", + "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", + "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", + "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" + }, + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--context-length", "262144"], + ["--cuda-graph-max-bs-decode", "1"], + ["--data-parallel-size", "1"], + ["--disable-attn-tp-gather"], + ["--disable-prefill-cuda-graph"], + ["--disable-shared-experts-fusion"], + ["--enable-hierarchical-cache"], + ["--enable-linear-replayssm-spec"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-size", "32"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "triton"], + ["--load-balance-method", "round_robin"], + ["--mamba-max-states-per-path", "3"], + ["--mamba-radix-cache-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "8192"], + ["--max-mamba-cache-size", "1536"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "1"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "none"], + ["--moe-dense-tp-size", "4"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "1"], + ["--quantization", "fp8"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-moe-a2a-backend", "none"], + ["--speculative-moe-runner-backend", "flashinfer_trtllm"], + ["--speculative-num-draft-tokens", "7"], + ["--speculative-num-steps", "6"], + ["--stream-interval", "20"], + ["--tensor-parallel-size", "4"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"], + ["--weight-loader-prefetch-checkpoints"], + ["--weight-loader-prefetch-num-threads", "4"] + ] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto#2", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP4", + "detail": "MTP 6-1-7 · --max-running-requests 1", + "gpus": 4, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "582e19ab0829eadb81104b36df154741ceec56b0", + "run": "37052235829/attempts/2", + "pr": "3231", + "status": "verified", + "date": "2026-10-05" + }, + "points": [ + { + "concs": [1], + "item": 3, + "goldenAL": "4.8", + "kv": "hicache", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml" + }, + { + "concs": [44], + "item": 2, + "goldenAL": "4.8", + "kv": "hicache", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "ref": [0, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve#2", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP8 (2 nodes)", + "detail": "MTP 6-1-7 · qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "gpus": 8, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml", + "pr": "3840", + "status": "in-progress" + }, + "points": [ + { + "concs": [1], + "item": 2, + "goldenAL": "4.8", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "nodes_per_worker": 2, + "diff": { + "args": { + "--dist-init-addr": ["$AGG_HEAD_IP:20000"], + "--nnodes": ["2"], + "--node-rank": ["$NODE_RANK"], + "--moe-dense-tp-size": ["8"], + "--tensor-parallel-size": ["8"] + } + }, + "ref": [0, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://$AGG_0_HEAD_IP:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto#4", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP8 (2 nodes)", + "detail": "MTP 6-1-7 · qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto", + "gpus": 8, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "582e19ab0829eadb81104b36df154741ceec56b0", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml", + "run": "37052235829/attempts/2", + "pr": "3231", + "status": "verified", + "date": "2026-10-05" + }, + "points": [ + { + "concs": [1], + "item": 4, + "goldenAL": "4.8", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "ref": [2, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://$AGG_0_HEAD_IP:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto#0", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP4", + "detail": "MTP 6-1-7 · --max-running-requests 4", + "gpus": 4, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "582e19ab0829eadb81104b36df154741ceec56b0", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml", + "run": "37052235829/attempts/2", + "pr": "3231", + "status": "verified", + "date": "2026-10-05" + }, + "points": [ + { + "concs": [44], + "item": 0, + "goldenAL": "4.8", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--hicache-size": ["128"], + "--max-running-requests": ["4"] + } + }, + "ref": [0, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto#1", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP4", + "detail": "MTP 6-1-7 · --max-running-requests 2", + "gpus": 4, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "582e19ab0829eadb81104b36df154741ceec56b0", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml", + "run": "37052235829/attempts/2", + "pr": "3231", + "status": "verified", + "date": "2026-10-05" + }, + "points": [ + { + "concs": [44], + "item": 1, + "goldenAL": "4.8", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["2"], + "--max-running-requests": ["2"] + } + }, + "ref": [4, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto#2", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 1P1D · prefill TP1 · decode DEP4", + "detail": "MTP 5-1-6 · qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto", + "gpus": 8, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "582e19ab0829eadb81104b36df154741ceec56b0", + "run": "37052235829/attempts/2", + "pr": "3231", + "status": "verified", + "date": "2026-10-05" + }, + "points": [ + { + "concs": [100], + "item": 3, + "goldenAL": "4.41", + "kv": "hicache", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c100-mtp-hicache-nightly-20260913.yaml" + }, + { + "concs": [200], + "item": 2, + "goldenAL": "4.41", + "kv": "hicache", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913.yaml" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/ad6f7d54ce7b99c53576b10482755c93c921fbac"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "1.0"] + ] + }, + { + "kind": "worker", + "role": "prefill", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m dynamo.sglang", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "MC_FORCE_MNNVL": "1", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTHONOPTIMIZE": "1", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", + "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", + "SGLANG_DISAGG_STAGING_BUFFER": "1", + "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_FLASHINFER_AUTOTUNE_EXTEND": "1", + "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "8192", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", + "SGLANG_PP_LAYER_PARTITION": "16,16,16,12", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" + }, + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "prefill"], + ["--disaggregation-bootstrap-port", "8998"], + ["--request-plane", "tcp"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "32768"], + ["--cuda-graph-backend-prefill", "breakable"], + ["--cuda-graph-max-bs-prefill", "32768"], + ["--data-parallel-size", "1"], + ["--disable-shared-experts-fusion"], + ["--enable-hierarchical-cache"], + ["--expert-parallel-size", "1"], + ["--hicache-host-memory-mode", "cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "1.0"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "triton"], + ["--load-balance-method", "round_robin"], + ["--mamba-max-states-per-path", "3"], + ["--mamba-radix-cache-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "1048576"], + ["--max-mamba-cache-size", "1536"], + ["--max-prefill-tokens", "32768"], + ["--max-running-requests", "128"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "none"], + ["--moe-dense-tp-size", "1"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "4"], + ["--pp-async-batch-depth", "1"], + ["--quantization", "fp8"], + ["--scheduler-recv-interval", "1"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-moe-a2a-backend", "none"], + ["--speculative-moe-runner-backend", "flashinfer_trtllm"], + ["--speculative-num-draft-tokens", "6"], + ["--speculative-num-steps", "5"], + ["--tensor-parallel-size", "1"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"], + ["--weight-loader-prefetch-checkpoints"], + ["--weight-loader-prefetch-num-threads", "4"] + ] + }, + { + "kind": "worker", + "role": "decode", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m dynamo.sglang", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "MC_FORCE_MNNVL": "1", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", + "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", + "SGLANG_DISAGG_STAGING_BUFFER": "1", + "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "SGLANG_USE_SYMM_MEM_DP_SYNC": "true", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" + }, + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "decode"], + ["--disaggregation-bootstrap-port", "8998"], + ["--request-plane", "tcp"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "4096"], + ["--cuda-graph-max-bs-decode", "128"], + ["--data-parallel-size", "4"], + ["--disable-radix-cache"], + ["--disable-shared-experts-fusion"], + ["--disaggregation-decode-extra-slots", "2"], + ["--enable-dp-attention"], + ["--enable-dp-lm-head"], + ["--enable-linear-replayssm-spec"], + ["--ep-dispatch-algorithm", "static"], + ["--eplb-algorithm", "deepseek"], + ["--expert-parallel-size", "4"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "triton"], + ["--linear-replayssm-cache-len", "32"], + ["--load-balance-method", "round_robin"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "128"], + ["--max-mamba-cache-size", "468"], + ["--max-running-requests", "640"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "flashinfer"], + ["--moe-dense-tp-size", "1"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "1"], + ["--quantization", "fp8"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-moe-a2a-backend", "flashinfer"], + ["--speculative-moe-runner-backend", "flashinfer_trtllm_routed"], + ["--speculative-num-draft-tokens", "6"], + ["--speculative-num-steps", "5"], + ["--stream-interval", "30"], + ["--tensor-parallel-size", "4"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"], + ["--weight-loader-prefetch-checkpoints"], + ["--weight-loader-prefetch-num-threads", "4"] + ] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve#4", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 1P1D · prefill TP1 · decode DEP16 (4 nodes)", + "detail": "MTP 6-1-7", + "gpus": 20, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-dep16-mtp6-variants.yaml", + "pr": "3840", + "status": "in-progress" + }, + "points": [ + { + "concs": [180], + "item": 4, + "override": "override_1p1d", + "goldenAL": "4.8", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/ad6f7d54ce7b99c53576b10482755c93c921fbac"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "1.0"] + ] + }, + { + "diff": { + "args": { + "--speculative-num-draft-tokens": ["7"], + "--speculative-num-steps": ["6"] + } + }, + "ref": [6, "dynamo", 4] + }, + { + "nodes_per_worker": 4, + "diff": { + "args": { + "--dist-init-addr": ["$DECODE_HEAD_IP:20000"], + "--nnodes": ["4"], + "--node-rank": ["$NODE_RANK"], + "--data-parallel-size": ["16"], + "--expert-parallel-size": ["16"], + "--max-mamba-cache-size": ["1872"], + "--speculative-num-draft-tokens": ["7"], + "--speculative-num-steps": ["6"], + "--tensor-parallel-size": ["16"] + }, + "env": { + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1" + } + }, + "ref": [6, "dynamo", 5] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve#1", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 1P1D · prefill TP1 · decode DEP4", + "detail": "MTP 5-1-6 · qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "gpus": 8, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913.yaml", + "pr": "3840", + "status": "in-progress" + }, + "points": [ + { + "concs": [200], + "item": 1, + "goldenAL": "4.41", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/ad6f7d54ce7b99c53576b10482755c93c921fbac"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "1.0"] + ] + }, + { + "ref": [6, "dynamo", 4] + }, + { + "ref": [6, "dynamo", 5] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve#3", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 2P1D · prefill TP1 · decode DEP16 (4 nodes)", + "detail": "MTP 6-1-7", + "gpus": 24, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-dep16-mtp6-variants.yaml", + "pr": "3840", + "status": "in-progress" + }, + "points": [ + { + "concs": [300], + "item": 3, + "override": "override_2p1d", + "goldenAL": "4.8", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/ad6f7d54ce7b99c53576b10482755c93c921fbac"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "1.0"] + ] + }, + { + "workers": 2, + "ref": [7, "dynamo", 4] + }, + { + "ref": [7, "dynamo", 5] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve#2", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 3P1D · prefill TP1 · decode DEP16 (4 nodes)", + "detail": "MTP 6-1-7", + "gpus": 28, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-dep16-mtp6-variants.yaml", + "pr": "3840", + "status": "in-progress" + }, + "points": [ + { + "concs": [400], + "item": 2, + "override": "override_3p1d", + "goldenAL": "4.8", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/ad6f7d54ce7b99c53576b10482755c93c921fbac"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "1.0"] + ] + }, + { + "workers": 3, + "ref": [7, "dynamo", 4] + }, + { + "ref": [7, "dynamo", 5] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto#1", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 3P2D · prefill TP1 · decode DEP4", + "detail": "MTP 5-1-6", + "gpus": 20, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "582e19ab0829eadb81104b36df154741ceec56b0", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-3p2d-pp4-dep4-c600-mtp-hicache-nightly-20260913.yaml", + "run": "37052235829/attempts/2", + "pr": "3231", + "status": "verified", + "date": "2026-10-05" + }, + "points": [ + { + "concs": [600], + "item": 1, + "goldenAL": "4.41", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/ad6f7d54ce7b99c53576b10482755c93c921fbac"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "1.0"] + ] + }, + { + "workers": 3, + "ref": [6, "dynamo", 4] + }, + { + "workers": 2, + "ref": [6, "dynamo", 5] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve#0", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 5P3D · prefill TP1 · decode DEP4", + "detail": "MTP 5-1-6 · qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "gpus": 32, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913.yaml", + "pr": "3840", + "status": "in-progress" + }, + "points": [ + { + "concs": [1000], + "item": 0, + "goldenAL": "4.41", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/ad6f7d54ce7b99c53576b10482755c93c921fbac"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "1.0"] + ] + }, + { + "workers": 5, + "ref": [6, "dynamo", 4] + }, + { + "workers": 3, + "ref": [6, "dynamo", 5] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_2_HEAD_IP:30000"], + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_3_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto#0", + "hw": "gb300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 5P3D · prefill TP1 · decode DEP4", + "detail": "MTP 5-1-6 · qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto", + "gpus": 32, + "configKey": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72", + "imageStatus": "pruned", + "imageCommit": "fa663e72", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "582e19ab0829eadb81104b36df154741ceec56b0", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913.yaml", + "run": "37052235829/attempts/2", + "pr": "3231", + "status": "verified", + "date": "2026-10-05" + }, + "points": [ + { + "concs": [1000], + "item": 0, + "goldenAL": "4.41", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/ad6f7d54ce7b99c53576b10482755c93c921fbac"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "1.0"] + ] + }, + { + "ref": [12, "dynamo", 4] + }, + { + "ref": [12, "dynamo", 5] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_2_HEAD_IP:30000"], + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_3_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg#0", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2", + "detail": "MTP 3-1-4", + "gpus": 4, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1", + "imageStatus": "pruned", + "imageCommit": "433429b1", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "eb1e4107ac078fa736735066c90b2ef457babab0", + "run": "31042542308/attempts/1", + "pr": "2477", + "status": "verified", + "date": "2026-08-07" + }, + "points": [ + { + "concs": [1], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml" + }, + { + "concs": [24], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml" + }, + { + "concs": [32], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml" + }, + { + "concs": [40], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml" + }, + { + "concs": [48], + "item": 4, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml" + }, + { + "concs": [52], + "item": 5, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml" + }, + { + "concs": [64], + "item": 6, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "NCCL_NVLS_ENABLE": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_CUDA_ARCH_LIST": "10.0" + }, + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--allow-auto-truncate"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--cuda-graph-max-bs", "80"], + ["--data-parallel-size", "1"], + ["--enable-hierarchical-cache"], + ["--enable-symm-mem"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--mamba-max-states-per-path", "1"], + ["--mamba-scheduler-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "8192"], + ["--max-mamba-cache-size", "360"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "72"], + ["--mem-fraction-static", "0.75"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--quantization", "modelopt_fp4"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tensor-parallel-size", "2"], + ["--tokenizer-worker-num", "6"], + ["--trust-remote-code"] + ] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8180"], + ["--worker-urls", "http://$AGG_0_HEAD_IP:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto#2", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP8 (2 nodes)", + "detail": "MTP 6-1-7", + "gpus": 8, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198", + "imageStatus": "ok", + "imageCommit": "bb5e6198", + "submitted": "sglang", + "setup": [], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml", + "run": "34345050756/attempts/1", + "pr": "2812", + "status": "verified", + "date": "2026-09-09" + }, + "points": [ + { + "concs": [7], + "item": 2, + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 2, + "cmd": "python3 -m sglang.launch_server", + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--dist-init-addr", "$AGG_HEAD_IP:20000"], + ["--nnodes", "2"], + ["--node-rank", "$NODE_RANK"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "8192"], + ["--context-length", "262144"], + ["--cuda-graph-bs-decode", "1"], + ["--cuda-graph-max-bs-decode", "1"], + ["--data-parallel-size", "1"], + ["--disable-attn-tp-gather"], + ["--disable-prefill-cuda-graph"], + ["--disable-shared-experts-fusion"], + ["--enable-hierarchical-cache"], + ["--enable-linear-replayssm-spec"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-size", "32"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "triton"], + ["--load-balance-method", "round_robin"], + ["--mamba-max-states-per-path", "3"], + ["--mamba-radix-cache-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "8192"], + ["--max-mamba-cache-size", "1536"], + ["--max-prefill-tokens", "8192"], + ["--max-running-requests", "1"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "none"], + ["--moe-dense-tp-size", "8"], + ["--moe-runner-backend", "flashinfer_cutedsl"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "1"], + ["--pp-max-micro-batch-size", "1"], + ["--prefill-max-requests", "1"], + ["--quantization", "modelopt_fp4"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-moe-a2a-backend", "none"], + ["--speculative-moe-runner-backend", "flashinfer_cutedsl"], + ["--speculative-num-draft-tokens", "7"], + ["--speculative-num-steps", "6"], + ["--stream-interval", "20"], + ["--tensor-parallel-size", "8"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"], + ["--weight-loader-prefetch-checkpoints"], + ["--weight-loader-prefetch-num-threads", "4"] + ], + "envRef": 0 + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://$AGG_0_HEAD_IP:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#0", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Disagg 1P1D · prefill TP4 · decode TP4", + "detail": "no spec", + "gpus": 8, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1", + "imageStatus": "pruned", + "imageCommit": "433429b1", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "eb1e4107ac078fa736735066c90b2ef457babab0", + "run": "31042542308/attempts/1", + "pr": "2477", + "status": "verified", + "date": "2026-08-07" + }, + "points": [ + { + "concs": [8], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml" + }, + { + "concs": [16], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml" + }, + { + "concs": [32], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml" + }, + { + "concs": [64], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["80"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs": ["80"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml" + }, + { + "concs": [96], + "item": 4, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["128"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs": ["128"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml" + }, + { + "concs": [128], + "item": 5, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["160"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs": ["160"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/5a638087d82c990d35c69cb8e41c2c2582e9ef9a"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ] + }, + { + "kind": "worker", + "role": "prefill", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m dynamo.sglang", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "MC_FORCE_MNNVL": "1", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "1", + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", + "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", + "SGLANG_DISAGG_STAGING_BUFFER": "1", + "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": "1024", + "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" + }, + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "prefill"], + ["--disaggregation-bootstrap-port", "8998"], + ["--request-plane", "tcp"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "65536"], + ["--data-parallel-size", "1"], + ["--disable-cuda-graph"], + ["--enable-hierarchical-cache"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--load-balance-method", "round_robin"], + ["--mamba-max-states-per-path", "3"], + ["--mamba-scheduler-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "2048"], + ["--max-mamba-cache-size", "1100"], + ["--mem-fraction-static", "0.8"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--quantization", "modelopt_fp4"], + ["--tensor-parallel-size", "4"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"] + ] + }, + { + "kind": "worker", + "role": "decode", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m dynamo.sglang", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "MC_FORCE_MNNVL": "1", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "1", + "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1", + "SGLANG_DECODE_BOOTSTRAP_TIMEOUT": "1000", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", + "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", + "SGLANG_DISAGG_STAGING_BUFFER": "1", + "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": "1024", + "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", + "SGLANG_ENABLE_JIT_DEEPGEMM": "true", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", + "SGLANG_HACK_SEQ_BOOTSTRAP_ROOM": "1", + "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", + "SGLANG_HEALTH_STARTING_OK": "1", + "SGLANG_MOE_NVFP4_DISPATCH": "1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", + "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "1", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" + }, + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "decode"], + ["--disaggregation-bootstrap-port", "8998"], + ["--request-plane", "tcp"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "4096"], + ["--cuda-graph-max-bs", "64"], + ["--data-parallel-size", "1"], + ["--disable-radix-cache"], + ["--disable-shared-experts-fusion"], + ["--ep-dispatch-algorithm", "static"], + ["--eplb-algorithm", "deepseek"], + ["--expert-parallel-size", "1"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--mamba-scheduler-strategy", "no_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "128"], + ["--max-mamba-cache-size", "1024"], + ["--max-running-requests", "512"], + ["--mem-fraction-static", "0.75"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--prefill-round-robin-balance"], + ["--quantization", "modelopt_fp4"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--tensor-parallel-size", "4"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"] + ] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto#0", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2", + "detail": "MTP 6-1-7 · --max-running-requests 2", + "gpus": 4, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198", + "imageStatus": "ok", + "imageCommit": "bb5e6198", + "submitted": "sglang", + "setup": [], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml", + "run": "34345050756/attempts/1", + "pr": "2812", + "status": "verified", + "date": "2026-09-09" + }, + "points": [ + { + "concs": [44], + "item": 0, + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "nodes_per_worker": 1, + "diff": { + "args": { + "--cuda-graph-bs-decode": ["1", "2"], + "--cuda-graph-max-bs-decode": ["2"], + "--hicache-size": ["128"], + "--max-running-requests": ["2"], + "--moe-dense-tp-size": ["2"], + "--moe-runner-backend": ["flashinfer_trtllm"], + "--pp-max-micro-batch-size": ["2"], + "--prefill-decode-interval": ["0"], + "--prefill-max-requests": ["2"], + "--speculative-moe-runner-backend": ["flashinfer_trtllm"], + "--tensor-parallel-size": ["2"], + "--dist-init-addr": null, + "--nnodes": null, + "--node-rank": null, + "--disable-prefill-cuda-graph": null + } + }, + "ref": [15, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto#1", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2", + "detail": "MTP 6-1-7 · --max-running-requests 1", + "gpus": 4, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198", + "imageStatus": "ok", + "imageCommit": "bb5e6198", + "submitted": "sglang", + "setup": [], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml", + "run": "34345050756/attempts/1", + "pr": "2812", + "status": "verified", + "date": "2026-09-09" + }, + "points": [ + { + "concs": [44], + "item": 1, + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-bs-decode": ["1"], + "--cuda-graph-max-bs-decode": ["1"], + "--hicache-size": ["32"], + "--max-running-requests": ["1"], + "--pp-max-micro-batch-size": ["1"], + "--prefill-max-requests": ["1"] + } + }, + "ref": [17, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#6", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Disagg 1P1D · prefill TP2 · decode TP2", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1", + "imageStatus": "pruned", + "imageCommit": "433429b1", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "eb1e4107ac078fa736735066c90b2ef457babab0", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml", + "run": "31042542308/attempts/1", + "pr": "2477", + "status": "verified", + "date": "2026-08-07" + }, + "points": [ + { + "concs": [72], + "item": 6, + "goldenAL": "3.39", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/5a638087d82c990d35c69cb8e41c2c2582e9ef9a"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ] + }, + { + "diff": { + "args": { + "--mamba-max-states-per-path": ["1"], + "--max-mamba-cache-size": ["320"], + "--speculative-algorithm": ["NEXTN"], + "--speculative-eagle-topk": ["1"], + "--speculative-num-draft-tokens": ["4"], + "--speculative-num-steps": ["3"], + "--tensor-parallel-size": ["2"] + } + }, + "ref": [16, "dynamo", 4] + }, + { + "diff": { + "args": { + "--cuda-graph-max-bs": ["128"], + "--mamba-max-states-per-path": ["-1"], + "--max-mamba-cache-size": ["80"], + "--max-running-requests": ["80"], + "--tensor-parallel-size": ["2"] + } + }, + "ref": [16, "dynamo", 5] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto#1", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Disagg 3P4D · prefill TP1 · decode DEP4", + "detail": "MTP 4-1-5", + "gpus": 28, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198", + "imageStatus": "ok", + "imageCommit": "bb5e6198", + "submitted": "dynamo", + "setup": [], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml", + "run": "34345050756/attempts/1", + "pr": "2812", + "status": "verified", + "date": "2026-09-09" + }, + "points": [ + { + "concs": [565], + "item": 1, + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/61c37bc33c3e88da3facaa39fa2b5caf616740ba"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "1.0"] + ] + }, + { + "workers": 3, + "diff": { + "args": { + "--chunked-prefill-size": ["32768"], + "--disable-overlap-schedule": [], + "--disable-shared-experts-fusion": [], + "--linear-attn-decode-backend": ["triton"], + "--mamba-max-states-per-path": ["3"], + "--mamba-radix-cache-strategy": ["extra_buffer"], + "--mamba-track-interval": ["1048576"], + "--max-mamba-cache-size": ["1536"], + "--max-prefill-tokens": ["32768"], + "--max-running-requests": ["128"], + "--mem-fraction-static": ["0.85"], + "--moe-a2a-backend": ["none"], + "--moe-dense-tp-size": ["1"], + "--moe-runner-backend": ["flashinfer_cutedsl"], + "--pipeline-parallel-size": ["4"], + "--pp-async-batch-depth": ["1"], + "--scheduler-recv-interval": ["1"], + "--speculative-moe-a2a-backend": ["none"], + "--speculative-moe-runner-backend": ["flashinfer_trtllm"], + "--speculative-num-draft-tokens": ["5"], + "--speculative-num-steps": ["4"], + "--tensor-parallel-size": ["1"], + "--weight-loader-prefetch-checkpoints": [], + "--weight-loader-prefetch-num-threads": ["4"], + "--mamba-scheduler-strategy": null + }, + "env": { + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTHONOPTIMIZE": "1", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_FLASHINFER_AUTOTUNE_EXTEND": "1", + "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "8192", + "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", + "SGLANG_PP_LAYER_PARTITION": "16,16,16,12", + "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": null, + "SGLANG_ENABLE_SPEC_V2": null + } + }, + "ref": [19, "dynamo", 4] + }, + { + "workers": 4, + "diff": { + "args": { + "--cuda-graph-max-bs": ["80"], + "--data-parallel-size": ["4"], + "--disaggregation-decode-extra-slots": ["2"], + "--enable-dp-attention": [], + "--enable-dp-lm-head": [], + "--enable-linear-replayssm-spec": [], + "--expert-parallel-size": ["4"], + "--linear-attn-decode-backend": ["triton"], + "--linear-replayssm-cache-len": ["32"], + "--load-balance-method": ["round_robin"], + "--max-mamba-cache-size": ["468"], + "--max-running-requests": ["320"], + "--mem-fraction-static": ["0.7"], + "--moe-a2a-backend": ["flashinfer"], + "--moe-dense-tp-size": ["1"], + "--moe-runner-backend": ["flashinfer_cutedsl"], + "--pipeline-parallel-size": ["1"], + "--speculative-draft-model-quantization": ["nvfp4_online"], + "--speculative-moe-a2a-backend": ["flashinfer"], + "--speculative-moe-runner-backend": ["flashinfer_trtllm_routed"], + "--speculative-num-draft-tokens": ["5"], + "--speculative-num-steps": ["4"], + "--stream-interval": ["30"], + "--weight-loader-prefetch-checkpoints": [], + "--weight-loader-prefetch-num-threads": ["4"], + "--prefill-round-robin-balance": null + }, + "env": { + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4", + "SGLANG_USE_SYMM_MEM_DP_SYNC": "true", + "SGLANG_DECODE_BOOTSTRAP_TIMEOUT": null, + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": null, + "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": null, + "SGLANG_HACK_SEQ_BOOTSTRAP_ROOM": null, + "SGLANG_HEALTH_STARTING_OK": null + } + }, + "ref": [16, "dynamo", 5] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_3_HEAD_IP:30000"], + ["--decode", "http://$DECODE_2_HEAD_IP:30000"], + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto#0", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Disagg 3P2D · prefill TP1 · decode DEP4", + "detail": "MTP 5-1-6", + "gpus": 20, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198", + "imageStatus": "ok", + "imageCommit": "bb5e6198", + "submitted": "dynamo", + "setup": [], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml", + "run": "34345050756/attempts/1", + "pr": "2812", + "status": "verified", + "date": "2026-09-09" + }, + "points": [ + { + "concs": [704], + "item": 0, + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/61c37bc33c3e88da3facaa39fa2b5caf616740ba"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "1.0"] + ] + }, + { + "diff": { + "args": { + "--speculative-num-draft-tokens": ["6"], + "--speculative-num-steps": ["5"] + } + }, + "ref": [20, "dynamo", 4] + }, + { + "workers": 2, + "diff": { + "args": { + "--speculative-num-draft-tokens": ["6"], + "--speculative-num-steps": ["5"] + } + }, + "ref": [20, "dynamo", 5] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp#0", + "hw": "gb200", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4-V2", + "label": "Agg TP4", + "detail": "MTP 5-1-6", + "gpus": 4, + "configKey": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-20260818-c0b6474b", + "imageStatus": "ok", + "imageCommit": "c0b6474b", + "submitted": "dynamo", + "setup": [ + { + "kind": "setup", + "command": "pip install torchao", + "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed." + } + ], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2.5"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", + "donorHardware": "gb200" + } + }, + "prov": { + "sha": "cc865fb3945848e9d0627ad3b94df0cface6a6fc", + "run": "34436606041/attempts/2", + "pr": "2945", + "status": "verified", + "date": "2026-09-11" + }, + "points": [ + { + "concs": [1, 4, 8, 12, 20], + "item": 0, + "kv": "none", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml" + }, + { + "concs": [2], + "item": 1, + "kv": "none", + "patch": { + "dynamo": [ + null, + null, + null, + { + "args": { + "--enable-symm-mem": null + } + } + ], + "sglang": [ + { + "args": { + "--enable-symm-mem": null + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-no-symm.yaml" + }, + { + "concs": [24], + "item": 2, + "kv": "none", + "patch": { + "dynamo": [ + null, + null, + null, + { + "args": { + "--enable-linear-replayssm-spec": [], + "--linear-attn-prefill-backend": ["flashinfer"], + "--linear-replayssm-cache-len": ["8"], + "--mamba-scheduler-strategy": ["extra_buffer_lazy"], + "--mamba-track-interval": ["1048576"] + } + } + ], + "sglang": [ + { + "args": { + "--enable-linear-replayssm-spec": [], + "--linear-attn-prefill-backend": ["flashinfer"], + "--linear-replayssm-cache-len": ["8"], + "--mamba-scheduler-strategy": ["extra_buffer_lazy"], + "--mamba-track-interval": ["1048576"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-parity.yaml" + } + ], + "routers": { + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "round-robin"], + ["--router-session-affinity-ttl-secs", "3600"] + ] + }, + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m dynamo.sglang", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_CUDA_ARCH_LIST": "10.0" + }, + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4-V2"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4-V2"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--request-plane", "tcp"], + ["--allow-auto-truncate"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--cuda-graph-max-bs", "64"], + ["--data-parallel-size", "1"], + ["--enable-symm-mem"], + ["--expert-parallel-size", "1"], + ["--fp4-gemm-backend", "flashinfer_cutlass"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--mamba-max-states-per-path", "1"], + ["--mamba-scheduler-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "8192"], + ["--max-mamba-cache-size", "360"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "80"], + ["--mem-fraction-static", "0.8"], + ["--model-loader-extra-config", "{\"enable_multithread_load\":true}"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--quantization", "modelopt_fp4"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "6"], + ["--speculative-num-steps", "5"], + ["--stream-interval", "50"], + ["--tensor-parallel-size", "4"], + ["--trust-remote-code"], + ["--weight-loader-prefetch-checkpoints"], + ["--weight-loader-prefetch-num-threads", "4"] + ] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp#3", + "hw": "gb200", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4-V2", + "label": "Agg TP2·EP2", + "detail": "MTP 5-1-6", + "gpus": 4, + "configKey": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-20260818-c0b6474b", + "imageStatus": "ok", + "imageCommit": "c0b6474b", + "submitted": "dynamo", + "setup": [ + { + "kind": "setup", + "command": "pip install torchao", + "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed." + } + ], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2.5"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", + "donorHardware": "gb200" + } + }, + "prov": { + "sha": "cc865fb3945848e9d0627ad3b94df0cface6a6fc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache.yaml", + "run": "34436606041/attempts/2", + "pr": "2945", + "status": "verified", + "date": "2026-09-11" + }, + "points": [ + { + "concs": [22], + "item": 3, + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "round-robin"], + ["--router-session-affinity-ttl-secs", "3600"] + ] + }, + { + "diff": { + "args": { + "--enable-hierarchical-cache": [], + "--enable-linear-replayssm-spec": [], + "--expert-parallel-size": ["2"], + "--hicache-io-backend": ["direct"], + "--hicache-mem-layout": ["page_first_direct"], + "--hicache-ratio": ["2.5"], + "--hicache-write-policy": ["write_through_selective"], + "--linear-attn-prefill-backend": ["flashinfer"], + "--linear-replayssm-cache-len": ["8"], + "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], + "--mamba-track-interval": ["1048576"], + "--max-mamba-cache-size": ["320"], + "--mem-fraction-static": ["0.88"], + "--page-size": ["64"], + "--pipeline-parallel-size": ["1"], + "--speculative-moe-runner-backend": ["flashinfer_trtllm"], + "--tensor-parallel-size": ["2"], + "--watchdog-timeout": ["1000000"], + "--enable-symm-mem": null, + "--mamba-scheduler-strategy": null + } + }, + "ref": [22, "dynamo", 3] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp#4", + "hw": "gb200", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4-V2", + "label": "Agg TP2·EP2", + "detail": "MTP 3-1-4", + "gpus": 4, + "configKey": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-20260818-c0b6474b", + "imageStatus": "ok", + "imageCommit": "c0b6474b", + "submitted": "dynamo", + "setup": [ + { + "kind": "setup", + "command": "pip install torchao", + "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed." + } + ], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2.5"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", + "donorHardware": "gb200" + } + }, + "prov": { + "sha": "cc865fb3945848e9d0627ad3b94df0cface6a6fc", + "run": "34436606041/attempts/2", + "pr": "2945", + "status": "verified", + "date": "2026-09-11" + }, + "points": [ + { + "concs": [32], + "item": 4, + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-k3-baseline.yaml" + }, + { + "concs": [40, 48], + "item": 5, + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-mamba-cache-size": ["192"], + "--max-running-requests": ["48"], + "--tokenizer-worker-num": null + } + } + ], + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-mamba-cache-size": ["192"], + "--max-running-requests": ["48"], + "--tokenizer-worker-num": null + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-cap48.yaml" + } + ], + "routers": { + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "round-robin"], + ["--router-session-affinity-ttl-secs", "3600"] + ] + }, + { + "diff": { + "args": { + "--speculative-num-draft-tokens": ["4"], + "--speculative-num-steps": ["3"], + "--tokenizer-worker-num": ["1"] + } + }, + "ref": [23, "dynamo", 3] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b300-sglang-agentic-mtp#1", + "hw": "b300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP2·EP2", + "detail": "MTP 3-1-4 · measured with no KV offload", + "gpus": 8, + "configKey": "qwen3.5-fp8-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.16-cu130", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "997e74f4d7131f7c8bd63513879d3b474399ec24", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh", + "run": "31152055079/attempts/2", + "pr": "2515", + "status": "verified", + "date": "2026-08-07" + }, + "points": [ + { + "concs": [1], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "job": "31152055079/job/92879195874" + }, + { + "concs": [4], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "31152055079/job/92879197671" + }, + { + "concs": [8], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "31152055079/job/92879196905" + }, + { + "concs": [12], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ] + }, + "job": "31152055079/job/92879196947" + }, + { + "concs": [16], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ] + }, + "job": "31152055079/job/92879197571" + }, + { + "concs": [20], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--max-running-requests": ["40"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--max-running-requests": ["40"] + } + } + ] + }, + "job": "31152055079/job/92879197321" + }, + { + "concs": [24], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["48"] + } + } + ] + }, + "job": "31152055079/job/92879196928" + }, + { + "concs": [28], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--max-running-requests": ["56"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--max-running-requests": ["56"] + } + } + ] + }, + "job": "31152055079/job/92879196478" + }, + { + "concs": [32], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-running-requests": ["64"] + } + } + ] + }, + "job": "31152055079/job/92879197211" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--cuda-graph-max-bs", "1"], + ["--dp", "1"], + ["--enable-symm-mem"], + ["--ep-size", "2"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.80"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--quantization", "fp8"], + ["--reasoning-parser", "qwen3"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tokenizer-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--tool-call-parser", "qwen3_coder"], + ["--tp", "2"], + ["--trust-remote-code"] + ], + "envRef": 1 + } + ] + }, + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b300-sglang-agentic-mtp#0", + "hw": "b300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP4", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "qwen3.5-fp8-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.16-cu130", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "997e74f4d7131f7c8bd63513879d3b474399ec24", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh", + "run": "31152055079/attempts/2", + "pr": "2515", + "status": "verified", + "date": "2026-08-07" + }, + "points": [ + { + "concs": [1], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "31152055079/job/92879196571" + }, + { + "concs": [4], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "31152055079/job/92879197228" + }, + { + "concs": [8], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "31152055079/job/92879196359" + }, + { + "concs": [12], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ] + }, + "job": "31152055079/job/92879197180" + }, + { + "concs": [16], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ] + }, + "job": "31152055079/job/92879196920" + }, + { + "concs": [20], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--max-running-requests": ["40"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--max-running-requests": ["40"] + } + } + ] + }, + "job": "31152055079/job/92879197029" + }, + { + "concs": [24], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["48"] + } + } + ] + }, + "job": "31152055079/job/92879196244" + }, + { + "concs": [28], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--max-running-requests": ["56"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--max-running-requests": ["56"] + } + } + ] + }, + "job": "31152055079/job/92879196684" + }, + { + "concs": [32], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-running-requests": ["64"] + } + } + ] + }, + "job": "31152055079/job/92879196802" + }, + { + "concs": [40], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["40"], + "--max-running-requests": ["80"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["40"], + "--max-running-requests": ["80"] + } + } + ] + }, + "job": "31152055079/job/92879196909" + }, + { + "concs": [48], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-running-requests": ["96"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-running-requests": ["96"] + } + } + ] + }, + "job": "31152055079/job/92879196274" + }, + { + "concs": [56], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["56"], + "--max-running-requests": ["112"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["56"], + "--max-running-requests": ["112"] + } + } + ] + }, + "job": "31152055079/job/92879196775" + }, + { + "concs": [60], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["60"], + "--max-running-requests": ["120"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["60"], + "--max-running-requests": ["120"] + } + } + ] + }, + "job": "31152055079/job/92879197203" + }, + { + "concs": [64], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["128"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["128"] + } + } + ] + }, + "job": "31152055079/job/92879197043" + }, + { + "concs": [68], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["136"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["136"] + } + } + ] + }, + "job": "31152055079/job/92879197381" + }, + { + "concs": [72], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["144"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["144"] + } + } + ] + }, + "job": "31152055079/job/92879197214" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--ep-size": ["1"], + "--tokenizer-worker-num": ["6"], + "--tp": ["4"] + } + }, + "ref": [25, "sglang", 0] + } + ] + }, + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg#0", + "hw": "b300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 1P1D · prefill TP4 · decode TP4", + "detail": "no spec", + "gpus": 16, + "configKey": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85", + "imageStatus": "pruned", + "imageCommit": "20518d85", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "298"], + ["--hicache-write-policy", "write_through"] + ], + "donor": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "8eb9a40f3bdb57d9ffaaee56eb86e8aef213a6f5", + "run": "35628368631/attempts/1", + "pr": "3268", + "status": "verified", + "date": "2026-09-21" + }, + "points": [ + { + "concs": [4], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml" + }, + { + "concs": [12], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml" + }, + { + "concs": [24], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/805a77f053d807b0d8def5d27f674a6df0ed839e"], + "files": ["nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ] + }, + { + "kind": "worker", + "role": "prefill", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m dynamo.sglang", + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "prefill"], + ["--disaggregation-bootstrap-port", "8998"], + ["--request-plane", "tcp"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--context-length", "262144"], + ["--data-parallel-size", "1"], + ["--disable-cuda-graph"], + ["--disaggregation-transfer-backend", "mooncake"], + ["--enable-hierarchical-cache"], + ["--enable-symm-mem"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "72"], + ["--hicache-write-policy", "write_through_selective"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--linear-attn-prefill-backend", "flashinfer"], + ["--mamba-radix-cache-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "2048"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "32"], + ["--mem-fraction-static", "0.8"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--quantization", "fp8"], + ["--reasoning-parser", "qwen3"], + ["--scheduler-recv-interval", "10"], + ["--stream-interval", "50"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "qwen3_coder"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ], + "envRef": 2 + }, + { + "kind": "worker", + "role": "decode", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m dynamo.sglang", + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "decode"], + ["--request-plane", "tcp"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--context-length", "262144"], + ["--cuda-graph-max-bs-decode", "16"], + ["--data-parallel-size", "1"], + ["--disable-radix-cache"], + ["--disaggregation-decode-extra-slots", "0"], + ["--disaggregation-decode-retraction-backup", "cpu_tensor"], + ["--disaggregation-transfer-backend", "mooncake"], + ["--enable-symm-mem"], + ["--expert-parallel-size", "1"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--linear-attn-prefill-backend", "flashinfer"], + ["--mamba-radix-cache-strategy", "no_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "32"], + ["--mem-fraction-static", "0.8"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--quantization", "fp8"], + ["--reasoning-parser", "qwen3"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "qwen3_coder"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ], + "envRef": 2 + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b300-sglang-agentic-mtp#2", + "hw": "b300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP2·EP2", + "detail": "MTP 3-1-4 · measured with HiCache", + "gpus": 8, + "configKey": "qwen3.5-fp8-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.16-cu130", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "997e74f4d7131f7c8bd63513879d3b474399ec24", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh", + "run": "31152055079/attempts/2", + "pr": "2515", + "status": "verified", + "date": "2026-08-07" + }, + "points": [ + { + "concs": [32], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "job": "31152055079/job/92879197343" + }, + { + "concs": [34], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["34"], + "--max-running-requests": ["68"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["34"], + "--max-running-requests": ["68"] + } + } + ] + }, + "job": "31152055079/job/92879196408" + }, + { + "concs": [36], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["36"], + "--max-running-requests": ["72"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["36"], + "--max-running-requests": ["72"] + } + } + ] + }, + "job": "31152055079/job/92879196451" + }, + { + "concs": [38], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["38"], + "--max-running-requests": ["76"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["38"], + "--max-running-requests": ["76"] + } + } + ] + }, + "job": "31152055079/job/92879197765" + }, + { + "concs": [40], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["40"], + "--max-running-requests": ["80"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["40"], + "--max-running-requests": ["80"] + } + } + ] + }, + "job": "31152055079/job/92879197103" + }, + { + "concs": [44], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["44"], + "--max-running-requests": ["88"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["44"], + "--max-running-requests": ["88"] + } + } + ] + }, + "job": "31152055079/job/92879196559" + }, + { + "concs": [48], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-running-requests": ["96"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-running-requests": ["96"] + } + } + ] + }, + "job": "31152055079/job/92879196495" + }, + { + "concs": [52], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["52"], + "--max-running-requests": ["104"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["52"], + "--max-running-requests": ["104"] + } + } + ] + }, + "job": "31152055079/job/92879196293" + }, + { + "concs": [56], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["56"], + "--max-running-requests": ["112"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["56"], + "--max-running-requests": ["112"] + } + } + ] + }, + "job": "31152055079/job/92879196876" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs": ["32"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["64"], + "--page-size": ["64"] + } + }, + "ref": [25, "sglang", 0] + } + ] + }, + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg#4", + "hw": "b300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 1P1D · prefill TP2·EP2 · decode TP2·EP2", + "detail": "no spec", + "gpus": 16, + "configKey": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85", + "imageStatus": "pruned", + "imageCommit": "20518d85", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "298"], + ["--hicache-write-policy", "write_through"] + ], + "donor": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "8eb9a40f3bdb57d9ffaaee56eb86e8aef213a6f5", + "run": "35628368631/attempts/1", + "pr": "3268", + "status": "verified", + "date": "2026-09-21" + }, + "points": [ + { + "concs": [32], + "item": 4, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml" + }, + { + "concs": [40], + "item": 5, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml" + }, + { + "concs": [44], + "item": 6, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml" + }, + { + "concs": [48], + "item": 7, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--mem-fraction-static": ["0.92"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--mem-fraction-static": ["0.92"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml" + }, + { + "concs": [56], + "item": 8, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--enable-linear-replayssm-spec": [], + "--mem-fraction-static": ["0.92"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--enable-linear-replayssm-spec": [], + "--mem-fraction-static": ["0.92"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/805a77f053d807b0d8def5d27f674a6df0ed839e"], + "files": ["nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ] + }, + { + "diff": { + "args": { + "--expert-parallel-size": ["2"], + "--hicache-size": ["298"], + "--hicache-write-policy": ["write_through"], + "--max-running-requests": ["64"], + "--tensor-parallel-size": ["2"] + } + }, + "ref": [27, "dynamo", 3] + }, + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--expert-parallel-size": ["2"], + "--max-running-requests": ["64"], + "--mem-fraction-static": ["0.88"], + "--tensor-parallel-size": ["2"] + } + }, + "ref": [27, "dynamo", 4] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg#3", + "hw": "b300", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 1P1D · prefill TP4·EP4 · decode TP4", + "detail": "no spec", + "gpus": 16, + "configKey": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85", + "imageStatus": "pruned", + "imageCommit": "20518d85", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "298"], + ["--hicache-write-policy", "write_through"] + ], + "donor": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "8eb9a40f3bdb57d9ffaaee56eb86e8aef213a6f5", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml", + "run": "35628368631/attempts/1", + "pr": "3268", + "status": "verified", + "date": "2026-09-21" + }, + "points": [ + { + "concs": [32], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/805a77f053d807b0d8def5d27f674a6df0ed839e"], + "files": ["nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ] + }, + { + "diff": { + "args": { + "--expert-parallel-size": ["4"] + } + }, + "ref": [27, "dynamo", 3] + }, + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["32"] + } + }, + "ref": [27, "dynamo", 4] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b300-sglang-agentic-mtp#1", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2·EP2", + "detail": "MTP 3-1-4 · measured with no KV offload", + "gpus": 8, + "configKey": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.16-cu130", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "22265bb54f47ec520a7ca67078b9c5d1129ef48d", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh", + "run": "30730541420/attempts/1", + "pr": "2421", + "status": "verified", + "date": "2026-08-03" + }, + "points": [ + { + "concs": [1], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "job": "30730541420/job/91450169820" + }, + { + "concs": [4], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "30730541420/job/91450169638" + }, + { + "concs": [8], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "30730541420/job/91450169806" + }, + { + "concs": [12], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ] + }, + "job": "30730541420/job/91450169745" + }, + { + "concs": [16], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ] + }, + "job": "30730541420/job/91450169761" + }, + { + "concs": [20], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--max-running-requests": ["40"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--max-running-requests": ["40"] + } + } + ] + }, + "job": "30730541420/job/91450169845" + }, + { + "concs": [24], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["48"] + } + } + ] + }, + "job": "30730541420/job/91450169831" + }, + { + "concs": [28], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--max-running-requests": ["56"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--max-running-requests": ["56"] + } + } + ] + }, + "job": "30730541420/job/91450169790" + }, + { + "concs": [32], + "item": 1, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-running-requests": ["64"] + } + } + ] + }, + "job": "30730541420/job/91450169769" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--cuda-graph-max-bs", "1"], + ["--dp", "1"], + ["--enable-symm-mem"], + ["--ep-size", "2"], + ["--fp4-gemm-backend", "flashinfer_cutlass"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.80"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--quantization", "modelopt_fp4"], + ["--reasoning-parser", "qwen3"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tokenizer-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--tool-call-parser", "qwen3_coder"], + ["--tp", "2"], + ["--trust-remote-code"] + ], + "envRef": 1 + } + ] + }, + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b300-sglang-agentic-mtp#0", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP4", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.16-cu130", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "22265bb54f47ec520a7ca67078b9c5d1129ef48d", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh", + "run": "30730541420/attempts/1", + "pr": "2421", + "status": "verified", + "date": "2026-08-03" + }, + "points": [ + { + "concs": [1], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "30730541420/job/91450169626" + }, + { + "concs": [4], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "30730541420/job/91450169635" + }, + { + "concs": [8], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "30730541420/job/91450169667" + }, + { + "concs": [12], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ] + }, + "job": "30730541420/job/91450169627" + }, + { + "concs": [16], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ] + }, + "job": "30730541420/job/91450169864" + }, + { + "concs": [20], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--max-running-requests": ["40"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--max-running-requests": ["40"] + } + } + ] + }, + "job": "30730541420/job/91450169690" + }, + { + "concs": [24], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["48"] + } + } + ] + }, + "job": "30730541420/job/91450169746" + }, + { + "concs": [28], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--max-running-requests": ["56"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--max-running-requests": ["56"] + } + } + ] + }, + "job": "30730541420/job/91450169774" + }, + { + "concs": [32], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-running-requests": ["64"] + } + } + ] + }, + "job": "30730541420/job/91450169752" + }, + { + "concs": [40], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["40"], + "--max-running-requests": ["80"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["40"], + "--max-running-requests": ["80"] + } + } + ] + }, + "job": "30730541420/job/91450169652" + }, + { + "concs": [48], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-running-requests": ["96"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-running-requests": ["96"] + } + } + ] + }, + "job": "30730541420/job/91450169657" + }, + { + "concs": [56], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["56"], + "--max-running-requests": ["112"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["56"], + "--max-running-requests": ["112"] + } + } + ] + }, + "job": "30730541420/job/91450169694" + }, + { + "concs": [60], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["60"], + "--max-running-requests": ["120"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["60"], + "--max-running-requests": ["120"] + } + } + ] + }, + "job": "30730541420/job/91450169848" + }, + { + "concs": [64], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["128"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["128"] + } + } + ] + }, + "job": "30730541420/job/91450169788" + }, + { + "concs": [68], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["136"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["136"] + } + } + ] + }, + "job": "30730541420/job/91450169814" + }, + { + "concs": [72], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["144"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["144"] + } + } + ] + }, + "job": "30730541420/job/91450169840" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--ep-size": ["1"], + "--tokenizer-worker-num": ["6"], + "--tp": ["4"] + } + }, + "ref": [31, "sglang", 0] + } + ] + }, + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#4", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP4", + "detail": "MTP 6-1-7", + "gpus": 8, + "configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729", + "imageStatus": "ok", + "imageCommit": "07c8f729", + "submitted": "sglang", + "setup": [], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "6b7dcd07ab0c8c0805ac7fbec2f2ded38d450c20", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml", + "run": "34548026742", + "pr": "2978", + "comment": "5719060171", + "status": "verified-pr" + }, + "points": [ + { + "concs": [4], + "item": 4, + "override": "base", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "8192"], + ["--context-length", "262144"], + ["--cuda-graph-bs-decode", "1"], + ["--cuda-graph-max-bs-decode", "1"], + ["--data-parallel-size", "1"], + ["--disable-attn-tp-gather"], + ["--disable-prefill-cuda-graph"], + ["--disable-shared-experts-fusion"], + ["--enable-hierarchical-cache"], + ["--enable-linear-replayssm-spec"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-size", "32"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "triton"], + ["--load-balance-method", "round_robin"], + ["--mamba-max-states-per-path", "3"], + ["--mamba-radix-cache-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "8192"], + ["--max-mamba-cache-size", "512"], + ["--max-prefill-tokens", "8192"], + ["--max-running-requests", "1"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "none"], + ["--moe-dense-tp-size", "4"], + ["--moe-runner-backend", "flashinfer_cutedsl"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "1"], + ["--pp-max-micro-batch-size", "1"], + ["--prefill-max-requests", "1"], + ["--quantization", "modelopt_fp4"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-moe-a2a-backend", "none"], + ["--speculative-moe-runner-backend", "flashinfer_cutedsl"], + ["--speculative-num-draft-tokens", "7"], + ["--speculative-num-steps", "6"], + ["--stream-interval", "20"], + ["--tensor-parallel-size", "4"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"], + ["--weight-loader-prefetch-checkpoints"], + ["--weight-loader-prefetch-num-threads", "4"] + ], + "envRef": 0 + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b300-sglang-agentic-mtp#2", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2·EP2", + "detail": "MTP 3-1-4 · measured with HiCache", + "gpus": 8, + "configKey": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.16-cu130", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "22265bb54f47ec520a7ca67078b9c5d1129ef48d", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh", + "run": "30730541420/attempts/1", + "pr": "2421", + "status": "verified", + "date": "2026-08-03" + }, + "points": [ + { + "concs": [32], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "job": "30730541420/job/91450169759" + }, + { + "concs": [34], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["34"], + "--max-running-requests": ["68"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["34"], + "--max-running-requests": ["68"] + } + } + ] + }, + "job": "30730541420/job/91450169641" + }, + { + "concs": [36], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["36"], + "--max-running-requests": ["72"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["36"], + "--max-running-requests": ["72"] + } + } + ] + }, + "job": "30730541420/job/91450169795" + }, + { + "concs": [38], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["38"], + "--max-running-requests": ["76"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["38"], + "--max-running-requests": ["76"] + } + } + ] + }, + "job": "30730541420/job/91450169781" + }, + { + "concs": [40], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["40"], + "--max-running-requests": ["80"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["40"], + "--max-running-requests": ["80"] + } + } + ] + }, + "job": "30730541420/job/91450169825" + }, + { + "concs": [44], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["44"], + "--max-running-requests": ["88"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["44"], + "--max-running-requests": ["88"] + } + } + ] + }, + "job": "30730541420/job/91450169828" + }, + { + "concs": [48], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-running-requests": ["96"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-running-requests": ["96"] + } + } + ] + }, + "job": "30730541420/job/91450169610" + }, + { + "concs": [52], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["52"], + "--max-running-requests": ["104"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["52"], + "--max-running-requests": ["104"] + } + } + ] + }, + "job": "30730541420/job/91450169749" + }, + { + "concs": [56], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["56"], + "--max-running-requests": ["112"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["56"], + "--max-running-requests": ["112"] + } + } + ] + }, + "job": "30730541420/job/91450169853" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs": ["32"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["64"], + "--page-size": ["64"] + } + }, + "ref": [31, "sglang", 0] + } + ] + }, + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#1", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2", + "detail": "MTP 7-1-8", + "gpus": 8, + "configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729", + "imageStatus": "ok", + "imageCommit": "07c8f729", + "submitted": "sglang", + "setup": [], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "6b7dcd07ab0c8c0805ac7fbec2f2ded38d450c20", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml", + "run": "34548026742", + "pr": "2978", + "comment": "5719060171", + "status": "verified-pr" + }, + "points": [ + { + "concs": [36], + "item": 2, + "override": "zip_override_running_cap[0]", + "kv": "hicache" + }, + { + "concs": [44], + "item": 1, + "override": "zip_override_running_cap[2]", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8"], + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"], + "--pp-max-micro-batch-size": ["8"], + "--prefill-max-requests": ["8"] + } + }, + null + ], + "dynamo": [ + null, + null, + null, + { + "args": { + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8"], + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"], + "--pp-max-micro-batch-size": ["8"], + "--prefill-max-requests": ["8"] + } + } + ] + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-bs-decode": ["1", "2", "3", "4"], + "--cuda-graph-max-bs-decode": ["4"], + "--hicache-size": ["128"], + "--max-mamba-cache-size": ["1536"], + "--max-running-requests": ["4"], + "--moe-dense-tp-size": ["2"], + "--moe-runner-backend": ["flashinfer_trtllm"], + "--pp-max-micro-batch-size": ["4"], + "--prefill-decode-interval": ["0"], + "--prefill-max-requests": ["4"], + "--speculative-moe-runner-backend": ["flashinfer_trtllm"], + "--speculative-num-draft-tokens": ["8"], + "--speculative-num-steps": ["7"], + "--tensor-parallel-size": ["2"], + "--disable-prefill-cuda-graph": null + } + }, + "ref": [33, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#3", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2", + "detail": "MTP 5-1-6", + "gpus": 8, + "configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729", + "imageStatus": "ok", + "imageCommit": "07c8f729", + "submitted": "sglang", + "setup": [], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "6b7dcd07ab0c8c0805ac7fbec2f2ded38d450c20", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml", + "run": "34548026742", + "pr": "2978", + "comment": "5719060171", + "status": "verified-pr" + }, + "points": [ + { + "concs": [40], + "item": 3, + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-bs-decode": ["1", "2"], + "--cuda-graph-max-bs-decode": ["2"], + "--max-running-requests": ["2"], + "--pp-max-micro-batch-size": ["2"], + "--prefill-max-requests": ["2"], + "--speculative-num-draft-tokens": ["6"], + "--speculative-num-steps": ["5"] + } + }, + "ref": [35, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#0", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2·EP2", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729", + "imageStatus": "ok", + "imageCommit": "07c8f729", + "submitted": "sglang", + "setup": [], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "6b7dcd07ab0c8c0805ac7fbec2f2ded38d450c20", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml", + "run": "34548026742", + "pr": "2978", + "comment": "5719060171", + "status": "verified-pr" + }, + "points": [ + { + "concs": [64], + "item": 0, + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--allow-auto-truncate": [], + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs": ["224"], + "--expert-parallel-size": ["2"], + "--hicache-ratio": ["1.8"], + "--linear-attn-prefill-backend": ["flashinfer"], + "--linear-replayssm-cache-len": ["8"], + "--mamba-max-states-per-path": ["1"], + "--mamba-track-interval": ["1048576"], + "--max-mamba-cache-size": ["512"], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["224"], + "--moe-runner-backend": ["flashinfer_cutedsl"], + "--scheduler-recv-interval": ["10"], + "--speculative-num-draft-tokens": ["4"], + "--speculative-num-steps": ["3"], + "--stream-interval": ["50"], + "--tokenizer-worker-num": ["6"], + "--weight-loader-drop-cache-after-load": [], + "--cuda-graph-bs-decode": null, + "--cuda-graph-max-bs-decode": null, + "--disable-attn-tp-gather": null, + "--disable-shared-experts-fusion": null, + "--hicache-size": null, + "--linear-attn-decode-backend": null, + "--load-balance-method": null, + "--moe-a2a-backend": null, + "--moe-dense-tp-size": null, + "--pp-max-micro-batch-size": null, + "--prefill-decode-interval": null, + "--prefill-max-requests": null, + "--speculative-moe-a2a-backend": null, + "--weight-loader-prefetch-checkpoints": null, + "--weight-loader-prefetch-num-threads": null + }, + "env": { + "SGLANG_ENABLE_JIT_DEEPGEMM": "false", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "2" + } + }, + "ref": [35, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b200-sglang-agentic-mtp#2", + "hw": "b200", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP4", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "qwen3.5-fp8-b200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161", + "imageStatus": "pruned", + "imageCommit": "4358a161", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b200-sglang-agentic-mtp", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "21a63c471574e0cda725d4a61d3837e91405b522", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh", + "run": "35032235397/attempts/1", + "pr": "3142", + "status": "verified", + "date": "2026-09-16" + }, + "points": [ + { + "concs": [1], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "job": "35032235397/job/104593683554" + }, + { + "concs": [4], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "35032235397/job/104593683369" + }, + { + "concs": [8], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "35032235397/job/104593684311" + }, + { + "concs": [12], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["24"] + } + } + ] + }, + "job": "35032235397/job/104593684310" + }, + { + "concs": [14], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["14"], + "--max-running-requests": ["28"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["14"], + "--max-running-requests": ["28"] + } + } + ] + }, + "job": "35032235397/job/104593684304" + }, + { + "concs": [16], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["32"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["32"], + "--page-size": ["64"] + } + } + ] + }, + "job": "35032235397/job/104593684012" + }, + { + "concs": [18], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["18"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["36"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["18"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["36"], + "--page-size": ["64"] + } + } + ] + }, + "job": "35032235397/job/104593684349" + }, + { + "concs": [20], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["20"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["40"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["20"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["40"], + "--page-size": ["64"] + } + } + ] + }, + "job": "35032235397/job/104593683966" + }, + { + "concs": [22], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["22"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["44"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["22"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["44"], + "--page-size": ["64"] + } + } + ] + }, + "job": "35032235397/job/104593683751" + }, + { + "concs": [24], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["48"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["48"], + "--page-size": ["64"] + } + } + ] + }, + "job": "35032235397/job/104593684216" + }, + { + "concs": [28], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["28"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["56"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["28"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["56"], + "--page-size": ["64"] + } + } + ] + }, + "job": "35032235397/job/104593683411" + }, + { + "concs": [32], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["64"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["64"], + "--page-size": ["64"] + } + } + ] + }, + "job": "35032235397/job/104593683809" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--cuda-graph-max-bs-decode", "1"], + ["--dp", "1"], + ["--enable-symm-mem"], + ["--ep-size", "1"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.80"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--quantization", "fp8"], + ["--reasoning-parser", "qwen3"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tokenizer-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--tokenizer-worker-num", "6"], + ["--tool-call-parser", "qwen3_coder"], + ["--tp", "4"], + ["--trust-remote-code"] + ], + "envRef": 1 + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b200-sglang-agentic-mtp#0", + "hw": "b200", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP8", + "detail": "MTP 3-1-4 · measured with no KV offload", + "gpus": 8, + "configKey": "qwen3.5-fp8-b200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161", + "imageStatus": "pruned", + "imageCommit": "4358a161", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b200-sglang-agentic-mtp", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "21a63c471574e0cda725d4a61d3837e91405b522", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh", + "run": "35032235397/attempts/1", + "pr": "3142", + "status": "verified", + "date": "2026-09-16" + }, + "points": [ + { + "concs": [1], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "35032235397/job/104593684082" + }, + { + "concs": [4], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "35032235397/job/104593684181" + }, + { + "concs": [8], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "35032235397/job/104593683585" + }, + { + "concs": [12], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["24"] + } + } + ] + }, + "job": "35032235397/job/104593684113" + }, + { + "concs": [16], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["32"] + } + } + ] + }, + "job": "35032235397/job/104593683289" + }, + { + "concs": [20], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["20"], + "--max-running-requests": ["40"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["20"], + "--max-running-requests": ["40"] + } + } + ] + }, + "job": "35032235397/job/104593684136" + }, + { + "concs": [24], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--max-running-requests": ["48"] + } + } + ] + }, + "job": "35032235397/job/104593683532" + }, + { + "concs": [28], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["28"], + "--max-running-requests": ["56"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["28"], + "--max-running-requests": ["56"] + } + } + ] + }, + "job": "35032235397/job/104593683645" + }, + { + "concs": [32], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["64"] + } + } + ] + }, + "job": "35032235397/job/104593683606" + }, + { + "concs": [40], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["40"], + "--max-running-requests": ["80"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["40"], + "--max-running-requests": ["80"] + } + } + ] + }, + "job": "35032235397/job/104593684158" + }, + { + "concs": [48], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["96"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["96"] + } + } + ] + }, + "job": "35032235397/job/104593684375" + }, + { + "concs": [56], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["56"], + "--max-running-requests": ["112"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["56"], + "--max-running-requests": ["112"] + } + } + ] + }, + "job": "35032235397/job/104593683940" + }, + { + "concs": [60], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["60"], + "--max-running-requests": ["120"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["60"], + "--max-running-requests": ["120"] + } + } + ] + }, + "job": "35032235397/job/104593683635" + }, + { + "concs": [62], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["62"], + "--max-running-requests": ["124"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["62"], + "--max-running-requests": ["124"] + } + } + ] + }, + "job": "35032235397/job/104593684023" + }, + { + "concs": [64], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["128"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["128"] + } + } + ] + }, + "job": "35032235397/job/104593684272" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--tp": ["8"] + } + }, + "ref": [38, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp#0", + "hw": "b200", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Disagg 1P1D · prefill TP4 · decode TP4", + "detail": "no spec", + "gpus": 16, + "configKey": "qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85", + "imageStatus": "pruned", + "imageCommit": "20518d85", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "104"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "0b75e138be098de0122959460c765860f79b95fd", + "run": "35404276619/attempts/2", + "pr": "3269", + "status": "verified", + "date": "2026-09-19" + }, + "points": [ + { + "concs": [8], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml" + }, + { + "concs": [16], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml" + }, + { + "concs": [24], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + { + "args": { + "--max-running-requests": ["48"] + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--max-running-requests": ["48"] + } + } + ], + "sglang": [ + { + "args": { + "--max-running-requests": ["48"] + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--max-running-requests": ["48"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml" + }, + { + "concs": [32], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + { + "args": { + "--max-running-requests": ["64"] + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["64"] + } + } + ], + "sglang": [ + { + "args": { + "--max-running-requests": ["64"] + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["64"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml" + }, + { + "concs": [48], + "item": 4, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + { + "args": { + "--hicache-size": ["215"], + "--hicache-write-policy": ["write_through"], + "--max-running-requests": ["96"] + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["96"], + "--mem-fraction-static": ["0.88"] + } + } + ], + "sglang": [ + { + "args": { + "--hicache-size": ["215"], + "--hicache-write-policy": ["write_through"], + "--max-running-requests": ["96"] + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["96"], + "--mem-fraction-static": ["0.88"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml" + }, + { + "concs": [64], + "item": 5, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + { + "args": { + "--hicache-size": ["215"], + "--hicache-write-policy": ["write_through"], + "--max-running-requests": ["128"] + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["128"], + "--mem-fraction-static": ["0.88"] + } + } + ], + "sglang": [ + { + "args": { + "--hicache-size": ["215"], + "--hicache-write-policy": ["write_through"], + "--max-running-requests": ["128"] + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["128"], + "--mem-fraction-static": ["0.88"] + } + }, + null + ] + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/805a77f053d807b0d8def5d27f674a6df0ed839e"], + "files": ["nginx.conf#033bddfb"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"] + ] + }, + { + "kind": "worker", + "role": "prefill", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m dynamo.sglang", + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "prefill"], + ["--disaggregation-bootstrap-port", "8998"], + ["--request-plane", "tcp"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--context-length", "262144"], + ["--data-parallel-size", "1"], + ["--disable-cuda-graph"], + ["--disaggregation-transfer-backend", "mooncake"], + ["--enable-hierarchical-cache"], + ["--enable-symm-mem"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "104"], + ["--hicache-write-policy", "write_through_selective"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--mamba-radix-cache-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "2048"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "32"], + ["--mem-fraction-static", "0.8"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--quantization", "fp8"], + ["--reasoning-parser", "qwen3"], + ["--scheduler-recv-interval", "10"], + ["--stream-interval", "50"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "qwen3_coder"], + ["--trust-remote-code"] + ], + "envRef": 3 + }, + { + "kind": "worker", + "role": "decode", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m dynamo.sglang", + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "decode"], + ["--request-plane", "tcp"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--context-length", "262144"], + ["--cuda-graph-max-bs-decode", "8"], + ["--data-parallel-size", "1"], + ["--disable-radix-cache"], + ["--disaggregation-transfer-backend", "mooncake"], + ["--enable-symm-mem"], + ["--expert-parallel-size", "1"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--mamba-radix-cache-strategy", "no_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "128"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "32"], + ["--mem-fraction-static", "0.8"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--quantization", "fp8"], + ["--reasoning-parser", "qwen3"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "qwen3_coder"], + ["--trust-remote-code"] + ], + "envRef": 3 + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b200-sglang-agentic-mtp#1", + "hw": "b200", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP8", + "detail": "MTP 3-1-4 · measured with HiCache", + "gpus": 8, + "configKey": "qwen3.5-fp8-b200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161", + "imageStatus": "pruned", + "imageCommit": "4358a161", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b200-sglang-agentic-mtp", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "21a63c471574e0cda725d4a61d3837e91405b522", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh", + "run": "35032235397/attempts/1", + "pr": "3142", + "status": "verified", + "date": "2026-09-16" + }, + "points": [ + { + "concs": [64], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35032235397/job/104593683504" + }, + { + "concs": [66], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["132"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["132"] + } + } + ] + }, + "job": "35032235397/job/104593683262" + }, + { + "concs": [68], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["136"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["136"] + } + } + ] + }, + "job": "35032235397/job/104593684193" + }, + { + "concs": [70], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["140"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["140"] + } + } + ] + }, + "job": "35032235397/job/104593684126" + }, + { + "concs": [72], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["144"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["144"] + } + } + ] + }, + "job": "35032235397/job/104593684325" + }, + { + "concs": [76], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["152"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["152"] + } + } + ] + }, + "job": "35032235397/job/104593684071" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["104"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["128"], + "--page-size": ["64"] + } + }, + "ref": [39, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b200-sglang-agentic-mtp#2", + "hw": "b200", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "qwen3.5-fp4-b200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.16-cu130", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b200-sglang-agentic-mtp", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "5e3d34644ecd62b0fb87cd2f2b909a814b4f74d7", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh", + "run": "30758866378/attempts/1", + "pr": "2420", + "status": "verified", + "date": "2026-08-03" + }, + "points": [ + { + "concs": [1], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "job": "30758866378/job/91525887738" + }, + { + "concs": [4], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "30758866378/job/91525887637" + }, + { + "concs": [8], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "30758866378/job/91525887851" + }, + { + "concs": [12], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ] + }, + "job": "30758866378/job/91525887713" + }, + { + "concs": [14], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["14"], + "--max-running-requests": ["28"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["14"], + "--max-running-requests": ["28"] + } + } + ] + }, + "job": "30758866378/job/91525887789" + }, + { + "concs": [16], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["32"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["32"], + "--page-size": ["64"] + } + } + ] + }, + "job": "30758866378/job/91525887783" + }, + { + "concs": [18], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["18"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["36"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["18"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["36"], + "--page-size": ["64"] + } + } + ] + }, + "job": "30758866378/job/91525887539" + }, + { + "concs": [20], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["40"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["40"], + "--page-size": ["64"] + } + } + ] + }, + "job": "30758866378/job/91525887812" + }, + { + "concs": [22], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["22"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["44"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["22"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["44"], + "--page-size": ["64"] + } + } + ] + }, + "job": "30758866378/job/91525887723" + }, + { + "concs": [24], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["48"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["48"], + "--page-size": ["64"] + } + } + ] + }, + "job": "30758866378/job/91525887826" + }, + { + "concs": [28], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["56"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["56"], + "--page-size": ["64"] + } + } + ] + }, + "job": "30758866378/job/91525887821" + }, + { + "concs": [32], + "item": 3, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["64"], + "--page-size": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["64"], + "--page-size": ["64"] + } + } + ] + }, + "job": "30758866378/job/91525887735" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--cuda-graph-max-bs", "1"], + ["--dp", "1"], + ["--enable-symm-mem"], + ["--ep-size", "1"], + ["--fp4-gemm-backend", "flashinfer_cutlass"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.80"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--quantization", "modelopt_fp4"], + ["--reasoning-parser", "qwen3"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tokenizer-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--tool-call-parser", "qwen3_coder"], + ["--tp", "2"], + ["--trust-remote-code"] + ], + "envRef": 1 + } + ] + }, + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b200-sglang-agentic-mtp#0", + "hw": "b200", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP4", + "detail": "MTP 3-1-4 · measured with no KV offload", + "gpus": 8, + "configKey": "qwen3.5-fp4-b200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.16-cu130", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b200-sglang-agentic-mtp", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "5e3d34644ecd62b0fb87cd2f2b909a814b4f74d7", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh", + "run": "30758866378/attempts/1", + "pr": "2420", + "status": "verified", + "date": "2026-08-03" + }, + "points": [ + { + "concs": [1], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "30758866378/job/91525887620" + }, + { + "concs": [4], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "30758866378/job/91525887677" + }, + { + "concs": [8], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "30758866378/job/91525887661" + }, + { + "concs": [12], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ] + }, + "job": "30758866378/job/91525887638" + }, + { + "concs": [16], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ] + }, + "job": "30758866378/job/91525887675" + }, + { + "concs": [20], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--max-running-requests": ["40"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["20"], + "--max-running-requests": ["40"] + } + } + ] + }, + "job": "30758866378/job/91525887796" + }, + { + "concs": [24], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["48"] + } + } + ] + }, + "job": "30758866378/job/91525887714" + }, + { + "concs": [28], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--max-running-requests": ["56"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["28"], + "--max-running-requests": ["56"] + } + } + ] + }, + "job": "30758866378/job/91525887589" + }, + { + "concs": [32], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-running-requests": ["64"] + } + } + ] + }, + "job": "30758866378/job/91525887709" + }, + { + "concs": [40], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["40"], + "--max-running-requests": ["80"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["40"], + "--max-running-requests": ["80"] + } + } + ] + }, + "job": "30758866378/job/91525887768" + }, + { + "concs": [48], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-running-requests": ["96"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["48"], + "--max-running-requests": ["96"] + } + } + ] + }, + "job": "30758866378/job/91525887575" + }, + { + "concs": [56], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["56"], + "--max-running-requests": ["112"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["56"], + "--max-running-requests": ["112"] + } + } + ] + }, + "job": "30758866378/job/91525887625" + }, + { + "concs": [60], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["60"], + "--max-running-requests": ["120"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["60"], + "--max-running-requests": ["120"] + } + } + ] + }, + "job": "30758866378/job/91525887818" + }, + { + "concs": [62], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["62"], + "--max-running-requests": ["124"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["62"], + "--max-running-requests": ["124"] + } + } + ] + }, + "job": "30758866378/job/91525887690" + }, + { + "concs": [64], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["128"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["64"], + "--max-running-requests": ["128"] + } + } + ] + }, + "job": "30758866378/job/91525887678" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--tokenizer-worker-num": ["6"], + "--tp": ["4"] + } + }, + "ref": [42, "sglang", 0] + } + ] + }, + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b200-sglang-agentic-mtp#1", + "hw": "b200", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP4", + "detail": "MTP 3-1-4 · measured with HiCache", + "gpus": 8, + "configKey": "qwen3.5-fp4-b200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.16-cu130", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b200-sglang-agentic-mtp", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "5e3d34644ecd62b0fb87cd2f2b909a814b4f74d7", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh", + "run": "30758866378/attempts/1", + "pr": "2420", + "status": "verified", + "date": "2026-08-03" + }, + "points": [ + { + "concs": [64], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "job": "30758866378/job/91525887732" + }, + { + "concs": [66], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["132"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["132"] + } + } + ] + }, + "job": "30758866378/job/91525887681" + }, + { + "concs": [68], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["136"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["136"] + } + } + ] + }, + "job": "30758866378/job/91525887679" + }, + { + "concs": [70], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["140"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["140"] + } + } + ] + }, + "job": "30758866378/job/91525887866" + }, + { + "concs": [72], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["144"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["144"] + } + } + ] + }, + "job": "30758866378/job/91525887773" + }, + { + "concs": [76], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["152"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["152"] + } + } + ] + }, + "job": "30758866378/job/91525887544" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs": ["64"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["144"], + "--hicache-write-policy": ["write_through_selective"], + "--max-running-requests": ["128"], + "--page-size": ["64"] + } + }, + "ref": [43, "sglang", 0] + } + ] + }, + "dynamo": { + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp#0", + "hw": "h200", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP8", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9", + "imageStatus": "pruned", + "imageCommit": "c9a8fba9", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "77"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", + "donorHardware": "h200" + } + }, + "prov": { + "sha": "f137552f4851754d01946fadd46bbcfcbc90b222", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh", + "run": "35145000323/attempts/1", + "pr": "3200", + "status": "verified", + "date": "2026-09-21" + }, + "points": [ + { + "concs": [2], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35145000323/job/104959601340" + }, + { + "concs": [4], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35145000323/job/104959601216" + }, + { + "concs": [8], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35145000323/job/104959601193" + }, + { + "concs": [10], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35145000323/job/104959601626" + }, + { + "concs": [12], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35145000323/job/104959601321" + }, + { + "concs": [16], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35145000323/job/104959601300" + }, + { + "concs": [20], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35145000323/job/104959601348" + }, + { + "concs": [24], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35145000323/job/104959601480" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "SGLANG_ENABLE_SPEC_V2": "1" + }, + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--attention-backend", "flashinfer"], + ["--data-parallel-size", "1"], + ["--enable-flashinfer-allreduce-fusion"], + ["--enable-hierarchical-cache"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "77"], + ["--hicache-write-policy", "write_through_selective"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mem-fraction-static", "0.8"], + ["--page-size", "64"], + ["--quantization", "fp8"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "EAGLE"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tensor-parallel-size", "8"], + ["--tokenizer-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--tokenizer-worker-num", "6"], + ["--trust-remote-code"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-h200-sglang-agentic-mtp#1", + "hw": "h200", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP8·EP8", + "detail": "MTP 3-1-4 · measured with HiCache", + "gpus": 8, + "configKey": "qwen3.5-fp8-h200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "77"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", + "donorHardware": "h200" + } + }, + "prov": { + "sha": "69d978f24846de079e8cdcbbdcdc5c29629ca90e", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh", + "run": "35146004544/attempts/2", + "pr": "3198", + "status": "verified", + "date": "2026-09-17" + }, + "points": [ + { + "concs": [4], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35146004544/job/104988562901" + }, + { + "concs": [8], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35146004544/job/104988564135" + }, + { + "concs": [12], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35146004544/job/104988564737" + }, + { + "concs": [16], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35146004544/job/104988563015" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--expert-parallel-size": ["8"] + } + }, + "ref": [45, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-h200-sglang-agentic-mtp#0", + "hw": "h200", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP8·EP8", + "detail": "MTP 3-1-4 · measured with no KV offload", + "gpus": 8, + "configKey": "qwen3.5-fp8-h200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "77"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", + "donorHardware": "h200" + } + }, + "prov": { + "sha": "69d978f24846de079e8cdcbbdcdc5c29629ca90e", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh", + "run": "35146004544/attempts/2", + "pr": "3198", + "status": "verified", + "date": "2026-09-17" + }, + "points": [ + { + "concs": [4], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "35146004544/job/104988563924" + }, + { + "concs": [8], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "35146004544/job/104988563857" + }, + { + "concs": [12], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "35146004544/job/104988562507" + }, + { + "concs": [16], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "35146004544/job/104988563025" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--enable-hierarchical-cache": null, + "--hicache-io-backend": null, + "--hicache-mem-layout": null, + "--hicache-size": null, + "--hicache-write-policy": null, + "--page-size": null + } + }, + "ref": [46, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-h100-sglang-agentic-mtp#0", + "hw": "h100", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP8·EP8", + "detail": "MTP 3-1-4 · measured with no KV offload", + "gpus": 8, + "configKey": "qwen3.5-fp8-h100-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161", + "imageStatus": "pruned", + "imageCommit": "4358a161", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "108"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp8-h100-sglang-agentic-mtp", + "donorHardware": "h100" + } + }, + "prov": { + "sha": "dca6d8fce6e20b598e0d6a2bc3c6c3a7f286d0d2", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh", + "run": "35032265950/attempts/1", + "pr": "3150", + "status": "verified", + "date": "2026-09-16" + }, + "points": [ + { + "concs": [1], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "35032265950/job/104593779472" + }, + { + "concs": [4], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "35032265950/job/104593779420" + }, + { + "concs": [8], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "35032265950/job/104593779502" + }, + { + "concs": [12], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "35032265950/job/104593779480" + }, + { + "concs": [16], + "item": 0, + "goldenAL": "3.39", + "kv": "none", + "job": "35032265950/job/104593779459" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "SGLANG_ENABLE_SPEC_V2": "1" + }, + "args": [ + ["--model-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--served-model-name", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--attention-backend", "flashinfer"], + ["--data-parallel-size", "1"], + ["--enable-flashinfer-allreduce-fusion"], + ["--expert-parallel-size", "8"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mem-fraction-static", "0.75"], + ["--quantization", "fp8"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "EAGLE"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tensor-parallel-size", "8"], + ["--tokenizer-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--tokenizer-worker-num", "6"], + ["--trust-remote-code"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-h100-sglang-agentic-mtp#1", + "hw": "h100", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP8·EP8", + "detail": "MTP 3-1-4 · measured with HiCache", + "gpus": 8, + "configKey": "qwen3.5-fp8-h100-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161", + "imageStatus": "pruned", + "imageCommit": "4358a161", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "108"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp8-h100-sglang-agentic-mtp", + "donorHardware": "h100" + } + }, + "prov": { + "sha": "dca6d8fce6e20b598e0d6a2bc3c6c3a7f286d0d2", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh", + "run": "35032265950/attempts/1", + "pr": "3150", + "status": "verified", + "date": "2026-09-16" + }, + "points": [ + { + "concs": [4], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35032265950/job/104593779436" + }, + { + "concs": [8], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35032265950/job/104593779521" + }, + { + "concs": [12], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35032265950/job/104593779579" + }, + { + "concs": [16], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "job": "35032265950/job/104593779479" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["108"], + "--hicache-write-policy": ["write_through_selective"], + "--page-size": ["64"] + } + }, + "ref": [48, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + } + ] +}; diff --git a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx new file mode 100644 index 000000000000..98d6e57f21ef --- /dev/null +++ b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx @@ -0,0 +1,1046 @@ +// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. +// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions +// (SemiAnalysisAI/InferenceX main@ceb50ba21208 plus open PRs #2978, #3606, #3701, #3744, #3782, #3816, #3817, #3836, #3840), with SA-specific settings removed. +export const agentx = { + "model": "Qwen3.8-Flash-Next", + "generatedOn": "2026-10-08", + "source": { + "repo": "https://github.com/SemiAnalysisAI/InferenceX", + "main": "ceb50ba212080174e0563d2be1db9b0d654ab2ef", + "prs": { + "3816": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37714835620", + "sha": "e67ca6fc9ecf4cd57f27654bda8a3e9059923820", + "status": "verified-pr" + }, + "3840": { + "run": null, + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "status": "in-progress" + }, + "2978": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34548026742", + "sha": "6b7dcd07ab0c8c0805ac7fbec2f2ded38d450c20", + "status": "verified-pr" + }, + "3817": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", + "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", + "status": "verified-pr" + }, + "3606": { + "run": null, + "sha": "5b394d395eefcaacd5168e04aca95020adac5312", + "status": "in-progress" + }, + "3701": { + "run": null, + "sha": "d9185455e166dfd034dcdfaf9608bd8335618b54", + "status": "in-progress" + }, + "3782": { + "run": null, + "sha": "f00c81ab40297f98a68d2726d4301b44a2892419", + "status": "in-progress" + }, + "3744": { + "run": null, + "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "status": "in-progress" + }, + "3836": { + "run": null, + "sha": "03ffb35db157a0fecc78206bea748449daa1385a", + "status": "in-progress" + } + } + }, + "hardware": [ + { + "id": "gb300", + "label": "GB300" + }, + { + "id": "b300", + "label": "B300" + }, + { + "id": "b200", + "label": "B200" + }, + { + "id": "h200", + "label": "H200" + } + ], + "checkpoints": [ + { + "id": "Qwen/Qwen3.8-Flash-Next-FP8" + }, + { + "id": "RadixArk/Qwen3.8-Flash-Next-NVFP4" + }, + { + "id": "nvidia/Qwen3.8-Flash-Next-NVFP4" + } + ], + "envs": [ + { + "NCCL_NVLS_ENABLE": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_CUDA_ARCH_LIST": "10.0" + } + ], + "kvMooncake": { + "donor": "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300", + "master": { + "dynamo": { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": { + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + }, + "sglang": { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": {}, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + } + }, + "args": [ + ["--enable-unified-cache-external-linker"], + ["--hicache-storage-backend-extra-config", "{\"enable_group_semantics\":true}"], + ["--unified-cache-external-linker-backend", "mooncake"] + ], + "env": { + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", + "MC_TCP_BIND_ADDRESS": "$NODE_IP", + "MOONCAKE_DEVICE": "$IB_DEVICES", + "MOONCAKE_GLOBAL_SEGMENT_SIZE": "140gb", + "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", + "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", + "MOONCAKE_PROTOCOL": "rdma", + "MOONCAKE_STANDALONE_STORAGE": "0", + "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata" + }, + "decode_env": { + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", + "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", + "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", + "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata" + } + }, + "files": {}, + "cells": [ + { + "id": "qwen3.8next-fp4-gb300-sglang-agentic-mtp#0", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4", + "label": "Agg TP1", + "detail": "MTP 3-1-4", + "gpus": 4, + "configKey": "qwen3.8next-fp4-gb300-sglang-agentic-mtp", + "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:1f176b3a1dd1205d2a462ab80ea9fc17769a32e53cb6f3c690efa2dc2c384977", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "224"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.8next-fp4-gb300-sglang-agentic-mtp", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "e67ca6fc9ecf4cd57f27654bda8a3e9059923820", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml", + "run": "37714835620", + "pr": "3816", + "comment": "6064888765", + "status": "verified-pr" + }, + "points": [ + { + "concs": [1], + "item": 0, + "override": "override_c1", + "goldenAL": "2.32", + "kv": "hicache" + }, + { + "concs": [2], + "item": 0, + "override": "override_c2", + "goldenAL": "2.32", + "kv": "hicache" + }, + { + "concs": [4], + "item": 0, + "override": "override_c4", + "goldenAL": "2.32", + "kv": "hicache" + }, + { + "concs": [8], + "item": 0, + "override": "override_c8", + "goldenAL": "2.32", + "kv": "hicache" + }, + { + "concs": [16], + "item": 0, + "override": "override_c16", + "goldenAL": "2.32", + "kv": "hicache" + }, + { + "concs": [32], + "item": 0, + "override": "override_c32", + "goldenAL": "2.32", + "kv": "hicache" + }, + { + "concs": [48], + "item": 0, + "override": "override_c48", + "goldenAL": "2.32", + "kv": "hicache" + }, + { + "concs": [64], + "item": 0, + "override": "override_c64", + "goldenAL": "2.32", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800" + }, + "args": [ + ["--model-path", "nvidia/Qwen3.8-Flash-Next-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.8-Flash-Next-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--context-length", "262144"], + ["--cuda-graph-backend-decode", "full"], + ["--cuda-graph-backend-prefill", "disabled"], + ["--cuda-graph-bs-decode", "1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], + ["--cuda-graph-max-bs-decode", "96"], + ["--data-parallel-size", "1"], + ["--disable-shared-experts-fusion"], + ["--enable-hierarchical-cache"], + ["--enable-linear-replayssm-spec"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "224"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "bfloat16"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--linear-attn-prefill-backend", "flashinfer"], + ["--linear-attn-verify-backend", "triton"], + ["--mamba-radix-cache-strategy", "extra_buffer_lazy"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "256"], + ["--max-mamba-cache-size", "480"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "96"], + ["--mem-fraction-static", "0.88"], + ["--moe-a2a-backend", "none"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "1"], + ["--ple-offload-backend", "pinned"], + ["--ple-offload-embedding"], + ["--quantization", "modelopt_mixed"], + ["--random-seed", "35"], + ["--reasoning-parser", "auto"], + ["--revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], + ["--scheduler-recv-interval", "1"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-draft-kv-cache-dtype", "bfloat16"], + ["--speculative-draft-model-revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], + ["--speculative-eagle-topk", "1"], + ["--speculative-moe-runner-backend", "flashinfer_trtllm"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "20"], + ["--tensor-parallel-size", "1"], + ["--tokenizer-worker-num", "1"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1800"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#0", + "hw": "b300", + "ckpt": "RadixArk/Qwen3.8-Flash-Next-NVFP4", + "label": "Agg TP1", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang:qwen38flashnext", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none"], + "unavailable": { + "hicache": "no verified HiCache settings for this model yet", + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + } + }, + "prov": { + "sha": "7caed5a556ee1ae97f072328fb2c1befa338caa4", + "recipe": "benchmarks/single_node/agentic/qwen3.8next_fp4_b300_sglang_mtp.sh", + "run": "33039186365/attempts/1", + "pr": "2758", + "status": "verified", + "date": "2026-08-27" + }, + "points": [ + { + "concs": [1], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "job": "33039186365/job/98408899989" + }, + { + "concs": [4], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "33039186365/job/98408899982" + }, + { + "concs": [8], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "33039186365/job/98408899939" + }, + { + "concs": [12], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ] + }, + "job": "33039186365/job/98408899965" + }, + { + "concs": [16], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ] + }, + "job": "33039186365/job/98408899999" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "args": [ + ["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], + ["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--cuda-graph-max-bs", "1"], + ["--dp", "1"], + ["--ep-size", "1"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--linear-attn-prefill-backend", "flashinfer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.80"], + ["--reasoning-parser", "auto"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tokenizer-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], + ["--tp", "1"], + ["--trust-remote-code"] + ], + "envRef": 0 + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.8next-fp4-b200-sglang-agentic-mtp#0", + "hw": "b200", + "ckpt": "RadixArk/Qwen3.8-Flash-Next-NVFP4", + "label": "Agg TP1", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "qwen3.8next-fp4-b200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:qwen38flashnext", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none"], + "unavailable": { + "hicache": "no verified HiCache settings for this model yet", + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + } + }, + "prov": { + "sha": "4b9dddd090c913e0fc305bb37d88c28b32113b60", + "recipe": "benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh", + "run": "34879996995/attempts/1", + "pr": "2751", + "status": "verified", + "date": "2026-09-15" + }, + "points": [ + { + "concs": [1], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "job": "34879996995/job/104097491275" + }, + { + "concs": [4], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "34879996995/job/104097491564" + }, + { + "concs": [8], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "34879996995/job/104097491581" + }, + { + "concs": [16], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-mamba-cache-size": ["160"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.90"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["32"], + "--max-mamba-cache-size": ["160"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.90"] + } + } + ] + }, + "job": "34879996995/job/104097491534" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "args": [ + ["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], + ["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--cuda-graph-max-bs", "2"], + ["--dp", "1"], + ["--ep-size", "1"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--linear-attn-prefill-backend", "flashinfer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.80"], + ["--reasoning-parser", "auto"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tokenizer-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], + ["--tokenizer-worker-num", "6"], + ["--tp", "1"], + ["--trust-remote-code"] + ], + "envRef": 0 + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.8next-fp8-h200-sglang-agentic-mtp#0", + "hw": "h200", + "ckpt": "Qwen/Qwen3.8-Flash-Next-FP8", + "label": "Agg TP4·EP4", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "qwen3.8next-fp8-h200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:qwen38flashnext", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none"], + "unavailable": { + "hicache": "no verified HiCache settings for this model yet", + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + } + }, + "prov": { + "sha": "f1da353965f46c40465e51a49be402414644a414", + "recipe": "benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh", + "run": "33038487711/attempts/1", + "pr": "2753", + "status": "verified", + "date": "2026-08-27" + }, + "points": [ + { + "concs": [1], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "job": "33038487711/job/98406760958" + }, + { + "concs": [4], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "33038487711/job/98406761035" + }, + { + "concs": [8], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "33038487711/job/98406760957" + }, + { + "concs": [12], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] + } + } + ] + }, + "job": "33038487711/job/98406760930" + }, + { + "concs": [16], + "item": 0, + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] + } + } + ] + }, + "job": "33038487711/job/98406760985" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "SGLANG_ENABLE_SPEC_V2": "1" + }, + "args": [ + ["--model-path", "Qwen/Qwen3.8-Flash-Next-FP8"], + ["--served-model-name", "Qwen/Qwen3.8-Flash-Next-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--chunked-prefill-size", "8192"], + ["--cuda-graph-max-bs", "1"], + ["--dp-size", "1"], + ["--ep-size", "4"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--linear-attn-prefill-backend", "flashinfer"], + ["--mamba-ssm-dtype", "float32"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.85"], + ["--reasoning-parser", "auto"], + ["--scheduler-recv-interval", "10"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tokenizer-path", "Qwen/Qwen3.8-Flash-Next-FP8"], + ["--tokenizer-worker-num", "6"], + ["--tp-size", "4"], + ["--trust-remote-code"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + } + ] +}; diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx index 82eb9f91b74d..c29fb0e29e93 100644 --- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx +++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx @@ -1,37 +1,57 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@0259609ce99d plus open PRs #3187, #3598, #3631, #3659, #3745), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@ceb50ba21208 plus open PRs #2978, #3606, #3701, #3744, #3782, #3816, #3817, #3836, #3840), with SA-specific settings removed. export const agentx = { "model": "DeepSeek-V4", - "generatedOn": "2026-10-06", + "generatedOn": "2026-10-08", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "0259609ce99d0f4f7bd7180c3b3436e4f0334a46", + "main": "ceb50ba212080174e0563d2be1db9b0d654ab2ef", "prs": { - "3598": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37260940238", - "sha": "0f45d5f4c3d57f9804f7655d7b3b294acdd53441", + "3816": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37714835620", + "sha": "e67ca6fc9ecf4cd57f27654bda8a3e9059923820", "status": "verified-pr" }, - "3631": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/36974436489", - "sha": "3b798abfd43bc4260ab11e43d66c88a99e33461a", - "status": "verified-pr" + "3840": { + "run": null, + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "status": "in-progress" }, - "3659": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/36997573694", - "sha": "e79239bab1e1cb487dc74e27d693110f03384d41", + "2978": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34548026742", + "sha": "6b7dcd07ab0c8c0805ac7fbec2f2ded38d450c20", "status": "verified-pr" }, - "3187": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37270628136", - "sha": "34d06e54c7614bb33d2a271b010ffde3ce3fa563", + "3817": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", + "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "status": "verified-pr" }, - "3745": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37420473448", - "sha": "322bc37b368c85828357afb57dce6739d94ef955", - "status": "verified-pr" + "3606": { + "run": null, + "sha": "5b394d395eefcaacd5168e04aca95020adac5312", + "status": "in-progress" + }, + "3701": { + "run": null, + "sha": "d9185455e166dfd034dcdfaf9608bd8335618b54", + "status": "in-progress" + }, + "3782": { + "run": null, + "sha": "f00c81ab40297f98a68d2726d4301b44a2892419", + "status": "in-progress" + }, + "3744": { + "run": null, + "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "status": "in-progress" + }, + "3836": { + "run": null, + "sha": "03ffb35db157a0fecc78206bea748449daa1385a", + "status": "in-progress" } } }, @@ -101,6 +121,37 @@ export const agentx = { "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata", "NATS_SERVER": "nats://$NATS_IP:4222" }, + { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", + "MC_FORCE_MNNVL": "1", + "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", + "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", + "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_MNNVL_UUID_QUERY_TIMEOUT": "100000", + "NCCL_TIMEOUT": "100000", + "SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "8", + "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "900", + "SGLANG_DSV4_REASONING_EFFORT": "high", + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", + "SGLANG_JIT_DEEPGEMM_FAST_WARMUP": "1", + "SGLANG_JIT_DEEPGEMM_PRECOMPILE": "1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE": "1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", + "SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW": "1", + "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1", + "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", + "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", + "SGLANG_RAGGED_VERIFY_MODE": "static", + "SGLANG_REQUEST_STATE_WAIT_TIMEOUT": "60", + "WITH_NVIDIA_PEERMEM": "0" + }, { "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", "MC_TCP_BIND_ADDRESS": "$NODE_IP", @@ -120,7 +171,6 @@ export const agentx = { { "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1", "SGLANG_JIT_DEEPGEMM_FAST_WARMUP": "1", - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320", "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1", "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2": "1", @@ -128,7 +178,8 @@ export const agentx = { "SGLANG_OPT_USE_JIT_NORM": "1", "SGLANG_OPT_USE_TOPK_V2": "1", "SGLANG_TIMEOUT_KEEP_ALIVE": "900", - "TORCH_CUDA_ARCH_LIST": "10.0" + "TORCH_CUDA_ARCH_LIST": "10.0", + "TRITON_PTXAS_PATH": "/usr/local/cuda/bin/ptxas" } ], "kvMooncake": { @@ -253,10 +304,10 @@ export const agentx = { "prov": { "sha": "34d06e54c7614bb33d2a271b010ffde3ce3fa563", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "37270628136", + "run": "37270628136/attempts/1", "pr": "3187", - "comment": "5996444239", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -402,20 +453,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "8"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg", - "donorHardware": "gb200" + "donor": "dsv4-fp4-b200-dynamo-sglang-agentic-disagg", + "donorHardware": "b200" } }, "prov": { "sha": "34d06e54c7614bb33d2a271b010ffde3ce3fa563", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "37270628136", + "run": "37270628136/attempts/1", "pr": "3187", - "comment": "5996444239", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -597,37 +648,6 @@ export const agentx = { "workers": 1, "nodes_per_worker": 4, "cmd": "python3 -m dynamo.sglang", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", - "MC_FORCE_MNNVL": "1", - "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", - "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", - "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata", - "NATS_SERVER": "nats://$NATS_IP:4222", - "NCCL_CUMEM_ENABLE": "1", - "NCCL_MNNVL_ENABLE": "1", - "NCCL_MNNVL_UUID_QUERY_TIMEOUT": "100000", - "NCCL_TIMEOUT": "100000", - "SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "8", - "SGLANG_DEFAULT_THINKING": "1", - "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "900", - "SGLANG_DSV4_REASONING_EFFORT": "high", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", - "SGLANG_JIT_DEEPGEMM_FAST_WARMUP": "1", - "SGLANG_JIT_DEEPGEMM_PRECOMPILE": "1", - "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", - "SGLANG_OPT_DEEPGEMM_MEGA_MOE": "1", - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", - "SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW": "1", - "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1", - "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", - "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", - "SGLANG_RAGGED_VERIFY_MODE": "static", - "SGLANG_REQUEST_STATE_WAIT_TIMEOUT": "60", - "WITH_NVIDIA_PEERMEM": "0" - }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4-Pro-0813"], ["--served-model-name", "deepseek-ai/DeepSeek-V4-Pro-0813"], @@ -664,7 +684,8 @@ export const agentx = { ["--tp-size", "16"], ["--trust-remote-code"], ["--watchdog-timeout", "86400"] - ] + ], + "envRef": 3 } ] }, @@ -696,7 +717,7 @@ export const agentx = { "args": [ ["--port", "8800"] ], - "envRef": 3 + "envRef": 4 }, { "from": 6, @@ -769,20 +790,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "8"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg", - "donorHardware": "gb200" + "donor": "dsv4-fp4-b200-dynamo-sglang-agentic-disagg", + "donorHardware": "b200" } }, "prov": { "sha": "34d06e54c7614bb33d2a271b010ffde3ce3fa563", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "37270628136", + "run": "37270628136/attempts/1", "pr": "3187", - "comment": "5996444239", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -917,7 +938,7 @@ export const agentx = { "args": [ ["--port", "8800"] ], - "envRef": 3 + "envRef": 4 }, { "from": 6, @@ -991,20 +1012,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "8"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg", - "donorHardware": "gb200" + "donor": "dsv4-fp4-b200-dynamo-sglang-agentic-disagg", + "donorHardware": "b200" } }, "prov": { "sha": "34d06e54c7614bb33d2a271b010ffde3ce3fa563", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "37270628136", + "run": "37270628136/attempts/1", "pr": "3187", - "comment": "5996444239", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -1139,7 +1160,7 @@ export const agentx = { "args": [ ["--port", "8800"] ], - "envRef": 3 + "envRef": 4 }, { "from": 6, @@ -1214,20 +1235,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "8"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg", - "donorHardware": "gb200" + "donor": "dsv4-fp4-b200-dynamo-sglang-agentic-disagg", + "donorHardware": "b200" } }, "prov": { "sha": "34d06e54c7614bb33d2a271b010ffde3ce3fa563", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "37270628136", + "run": "37270628136/attempts/1", "pr": "3187", - "comment": "5996444239", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -1279,7 +1300,7 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 4 + "envRef": 5 }, { "kind": "mooncake-master", @@ -1364,7 +1385,7 @@ export const agentx = { "args": [ ["--port", "8800"] ], - "envRef": 3 + "envRef": 4 }, { "from": 6, @@ -1644,39 +1665,39 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], - ["--hicache-write-policy", "write_back"] + ["--hicache-ratio", "2.75"], + ["--hicache-write-policy", "write_through"] ], - "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg", + "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-agg", "donorHardware": "gb200" } }, "prov": { - "sha": "2a58623b5a441f1b5c190ba9666a70881301d1d3", - "run": "36833446190/attempts/1", - "pr": "3629", - "status": "verified", - "date": "2026-10-02" + "sha": "d9185455e166dfd034dcdfaf9608bd8335618b54", + "pr": "3701", + "status": "in-progress" }, "points": [ { "concs": [64], "item": 0, "goldenAL": "3.77", - "kv": "hicache", + "kv": "mooncake", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml" }, { "concs": [128], "item": 1, "goldenAL": "3.77", - "kv": "hicache", + "kv": "mooncake", "patch": { "dynamo": [ null, null, null, null, + null, + null, { "args": { "--mem-fraction-static": ["0.8"] @@ -1689,6 +1710,8 @@ export const agentx = { } ], "sglang": [ + null, + null, { "args": { "--mem-fraction-static": ["0.8"] @@ -1752,6 +1775,35 @@ export const agentx = { ["--active-prefill-tokens-threshold-frac", "None"] ] }, + { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": { + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + }, + { + "kind": "mooncake-store", + "count_nodes": 4, + "ports": ["8800", "8801", "8802", "8803"], + "preamble": ["ulimit -n 1048576", "ulimit -l unlimited"], + "cmd": "python -m mooncake.mooncake_store_service", + "args": [ + ["--port", "8800"] + ], + "envRef": 2 + }, { "kind": "worker", "role": "prefill", @@ -1761,7 +1813,16 @@ export const agentx = { "env": { "DYN_REQUEST_PLANE": "tcp", "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", "MC_FORCE_MNNVL": "1", + "MC_TCP_BIND_ADDRESS": "$NODE_IP", + "MOONCAKE_DEVICE": "$IB_DEVICES", + "MOONCAKE_GLOBAL_SEGMENT_SIZE": "140gb", + "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", + "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", + "MOONCAKE_PROTOCOL": "rdma", + "MOONCAKE_STANDALONE_STORAGE": "0", + "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata", "NATS_SERVER": "nats://$NATS_IP:4222", "NCCL_CUMEM_ENABLE": "1", "NCCL_MNNVL_ENABLE": "1", @@ -1785,7 +1846,8 @@ export const agentx = { "SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE": "1", "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", "SGLANG_RAGGED_VERIFY_MODE": "static", - "SGLANG_REQUEST_STATE_WAIT_TIMEOUT": "60" + "SGLANG_REQUEST_STATE_WAIT_TIMEOUT": "60", + "WITH_NVIDIA_PEERMEM": "0" }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4-Pro-0813"], @@ -1807,12 +1869,9 @@ export const agentx = { ["--enable-deepseek-v4-fp4-indexer"], ["--enable-dp-attention"], ["--enable-dp-lm-head"], - ["--enable-hierarchical-cache"], + ["--enable-unified-cache-external-linker"], ["--ep-size", "8"], - ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "8"], - ["--hicache-write-policy", "write_back"], + ["--hicache-storage-backend-extra-config", "{\"enable_group_semantics\":true}"], ["--load-balance-method", "total_tokens"], ["--max-running-requests", "256"], ["--mem-fraction-static", "0.85"], @@ -1829,6 +1888,7 @@ export const agentx = { ["--swa-full-tokens-ratio", "0.02"], ["--tp-size", "8"], ["--trust-remote-code"], + ["--unified-cache-external-linker-backend", "mooncake"], ["--watchdog-timeout", "86400"] ] }, @@ -1838,32 +1898,6 @@ export const agentx = { "workers": 1, "nodes_per_worker": 4, "cmd": "python3 -m dynamo.sglang", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "MC_FORCE_MNNVL": "1", - "NATS_SERVER": "nats://$NATS_IP:4222", - "NCCL_CUMEM_ENABLE": "1", - "NCCL_MNNVL_ENABLE": "1", - "NCCL_MNNVL_UUID_QUERY_TIMEOUT": "100000", - "NCCL_TIMEOUT": "100000", - "SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "8", - "SGLANG_DEFAULT_THINKING": "1", - "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "900", - "SGLANG_DSV4_REASONING_EFFORT": "high", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", - "SGLANG_JIT_DEEPGEMM_FAST_WARMUP": "1", - "SGLANG_JIT_DEEPGEMM_PRECOMPILE": "1", - "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", - "SGLANG_OPT_DEEPGEMM_MEGA_MOE": "1", - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", - "SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW": "1", - "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1", - "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", - "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", - "SGLANG_RAGGED_VERIFY_MODE": "static", - "SGLANG_REQUEST_STATE_WAIT_TIMEOUT": "60" - }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4-Pro-0813"], ["--served-model-name", "deepseek-ai/DeepSeek-V4-Pro-0813"], @@ -1900,7 +1934,8 @@ export const agentx = { ["--tp-size", "16"], ["--trust-remote-code"], ["--watchdog-timeout", "86400"] - ] + ], + "envRef": 3 } ] }, @@ -1909,7 +1944,33 @@ export const agentx = { "files": [], "blocks": [ { - "from": 4, + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": {}, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + }, + { + "kind": "mooncake-store", + "count_nodes": 4, + "ports": ["8800", "8801", "8802", "8803"], + "preamble": ["ulimit -n 1048576", "ulimit -l unlimited"], + "cmd": "python -m mooncake.mooncake_store_service", + "args": [ + ["--port", "8800"] + ], + "envRef": 4 + }, + { + "from": 6, "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -1925,7 +1986,7 @@ export const agentx = { } }, { - "from": 5, + "from": 7, "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -1985,27 +2046,25 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], - ["--hicache-write-policy", "write_back"] + ["--hicache-ratio", "2.75"], + ["--hicache-write-policy", "write_through"] ], - "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg", + "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-agg", "donorHardware": "gb200" } }, "prov": { - "sha": "2a58623b5a441f1b5c190ba9666a70881301d1d3", + "sha": "d9185455e166dfd034dcdfaf9608bd8335618b54", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml", - "run": "36833446190/attempts/1", - "pr": "3629", - "status": "verified", - "date": "2026-10-02" + "pr": "3701", + "status": "in-progress" }, "points": [ { "concs": [256], "item": 2, "goldenAL": "3.77", - "kv": "hicache" + "kv": "mooncake" } ], "routers": { @@ -2056,24 +2115,52 @@ export const agentx = { ] }, { - "nodes_per_worker": 4, - "diff": { - "args": { - "--nnodes": ["4"], - "--dp-size": ["16"], - "--ep-size": ["16"], - "--hicache-ratio": ["2"], - "--tp-size": ["16"] - } + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": { + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" }, - "ref": [6, "dynamo", 4] - }, - { - "nodes_per_worker": 8, - "diff": { - "args": { - "--nnodes": ["8"], - "--cuda-graph-max-bs-decode": ["64"], + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + }, + { + "kind": "mooncake-store", + "count_nodes": 8, + "ports": ["8800", "8801", "8802", "8803"], + "preamble": ["ulimit -n 1048576", "ulimit -l unlimited"], + "cmd": "python -m mooncake.mooncake_store_service", + "args": [ + ["--port", "8800"] + ], + "envRef": 2 + }, + { + "nodes_per_worker": 4, + "diff": { + "args": { + "--nnodes": ["4"], + "--dp-size": ["16"], + "--ep-size": ["16"], + "--tp-size": ["16"] + } + }, + "ref": [6, "dynamo", 6] + }, + { + "nodes_per_worker": 8, + "diff": { + "args": { + "--nnodes": ["8"], + "--cuda-graph-max-bs-decode": ["64"], "--dp-size": ["32"], "--ep-size": ["32"], "--max-running-requests": ["512"], @@ -2084,7 +2171,7 @@ export const agentx = { "SGLANG_DSV4_MHC_PREWARM": "1" } }, - "ref": [6, "dynamo", 5] + "ref": [6, "dynamo", 7] } ] }, @@ -2093,7 +2180,33 @@ export const agentx = { "files": [], "blocks": [ { - "from": 4, + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": {}, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + }, + { + "kind": "mooncake-store", + "count_nodes": 8, + "ports": ["8800", "8801", "8802", "8803"], + "preamble": ["ulimit -n 1048576", "ulimit -l unlimited"], + "cmd": "python -m mooncake.mooncake_store_service", + "args": [ + ["--port", "8800"] + ], + "envRef": 4 + }, + { + "from": 6, "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -2109,7 +2222,7 @@ export const agentx = { } }, { - "from": 5, + "from": 7, "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -2169,40 +2282,38 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], - ["--hicache-write-policy", "write_back"] + ["--hicache-ratio", "2.75"], + ["--hicache-write-policy", "write_through"] ], - "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg", + "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-agg", "donorHardware": "gb200" } }, "prov": { - "sha": "2a58623b5a441f1b5c190ba9666a70881301d1d3", - "run": "36833446190/attempts/1", - "pr": "3629", - "status": "verified", - "date": "2026-10-02" + "sha": "d9185455e166dfd034dcdfaf9608bd8335618b54", + "pr": "3701", + "status": "in-progress" }, "points": [ { "concs": [768], "item": 3, "goldenAL": "3.77", - "kv": "hicache", + "kv": "mooncake", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml" }, { "concs": [1024], "item": 4, "goldenAL": "3.77", - "kv": "hicache", + "kv": "mooncake", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml" }, { "concs": [1280], "item": 5, "goldenAL": "3.77", - "kv": "hicache", + "kv": "mooncake", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml" } ], @@ -2253,9 +2364,38 @@ export const agentx = { ["--active-prefill-tokens-threshold-frac", "None"] ] }, + { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": { + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + }, + { + "kind": "mooncake-store", + "count_nodes": 8, + "ports": ["8800", "8801", "8802", "8803"], + "preamble": ["ulimit -n 1048576", "ulimit -l unlimited"], + "cmd": "python -m mooncake.mooncake_store_service", + "args": [ + ["--port", "8800"] + ], + "envRef": 2 + }, { "workers": 2, - "ref": [7, "dynamo", 4] + "ref": [7, "dynamo", 6] }, { "diff": { @@ -2263,7 +2403,7 @@ export const agentx = { "--max-running-requests": ["1024"] } }, - "ref": [7, "dynamo", 5] + "ref": [7, "dynamo", 7] } ] }, @@ -2272,7 +2412,33 @@ export const agentx = { "files": [], "blocks": [ { - "from": 4, + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": {}, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + }, + { + "kind": "mooncake-store", + "count_nodes": 8, + "ports": ["8800", "8801", "8802", "8803"], + "preamble": ["ulimit -n 1048576", "ulimit -l unlimited"], + "cmd": "python -m mooncake.mooncake_store_service", + "args": [ + ["--port", "8800"] + ], + "envRef": 4 + }, + { + "from": 6, "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -2288,7 +2454,7 @@ export const agentx = { } }, { - "from": 5, + "from": 7, "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -2513,19 +2679,6 @@ export const agentx = { "workers": 1, "nodes_per_worker": 1, "cmd": "python3 -m sglang.launch_server", - "env": { - "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1", - "SGLANG_JIT_DEEPGEMM_FAST_WARMUP": "1", - "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1", - "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", - "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2": "1", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "1", - "SGLANG_OPT_USE_JIT_NORM": "1", - "SGLANG_OPT_USE_TOPK_V2": "1", - "SGLANG_TIMEOUT_KEEP_ALIVE": "900", - "TORCH_CUDA_ARCH_LIST": "10.0", - "TRITON_PTXAS_PATH": "/usr/local/cuda/bin/ptxas" - }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4-Pro"], ["--served-model-name", "deepseek-ai/DeepSeek-V4-Pro"], @@ -2551,7 +2704,8 @@ export const agentx = { ["--tool-call-parser", "deepseekv4"], ["--trust-remote-code"], ["--watchdog-timeout", "1800"] - ] + ], + "envRef": 6 } ] }, @@ -2863,10 +3017,10 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "3"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "donor": "dsv4-fp4-b300-sglang-agentic-hicache-eagle", "donorHardware": "b300" } }, @@ -3102,19 +3256,19 @@ export const agentx = { } }, { - "id": "dsv4-fp4-b300-dynamo-sglang-agentic-agg#0@c1", + "id": "dsv4-fp4-b300-sglang-agentic-hicache-mtp#0", "hw": "b300", "ckpt": "deepseek-ai/DeepSeek-V4-Pro-0813", - "label": "Agg TP8", - "detail": "DSpark 6 · dsv4-fp4-b300-dynamo-sglang-agentic-agg", + "label": "Agg TP4", + "detail": "DSpark 6 · dsv4-fp4-b300-sglang-agentic-hicache-mtp", "gpus": 8, - "configKey": "dsv4-fp4-b300-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9", - "imageStatus": "pruned", - "imageCommit": "c9a8fba9", - "submitted": "dynamo", + "configKey": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261008-7760d526@sha256:f991a9648fc0a00410bab27d96bf96a0263be9d9b178fe01ee95cffe622fd283", + "imageStatus": "ok", + "imageCommit": "7760d526", + "submitted": "sglang", "setup": [], - "disclosures": ["synthetic-acceptance"], + "disclosures": ["chat-template", "synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], "unavailable": {}, @@ -3131,112 +3285,361 @@ export const agentx = { } }, "prov": { - "sha": "3b798abfd43bc4260ab11e43d66c88a99e33461a", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-tp8-c1-mtp.yaml", - "run": "36974436489", - "pr": "3631", - "comment": "6010320587", - "status": "verified-pr" + "sha": "5b394d395eefcaacd5168e04aca95020adac5312", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml", + "pr": "3606", + "status": "in-progress" }, "points": [ { "concs": [1], "item": 0, + "override": "override_tp4_c1", "goldenAL": "3.77", "kv": "none" + }, + { + "concs": [2], + "item": 0, + "override": "override_tp4_c2", + "goldenAL": "3.77", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["4"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["4"] + } + } + ] + } + }, + { + "concs": [4], + "item": 0, + "override": "override_tp4_c4", + "goldenAL": "3.77", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["8"] + } + } + ] + } + }, + { + "concs": [8], + "item": 0, + "override": "override_tp4_c8", + "goldenAL": "3.77", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["16"] + } + } + ] + } + }, + { + "concs": [16], + "item": 0, + "override": "override_tp4_c16", + "goldenAL": "3.77", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["32"] + } + } + ] + } + }, + { + "concs": [32], + "item": 0, + "override": "override_tp4_c32", + "goldenAL": "3.77", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ] + } } ], "routers": { - "dynamo": { + "sglang": { "install": [], - "files": ["nats.conf#13771a85"], + "files": [], "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "nats", - "count_nodes": 1, - "cmd": "nats-server -c nats.conf", - "env": {}, - "args": [] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"], - ["--router-session-affinity-ttl-secs", "3600"], - ["--active-decode-blocks-threshold", "None"], - ["--active-prefill-tokens-threshold", "None"], - ["--active-prefill-tokens-threshold-frac", "None"] - ], - "envRef": 0 - }, { "kind": "worker", "role": "agg", "workers": 1, "nodes_per_worker": 1, - "cmd": "python3 -m dynamo.sglang", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "NATS_SERVER": "nats://$NATS_IP:4222", - "SGLANG_DEFAULT_THINKING": "1", - "SGLANG_DSV4_REASONING_EFFORT": "high", - "SGLANG_ENABLE_PREFILL_WAR_READ_DONE": "1", - "SGLANG_JIT_DEEPGEMM_FAST_WARMUP": "1", - "SGLANG_JIT_DEEPGEMM_PRECOMPILE": "1", - "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1", - "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", - "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2": "1", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "1", - "SGLANG_OPT_USE_JIT_NORM": "1", - "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", - "SGLANG_OPT_USE_TOPK_V2": "True", - "SGLANG_RAGGED_VERIFY_MODE": "static" - }, + "cmd": "python3 -m sglang.launch_server", "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4-Pro-0813"], ["--served-model-name", "deepseek-ai/DeepSeek-V4-Pro-0813"], ["--host", "0.0.0.0"], ["--port", "30000"], - ["--request-plane", "tcp"], + ["--allow-auto-truncate"], + ["--attention-backend", "dsv4"], ["--chunked-prefill-size", "8192"], - ["--cuda-graph-max-bs-decode", "2"], + ["--cuda-graph-max-bs-decode", "4"], ["--disable-flashinfer-autotune"], - ["--dp-size", "1"], - ["--enable-deepseek-v4-fp4-indexer"], - ["--ep-size", "1"], - ["--max-prefill-tokens", "8192"], + ["--disable-shared-experts-fusion"], ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.9"], + ["--mem-fraction-static", "0.88"], ["--moe-runner-backend", "flashinfer_mxfp4"], ["--page-size", "256"], - ["--scheduler-recv-interval", "30"], + ["--reasoning-parser", "deepseek-v4"], ["--speculative-algorithm", "DSPARK"], ["--speculative-dspark-block-size", "6"], ["--speculative-eagle-topk", "1"], ["--speculative-num-draft-tokens", "7"], ["--speculative-num-steps", "1"], - ["--stream-interval", "10"], ["--swa-full-tokens-ratio", "0.1"], - ["--tp-size", "8"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "deepseekv4"], ["--trust-remote-code"], - ["--watchdog-timeout", "1000000"], - ["--weight-loader-prefetch-checkpoints"] + ["--watchdog-timeout", "1800"] + ], + "envRef": 6 + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "dsv4-fp4-b300-dynamo-sglang-agentic-agg#0@c1", + "hw": "b300", + "ckpt": "deepseek-ai/DeepSeek-V4-Pro-0813", + "label": "Agg TP8", + "detail": "DSpark 6 · dsv4-fp4-b300-dynamo-sglang-agentic-agg", + "gpus": 8, + "configKey": "dsv4-fp4-b300-dynamo-sglang-agentic-agg", + "image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9", + "imageStatus": "pruned", + "imageCommit": "c9a8fba9", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2.75"], + ["--hicache-write-policy", "write_through"] + ], + "donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-tp8-c1-mtp.yaml", + "run": "37715955859", + "pr": "3817", + "comment": "6072539092", + "status": "verified-pr" + }, + "points": [ + { + "concs": [1], + "item": 0, + "goldenAL": "3.77", + "kv": "none" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": ["nats.conf#13771a85"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["2"], + "--dp-size": ["1"], + "--enable-deepseek-v4-fp4-indexer": [], + "--ep-size": ["1"], + "--max-prefill-tokens": ["8192"], + "--mem-fraction-static": ["0.9"], + "--scheduler-recv-interval": ["30"], + "--stream-interval": ["10"], + "--tp-size": ["8"], + "--watchdog-timeout": ["1000000"], + "--weight-loader-prefetch-checkpoints": [], + "--allow-auto-truncate": null, + "--attention-backend": null, + "--disable-shared-experts-fusion": null, + "--reasoning-parser": null, + "--tensor-parallel-size": null, + "--tool-call-parser": null + }, + "env": { + "NATS_SERVER": "nats://$NATS_IP:4222", + "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DSV4_REASONING_EFFORT": "high", + "SGLANG_ENABLE_PREFILL_WAR_READ_DONE": "1", + "SGLANG_JIT_DEEPGEMM_PRECOMPILE": "1", + "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", + "SGLANG_OPT_USE_TOPK_V2": "True", + "SGLANG_RAGGED_VERIFY_MODE": "static", + "SGLANG_ENABLE_UNIFIED_RADIX_TREE": null, + "SGLANG_TIMEOUT_KEEP_ALIVE": null, + "TORCH_CUDA_ARCH_LIST": null, + "TRITON_PTXAS_PATH": null + } + }, + "ref": [12, "dynamo", 2] } ] }, @@ -3274,24 +3677,22 @@ export const agentx = { } }, { - "id": "dsv4-fp4-b300-sglang-agentic-hicache-mtp#0", + "id": "dsv4-fp4-b300-sglang-agentic-hicache-mtp#1", "hw": "b300", "ckpt": "deepseek-ai/DeepSeek-V4-Pro-0813", "label": "Agg TP8", "detail": "DSpark 6 · dsv4-fp4-b300-sglang-agentic-hicache-mtp", "gpus": 8, "configKey": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", - "image": "lmsysorg/sglang:nightly-dev-20260901-07c8f729", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261008-7760d526@sha256:f991a9648fc0a00410bab27d96bf96a0263be9d9b178fe01ee95cffe622fd283", "imageStatus": "ok", - "imageCommit": "07c8f729", + "imageCommit": "7760d526", "submitted": "sglang", "setup": [], "disclosures": ["chat-template", "synthetic-acceptance"], "kv": { - "available": ["none", "hicache"], - "unavailable": { - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" - }, + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, "hicache": { "args": [ ["--enable-hierarchical-cache"], @@ -3305,31 +3706,57 @@ export const agentx = { } }, "prov": { - "sha": "3f52b08875691ed8eeb9d1834a6647a0d212c1ed", - "recipe": "benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh", - "run": "34595563156/attempts/1", - "pr": "2918", - "status": "verified", - "date": "2026-09-15" + "sha": "5b394d395eefcaacd5168e04aca95020adac5312", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml", + "pr": "3606", + "status": "in-progress" }, "points": [ { "concs": [1], - "item": 0, + "item": 1, + "override": "override_tp8_c1", + "goldenAL": "3.77", + "kv": "none" + }, + { + "concs": [2], + "item": 1, + "override": "override_tp8_c2", "goldenAL": "3.77", "kv": "none", - "job": "34595563156/job/103251129685" + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["4"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["4"] + } + } + ] + } }, { "concs": [4], - "item": 0, + "item": 1, + "override": "override_tp8_c4", "goldenAL": "3.77", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["16"], + "--cuda-graph-max-bs-decode": ["16"], "--max-running-requests": ["8"] } } @@ -3339,24 +3766,24 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["16"], + "--cuda-graph-max-bs-decode": ["16"], "--max-running-requests": ["8"] } } ] - }, - "job": "34595563156/job/103251129832" + } }, { "concs": [8], - "item": 0, + "item": 1, + "override": "override_tp8_c8", "goldenAL": "3.77", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["32"], + "--cuda-graph-max-bs-decode": ["32"], "--max-running-requests": ["16"] } } @@ -3366,24 +3793,24 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["32"], + "--cuda-graph-max-bs-decode": ["32"], "--max-running-requests": ["16"] } } ] - }, - "job": "34595563156/job/103251129768" + } }, { "concs": [16], - "item": 0, + "item": 1, + "override": "override_tp8_c16", "goldenAL": "3.77", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["64"], + "--cuda-graph-max-bs-decode": ["64"], "--max-running-requests": ["32"] } } @@ -3393,24 +3820,24 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["64"], + "--cuda-graph-max-bs-decode": ["64"], "--max-running-requests": ["32"] } } ] - }, - "job": "34595563156/job/103251129881" + } }, { "concs": [32], - "item": 0, + "item": 1, + "override": "override_tp8_c32", "goldenAL": "3.77", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["64"], + "--cuda-graph-max-bs-decode": ["64"], "--max-running-requests": ["64"] } } @@ -3420,13 +3847,12 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["64"], + "--cuda-graph-max-bs-decode": ["64"], "--max-running-requests": ["64"] } } ] - }, - "job": "34595563156/job/103251129678" + } } ], "routers": { @@ -3437,36 +3863,7 @@ export const agentx = { { "diff": { "args": { - "--allow-auto-truncate": [], - "--attention-backend": ["compressed"], - "--cuda-graph-max-bs": ["4"], - "--disable-shared-experts-fusion": [], - "--mem-fraction-static": ["0.88"], - "--reasoning-parser": ["deepseek-v4"], - "--tool-call-parser": ["deepseekv4"], - "--tp": ["8"], - "--watchdog-timeout": ["1800"], - "--cuda-graph-max-bs-decode": null, - "--dp-size": null, - "--enable-deepseek-v4-fp4-indexer": null, - "--ep-size": null, - "--max-prefill-tokens": null, - "--scheduler-recv-interval": null, - "--stream-interval": null, - "--tp-size": null, - "--weight-loader-prefetch-checkpoints": null - }, - "env": { - "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1", - "SGLANG_OPT_USE_TOPK_V2": "1", - "SGLANG_TIMEOUT_KEEP_ALIVE": "900", - "TORCH_CUDA_ARCH_LIST": "10.0", - "SGLANG_DEFAULT_THINKING": null, - "SGLANG_DSV4_REASONING_EFFORT": null, - "SGLANG_ENABLE_PREFILL_WAR_READ_DONE": null, - "SGLANG_JIT_DEEPGEMM_PRECOMPILE": null, - "SGLANG_OPT_USE_ONLINE_COMPRESS": null, - "SGLANG_RAGGED_VERIFY_MODE": null + "--tensor-parallel-size": ["8"] } }, "ref": [12, "sglang", 0] @@ -3523,7 +3920,7 @@ export const agentx = { "hw": "b300", "ckpt": "deepseek-ai/DeepSeek-V4-Pro-0813", "label": "Agg TP4", - "detail": "DSpark 6", + "detail": "DSpark 6 · dsv4-fp4-b300-dynamo-sglang-agentic-agg", "gpus": 8, "configKey": "dsv4-fp4-b300-dynamo-sglang-agentic-agg", "image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9", @@ -3548,10 +3945,10 @@ export const agentx = { } }, "prov": { - "sha": "3b798abfd43bc4260ab11e43d66c88a99e33461a", - "run": "36974436489", - "pr": "3631", - "comment": "6010320587", + "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", + "run": "37715955859", + "pr": "3817", + "comment": "6072539092", "status": "verified-pr" }, "points": [ @@ -3647,7 +4044,7 @@ export const agentx = { "--tp-size": ["4"] } }, - "ref": [12, "dynamo", 3] + "ref": [13, "dynamo", 3] } ] }, @@ -3685,55 +4082,52 @@ export const agentx = { } }, { - "id": "dsv4-fp4-b300-sglang-agentic-hicache-mtp#1", + "id": "dsv4-fp4-b300-sglang-agentic-hicache-mtp#2", "hw": "b300", "ckpt": "deepseek-ai/DeepSeek-V4-Pro-0813", "label": "Agg DEP8", "detail": "DSpark 6 · dsv4-fp4-b300-sglang-agentic-hicache-mtp", "gpus": 8, "configKey": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", - "image": "lmsysorg/sglang:nightly-dev-20260901-07c8f729", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261008-7760d526@sha256:f991a9648fc0a00410bab27d96bf96a0263be9d9b178fe01ee95cffe622fd283", "imageStatus": "ok", - "imageCommit": "07c8f729", + "imageCommit": "7760d526", "submitted": "sglang", "setup": [], "disclosures": ["chat-template", "synthetic-acceptance"], "kv": { - "available": ["none", "hicache"], - "unavailable": { - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" - }, + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, "hicache": { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "3"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "donor": "dsv4-fp4-b300-sglang-agentic-hicache-eagle", "donorHardware": "b300" } }, "prov": { - "sha": "3f52b08875691ed8eeb9d1834a6647a0d212c1ed", - "recipe": "benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh", - "run": "34595563156/attempts/1", - "pr": "2918", - "status": "verified", - "date": "2026-09-15" + "sha": "5b394d395eefcaacd5168e04aca95020adac5312", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml", + "pr": "3606", + "status": "in-progress" }, "points": [ { "concs": [32], - "item": 1, + "item": 2, + "override": "override_dep8_c32", "goldenAL": "3.77", - "kv": "hicache", - "job": "34595563156/job/103251129927" + "kv": "hicache" }, { "concs": [64], - "item": 1, + "item": 2, + "override": "override_dep8_c64", "goldenAL": "3.77", "kv": "hicache", "patch": { @@ -3754,12 +4148,12 @@ export const agentx = { } } ] - }, - "job": "34595563156/job/103251129864" + } }, { "concs": [128], - "item": 1, + "item": 2, + "override": "override_dep8_c128", "goldenAL": "3.77", "kv": "hicache", "patch": { @@ -3780,12 +4174,12 @@ export const agentx = { } } ] - }, - "job": "34595563156/job/103251129925" + } }, { "concs": [256], - "item": 1, + "item": 2, + "override": "override_dep8_c256", "goldenAL": "3.77", "kv": "hicache", "patch": { @@ -3806,20 +4200,19 @@ export const agentx = { } } ] - }, - "job": "34595563156/job/103251129840" + } }, { "concs": [384], - "item": 1, + "item": 2, + "override": "override_dep8_c384", "goldenAL": "3.77", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["768"], - "--mem-fraction-static": ["0.88"] + "--max-running-requests": ["768"] } }, null @@ -3829,25 +4222,23 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["768"], - "--mem-fraction-static": ["0.88"] + "--max-running-requests": ["768"] } } ] - }, - "job": "34595563156/job/103251129938" + } }, { "concs": [512], - "item": 1, + "item": 2, + "override": "override_dep8_c512", "goldenAL": "3.77", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["1024"], - "--mem-fraction-static": ["0.86"] + "--max-running-requests": ["768"] } }, null @@ -3857,25 +4248,23 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["1024"], - "--mem-fraction-static": ["0.86"] + "--max-running-requests": ["768"] } } ] - }, - "job": "34595563156/job/103251129788" + } }, { "concs": [576], - "item": 1, + "item": 2, + "override": "override_dep8_c576", "goldenAL": "3.77", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["1152"], - "--mem-fraction-static": ["0.86"] + "--max-running-requests": ["768"] } }, null @@ -3885,13 +4274,11 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["1152"], - "--mem-fraction-static": ["0.86"] + "--max-running-requests": ["768"] } } ] - }, - "job": "34595563156/job/103251129766" + } } ], "routers": { @@ -3902,59 +4289,62 @@ export const agentx = { { "diff": { "args": { - "--dist-init-addr": ["$AGG_HEAD_IP:20000"], "--chunked-prefill-size": ["65536"], "--cuda-graph-max-bs-decode": ["544"], - "--dp": ["8"], + "--data-parallel-size": ["8"], "--enable-deepseek-v4-fp4-indexer": [], "--enable-dp-attention": [], "--enable-dp-attention-local-control-broadcast": [], "--enable-dp-lm-head": [], "--enable-hierarchical-cache": [], + "--enable-mixed-chunk": [], "--enable-prefill-delayer": [], "--enable-w4a4-mxfp4-megamoe": [], - "--ep-size": ["8"], + "--expert-parallel-size": ["8"], "--hicache-io-backend": ["direct"], "--hicache-mem-layout": ["page_first_direct"], - "--hicache-ratio": ["3"], + "--hicache-ratio": ["2"], "--hicache-write-policy": ["write_back"], "--incremental-streaming-output": [], "--max-running-requests": ["64"], - "--mem-fraction-static": ["0.90"], + "--mem-fraction-static": ["0.84"], "--moe-a2a-backend": ["megamoe"], - "--prefill-decode-interval": ["20"], + "--prefill-decode-interval": ["32"], + "--schedule-policy": ["shortest-prefill-first"], "--skip-server-warmup": [], "--stream-interval": ["20"], "--swa-full-tokens-ratio": ["0.075"], "--tokenizer-worker-num": ["8"], - "--cuda-graph-max-bs": null, "--moe-runner-backend": null }, "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320" + "SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION": "1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320", + "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [13, "sglang", 0] + "ref": [14, "sglang", 0] }, { "kind": "sglang-router", "cmd": "python3 -m sglang_router.launch_router", + "env": {}, "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], ["--policy", "consistent_hashing"], ["--request-id-headers", "x-correlation-id"], ["--dp-aware"], - ["--host", "0.0.0.0"], - ["--port", "8000"], ["--connect-timeout-secs", "900"], ["--request-timeout-secs", "14400"], ["--disable-health-check"], + ["--disable-circuit-breaker"], ["--retry-max-retries", "8"], ["--retry-initial-backoff-ms", "500"], ["--retry-max-backoff-ms", "10000"], ["--retry-backoff-multiplier", "2"], ["--worker-urls", "http://127.0.0.1:30000"] - ], - "envRef": 5 + ] } ] }, @@ -3991,8 +4381,7 @@ export const agentx = { "cmd": "python3 -m dynamo.sglang", "diff": { "args": { - "--request-plane": ["tcp"], - "--dist-init-addr": null + "--request-plane": ["tcp"] }, "env": { "DYN_REQUEST_PLANE": "tcp", @@ -4016,13 +4405,7 @@ export const agentx = { "imageStatus": "pruned", "imageCommit": "c9a8fba9", "submitted": "dynamo", - "setup": [ - { - "kind": "note", - "script": "install-mooncake-efa-cu13.sh", - "reason": "Installs a Mooncake build with AWS EFA support on SA's B300 cluster. Needed only on EFA fabrics; on InfiniBand keep the image's Mooncake and set MOONCAKE_PROTOCOL=rdma." - } - ], + "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], @@ -4039,10 +4422,10 @@ export const agentx = { } }, "prov": { - "sha": "3b798abfd43bc4260ab11e43d66c88a99e33461a", - "run": "36974436489", - "pr": "3631", - "comment": "6010320587", + "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", + "run": "37715955859", + "pr": "3817", + "comment": "6072539092", "status": "verified-pr" }, "points": [ @@ -4125,7 +4508,7 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 4 + "envRef": 5 }, { "kind": "worker", @@ -4136,12 +4519,11 @@ export const agentx = { "env": { "DYN_REQUEST_PLANE": "tcp", "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "MC_MS_FILTERS": "$EFA_DEVICES", - "MOONCAKE_PROTOCOL": "efa", "NATS_SERVER": "nats://$NATS_IP:4222", "NCCL_CUMEM_ENABLE": "1", "NCCL_TIMEOUT": "100000", "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DISAGGREGATION_NIXL_BACKEND": "LIBFABRIC", "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "900", "SGLANG_DSV4_MHC_PREWARM": "1", "SGLANG_DSV4_REASONING_EFFORT": "high", @@ -4158,8 +4540,7 @@ export const agentx = { "SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE": "1", "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", "SGLANG_RAGGED_VERIFY_MODE": "static", - "SGLANG_REQUEST_STATE_WAIT_TIMEOUT": "60", - "WITH_NVIDIA_PEERMEM": "0" + "SGLANG_REQUEST_STATE_WAIT_TIMEOUT": "60" }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4-Pro-0813"], @@ -4173,7 +4554,7 @@ export const agentx = { ["--chunked-prefill-size", "65536"], ["--cuda-graph-max-bs-prefill", "256"], ["--disable-flashinfer-autotune"], - ["--disaggregation-transfer-backend", "mooncake"], + ["--disaggregation-transfer-backend", "nixl"], ["--dp-size", "8"], ["--enable-deepseek-v4-fp4-indexer"], ["--enable-dp-attention"], @@ -4211,13 +4592,12 @@ export const agentx = { "env": { "DYN_REQUEST_PLANE": "tcp", "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "MC_MS_FILTERS": "$EFA_DEVICES", - "MOONCAKE_PROTOCOL": "efa", "NATS_SERVER": "nats://$NATS_IP:4222", "NCCL_CUMEM_ENABLE": "1", "NCCL_TIMEOUT": "100000", "SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "8", "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DISAGGREGATION_NIXL_BACKEND": "LIBFABRIC", "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "900", "SGLANG_DSV4_REASONING_EFFORT": "high", "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", @@ -4230,8 +4610,7 @@ export const agentx = { "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", "SGLANG_RAGGED_VERIFY_MODE": "static", - "SGLANG_REQUEST_STATE_WAIT_TIMEOUT": "60", - "WITH_NVIDIA_PEERMEM": "0" + "SGLANG_REQUEST_STATE_WAIT_TIMEOUT": "60" }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4-Pro-0813"], @@ -4243,7 +4622,7 @@ export const agentx = { ["--kv-events-config", "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"], ["--cuda-graph-max-bs-decode", "256"], ["--disable-flashinfer-autotune"], - ["--disaggregation-transfer-backend", "mooncake"], + ["--disaggregation-transfer-backend", "nixl"], ["--dp-size", "8"], ["--enable-deepseek-v4-fp4-indexer"], ["--enable-dp-attention"], @@ -4335,13 +4714,7 @@ export const agentx = { "imageStatus": "pruned", "imageCommit": "c9a8fba9", "submitted": "dynamo", - "setup": [ - { - "kind": "note", - "script": "install-mooncake-efa-cu13.sh", - "reason": "Installs a Mooncake build with AWS EFA support on SA's B300 cluster. Needed only on EFA fabrics; on InfiniBand keep the image's Mooncake and set MOONCAKE_PROTOCOL=rdma." - } - ], + "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], @@ -4358,11 +4731,11 @@ export const agentx = { } }, "prov": { - "sha": "3b798abfd43bc4260ab11e43d66c88a99e33461a", + "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p2d-dep8-dep8-c64-mtp-kvoffload.yaml", - "run": "36974436489", - "pr": "3631", - "comment": "6010320587", + "run": "37715955859", + "pr": "3817", + "comment": "6072539092", "status": "verified-pr" }, "points": [ @@ -4414,14 +4787,14 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 4 + "envRef": 5 }, { - "ref": [16, "dynamo", 4] + "ref": [17, "dynamo", 4] }, { "workers": 2, - "ref": [16, "dynamo", 5] + "ref": [17, "dynamo", 5] } ] }, @@ -4501,19 +4874,19 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "3"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "donor": "dsv4-fp4-b300-sglang-agentic-hicache-eagle", "donorHardware": "b300" } }, "prov": { - "sha": "3b798abfd43bc4260ab11e43d66c88a99e33461a", + "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-dep8-c384-mtp-kvoffload.yaml", - "run": "36974436489", - "pr": "3631", - "comment": "6010320587", + "run": "37715955859", + "pr": "3817", + "comment": "6072539092", "status": "verified-pr" }, "points": [ @@ -4571,21 +4944,26 @@ export const agentx = { "diff": { "args": { "--kv-events-config": ["{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"], - "--attention-backend": ["dsv4"], "--cuda-graph-max-bs-decode": ["96"], "--dp-size": ["8"], + "--ep-size": ["8"], + "--hicache-ratio": ["3"], "--max-running-requests": ["768"], "--mem-fraction-static": ["0.88"], + "--prefill-decode-interval": ["20"], "--tp-size": ["8"], "--watchdog-timeout": ["1000000"], "--weight-loader-prefetch-checkpoints": [], - "--dp": null, + "--data-parallel-size": null, + "--enable-mixed-chunk": null, "--enable-w4a4-mxfp4-megamoe": null, + "--expert-parallel-size": null, "--reasoning-parser": null, + "--schedule-policy": null, "--skip-server-warmup": null, + "--tensor-parallel-size": null, "--tokenizer-worker-num": null, - "--tool-call-parser": null, - "--tp": null + "--tool-call-parser": null }, "env": { "NATS_SERVER": "nats://$NATS_IP:4222", @@ -4595,13 +4973,12 @@ export const agentx = { "SGLANG_JIT_DEEPGEMM_PRECOMPILE": "1", "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", "SGLANG_OPT_USE_TOPK_V2": "True", - "SGLANG_RAGGED_VERIFY_MODE": "static", - "TRITON_PTXAS_PATH": "/usr/local/cuda/bin/ptxas", + "SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION": null, "SGLANG_ENABLE_UNIFIED_RADIX_TREE": null, "SGLANG_TIMEOUT_KEEP_ALIVE": null } }, - "ref": [15, "dynamo", 2] + "ref": [16, "dynamo", 2] } ] }, @@ -5419,11 +5796,24 @@ export const agentx = { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320" } }, - "ref": [20, "sglang", 0] + "ref": [21, "sglang", 0] }, { "kind": "sglang-router", "cmd": "python3 -m sglang_router.launch_router", + "env": { + "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1", + "SGLANG_JIT_DEEPGEMM_FAST_WARMUP": "1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320", + "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1", + "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", + "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2": "1", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "1", + "SGLANG_OPT_USE_JIT_NORM": "1", + "SGLANG_OPT_USE_TOPK_V2": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "900", + "TORCH_CUDA_ARCH_LIST": "10.0" + }, "args": [ ["--policy", "cache_aware"], ["--request-id-headers", "x-correlation-id"], @@ -5435,8 +5825,7 @@ export const agentx = { ["--disable-health-check"], ["--disable-retries"], ["--worker-urls", "http://127.0.0.1:30000"] - ], - "envRef": 5 + ] } ] }, @@ -5905,7 +6294,7 @@ export const agentx = { "--swa-full-tokens-ratio": ["0.02"] } }, - "ref": [22, "dynamo", 4] + "ref": [23, "dynamo", 4] }, { "diff": { @@ -5918,7 +6307,7 @@ export const agentx = { "SGLANG_DSV4_MHC_PREWARM": "1" } }, - "ref": [22, "dynamo", 5] + "ref": [23, "dynamo", 5] } ] }, diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx index 04c9b1aea9a2..2f59ae137002 100644 --- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx +++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx @@ -1,37 +1,57 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@0259609ce99d plus open PRs #3187, #3598, #3631, #3659, #3745), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@ceb50ba21208 plus open PRs #2978, #3606, #3701, #3744, #3782, #3816, #3817, #3836, #3840), with SA-specific settings removed. export const agentx = { "model": "DeepSeek-V4.1", - "generatedOn": "2026-10-06", + "generatedOn": "2026-10-08", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "0259609ce99d0f4f7bd7180c3b3436e4f0334a46", + "main": "ceb50ba212080174e0563d2be1db9b0d654ab2ef", "prs": { - "3598": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37260940238", - "sha": "0f45d5f4c3d57f9804f7655d7b3b294acdd53441", + "3816": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37714835620", + "sha": "e67ca6fc9ecf4cd57f27654bda8a3e9059923820", "status": "verified-pr" }, - "3631": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/36974436489", - "sha": "3b798abfd43bc4260ab11e43d66c88a99e33461a", - "status": "verified-pr" + "3840": { + "run": null, + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "status": "in-progress" }, - "3659": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/36997573694", - "sha": "e79239bab1e1cb487dc74e27d693110f03384d41", + "2978": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34548026742", + "sha": "6b7dcd07ab0c8c0805ac7fbec2f2ded38d450c20", "status": "verified-pr" }, - "3187": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37270628136", - "sha": "34d06e54c7614bb33d2a271b010ffde3ce3fa563", + "3817": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", + "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "status": "verified-pr" }, - "3745": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37420473448", - "sha": "322bc37b368c85828357afb57dce6739d94ef955", - "status": "verified-pr" + "3606": { + "run": null, + "sha": "5b394d395eefcaacd5168e04aca95020adac5312", + "status": "in-progress" + }, + "3701": { + "run": null, + "sha": "d9185455e166dfd034dcdfaf9608bd8335618b54", + "status": "in-progress" + }, + "3782": { + "run": null, + "sha": "f00c81ab40297f98a68d2726d4301b44a2892419", + "status": "in-progress" + }, + "3744": { + "run": null, + "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "status": "in-progress" + }, + "3836": { + "run": null, + "sha": "03ffb35db157a0fecc78206bea748449daa1385a", + "status": "in-progress" } } }, @@ -183,10 +203,10 @@ export const agentx = { "prov": { "sha": "0f45d5f4c3d57f9804f7655d7b3b294acdd53441", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "37260940238", + "run": "37260940238/attempts/1", "pr": "3598", - "comment": "6006341782", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -318,9 +338,9 @@ export const agentx = { "detail": "DSpark 5 · dsv41flash-fp4-gb300-sglang-agentic-dspark", "gpus": 4, "configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-cu13-nightly-0924@sha256:d2c9929f8cc9889326203f6cbc46da48b69de42891b167592c9c76cc803d246e", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260929-79cafec0@sha256:0f075735a6bf913a7cc1cd7ece4f526ae8af70fc5b88fdc02d9efe92a61cf3ae", "imageStatus": "ok", - "imageCommit": null, + "imageCommit": "79cafec0", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -339,12 +359,10 @@ export const agentx = { } }, "prov": { - "sha": "68898926209230628f500819194999b8dd89f17d", + "sha": "f00c81ab40297f98a68d2726d4301b44a2892419", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "36490327584/attempts/1", - "pr": "3421", - "status": "verified", - "date": "2026-09-29" + "pr": "3782", + "status": "in-progress" }, "points": [ { @@ -529,7 +547,10 @@ export const agentx = { { "args": { "--chunked-prefill-size": ["8192"], - "--max-running-requests": ["64"], + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"], + "--mem-fraction-static": ["0.85"], + "--prefill-decode-interval": ["4"], "--swa-prefix-tails": ["4096"] } } @@ -540,7 +561,10 @@ export const agentx = { { "args": { "--chunked-prefill-size": ["8192"], - "--max-running-requests": ["64"], + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"], + "--mem-fraction-static": ["0.85"], + "--prefill-decode-interval": ["4"], "--swa-prefix-tails": ["4096"] } } @@ -560,6 +584,7 @@ export const agentx = { "--weight-loader-prefetch-checkpoints": null }, "env": { + "SGLANG_DSPARK_FOLDED_SAMPLING": "2", "SGLANG_RAGGED_VERIFY_MODE": null } }, @@ -643,10 +668,10 @@ export const agentx = { "prov": { "sha": "0f45d5f4c3d57f9804f7655d7b3b294acdd53441", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "37260940238", + "run": "37260940238/attempts/1", "pr": "3598", - "comment": "6006341782", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -749,9 +774,9 @@ export const agentx = { "detail": "DSpark 5 · dsv41flash-fp4-gb300-sglang-agentic-dspark", "gpus": 4, "configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-cu13-nightly-0924@sha256:d2c9929f8cc9889326203f6cbc46da48b69de42891b167592c9c76cc803d246e", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260929-79cafec0@sha256:0f075735a6bf913a7cc1cd7ece4f526ae8af70fc5b88fdc02d9efe92a61cf3ae", "imageStatus": "ok", - "imageCommit": null, + "imageCommit": "79cafec0", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -770,12 +795,10 @@ export const agentx = { } }, "prov": { - "sha": "68898926209230628f500819194999b8dd89f17d", + "sha": "f00c81ab40297f98a68d2726d4301b44a2892419", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "36490327584/attempts/1", - "pr": "3421", - "status": "verified", - "date": "2026-09-29" + "pr": "3782", + "status": "in-progress" }, "points": [ { @@ -825,6 +848,7 @@ export const agentx = { "--weight-loader-prefetch-checkpoints": null }, "env": { + "SGLANG_DSPARK_FOLDED_SAMPLING": "2", "SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT": null } }, @@ -885,9 +909,9 @@ export const agentx = { "detail": "DSpark 5", "gpus": 4, "configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-cu13-nightly-0924@sha256:d2c9929f8cc9889326203f6cbc46da48b69de42891b167592c9c76cc803d246e", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260929-79cafec0@sha256:0f075735a6bf913a7cc1cd7ece4f526ae8af70fc5b88fdc02d9efe92a61cf3ae", "imageStatus": "ok", - "imageCommit": null, + "imageCommit": "79cafec0", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -906,12 +930,10 @@ export const agentx = { } }, "prov": { - "sha": "68898926209230628f500819194999b8dd89f17d", + "sha": "f00c81ab40297f98a68d2726d4301b44a2892419", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "36490327584/attempts/1", - "pr": "3421", - "status": "verified", - "date": "2026-09-29" + "pr": "3782", + "status": "in-progress" }, "points": [ { @@ -947,7 +969,27 @@ export const agentx = { "item": 2, "override": "override_tp4_c64", "goldenAL": "3.51", - "kv": "none" + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--chunked-prefill-size": ["8192"], + "--prefill-decode-interval": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--chunked-prefill-size": ["8192"], + "--prefill-decode-interval": ["16"] + } + } + ] + } }, { "concs": [128], @@ -1057,10 +1099,10 @@ export const agentx = { "prov": { "sha": "0f45d5f4c3d57f9804f7655d7b3b294acdd53441", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "37260940238", + "run": "37260940238/attempts/1", "pr": "3598", - "comment": "6006341782", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -1309,10 +1351,10 @@ export const agentx = { "prov": { "sha": "0f45d5f4c3d57f9804f7655d7b3b294acdd53441", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "37260940238", + "run": "37260940238/attempts/1", "pr": "3598", - "comment": "6006341782", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -1505,10 +1547,10 @@ export const agentx = { "prov": { "sha": "0f45d5f4c3d57f9804f7655d7b3b294acdd53441", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "37260940238", + "run": "37260940238/attempts/1", "pr": "3598", - "comment": "6006341782", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -2260,13 +2302,13 @@ export const agentx = { "id": "dsv41flash-fp4-b300-sglang-agentic-dspark#1", "hw": "b300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Agg TP2·EP2", + "label": "Agg TP2", "detail": "DSpark 5", "gpus": 8, "configKey": "dsv41flash-fp4-b300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:35ea4d321b0735051dcce2599fc495853ad1ef30f6a1908f0a75e74204362d14", - "imageStatus": "digest-only", - "imageCommit": "582389ce", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34", + "imageStatus": "ok", + "imageCommit": "295a53c9", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2285,49 +2327,23 @@ export const agentx = { } }, "prov": { - "sha": "d4c38818ce6eb20093b7f6cc7ba860522ef41632", - "recipe": "benchmarks/single_node/agentic/dsv41flash_fp4_b300_sglang_mtp.sh", - "run": "35743388889/attempts/1", - "pr": "3342", - "status": "verified", - "date": "2026-09-22" + "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml", + "pr": "3744", + "status": "in-progress" }, "points": [ { "concs": [1], "item": 1, + "override": "override_tp2_c1", "goldenAL": "3.51", - "kv": "none", - "job": "35743388889/job/106799667235" - }, - { - "concs": [2], - "item": 1, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["4"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["4"] - } - } - ] - }, - "job": "35743388889/job/106799667317" + "kv": "none" }, { "concs": [4], "item": 1, + "override": "override_tp2_c4", "goldenAL": "3.51", "kv": "none", "patch": { @@ -2335,6 +2351,7 @@ export const agentx = { { "args": { "--max-running-requests": ["8"], + "--prefill-decode-interval": ["96"], "--swa-prefix-tails": ["256"] } } @@ -2345,43 +2362,17 @@ export const agentx = { { "args": { "--max-running-requests": ["8"], + "--prefill-decode-interval": ["96"], "--swa-prefix-tails": ["256"] } } ] - }, - "job": "35743388889/job/106799667390" - }, - { - "concs": [8], - "item": 1, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["16"], - "--swa-prefix-tails": ["512"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["16"], - "--swa-prefix-tails": ["512"] - } - } - ] - }, - "job": "35743388889/job/106799668150" + } }, { "concs": [16], "item": 1, + "override": "override_tp2_c16", "goldenAL": "3.51", "kv": "none", "patch": { @@ -2389,7 +2380,8 @@ export const agentx = { { "args": { "--max-running-requests": ["32"], - "--mem-fraction-static": ["0.80"], + "--mem-fraction-static": ["0.8"], + "--prefill-decode-interval": ["192"], "--swa-prefix-tails": ["1024"] } } @@ -2400,27 +2392,28 @@ export const agentx = { { "args": { "--max-running-requests": ["32"], - "--mem-fraction-static": ["0.80"], + "--mem-fraction-static": ["0.8"], + "--prefill-decode-interval": ["192"], "--swa-prefix-tails": ["1024"] } } ] - }, - "job": "35743388889/job/106799667458" + } }, { - "concs": [32], + "concs": [24], "item": 1, + "override": "override_tp2_c24", "goldenAL": "3.51", "kv": "none", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["64"], - "--mem-fraction-static": ["0.85"], - "--prefill-decode-interval": ["4"], - "--swa-prefix-tails": ["2048"] + "--max-running-requests": ["48"], + "--mem-fraction-static": ["0.8"], + "--prefill-decode-interval": ["96"], + "--swa-prefix-tails": ["1536"] } } ], @@ -2429,19 +2422,19 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["64"], - "--mem-fraction-static": ["0.85"], - "--prefill-decode-interval": ["4"], - "--swa-prefix-tails": ["2048"] + "--max-running-requests": ["48"], + "--mem-fraction-static": ["0.8"], + "--prefill-decode-interval": ["96"], + "--swa-prefix-tails": ["1536"] } } ] - }, - "job": "35743388889/job/106799667227" + } }, { - "concs": [64], + "concs": [32], "item": 1, + "override": "override_tp2_c32", "goldenAL": "3.51", "kv": "none", "patch": { @@ -2450,8 +2443,8 @@ export const agentx = { "args": { "--max-running-requests": ["64"], "--mem-fraction-static": ["0.85"], - "--prefill-decode-interval": ["4"], - "--swa-prefix-tails": ["4096"] + "--prefill-decode-interval": ["64"], + "--swa-prefix-tails": ["2048"] } } ], @@ -2462,17 +2455,17 @@ export const agentx = { "args": { "--max-running-requests": ["64"], "--mem-fraction-static": ["0.85"], - "--prefill-decode-interval": ["4"], - "--swa-prefix-tails": ["4096"] + "--prefill-decode-interval": ["64"], + "--swa-prefix-tails": ["2048"] } } ] - }, - "job": "35743388889/job/106799666217" + } }, { - "concs": [128], + "concs": [48], "item": 1, + "override": "override_tp2_c48", "goldenAL": "3.51", "kv": "none", "patch": { @@ -2480,9 +2473,9 @@ export const agentx = { { "args": { "--max-running-requests": ["64"], - "--mem-fraction-static": ["0.85"], - "--prefill-decode-interval": ["4"], - "--swa-prefix-tails": ["4096"] + "--mem-fraction-static": ["0.8"], + "--prefill-decode-interval": ["64"], + "--swa-prefix-tails": ["3072"] } } ], @@ -2492,14 +2485,13 @@ export const agentx = { { "args": { "--max-running-requests": ["64"], - "--mem-fraction-static": ["0.85"], - "--prefill-decode-interval": ["4"], - "--swa-prefix-tails": ["4096"] + "--mem-fraction-static": ["0.8"], + "--prefill-decode-interval": ["64"], + "--swa-prefix-tails": ["3072"] } } ] - }, - "job": "35743388889/job/106799667031" + } } ], "routers": { @@ -2518,18 +2510,18 @@ export const agentx = { ["--served-model-name", "deepseek-ai/DeepSeek-V4.1-Flash"], ["--host", "0.0.0.0"], ["--port", "30000"], - ["--chunked-prefill-size", "4096"], + ["--chunked-prefill-size", "16384"], ["--cuda-graph-max-bs-decode", "64"], - ["--ep-size", "2"], + ["--expert-parallel-size", "1"], ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.70"], + ["--mem-fraction-static", "0.7"], ["--prefill-decode-interval", "16"], ["--reasoning-parser", "auto"], ["--speculative-algorithm", "DSPARK"], ["--speculative-dspark-block-size", "5"], ["--swa-prefix-tails", "128"], + ["--tensor-parallel-size", "2"], ["--tool-call-parser", "auto"], - ["--tp", "2"], ["--trust-remote-code"], ["--watchdog-timeout", "3600"], ["--weight-loader-prefetch-checkpoints"] @@ -2587,13 +2579,13 @@ export const agentx = { "id": "dsv41flash-fp4-b300-sglang-agentic-dspark#0", "hw": "b300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Agg TP4·EP4", + "label": "Agg TP4", "detail": "DSpark 5", "gpus": 8, "configKey": "dsv41flash-fp4-b300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:35ea4d321b0735051dcce2599fc495853ad1ef30f6a1908f0a75e74204362d14", - "imageStatus": "digest-only", - "imageCommit": "582389ce", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34", + "imageStatus": "ok", + "imageCommit": "295a53c9", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2612,49 +2604,23 @@ export const agentx = { } }, "prov": { - "sha": "d4c38818ce6eb20093b7f6cc7ba860522ef41632", - "recipe": "benchmarks/single_node/agentic/dsv41flash_fp4_b300_sglang_mtp.sh", - "run": "35743388889/attempts/1", - "pr": "3342", - "status": "verified", - "date": "2026-09-22" + "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml", + "pr": "3744", + "status": "in-progress" }, "points": [ { "concs": [1], "item": 0, + "override": "override_tp4_c1", "goldenAL": "3.51", - "kv": "none", - "job": "35743388889/job/106799666410" - }, - { - "concs": [2], - "item": 0, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["4"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["4"] - } - } - ] - }, - "job": "35743388889/job/106799667236" + "kv": "none" }, { "concs": [4], "item": 0, + "override": "override_tp4_c4", "goldenAL": "3.51", "kv": "none", "patch": { @@ -2662,6 +2628,7 @@ export const agentx = { { "args": { "--max-running-requests": ["8"], + "--prefill-decode-interval": ["192"], "--swa-prefix-tails": ["256"] } } @@ -2672,147 +2639,115 @@ export const agentx = { { "args": { "--max-running-requests": ["8"], + "--prefill-decode-interval": ["192"], "--swa-prefix-tails": ["256"] } } ] - }, - "job": "35743388889/job/106799667229" - }, - { - "concs": [8], - "item": 0, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["16"], - "--swa-prefix-tails": ["512"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["16"], - "--swa-prefix-tails": ["512"] - } - } - ] - }, - "job": "35743388889/job/106799667206" - }, - { - "concs": [16], - "item": 0, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["32"], - "--swa-prefix-tails": ["1024"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["32"], - "--swa-prefix-tails": ["1024"] - } - } - ] - }, - "job": "35743388889/job/106799667446" - }, - { - "concs": [32], - "item": 0, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["64"], - "--swa-prefix-tails": ["2048"] - } - } - ], - "dynamo": [ - null, - null, - { + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { "args": { - "--max-running-requests": ["64"], - "--swa-prefix-tails": ["2048"] + "--tensor-parallel-size": ["4"] } - } - ] - }, - "job": "35743388889/job/106799667454" + }, + "ref": [10, "sglang", 0] + } + ] }, - { - "concs": [64], - "item": 0, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["64"], - "--swa-prefix-tails": ["4096"] - } - } - ], - "dynamo": [ - null, - null, - { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { "args": { - "--max-running-requests": ["64"], - "--swa-prefix-tails": ["4096"] + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" } } - ] - }, - "job": "35743388889/job/106799667008" - }, + } + ] + } + } + }, + { + "id": "dsv41flash-fp4-b300-sglang-agentic-dspark#2", + "hw": "b300", + "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", + "label": "Agg DEP4", + "detail": "DSpark 5", + "gpus": 8, + "configKey": "dsv41flash-fp4-b300-sglang-agentic-dspark", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34", + "imageStatus": "ok", + "imageCommit": "295a53c9", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-ratio", "1"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml", + "pr": "3744", + "status": "in-progress" + }, + "points": [ { - "concs": [128], - "item": 0, + "concs": [192], + "item": 2, + "override": "override_dp4_c192", "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["64"], - "--swa-prefix-tails": ["4096"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["64"], - "--swa-prefix-tails": ["4096"] - } - } - ] - }, - "job": "35743388889/job/106799667096" + "kv": "none" } ], "routers": { @@ -2823,11 +2758,37 @@ export const agentx = { { "diff": { "args": { - "--ep-size": ["4"], - "--tp": ["4"] + "--chunked-prefill-size": ["32768"], + "--data-parallel-size": ["4"], + "--enable-dp-attention": [], + "--enable-dp-attention-local-control-broadcast": [], + "--enable-dp-lm-head": [], + "--expert-parallel-size": ["4"], + "--load-balance-method": ["total_requests"], + "--max-running-requests": ["384"], + "--mem-fraction-static": ["0.76"], + "--swa-prefix-tails": ["3072"], + "--tokenizer-worker-num": ["8"] } }, - "ref": [10, "sglang", 0] + "ref": [11, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--request-id-headers", "x-correlation-id"], + ["--dp-aware"], + ["--connect-timeout-secs", "900"], + ["--request-timeout-secs", "14400"], + ["--disable-health-check"], + ["--disable-retries"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] } ] }, @@ -2907,10 +2868,10 @@ export const agentx = { "prov": { "sha": "322bc37b368c85828357afb57dce6739d94ef955", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml", - "run": "37420473448", + "run": "37420473448/attempts/1", "pr": "3745", - "comment": "6015313428", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -3182,10 +3143,10 @@ export const agentx = { "prov": { "sha": "322bc37b368c85828357afb57dce6739d94ef955", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml", - "run": "37420473448", + "run": "37420473448/attempts/1", "pr": "3745", - "comment": "6015313428", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -3243,7 +3204,7 @@ export const agentx = { "--schedule-policy": null } }, - "ref": [12, "sglang", 0] + "ref": [13, "sglang", 0] }, { "kind": "sglang-router", @@ -3843,7 +3804,7 @@ export const agentx = { "--tp": ["8"] } }, - "ref": [14, "sglang", 0] + "ref": [15, "sglang", 0] } ] }, @@ -4291,7 +4252,7 @@ export const agentx = { "--swa-prefix-tails": ["64"] } }, - "ref": [16, "sglang", 0] + "ref": [17, "sglang", 0] }, { "kind": "sglang-router", diff --git a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx index 715031c367d5..55a7a5a182e4 100644 --- a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx @@ -1,37 +1,57 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@0259609ce99d plus open PRs #3187, #3598, #3631, #3659, #3745), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@ceb50ba21208 plus open PRs #2978, #3606, #3701, #3744, #3782, #3816, #3817, #3836, #3840), with SA-specific settings removed. export const agentx = { "model": "GLM-5.2", - "generatedOn": "2026-10-06", + "generatedOn": "2026-10-08", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "0259609ce99d0f4f7bd7180c3b3436e4f0334a46", + "main": "ceb50ba212080174e0563d2be1db9b0d654ab2ef", "prs": { - "3598": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37260940238", - "sha": "0f45d5f4c3d57f9804f7655d7b3b294acdd53441", + "3816": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37714835620", + "sha": "e67ca6fc9ecf4cd57f27654bda8a3e9059923820", "status": "verified-pr" }, - "3631": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/36974436489", - "sha": "3b798abfd43bc4260ab11e43d66c88a99e33461a", - "status": "verified-pr" + "3840": { + "run": null, + "sha": "5948cad58f35cca9629151255e0be59d913144e7", + "status": "in-progress" }, - "3659": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/36997573694", - "sha": "e79239bab1e1cb487dc74e27d693110f03384d41", + "2978": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34548026742", + "sha": "6b7dcd07ab0c8c0805ac7fbec2f2ded38d450c20", "status": "verified-pr" }, - "3187": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37270628136", - "sha": "34d06e54c7614bb33d2a271b010ffde3ce3fa563", + "3817": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", + "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "status": "verified-pr" }, - "3745": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37420473448", - "sha": "322bc37b368c85828357afb57dce6739d94ef955", - "status": "verified-pr" + "3606": { + "run": null, + "sha": "5b394d395eefcaacd5168e04aca95020adac5312", + "status": "in-progress" + }, + "3701": { + "run": null, + "sha": "d9185455e166dfd034dcdfaf9608bd8335618b54", + "status": "in-progress" + }, + "3782": { + "run": null, + "sha": "f00c81ab40297f98a68d2726d4301b44a2892419", + "status": "in-progress" + }, + "3744": { + "run": null, + "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "status": "in-progress" + }, + "3836": { + "run": null, + "sha": "03ffb35db157a0fecc78206bea748449daa1385a", + "status": "in-progress" } } }, @@ -232,10 +252,10 @@ export const agentx = { }, "prov": { "sha": "e79239bab1e1cb487dc74e27d693110f03384d41", - "run": "36997573694", + "run": "36997573694/attempts/2", "pr": "3659", - "comment": "5987483060", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -473,10 +493,10 @@ export const agentx = { "prov": { "sha": "e79239bab1e1cb487dc74e27d693110f03384d41", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p4d-p-dep4-d-tp4-c45-mtp.yaml", - "run": "36997573694", + "run": "36997573694/attempts/2", "pr": "3659", - "comment": "5987483060", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -740,10 +760,10 @@ export const agentx = { "prov": { "sha": "e79239bab1e1cb487dc74e27d693110f03384d41", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p6d-p-dep4-d-tp4-c45-mtp.yaml", - "run": "36997573694", + "run": "36997573694/attempts/2", "pr": "3659", - "comment": "5987483060", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -897,10 +917,10 @@ export const agentx = { "prov": { "sha": "e79239bab1e1cb487dc74e27d693110f03384d41", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p2d-p-dep4-d-tp4-c48-mtp.yaml", - "run": "36997573694", + "run": "36997573694/attempts/2", "pr": "3659", - "comment": "5987483060", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -1050,10 +1070,10 @@ export const agentx = { "prov": { "sha": "e79239bab1e1cb487dc74e27d693110f03384d41", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-2p1d-p-dep8-d-dep16-c192-mtp.yaml", - "run": "36997573694", + "run": "36997573694/attempts/2", "pr": "3659", - "comment": "5987483060", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-08" }, "points": [ { @@ -2875,16 +2895,16 @@ export const agentx = { } }, { - "id": "glm5.2-fp8-b300-sglang-agentic-mtp#0", + "id": "glm5.2-fp8-b300-sglang-agentic-mtp#0@c1", "hw": "b300", "ckpt": "zai-org/GLM-5.2-FP8", "label": "Agg TP8", - "detail": "MTP 3-1-4", + "detail": "MTP 6-1-7", "gpus": 8, "configKey": "glm5.2-fp8-b300-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b", - "imageStatus": "pruned", - "imageCommit": "20ca564b", + "image": "lmsysorg/sglang@sha256:2d53500cb72184020c59fe16aab3512d53b1aa8ee6195bfb2aaf5855d58099df", + "imageStatus": "ok", + "imageCommit": null, "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2904,59 +2924,169 @@ export const agentx = { } }, "prov": { - "sha": "de588b628068061130059bf4ecd4c08877c74250", - "recipe": "benchmarks/single_node/agentic/glm5.2_fp8_b300_sglang_mtp.sh", - "run": "35015325736/attempts/1", - "pr": "3146", - "status": "verified", - "date": "2026-09-16" + "sha": "03ffb35db157a0fecc78206bea748449daa1385a", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml", + "pr": "3836", + "status": "in-progress" }, "points": [ { "concs": [1], "item": 0, - "goldenAL": "2.99", - "kv": "hicache", - "job": "35015325736/job/104538340182" + "override": "override_c1", + "goldenAL": "3.78", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "SGLANG_TIMEOUT_KEEP_ALIVE": "900", + "TORCH_CUDA_ARCH_LIST": "10.0" + }, + "args": [ + ["--model-path", "zai-org/GLM-5.2-FP8"], + ["--served-model-name", "zai-org/GLM-5.2-FP8"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--bf16-gemm-backend", "cutedsl"], + ["--chunked-prefill-size", "8192"], + ["--cuda-graph-max-bs", "2"], + ["--enable-hierarchical-cache"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-size", "270"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--max-prefill-tokens", "8192"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.85"], + ["--quantization", "fp8"], + ["--reasoning-parser", "glm45"], + ["--speculative-algorithm", "EAGLE"], + ["--speculative-eagle-topk", "1"], + ["--speculative-num-draft-tokens", "7"], + ["--speculative-num-steps", "6"], + ["--tensor-parallel-size", "8"], + ["--tool-call-parser", "glm47"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1800"] + ] + } + ] }, - { - "concs": [4], - "item": 0, - "goldenAL": "2.99", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs": ["8"], - "--max-running-requests": ["8"] - } - } - ], - "dynamo": [ - null, - null, - { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { "args": { - "--cuda-graph-max-bs": ["8"], - "--max-running-requests": ["8"] + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" } } - ] - }, - "job": "35015325736/job/104538340169" + } + ] + } + } + }, + { + "id": "glm5.2-fp8-b300-sglang-agentic-mtp#0@c4", + "hw": "b300", + "ckpt": "zai-org/GLM-5.2-FP8", + "label": "Agg TP8", + "detail": "MTP 4-1-5", + "gpus": 8, + "configKey": "glm5.2-fp8-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang@sha256:2d53500cb72184020c59fe16aab3512d53b1aa8ee6195bfb2aaf5855d58099df", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-size", "270"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "glm5.2-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "03ffb35db157a0fecc78206bea748449daa1385a", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml", + "pr": "3836", + "status": "in-progress" + }, + "points": [ + { + "concs": [4], + "item": 0, + "override": "override_c4", + "goldenAL": "3.33", + "kv": "hicache" }, { - "concs": [8], + "concs": [16], "item": 0, - "goldenAL": "2.99", + "override": "override_c16", + "goldenAL": "3.33", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["16"], - "--max-running-requests": ["16"] + "--cuda-graph-max-bs": ["32"], + "--hicache-write-policy": ["write_through"], + "--max-running-requests": ["7"], + "--prefill-decode-interval": ["8"] } } ], @@ -2965,67 +3095,228 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["16"], - "--max-running-requests": ["16"] + "--cuda-graph-max-bs": ["32"], + "--hicache-write-policy": ["write_through"], + "--max-running-requests": ["7"], + "--prefill-decode-interval": ["8"] } } ] - }, - "job": "35015325736/job/104538340279" - }, - { - "concs": [12], - "item": 0, - "goldenAL": "2.99", - "kv": "hicache", - "patch": { - "sglang": [ - { + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { "args": { - "--cuda-graph-max-bs": ["24"], - "--max-running-requests": ["24"] + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["3"], + "--speculative-num-draft-tokens": ["5"], + "--speculative-num-steps": ["4"] } - } - ], - "dynamo": [ - null, - null, - { + }, + "ref": [12, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { "args": { - "--cuda-graph-max-bs": ["24"], - "--max-running-requests": ["24"] + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" } } - ] - }, - "job": "35015325736/job/104538340368" - }, + } + ] + } + } + }, + { + "id": "glm5.2-fp8-b300-sglang-agentic-mtp#0@c8", + "hw": "b300", + "ckpt": "zai-org/GLM-5.2-FP8", + "label": "Agg TP8", + "detail": "MTP 3-1-4", + "gpus": 8, + "configKey": "glm5.2-fp8-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang@sha256:2d53500cb72184020c59fe16aab3512d53b1aa8ee6195bfb2aaf5855d58099df", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-size", "270"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "glm5.2-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "03ffb35db157a0fecc78206bea748449daa1385a", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml", + "pr": "3836", + "status": "in-progress" + }, + "points": [ { - "concs": [16], + "concs": [8], "item": 0, + "override": "override_c8", "goldenAL": "2.99", - "kv": "hicache", - "patch": { - "sglang": [ - { + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { "args": { - "--cuda-graph-max-bs": ["32"], - "--max-running-requests": ["32"] + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["16"], + "--speculative-num-draft-tokens": ["4"], + "--speculative-num-steps": ["3"] } - } - ], - "dynamo": [ - null, - null, - { + }, + "ref": [12, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { "args": { - "--cuda-graph-max-bs": ["32"], - "--max-running-requests": ["32"] + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" } } - ] - }, - "job": "35015325736/job/104538340285" + } + ] + } + } + }, + { + "id": "glm5.2-fp8-b300-sglang-agentic-mtp#0@c12", + "hw": "b300", + "ckpt": "zai-org/GLM-5.2-FP8", + "label": "Agg TP8", + "detail": "MTP 5-1-6", + "gpus": 8, + "configKey": "glm5.2-fp8-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang@sha256:2d53500cb72184020c59fe16aab3512d53b1aa8ee6195bfb2aaf5855d58099df", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-size", "270"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "glm5.2-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "03ffb35db157a0fecc78206bea748449daa1385a", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml", + "pr": "3836", + "status": "in-progress" + }, + "points": [ + { + "concs": [12], + "item": 0, + "override": "override_c12", + "goldenAL": "3.61", + "kv": "hicache" } ], "routers": { @@ -3034,44 +3325,15 @@ export const agentx = { "files": [], "blocks": [ { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, - "cmd": "python3 -m sglang.launch_server", - "env": { - "SGLANG_TIMEOUT_KEEP_ALIVE": "900", - "TORCH_CUDA_ARCH_LIST": "10.0" + "diff": { + "args": { + "--cuda-graph-max-bs": ["24"], + "--max-running-requests": ["5"], + "--speculative-num-draft-tokens": ["6"], + "--speculative-num-steps": ["5"] + } }, - "args": [ - ["--model-path", "zai-org/GLM-5.2-FP8"], - ["--served-model-name", "zai-org/GLM-5.2-FP8"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--bf16-gemm-backend", "cutedsl"], - ["--chunked-prefill-size", "8192"], - ["--cuda-graph-max-bs", "2"], - ["--enable-hierarchical-cache"], - ["--ep-size", "1"], - ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "270"], - ["--hicache-write-policy", "write_back"], - ["--kv-cache-dtype", "fp8_e4m3"], - ["--max-prefill-tokens", "8192"], - ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.85"], - ["--quantization", "fp8"], - ["--reasoning-parser", "glm45"], - ["--speculative-algorithm", "EAGLE"], - ["--speculative-eagle-topk", "1"], - ["--speculative-num-draft-tokens", "4"], - ["--speculative-num-steps", "3"], - ["--tool-call-parser", "glm47"], - ["--tp", "8"], - ["--trust-remote-code"], - ["--watchdog-timeout", "1800"] - ] + "ref": [12, "sglang", 0] } ] }, @@ -4002,7 +4264,7 @@ export const agentx = { "SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT": "1800" } }, - "ref": [15, "dynamo", 4] + "ref": [18, "dynamo", 4] }, { "drop": ["gpu_pinning"], @@ -4031,7 +4293,7 @@ export const agentx = { "SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT": "1800" } }, - "ref": [15, "dynamo", 5] + "ref": [18, "dynamo", 5] } ] }, @@ -4990,7 +5252,7 @@ export const agentx = { "--page-size": null } }, - "ref": [19, "dynamo", 4] + "ref": [22, "dynamo", 4] }, { "workers": 2, @@ -5000,7 +5262,7 @@ export const agentx = { "--speculative-num-steps": ["3"] } }, - "ref": [19, "dynamo", 5] + "ref": [22, "dynamo", 5] } ] }, diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-flash-next.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-flash-next.jsx index ea06946634ca..0b3f6eb8703a 100644 --- a/docs/src/snippets/configs/Qwen/qwen3.8-flash-next.jsx +++ b/docs/src/snippets/configs/Qwen/qwen3.8-flash-next.jsx @@ -812,7 +812,7 @@ export const config = { match: { hw: "gb300", variant: "default", quant: "nvfp4-nvda", strategy: "low-latency", nodes: "single" }, verified: true, verificationStatus: (sel) => sel.pleOffload === "off" ? "unverified" : "verified", - warn: "Long-context recipe: reserve 224 GiB of host RAM for HiCache, approximately 48 GiB for pinned PLE offload, and loading headroom. See [GB300 configuration notes](#23-gb300-long-context-serving-with-hicache-and-mtp). The Docker wrapper is adapted from the qualified Slurm/Enroot launch.", + warn: "Long-context recipe: reserve 224 GiB of host RAM for HiCache, approximately 48 GiB for pinned PLE offload, and loading headroom. See the [Agentic Long-Context Deployment](#agentic-long-context-deployment) section for the qualified launch. The Docker wrapper is adapted from the qualified Slurm/Enroot launch.", env: [ "PYTHONNOUSERSITE=1", "PYTHONUNBUFFERED=1",