diff --git a/.agents/skills/debug-runs/SKILL.md b/.agents/skills/debug-runs/SKILL.md index ab193410fb..0bb77eb3ec 100644 --- a/.agents/skills/debug-runs/SKILL.md +++ b/.agents/skills/debug-runs/SKILL.md @@ -109,7 +109,7 @@ Steps: the canvas, then SSH in with `ssh -A` when a jumpbox or agent forwarding is involved. 2. Reproduce the exact benchmark the launcher runs. Single-node jobs go through the srt driver of `python -m infx.launch run` (`inferencex-e2e/infx/launch/drivers/srt/`): it binds the - master row's `srt-recipe:` (`inferencex-e2e/benchmarks/single_node/srt-slurm-recipes///-[-mtp]/.yaml`) + matrix row's `srt-recipe` (`inferencex-e2e/benchmarks/single_node/srt-slurm-recipes//`) with `python -m infx.srt_slurm.single_node prepare`, renders the job-local `srtslurm.yaml` from the cluster's record, then submits it through srtctl. Read the recipe for the image (`model.container`), server args and env, and the cluster record for mounts and the job env diff --git a/.claude/commands/add-model-hardware.md b/.claude/commands/add-model-hardware.md index b280f907cc..18fa46bc43 100644 --- a/.claude/commands/add-model-hardware.md +++ b/.claude/commands/add-model-hardware.md @@ -75,7 +75,8 @@ This research directly feeds Step 2 (recipe args/env) and Step 3 (search space). 1. `inferencex-e2e/benchmarks/single_node/srt-slurm-recipes///-[-mtp]/8k1k.yaml` (an `agentic.yaml` beside it for AgentX) 2. an entry in either master config, **`inferencex-e2e/configs/nvidia-master.yaml`** (b*/h*/gb* SKUs) or - **`inferencex-e2e/configs/amd-master.yaml`** (mi* SKUs), with `srt-recipe:` on every search-space row + **`inferencex-e2e/configs/amd-master.yaml`** (mi* SKUs), with `srt-recipe-dir:` (the directory above, + relative to `srt-slurm-recipes/`) on the entry and `srt-recipe:` (the file name) on every search-space row 3. a `inferencex-e2e/perf-changelog.yaml` entry (this diff vs main is what selects the sweep) ## Step 1 — branch + find the sibling to copy diff --git a/.claude/commands/nuke.md b/.claude/commands/nuke.md index b80d88865f..2c5556421e 100644 --- a/.claude/commands/nuke.md +++ b/.claude/commands/nuke.md @@ -113,18 +113,22 @@ recipe each master search-space row names in `srt-recipe:`, and # Usage: edit_recipe_container.py [key2 ...] import os, re, sys f, new_image, keys = sys.argv[1], sys.argv[2], sys.argv[3:] -# srt-recipe paths are relative to inferencex-e2e/, the parent of configs/ -root = os.path.dirname(os.path.dirname(os.path.abspath(f))) +# srt-recipe names a file under the key's srt-recipe-dir, below this root +root = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(f))), "benchmarks/single_node/srt-slurm-recipes") master = open(f).read().split('\n') recipes = set() for key in keys: kre = re.compile(r'^' + re.escape(key) + r':\s*$') start = next((i for i,l in enumerate(master) if kre.match(l)), None) if start is None: sys.exit(f"ERROR: key not found: {key}") + directory, names = None, set() for j in range(start+1, len(master)): if re.match(r'^[A-Za-z0-9._-]+:\s*$', master[j]): break # next top-level key - recipes.update(re.findall(r'srt-recipe:\s*([^\s,}]+)', master[j])) -if not recipes: sys.exit(f"ERROR: no srt-recipe for keys {keys}") + d = re.match(r'^\s+srt-recipe-dir:\s*(\S+)\s*$', master[j]) + if d: directory = d.group(1) + names.update(re.findall(r'(?' inferencex-e2e/configs/*-master.yaml` shows a recipe is also -referenced by a key outside this family, stop and ask the user: bumping it would -break that other key's `model.container == image` check. +If another key's `srt-recipe-dir` plus `srt-recipe` resolves to the same recipe file, stop and ask the user: +bumping it would break that other key's `model.container == image` check. For each family, run strictly sequentially because git checkouts can't be parallel: diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 257ac01aaf..41952008e6 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -123,6 +123,8 @@ env: HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} EXP_NAME: ${{ fromJSON(inputs.config).exp-name }} RECIPE_FINGERPRINT: ${{ fromJSON(inputs.config)['recipe-fingerprint'] || '' }} + SRT_RECIPE: ${{ fromJSON(inputs.config)['srt-recipe'] || '' }} + EVAL_SRT_RECIPE: ${{ fromJSON(inputs.config)['eval-srt-recipe'] || '' }} IMAGE: ${{ fromJSON(inputs.config).image }} THINKING_MODE: thinking_on MODEL_PREFIX: ${{ fromJSON(inputs.config).model-prefix }} diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index 22cedceac7..6a2ca2db39 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -20,8 +20,8 @@ qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml - Workloads are `1k1k`, `8k1k`, or `agentx`. Existing bundles spanning several fixed sequence lengths use `fixed-seq-len`; keep their override selectors intact. - Use lowercase, hyphen-separated filenames beginning with `agg` or `disagg`. Include topology and the settings that distinguish sibling recipes, such as parallelism, batch size, concurrency, MTP, offload, or cache configuration. Avoid dates, numbered latency/throughput labels, and repeating the model or hardware already in the path. - In topology names, `1p4d` denotes prefill/decode worker counts, not necessarily physical nodes. Role-qualified `p-tp4` and `d-tp8` identify prefill/decode TP; `b` denotes batch size and `c` concurrency. The YAML is authoritative for runtime settings. -- Name override bundles `*-variants.yaml`. Multi-node AgentX recipes that differ only per configuration share one bundle per master-config entry, usually `agg-variants.yaml` or `disagg-variants.yaml`: `base` holds the shared settings and each former recipe becomes a named `override_` block holding only its differences (plain overrides, not `zip_override_*`). Master entries select one with `CONFIG_FILE=recipes//.yaml:override_`. Recipes read as text by a launcher, such as power recipes with top-level `telemetry:`, stay standalone. Keep distinct sweep entry files separate even when their contents match: recipe paths participate in eval grouping. The Qwen3.5 `*-stp-sweep.yaml` and `*-mtp-sweep.yaml` pair preserves that existing distinction. -- Update `CONFIG_FILE` and `EVAL_CONFIG_FILE` references in active and deprecated master configs, launcher path rules, workflow filters, and local documentation together when moving a file. Preserve upstream source URLs as provenance and leave historical performance-changelog entries unchanged. No aliases for the old layout are provided. +- Name override bundles `*-variants.yaml`. Multi-node AgentX recipes that differ only per configuration share one bundle per master-config entry, usually `agg-variants.yaml` or `disagg-variants.yaml`: `base` holds the shared settings and each former recipe becomes a named `override_` block holding only its differences (plain overrides, not `zip_override_*`). Master rows select one with `srt-recipe: .yaml:override_`, relative to the entry's `srt-recipe-dir`. Recipes read as text by a launcher, such as power recipes with top-level `telemetry:`, stay standalone. Keep distinct sweep entry files separate even when their contents match: recipe paths participate in eval grouping. The Qwen3.5 `*-stp-sweep.yaml` and `*-mtp-sweep.yaml` pair preserves that existing distinction. +- Update `srt-recipe-dir`, `srt-recipe`, and `eval-srt-recipe` references in master configs, launcher path rules, workflow filters, and local documentation together when moving a file. Preserve upstream source URLs as provenance and leave historical performance-changelog entries unchanged. No aliases for the old layout are provided. Shared runtime assets stay under `configs/` beside the model directories; they are not standalone recipes. The four files in `configs/dsv4-moe-load-balancer-configs/` are copied verbatim from NVIDIA/srt-slurm commit `deb1dfd9934398664f92d194169c183e009da83b`, preserving the EPLB initial expert assignments formerly used by the DSV4 TRT recipes; no checked-in recipe currently references them. The srt driver ([`infx/launch/drivers/srt/checkout.py`](../../../infx/launch/drivers/srt/checkout.py)) stages them into the job checkout's `configs/` directory for the recipes' bind mounts. Keeping a recipe in this tree does not activate it; the master configs determine the benchmark matrix. @@ -41,7 +41,8 @@ Recipes use `schema: 2`, `engine`, and `roles`. Each worker role owns its node c | `roles.prefill.args.ep-size` (SGLang) | `prefill.ep` | | `roles.prefill.args.enable-dp-attention` | `prefill.dp-attn` | | `benchmark.concurrencies` | `conc-list` | -| Recipe path, optionally with an override selector | `additional-settings: CONFIG_FILE=recipes/...yaml` | +| Recipe directory, relative to this tree | entry-level `srt-recipe-dir` | +| Recipe file, optionally with a `base`, `override_`, or `zip_override_[]` selector | search-space `srt-recipe`; `eval-srt-recipe` for eval-only real verification | Keep the recipe and master configuration synchronized. The launcher executes the recipe; the master configuration supplies result labels and scheduling metadata. For aggregate recipes use `roles.agg`; `roles.decode.nodes: colocate` shares prefill nodes and contributes no additional worker nodes to scheduling. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml index 3e302a5ac7..efa41b3216 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 sglang b200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml index 9b729ab3fd..7711a95165 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 sglang b200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml index 78f8c99a4e..8cf0b086da 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 sglang gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml index ec87795339..c7512f7adc 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 sglang gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml index d78570c07f..353df44501 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 trtllm gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml index 87fa054566..c2a6430112 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml index bf9c7321ba..81410efe2d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml index 57a3d6a417..5ff4e6f3e8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml index bfe51e70f6..c6308fadc7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml index 69ec5c40ff..b3fa74c8c5 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 vllm gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml index c91f01d08a..a333f36d0b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 vllm gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml index fc0aae1a06..2978805ee7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv4 vllm gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml index f0655b70be..4534849c7f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -1,6 +1,5 @@ # AgentX dsv41flash sglang gb300-fp4 recipes (Dynamo frontend + SGLang, AGGREGATED -# topology): shared settings in base, one override per benchmark point. Select one -# with CONFIG_FILE=recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml:override_. +# topology): shared settings in base, one override per benchmark point. # # One SGLang worker serves prefill and decode with the checkpoint's bundled DSpark # draft (block size 5); the Dynamo frontend routes with the KV-aware router and diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml index 3f924eed8f..98a9a5a03f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -1,7 +1,6 @@ # AgentX dsv41flash sglang gb300-fp4 recipes (Dynamo frontend + SGLang, DISAGGREGATED # topology with Mooncake KV transfer): shared settings in base, one override per -# benchmark point. Select one with -# CONFIG_FILE=recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_. +# benchmark point. # # Prefill and decode are separate TP2/EP2 SGLang workers (two GPUs each) behind the # Dynamo KV router; with the DSpark draft in PD-disaggregated mode both sides must diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml index baefd1fd34..2d86f69c78 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml @@ -1,6 +1,5 @@ # AgentX glm5.2 sglang b200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml index d1c5719dc5..bc5b0dd7a5 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX glm5.2 sglang b200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml index 858502c910..59357ed961 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml @@ -1,6 +1,5 @@ # AgentX glm5.2 sglang gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml index c439698894..2525c8b0b9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml @@ -1,6 +1,5 @@ # AgentX glm5.2 sglang gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml index 38902b95f1..f9497d8cb1 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml @@ -1,9 +1,7 @@ # Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on GB300 # (single aggregated worker, TP8, EAGLE MTP + hierarchical-cache KV). # -# Flat single-variant schema the agentic CI flow expects: applied via -# CONFIG_FILE= with no zip_override selector, so it must NOT be -# base:-wrapped. Concurrency is not a recipe field; the GHA matrix fans out +# Concurrency is not a recipe field; the GHA matrix fans out # one job per concurrency from the master-config conc-list into srt_agentic.sh. schema: 2 name: agg-tp8-c1-mtp diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml index c8e5cfd844..1bd1ce0600 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX glm5.2 trtllm gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml index 4afb5e8a62..5a38f180c0 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -1,6 +1,5 @@ # AgentX kimik3 vllm gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml index 4ccc57c455..735ae0b008 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml @@ -1,6 +1,5 @@ # AgentX minimaxm3 trtllm gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml index bd8c7bc9b6..1e65fda9de 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml @@ -1,6 +1,5 @@ # AgentX minimaxm3 vllm gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml index 8899d709fc..67870f9257 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX minimaxm3 vllm gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml index eefda7020c..51e581e71b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX minimaxm3 vllm gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml index 8a65ef16c1..cffec07d8e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX qwen3.5 sglang b300-fp8 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml index 91f364e773..6333e2619c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml @@ -1,6 +1,5 @@ # AgentX qwen3.5 sglang gb200-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml index c65a482875..5ed9aaa501 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml @@ -1,6 +1,5 @@ # AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml index a5b0fb407e..6c6314f3e8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -1,6 +1,5 @@ # AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml index 3caef43cbc..cf21ca15f3 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml @@ -1,6 +1,5 @@ # AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml index b38e93698f..557e76fcb4 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml index d607b6abae..ab164ce4c3 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -1,6 +1,5 @@ # AgentX qwen3.5 trtllm gb300-fp4 recipes: shared settings in base, one -# override per benchmark configuration. Select one with -# CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_. +# override per benchmark configuration. schema: 2 diff --git a/inferencex-e2e/configs/CONFIGS.md b/inferencex-e2e/configs/CONFIGS.md index 19154f6690..151a6e37ff 100644 --- a/inferencex-e2e/configs/CONFIGS.md +++ b/inferencex-e2e/configs/CONFIGS.md @@ -15,12 +15,14 @@ entry-name: # Optional defaults for every search-space entry in this config. router: { name: string, version: string } kv-p2p-transfer: string + # Recipe directory under benchmarks/_node/srt-slurm-recipes/. + srt-recipe-dir: string scenarios: fixed-seq-len: - isl: int osl: int search-space: - - { tp: int, conc-start: int, conc-end: int } + - { tp: int, conc-start: int, conc-end: int, srt-recipe: string } # Optionally, specify pipeline/expert/data-attention/context parallelism. - { tp: int, pp: int, ep: int, dp-attn: bool, dcp-size: int, pcp-size: int, conc-start: int, conc-end: int } # Optionally, declare router metadata and the P2P KV transfer engine. @@ -142,7 +144,8 @@ jobs to 3600 seconds. Reusable workflow callers may override the `duration` input. Notes: -- The fields above are the common ones, not the full schema. The Pydantic models in [`infx/matrix/validation.py`](../infx/matrix/validation.py) are the authoritative contract; they also accept fields such as `spec-decoding`, `srt-recipe`, `num-nodes`, and `require-power`, and they reject any field they do not define, which fails matrix generation. +- The fields above are the common ones, not the full schema. The Pydantic models in [`infx/matrix/validation.py`](../infx/matrix/validation.py) are the authoritative contract; they also accept fields such as `spec-decoding`, `num-nodes`, and `require-power`, and they reject any field they do not define, which fails matrix generation. +- srt-slurm entries require `srt-recipe-dir`. Every row requires `srt-recipe: [:]` under it (selector: `base`, `override_` or `zip_override_[]`). Multinode rows may add `eval-srt-recipe`. Validation rejects missing files or variants, and `CONFIG_FILE`/`EVAL_CONFIG_FILE` settings. Only `llmd-vllm` keeps `CONFIG_FILE`. Generated rows carry the path from `inferencex-e2e/`. - Setting the fields above only guarantees that their values are passed as environment variables to benchmark scripts. Single-node jobs receive `PP_SIZE`, `DCP_SIZE`, and `PCP_SIZE`. Multinode jobs receive `PREFILL_PP_SIZE`, `PREFILL_DCP_SIZE`, `PREFILL_PCP_SIZE`, `DECODE_PP_SIZE`, `DECODE_DCP_SIZE`, and `DECODE_PCP_SIZE`. Actually using those variables is an implementation detail of the benchmark Bash script. ## Runners diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index e0ed3d6b51..66103f4c57 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -6,6 +6,7 @@ dsr1-fp4-mi355x-sglang: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/mi355x-fp4 scenarios: fixed-seq-len: - isl: 8192 @@ -14,11 +15,11 @@ dsr1-fp4-mi355x-sglang: - tp: 4 conc-start: 4 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 conc-start: 4 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp4-mi355x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm700-mi35x @@ -28,6 +29,7 @@ dsr1-fp4-mi355x-sglang-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/mi355x-fp4-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -38,7 +40,7 @@ dsr1-fp4-mi355x-sglang-mtp: conc-start: 4 conc-end: 64 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp4-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 @@ -48,6 +50,7 @@ dsr1-fp4-mi355x-atom: precision: fp4 framework: atom multinode: false + srt-recipe-dir: dsr1/atom/mi355x-fp4 scenarios: fixed-seq-len: - isl: 8192 @@ -57,12 +60,12 @@ dsr1-fp4-mi355x-atom: ep: 1 conc-start: 4 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/atom/mi355x-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 1 conc-start: 4 conc-end: 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/atom/mi355x-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp4-mi355x-atom-mtp: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 @@ -73,6 +76,7 @@ dsr1-fp4-mi355x-atom-mtp: # WIP framework (no customers yet) framework: atom multinode: false + srt-recipe-dir: dsr1/atom/mi355x-fp4-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -83,7 +87,7 @@ dsr1-fp4-mi355x-atom-mtp: conc-start: 4 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/atom/mi355x-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm700-mi30x @@ -93,6 +97,7 @@ dsr1-fp8-mi300x-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/mi300x-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -101,7 +106,7 @@ dsr1-fp8-mi300x-sglang: - tp: 8 conc-start: 4 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/mi300x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-mi325x-sglang: image: lmsysorg/sglang:v0.5.19-rocm700-mi30x @@ -111,6 +116,7 @@ dsr1-fp8-mi325x-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/mi325x-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -119,7 +125,7 @@ dsr1-fp8-mi325x-sglang: - tp: 8 conc-start: 4 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/mi325x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-mi355x-sglang: image: lmsysorg/sglang:v0.5.12-rocm700-mi35x @@ -129,6 +135,7 @@ dsr1-fp8-mi355x-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/mi355x-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -137,11 +144,11 @@ dsr1-fp8-mi355x-sglang: - tp: 4 conc-start: 32 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 conc-start: 4 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-mi355x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm700-mi35x @@ -151,6 +158,7 @@ dsr1-fp8-mi355x-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/mi355x-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -161,7 +169,7 @@ dsr1-fp8-mi355x-sglang-mtp: conc-start: 4 conc-end: 64 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-mi325x-sglang: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x @@ -171,6 +179,7 @@ qwen3.5-fp8-mi325x-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi325x-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -179,7 +188,7 @@ qwen3.5-fp8-mi325x-sglang: - tp: 8 conc-start: 4 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 @@ -189,6 +198,7 @@ qwen3.5-fp8-mi355x-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi355x-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -198,7 +208,7 @@ qwen3.5-fp8-mi355x-sglang: ep: 1 conc-start: 4 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-mi355x-sglang-mtp: image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 @@ -208,6 +218,7 @@ qwen3.5-fp8-mi355x-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi355x-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -218,7 +229,7 @@ qwen3.5-fp8-mi355x-sglang-mtp: conc-start: 4 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml # MI355X FP8 AgentX, TP4 only. The grid follows the TP4 band of the B200 FP8 # AgentX arm so the two SKUs are directly comparable point for point, with the @@ -234,12 +245,13 @@ qwen3.5-fp8-mi355x-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi355x-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32, 36, 40, 48, 56, 64, 72, 80], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32, 36, 40, 48, 56, 64, 72, 80], srt-recipe: agentic.yaml } # MI325X official matrix selected from the complete 62-point fast sweep. TP2 # peaks at c4, TP4/TEP4 at c40, and TP8/TEP8 at c64; the next point beyond each @@ -252,15 +264,16 @@ qwen3.5-fp8-mi325x-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi325x-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 6], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 32, 40, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } - - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 32, 40, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 24, 32, 40, 48, 64, 80], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 24, 32, 40, 48, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 6], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 32, 40, 48], srt-recipe: agentic.yaml } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 32, 40, 48], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 24, 32, 40, 48, 64, 80], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 24, 32, 40, 48, 64], srt-recipe: agentic.yaml } qwen3.5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 @@ -270,6 +283,7 @@ qwen3.5-fp8-mi355x-atom: precision: fp8 framework: atom multinode: false + srt-recipe-dir: qwen3.5/atom/mi355x-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -279,17 +293,17 @@ qwen3.5-fp8-mi355x-atom: ep: 1 conc-start: 4 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/atom/mi355x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 1 conc-start: 4 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/atom/mi355x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 1 conc-start: 4 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/atom/mi355x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-mi355x-atom-mtp: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post @@ -299,6 +313,7 @@ qwen3.5-fp8-mi355x-atom-mtp: precision: fp8 framework: atom multinode: false + srt-recipe-dir: qwen3.5/atom/mi355x-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -309,13 +324,13 @@ qwen3.5-fp8-mi355x-atom-mtp: conc-start: 4 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/atom/mi355x-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 1 conc-start: 4 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/atom/mi355x-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-mi355x-sglang-disagg: image: lmsysorg/sglang:v0.5.16-rocm720-mi35x @@ -328,6 +343,7 @@ qwen3.5-fp8-mi355x-sglang-disagg: kv-p2p-transfer: mori multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/mi355x-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -345,13 +361,12 @@ qwen3.5-fp8-mi355x-sglang-disagg: # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). - spec-decoding: "none" conc-list: [8, 16, 32, 64, 128] + srt-recipe: disagg-1p1d-p-tp4-d-tp8.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/mi355x-fp8/8k1k/disagg-1p1d-p-tp4-d-tp8.yaml" decode: num-worker: 1 tp: 8 @@ -369,6 +384,7 @@ qwen3.5-fp4-mi355x-sglang: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi355x-fp4 scenarios: fixed-seq-len: - isl: 8192 @@ -377,11 +393,11 @@ qwen3.5-fp4-mi355x-sglang: - tp: 2 conc-start: 4 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 conc-start: 4 conc-end: 16 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp4-mi355x-atom: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post @@ -391,6 +407,7 @@ qwen3.5-fp4-mi355x-atom: precision: fp4 framework: atom multinode: false + srt-recipe-dir: qwen3.5/atom/mi355x-fp4 scenarios: fixed-seq-len: - isl: 8192 @@ -399,11 +416,11 @@ qwen3.5-fp4-mi355x-atom: - tp: 2 conc-start: 4 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/atom/mi355x-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 conc-start: 4 conc-end: 16 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/atom/mi355x-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp4-mi355x-sglang-mtp: image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260927 @@ -413,6 +430,7 @@ qwen3.5-fp4-mi355x-sglang-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi355x-fp4-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -422,12 +440,12 @@ qwen3.5-fp4-mi355x-sglang-mtp: conc-start: 4 conc-end: 128 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 conc-start: 4 conc-end: 16 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260927 @@ -437,13 +455,14 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi355x-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 20, 24, 28, 32, 36, 40], srt-recipe: agentic.yaml } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 @@ -456,6 +475,7 @@ qwen3.5-fp4-mi355x-sglang-disagg: kv-p2p-transfer: mori multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/mi355x-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -464,13 +484,12 @@ qwen3.5-fp4-mi355x-sglang-disagg: - spec-decoding: "none" # The recipe owns the complete production search in one allocation. conc-list: [8, 16, 32, 64, 128, 256, 512] + srt-recipe: disagg-1p1d-tp8.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/mi355x-fp4/8k1k/disagg-1p1d-tp8.yaml" decode: num-worker: 1 tp: 8 @@ -485,6 +504,7 @@ qwen3.5-fp8-mi300x-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi300x-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -493,7 +513,7 @@ qwen3.5-fp8-mi300x-sglang: - tp: 8 conc-start: 4 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml # AgentX Pareto sweep for Qwen3.5 FP8 on MI300X. The fast discovery run peaks # at c24; c32 records the post-knee throughput and latency cliff. @@ -505,12 +525,13 @@ qwen3.5-fp8-mi300x-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi300x-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32], srt-recipe: agentic.yaml } dsr1-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 @@ -521,6 +542,7 @@ dsr1-fp8-mi355x-atom: # WIP framework (no customers yet) framework: atom multinode: false + srt-recipe-dir: dsr1/atom/mi355x-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -529,7 +551,7 @@ dsr1-fp8-mi355x-atom: - tp: 8 conc-start: 4 conc-end: 128 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/atom/mi355x-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-mi355x-atom-mtp: image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 @@ -539,6 +561,7 @@ dsr1-fp8-mi355x-atom-mtp: precision: fp8 framework: atom multinode: false + srt-recipe-dir: dsr1/atom/mi355x-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -548,7 +571,7 @@ dsr1-fp8-mi355x-atom-mtp: conc-start: 4 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/atom/mi355x-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-mi355x-sglang-disagg: # Pure TP variants retain SGLang's dense-TP default; dense TP1 is DP-attention-only. @@ -562,6 +585,7 @@ dsr1-fp8-mi355x-sglang-disagg: kv-p2p-transfer: mori multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/mi355x-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -572,13 +596,12 @@ dsr1-fp8-mi355x-sglang-disagg: # single allocation so model initialization is paid once per topology. - spec-decoding: "none" conc-list: [1024, 2048] + srt-recipe: disagg-variants.yaml:override_stp_2p1d_dep8 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp8/8k1k/disagg-variants.yaml:override_stp_2p1d_dep8" decode: num-worker: 1 tp: 8 @@ -586,13 +609,12 @@ dsr1-fp8-mi355x-sglang-disagg: dp-attn: true - spec-decoding: "none" conc-list: [4, 8, 16, 32, 64, 128, 256] + srt-recipe: disagg-variants.yaml:override_stp_1p2d_tp8 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp8/8k1k/disagg-variants.yaml:override_stp_1p2d_tp8" decode: num-worker: 2 tp: 8 @@ -600,13 +622,12 @@ dsr1-fp8-mi355x-sglang-disagg: dp-attn: false - spec-decoding: "none" conc-list: [1, 2, 4, 8, 16, 32, 64] + srt-recipe: disagg-variants.yaml:override_stp_1p1d_tp4_tp8 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp8/8k1k/disagg-variants.yaml:override_stp_1p1d_tp4_tp8" decode: num-worker: 1 tp: 8 @@ -625,6 +646,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: kv-p2p-transfer: mori multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/mi355x-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -632,13 +654,12 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: search-space: - spec-decoding: "mtp" conc-list: [1024, 2048] + srt-recipe: disagg-variants.yaml:override_mtp_2p1d_dep8 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp8/8k1k/disagg-variants.yaml:override_mtp_2p1d_dep8" decode: num-worker: 1 tp: 8 @@ -646,13 +667,12 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: dp-attn: true - spec-decoding: "mtp" conc-list: [2, 4, 8, 16, 32, 64, 128, 256] + srt-recipe: disagg-variants.yaml:override_mtp_1p2d_tp8 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp8/8k1k/disagg-variants.yaml:override_mtp_1p2d_tp8" decode: num-worker: 2 tp: 8 @@ -660,13 +680,12 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: dp-attn: false - spec-decoding: "mtp" conc-list: [1, 2, 4, 8, 16, 32, 64] + srt-recipe: disagg-variants.yaml:override_mtp_1p1d_tp4_tp8 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp8/8k1k/disagg-variants.yaml:override_mtp_1p1d_tp4_tp8" decode: num-worker: 1 tp: 8 @@ -702,15 +721,16 @@ kimik3-fp4-mi355x-atom-agentic-mtp: precision: fp4 framework: atom multinode: false + srt-recipe-dir: kimik3/atom/mi355x-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.343 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [14, 16, 48], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [14, 16, 48], spec-decoding: mtp, srt-recipe: agentic.yaml } - dram-utilization: 0.513 search-space: - - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [56, 72], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [56, 72], spec-decoding: mtp, srt-recipe: agentic.yaml } # Kimi-K3 MXFP4 agentic-coding on MI355X via vLLM with DSpark speculative # decoding. Companion to kimik3-fp4-mi355x-atom-agentic-mtp: same checkpoint and @@ -729,13 +749,14 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: kimik3/vllm/mi355x-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.6 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 10, 12, 14], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48, 70], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 10, 12, 14], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48, 70], spec-decoding: mtp, srt-recipe: agentic.yaml } minimaxm3-fp4-mi355x-atom-agentic-mtp: image: rocm/atom-dev:nightly_202610070333 @@ -745,16 +766,17 @@ minimaxm3-fp4-mi355x-atom-agentic-mtp: precision: fp4 framework: atom multinode: false + srt-recipe-dir: minimaxm3/atom/mi355x-fp4-mtp scenarios: agentic-coding: # 0.687 targets ~256 GB/rank on the dram-offload arms: LMCACHE_MAX_LOCAL_CPU_SIZE # = TOTAL_CPU_DRAM_GB / TP = node_DRAM * dram-utilization / 8 (TP-independent). - dram-utilization: 0.687 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 5, 8, 10, 12, 15, 20, 24, 28], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, conc-list: [20, 25, 30], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, conc-list: [40, 48], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 5, 8, 10, 12, 15, 20, 24, 28], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 2, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 2, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, conc-list: [20, 25, 30], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, conc-list: [40, 48], spec-decoding: mtp, srt-recipe: agentic.yaml } dsr1-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 @@ -767,6 +789,7 @@ dsr1-fp4-mi355x-sglang-disagg: kv-p2p-transfer: mori multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/mi355x-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -774,53 +797,48 @@ dsr1-fp4-mi355x-sglang-disagg: search-space: - spec-decoding: "none" conc-list: [1, 2, 4, 8] + srt-recipe: disagg-variants.yaml:override_stp_1p1d_tp8 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_stp_1p1d_tp8" decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "none" conc-list: [2, 4, 8, 16, 32, 64, 128, 256] + srt-recipe: disagg-variants.yaml:override_stp_1p2d_tp8 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_stp_1p2d_tp8" decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "none" conc-list: [64, 128, 256] + srt-recipe: disagg-variants.yaml:override_stp_1p2d_tp4_tp8 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_stp_1p2d_tp4_tp8" decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "none" conc-list: [128, 256, 512] + srt-recipe: disagg-variants.yaml:override_stp_1p1d_dep8 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_stp_1p1d_dep8" decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} - spec-decoding: "none" conc-list: [1024, 2048, 4096] + srt-recipe: disagg-variants.yaml:override_stp_2p1d_dep8 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_stp_2p1d_dep8" decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: @@ -834,6 +852,7 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: kv-p2p-transfer: mori multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/mi355x-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -841,43 +860,39 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: search-space: - spec-decoding: "mtp" conc-list: [1, 2, 4, 8] + srt-recipe: disagg-variants.yaml:override_mtp3_1p1d_tp8 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_mtp3_1p1d_tp8" decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" conc-list: [2, 4, 8, 16, 32, 64] + srt-recipe: disagg-variants.yaml:override_mtp3_1p2d_tp8_wide prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_mtp3_1p2d_tp8_wide" decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" conc-list: [64, 128, 256, 512, 640] + srt-recipe: disagg-variants.yaml:override_mtp3_1p1d_dep8 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_mtp3_1p1d_dep8" decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} - spec-decoding: "mtp" conc-list: [1024, 2048, 4096] + srt-recipe: disagg-variants.yaml:override_mtp1_2p1d_dep8 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_mtp1_2p1d_dep8" decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} dsr1-fp8-mi325x-sglang-mtp: @@ -888,6 +903,7 @@ dsr1-fp8-mi325x-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/mi325x-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -898,7 +914,7 @@ dsr1-fp8-mi325x-sglang-mtp: conc-start: 4 conc-end: 64 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/mi325x-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-mi325x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x @@ -908,6 +924,7 @@ qwen3.5-fp8-mi325x-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/mi325x-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -918,7 +935,7 @@ qwen3.5-fp8-mi325x-sglang-mtp: conc-start: 4 conc-end: 64 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml # DeepSeek-V4-Pro-0813 FP4 AgentX on one MI355X node using vLLM DSpark K6. # Keep the historical config key for changelog compatibility. @@ -930,14 +947,15 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv4/vllm/mi355x-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32, 40, 48, 56, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32, 40, 48, 56, 64], srt-recipe: agentic.yaml } # Single-node validation found c128 to be the DPA throughput peak among # c64/c128/c256. Add c192 to locate the knee before the c256 regression. - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } # LMCache invalid-block recovery currently assumes one KV-cache group, # while speculative decoding creates two. Restore these points after the # upstream hybrid KV recovery fix lands: @@ -954,11 +972,12 @@ dsv4-fp4-mi355x-atom-agentic-mtp: precision: fp4 framework: atom multinode: false + srt-recipe-dir: dsv4/atom/mi355x-fp4-mtp scenarios: agentic-coding: - search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128, 256], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128, 256], srt-recipe: agentic.yaml } dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 @@ -971,6 +990,7 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: kv-p2p-transfer: mori multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/mi355x-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -978,53 +998,48 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: search-space: - spec-decoding: "mtp" conc-list: [1, 2, 4, 8] + srt-recipe: disagg-variants.yaml:override_mtp3_1p1d_tp8 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_mtp3_1p1d_tp8" decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" conc-list: [2, 4, 8, 16, 32] + srt-recipe: disagg-variants.yaml:override_mtp3_1p2d_tp8_narrow prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_mtp3_1p2d_tp8_narrow" decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" conc-list: [64, 128, 256] + srt-recipe: disagg-variants.yaml:override_mtp2_1p2d_tp8 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_mtp2_1p2d_tp8" decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" conc-list: [64, 128, 256, 512] + srt-recipe: disagg-variants.yaml:override_mtp1_1p1d_dep8 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_mtp1_1p1d_dep8" decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} - spec-decoding: "mtp" conc-list: [1024, 2048, 4096] + srt-recipe: disagg-variants.yaml:override_mtp1_2p1d_dep8 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml:override_mtp1_2p1d_dep8" decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} minimaxm3-fp8-mi300x-vllm-agentic-mtp: @@ -1035,12 +1050,13 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: precision: fp8 framework: vllm multinode: false + srt-recipe-dir: minimaxm3/vllm/mi300x-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 6, 8, 10], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 6, 8, 10], srt-recipe: agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [16], srt-recipe: agentic.yaml } # DeepSeek-V4.1-Flash AgentX on MI300X (gfx942), a copy of the validated MI355X vLLM # arm (run 34710937012) on the same ROCm nightly. gfx942 is not in the upstream @@ -1057,6 +1073,7 @@ dsv41flash-fp4-mi300x-vllm-agentic-dspark: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv41flash/vllm/mi300x-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.60 @@ -1066,10 +1083,10 @@ dsv41flash-fp4-mi300x-vllm-agentic-dspark: # the tables would not fit beside the checkpoint share on a 192 GB card. # vllm-project/vllm#57491 widened the two is_cuda() gates to # is_cuda_alike(), so engram_config resolves on gfx942 in this image. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } # TP4 halves the GPUs per server. With the Engram tables in pinned host # memory the resident weights are ~81 GiB per rank. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } # No TP2 arm on this SKU. Measured in run 35671005506: at TP2 the resident # weights are ~145 GiB per rank, and even with the indexer buffer halved # to 4096 batched tokens the engine reported "Available KV cache memory: @@ -1087,11 +1104,12 @@ glm5.2-fp8-mi325x-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: glm5.2/sglang/mi325x-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5, 6, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5, 6, 8], srt-recipe: agentic.yaml } minimaxm3-fp8-mi325x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:v0.27.1 @@ -1101,11 +1119,12 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: precision: fp8 framework: vllm multinode: false + srt-recipe-dir: minimaxm3/vllm/mi325x-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.20 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18], srt-recipe: agentic.yaml } # DeepSeek-V4.1-Flash AgentX on MI325X (gfx942), a copy of the validated MI355X vLLM # arm (run 34710937012) on the same ROCm nightly. gfx942 is not in the upstream @@ -1122,6 +1141,7 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv41flash/vllm/mi325x-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.60 @@ -1131,13 +1151,13 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark: # the tables would not fit beside the checkpoint share on a 256 GB card. # vllm-project/vllm#57491 widened the two is_cuda() gates to # is_cuda_alike(), so engram_config resolves on gfx942 in this image. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } # TP4 halves the GPUs per server. With the Engram tables in pinned host # memory the resident weights are ~81 GiB per rank. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } # TP2 quarters them. Resident weights rise to ~145 GiB per rank, so the # recipe halves the indexer buffer and caps the scheduler here. - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: image: lmsysorg/sglang-rocm:v0.5.21-rocm724-mi35x-20261001 @@ -1149,6 +1169,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: kv-p2p-transfer: mori multinode: true disagg: true + srt-recipe-dir: dsv4/sglang/mi355x-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -1159,13 +1180,12 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: conc-list: [4] kv-offloading: dram kv-offload-backend: { name: umbp-linker } + srt-recipe: disagg-umbp-dspark.yaml:override_c4 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c4" decode: num-worker: 1 tp: 4 @@ -1175,13 +1195,12 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: conc-list: [16] kv-offloading: dram kv-offload-backend: { name: umbp-linker } + srt-recipe: disagg-umbp-dspark.yaml:override_c16 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c16" decode: num-worker: 1 tp: 8 @@ -1191,13 +1210,12 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: conc-list: [32] kv-offloading: dram kv-offload-backend: { name: umbp-linker } + srt-recipe: disagg-umbp-dspark.yaml:override_c32 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c32" decode: num-worker: 1 tp: 8 @@ -1207,13 +1225,12 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: conc-list: [48] kv-offloading: dram kv-offload-backend: { name: umbp-linker } + srt-recipe: disagg-umbp-dspark.yaml:override_c48 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c48" decode: num-worker: 1 tp: 8 @@ -1225,13 +1242,12 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: conc-list: [192] kv-offloading: dram kv-offload-backend: { name: umbp-linker } + srt-recipe: disagg-umbp-dspark.yaml:override_c192 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c192" decode: num-worker: 1 tp: 8 @@ -1241,13 +1257,12 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: conc-list: [256] kv-offloading: dram kv-offload-backend: { name: umbp-linker } + srt-recipe: disagg-umbp-dspark.yaml:override_c256 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c256" decode: num-worker: 1 tp: 8 @@ -1259,13 +1274,12 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: conc-list: [384] kv-offloading: dram kv-offload-backend: { name: umbp-linker } + srt-recipe: disagg-umbp-dspark.yaml:override_c384 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c384" decode: num-worker: 1 tp: 8 @@ -1275,13 +1289,12 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: conc-list: [512] kv-offloading: dram kv-offload-backend: { name: umbp-linker } + srt-recipe: disagg-umbp-dspark.yaml:override_c512 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c512" decode: num-worker: 1 tp: 8 @@ -1301,6 +1314,7 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: dsv4/atom/mi355x-fp4/agentx scenarios: agentic-coding: # srt-slurm recipe benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml @@ -1311,13 +1325,12 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: - spec-decoding: "draft_model" conc-list: [ 1 ] kv-offloading: none + srt-recipe: disagg-lmcache-dspark.yaml:override_tp8_c1 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml:override_tp8_c1" decode: num-worker: 1 tp: 8 @@ -1326,13 +1339,12 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: - spec-decoding: "draft_model" conc-list: [ 16 ] kv-offloading: none + srt-recipe: disagg-lmcache-dspark.yaml:override_tp8_c16 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml:override_tp8_c16" decode: num-worker: 1 tp: 8 @@ -1341,13 +1353,12 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: - spec-decoding: "draft_model" conc-list: [ 64 ] kv-offloading: none + srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c64 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml:override_dpa8_c64" decode: num-worker: 1 tp: 8 @@ -1356,13 +1367,12 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: - spec-decoding: "draft_model" conc-list: [ 128 ] kv-offloading: none + srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c128 prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml:override_dpa8_c128" decode: num-worker: 1 tp: 8 @@ -1377,13 +1387,12 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: conc-list: [ 256 ] kv-offloading: dram kv-offload-backend: { name: lmcache } + srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c256_lmcache prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml:override_dpa8_c256_lmcache" decode: num-worker: 1 tp: 8 @@ -1407,12 +1416,13 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: glm5.2/sglang/mi355x-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.85 search-space: - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12, 14, 16], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12, 14, 16], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } # GLM-5.2 FP8 agentic-coding benchmark on MI355X via SGLang with MTP speculative # decoding: the FP8 precision sibling of glm5.2-fp4-mi355x-sglang-agentic-mtp, @@ -1437,12 +1447,13 @@ glm5.2-fp8-mi355x-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: glm5.2/sglang/mi355x-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 12, 16], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 12, 16], spec-decoding: mtp, srt-recipe: agentic.yaml } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative # decoding, matching the standalone recipe (ROCm/ATOM PR 2345). Small # concurrency (C2-C10) is TP4-only on the GPU prefix cache with no KV offload; @@ -1458,14 +1469,15 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: precision: fp4 framework: atom multinode: false + srt-recipe-dir: glm5.2/atom/mi355x-fp4-mtp scenarios: agentic-coding: # dram-utilization 0.171 -> TOTAL_CPU_DRAM_GB=256 per TP4 rank (recipe's # LMCACHE_MAX_LOCAL_CPU_SIZE=256): min(3095781,2861022)MiB*0.171*4/8 = 256. - dram-utilization: 0.171 search-space: - - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } dsv4-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261007 @@ -1475,14 +1487,15 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv4/sglang/mi355x-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 16], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256, 384], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 16], spec-decoding: draft_model, srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: draft_model, srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128], spec-decoding: draft_model, srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256, 384], spec-decoding: draft_model, srt-recipe: agentic.yaml } # Upstream ROCm nightly rather than the deepseekv41-flash-0909 release tag: that # tag predates vllm-project/vllm#56503, which moves the mHC delayed pre block off @@ -1503,6 +1516,7 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv41flash/vllm/mi355x-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.60 @@ -1512,8 +1526,8 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: # and TP=4. Every point is measured here rather than combined from the # earlier run: the image move leaves concurrency 1-32 measured only on # the superseded nightly-eed1f3d0 pin. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: agentic.yaml } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: agentic.yaml } # DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. @@ -1525,11 +1539,12 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: precision: fp4 framework: atom multinode: false + srt-recipe-dir: dsv41flash/atom/mi355x-fp4-mtp scenarios: agentic-coding: - search-space: - - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 5, 6, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 5, 6, 8, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 5, 6, 8, 16, 32, 64, 128], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 5, 6, 8, 32], srt-recipe: agentic.yaml } # The golden acceptance curve remains the provisional GLM-5.2-derived curve # documented in infx/golden_al_distribution/glm5.3_mtp.yaml. @@ -1544,11 +1559,13 @@ glm5.3-fp8-mi355x-tilert-agentic: multinode: true disagg: true kv-p2p-transfer: mooncake + srt-recipe-dir: glm5.3/tilert/mi355x-fp8/agentx scenarios: agentic-coding: - search-space: - spec-decoding: "mtp" conc-list: [1] + srt-recipe: disagg-1p1d-tp8-mtp.yaml prefill: num-worker: 1 tp: 8 @@ -1556,7 +1573,6 @@ glm5.3-fp8-mi355x-tilert-agentic: dp-attn: false additional-settings: - "PREFILL_IMAGE=ghcr.io/tile-ai/tilert-rocm-prefill:0.1.6.post1" - - "CONFIG_FILE=recipes/glm5.3/tilert/mi355x-fp8/agentx/disagg-1p1d-tp8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -1573,9 +1589,10 @@ dsv41flash-fp4-mi355x-sglang-agentic-dspark: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv41flash/sglang/mi355x-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 12c4cc160e..09b777891a 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9,6 +9,7 @@ dsr1-fp4-b200-dynamo-trt: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/trtllm/b200-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -16,14 +17,13 @@ dsr1-fp4-b200-dynamo-trt: search-space: - spec-decoding: "mtp" conc-list: [90] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen1_dep8_batch8_eplb0_mtp3.yaml + srt-recipe: disagg-1p1d-dep8-b8-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen1_dep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p1d-dep8-b8-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 8 @@ -31,14 +31,13 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [66] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen3_tep8_batch16_eplb0_mtp3.yaml + srt-recipe: disagg-1p3d-tep8-b16-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen3_tep8_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp3.yaml" decode: num-worker: 3 tp: 8 @@ -46,14 +45,13 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [6] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch1_eplb0_mtp3.yaml + srt-recipe: disagg-1p5d-tep8-b1-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b1-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 8 @@ -61,14 +59,13 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [10, 15, 30, 60] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch8_eplb0_mtp3.yaml + srt-recipe: disagg-1p5d-tep8-b8-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b8-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 8 @@ -76,14 +73,13 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [548] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3.yaml + srt-recipe: disagg-3p1d-dep8-b64-eplb0-mtp3.yaml prefill: num-worker: 3 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 8 @@ -91,14 +87,13 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [1096, 1691] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen1_dep8_batch192_eplb0_mtp1.yaml + srt-recipe: disagg-5p1d-dep8-b192-eplb0-mtp1.yaml prefill: num-worker: 5 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen1_dep8_batch192_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-5p1d-dep8-b192-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -106,14 +101,13 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [658] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen2_dep8_batch32_eplb0_mtp3.yaml + srt-recipe: disagg-5p2d-dep8-b32-eplb0-mtp3.yaml prefill: num-worker: 5 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen2_dep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-5p2d-dep8-b32-eplb0-mtp3.yaml" decode: num-worker: 2 tp: 8 @@ -122,84 +116,78 @@ dsr1-fp4-b200-dynamo-trt: # Non-MTP configurations - conc-list: [6] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch1_eplb0_mtp0.yaml + srt-recipe: disagg-1p5d-tep8-b1-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b1-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 8 ep: 8 dp-attn: false - conc-list: [10, 15, 25, 50, 100] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch8_eplb0_mtp0.yaml + srt-recipe: disagg-1p5d-tep8-b16-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b16-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 8 ep: 8 dp-attn: false - conc-list: [370] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx2_gen5_tep8_batch64_eplb0_mtp0.yaml + srt-recipe: disagg-2p5d-tep8-b64-eplb0-mtp0.yaml prefill: num-worker: 2 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx2_gen5_tep8_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-2p5d-tep8-b64-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 8 ep: 8 dp-attn: false - conc-list: [1606] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen1_dep8_batch192_eplb0_mtp0.yaml + srt-recipe: disagg-4p1d-dep8-b192-eplb0-mtp0.yaml prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen1_dep8_batch192_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-4p1d-dep8-b192-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - conc-list: [837] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen3_dep8_batch32_eplb0_mtp0.yaml + srt-recipe: disagg-4p3d-dep8-b32-eplb0-mtp0.yaml prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen3_dep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-4p3d-dep8-b32-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 8 ep: 8 dp-attn: true - conc-list: [2222] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx7_gen2_dep8_batch128_eplb0_mtp0.yaml + srt-recipe: disagg-7p2d-dep8-b128-eplb0-mtp0.yaml prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx7_gen2_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-7p2d-dep8-b128-eplb0-mtp0.yaml" decode: num-worker: 2 tp: 8 @@ -217,6 +205,7 @@ dsr1-fp8-b200-dynamo-trt: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/trtllm/b200-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -225,14 +214,13 @@ dsr1-fp8-b200-dynamo-trt: # MTP configurations - Low latency (TP attention) - spec-decoding: "mtp" conc-list: [8] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_8.yaml + srt-recipe: disagg-1p6d-tp8-b1-eplb0-mtp3-c8.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b1-eplb0-mtp3-c8.yaml" decode: num-worker: 6 tp: 8 @@ -240,14 +228,13 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [8] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen2_tp8_batch32_eplb0_mtp3_8.yaml + srt-recipe: disagg-1p2d-tp8-b4-eplb0-mtp3-c8.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen2_tp8_batch32_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p2d-tp8-b4-eplb0-mtp3-c8.yaml" decode: num-worker: 2 tp: 8 @@ -255,14 +242,13 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [48] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_48.yaml + srt-recipe: disagg-1p6d-tp8-b8-eplb0-mtp3-c48.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_48.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b8-eplb0-mtp3-c48.yaml" decode: num-worker: 6 tp: 8 @@ -270,14 +256,13 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [64] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen4_tp8_batch16_eplb0_mtp3_64.yaml + srt-recipe: disagg-1p4d-tp8-b16-eplb0-mtp3-c64.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen4_tp8_batch16_eplb0_mtp3_64.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b16-eplb0-mtp3-c64.yaml" decode: num-worker: 4 tp: 8 @@ -286,14 +271,13 @@ dsr1-fp8-b200-dynamo-trt: # MTP configurations - High throughput (DP attention) - spec-decoding: "mtp" conc-list: [224] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen3_dep8_batch8_eplb0_mtp3_224.yaml + srt-recipe: disagg-2p3d-dep8-b8-eplb0-mtp3-c224.yaml prefill: num-worker: 2 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen3_dep8_batch8_eplb0_mtp3_224.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p3d-dep8-b8-eplb0-mtp3-c224.yaml" decode: num-worker: 3 tp: 8 @@ -301,14 +285,13 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [288] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen1_dep8_batch32_eplb0_mtp3_288.yaml + srt-recipe: disagg-2p1d-dep8-b32-eplb0-mtp3-c288.yaml prefill: num-worker: 2 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen1_dep8_batch32_eplb0_mtp3_288.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p1d-dep8-b32-eplb0-mtp3-c288.yaml" decode: num-worker: 1 tp: 8 @@ -316,14 +299,13 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [1088] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx4_gen1_dep8_batch128_eplb0_mtp2_1088.yaml + srt-recipe: disagg-4p1d-dep8-b128-eplb0-mtp2-c1088.yaml prefill: num-worker: 4 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx4_gen1_dep8_batch128_eplb0_mtp2_1088.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-4p1d-dep8-b128-eplb0-mtp2-c1088.yaml" decode: num-worker: 1 tp: 8 @@ -332,56 +314,52 @@ dsr1-fp8-b200-dynamo-trt: # Non-MTP (STP) configurations - Low latency (TP attention) - conc-list: [1] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_tp8_batch1_eplb0_mtp0_1.yaml + srt-recipe: disagg-1p1d-tp8-b1-eplb0-mtp0-c1.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_tp8_batch1_eplb0_mtp0_1.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-tp8-b1-eplb0-mtp0-c1.yaml" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - conc-list: [32] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_32.yaml + srt-recipe: disagg-1p4d-tp8-b8-eplb0-mtp0-c32.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_32.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b8-eplb0-mtp0-c32.yaml" decode: num-worker: 4 tp: 8 ep: 1 dp-attn: false - conc-list: [128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_128.yaml + srt-recipe: disagg-1p4d-tp8-b32-eplb0-mtp0-c128.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b32-eplb0-mtp0-c128.yaml" decode: num-worker: 4 tp: 8 ep: 1 dp-attn: false - conc-list: [96] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen6_tp8_batch16_eplb0_mtp0_96.yaml + srt-recipe: disagg-1p6d-tp8-b16-eplb0-mtp0-c96.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen6_tp8_batch16_eplb0_mtp0_96.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b16-eplb0-mtp0-c96.yaml" decode: num-worker: 6 tp: 8 @@ -389,56 +367,52 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: false # Non-MTP (STP) configurations - High throughput (DP attention) - conc-list: [128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch128_eplb0_mtp0_128.yaml + srt-recipe: disagg-1p1d-dep8-b128-eplb0-mtp0-c128.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch128_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-dep8-b128-eplb0-mtp0-c128.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - conc-list: [128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen2_dep8_batch64_eplb0_mtp0_128.yaml + srt-recipe: disagg-1p2d-dep8-b64-eplb0-mtp0-c128.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen2_dep8_batch64_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p2d-dep8-b64-eplb0-mtp0-c128.yaml" decode: num-worker: 2 tp: 8 ep: 8 dp-attn: true - conc-list: [256] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch256_eplb0_mtp0_256.yaml + srt-recipe: disagg-1p1d-dep8-b256-eplb0-mtp0-c256.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch256_eplb0_mtp0_256.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-dep8-b256-eplb0-mtp0-c256.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - conc-list: [640] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx2_gen1_dep8_batch640_eplb0_mtp0_640.yaml + srt-recipe: disagg-2p1d-dep8-b640-eplb0-mtp0-c640.yaml prefill: num-worker: 2 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx2_gen1_dep8_batch640_eplb0_mtp0_640.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p1d-dep8-b640-eplb0-mtp0-c640.yaml" decode: num-worker: 1 tp: 8 @@ -456,6 +430,7 @@ dsr1-fp4-b300-dynamo-trt: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/trtllm/b300-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -463,14 +438,13 @@ dsr1-fp4-b300-dynamo-trt: search-space: - spec-decoding: "mtp" conc-list: [2198] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml + srt-recipe: disagg-10p1d-dep8-b256-eplb0-mtp1.yaml prefill: num-worker: 10 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -478,14 +452,13 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [52] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep4_batch8_eplb0_mtp3.yaml + srt-recipe: disagg-1p4d-tep4-b8-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep4_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep4-b8-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 4 @@ -493,14 +466,13 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [8] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml + srt-recipe: disagg-1p4d-tep8-b1-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -508,14 +480,13 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [32] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml + srt-recipe: disagg-1p4d-tep8-b4-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -523,14 +494,13 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [181] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx3_gen1_dep8_batch16_eplb0_mtp3.yaml + srt-recipe: disagg-3p1d-dep8-b16-eplb0-mtp3.yaml prefill: num-worker: 3 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx3_gen1_dep8_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-3p1d-dep8-b16-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 8 @@ -538,14 +508,13 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [1197] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx9_gen1_dep8_batch128_eplb0_mtp1.yaml + srt-recipe: disagg-9p1d-dep8-b128-eplb0-mtp1.yaml prefill: num-worker: 9 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx9_gen1_dep8_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-9p1d-dep8-b128-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -554,98 +523,91 @@ dsr1-fp4-b300-dynamo-trt: # Non-MTP configurations - conc-list: [105] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep4_batch32_eplb0_mtp0.yaml + srt-recipe: disagg-1p3d-tep4-b32-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep4_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep4-b32-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 4 ep: 4 dp-attn: false - conc-list: [63] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml + srt-recipe: disagg-1p3d-tep8-b16-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 8 ep: 8 dp-attn: false - conc-list: [4] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0.yaml + srt-recipe: disagg-1p3d-tep8-b1-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 8 ep: 8 dp-attn: false - conc-list: [12] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen4_tep4_batch2_eplb0_mtp0.yaml + srt-recipe: disagg-1p4d-tep4-b2-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen4_tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep4-b2-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 4 ep: 4 dp-attn: false - conc-list: [589] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx5_gen2_dep8_batch32_eplb0_mtp0.yaml + srt-recipe: disagg-5p2d-dep8-b32-eplb0-mtp0.yaml prefill: num-worker: 5 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx5_gen2_dep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-5p2d-dep8-b32-eplb0-mtp0.yaml" decode: num-worker: 2 tp: 8 ep: 8 dp-attn: true - conc-list: [1093] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx6_gen1_dep8_batch128_eplb0_mtp0.yaml + srt-recipe: disagg-6p1d-dep8-b128-eplb0-mtp0.yaml prefill: num-worker: 6 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx6_gen1_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-6p1d-dep8-b128-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - conc-list: [2048] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx8_gen1_dep8_batch256_eplb0_mtp0.yaml + srt-recipe: disagg-8p1d-dep8-b256-eplb0-mtp0.yaml prefill: num-worker: 8 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx8_gen1_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-8p1d-dep8-b256-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 8 @@ -662,6 +624,7 @@ dsr1-fp8-b300-dynamo-trt: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/trtllm/b300-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -669,14 +632,13 @@ dsr1-fp8-b300-dynamo-trt: search-space: - spec-decoding: "mtp" conc-list: [40] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen2_tp8_batch16_eplb0_mtp3_40.yaml + srt-recipe: disagg-1p2d-tp8-b16-eplb0-mtp3-c40.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen2_tp8_batch16_eplb0_mtp3_40.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p2d-tp8-b16-eplb0-mtp3-c40.yaml" decode: num-worker: 2 tp: 8 @@ -684,14 +646,13 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [8] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch1_eplb0_mtp3_8.yaml + srt-recipe: disagg-1p4d-tp8-b1-eplb0-mtp3-c8.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch1_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b1-eplb0-mtp3-c8.yaml" decode: num-worker: 4 tp: 8 @@ -699,14 +660,13 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [20] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch4_eplb0_mtp3_20.yaml + srt-recipe: disagg-1p4d-tp8-b4-eplb0-mtp3-c20.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch4_eplb0_mtp3_20.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b4-eplb0-mtp3-c20.yaml" decode: num-worker: 4 tp: 8 @@ -714,14 +674,13 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [72] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen1_dp8_batch8_eplb0_mtp3_72.yaml + srt-recipe: disagg-1p1d-dp8-b8-eplb0-mtp3-c72.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen1_dp8_batch8_eplb0_mtp3_72.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p1d-dp8-b8-eplb0-mtp3-c72.yaml" decode: num-worker: 1 tp: 8 @@ -729,14 +688,13 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [144] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx2_gen1_dp8_batch16_eplb0_mtp3_144.yaml + srt-recipe: disagg-2p1d-dp8-b16-eplb0-mtp3-c144.yaml prefill: num-worker: 2 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx2_gen1_dp8_batch16_eplb0_mtp3_144.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-2p1d-dp8-b16-eplb0-mtp3-c144.yaml" decode: num-worker: 1 tp: 8 @@ -744,14 +702,13 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [512] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx4_gen1_dp8_batch64_eplb0_mtp2_512.yaml + srt-recipe: disagg-4p1d-dp8-b64-eplb0-mtp2-c512.yaml prefill: num-worker: 4 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx4_gen1_dp8_batch64_eplb0_mtp2_512.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-4p1d-dp8-b64-eplb0-mtp2-c512.yaml" decode: num-worker: 1 tp: 8 @@ -762,98 +719,91 @@ dsr1-fp8-b300-dynamo-trt: osl: 1024 search-space: - conc-list: [64] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen4_tp8_batch16_eplb0_mtp0_64.yaml + srt-recipe: disagg-1p4d-tp8-b16-eplb0-mtp0-c64.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen4_tp8_batch16_eplb0_mtp0_64.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b16-eplb0-mtp0-c64.yaml" decode: num-worker: 4 tp: 8 ep: 1 dp-attn: false - conc-list: [16] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen8_tp8_batch2_eplb0_mtp0_16.yaml + srt-recipe: disagg-1p8d-tp8-b1-eplb0-mtp0-c16.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen8_tp8_batch2_eplb0_mtp0_16.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p8d-tp8-b1-eplb0-mtp0-c16.yaml" decode: num-worker: 8 tp: 8 ep: 1 dp-attn: false - conc-list: [256] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx2_gen1_dp8_batch32_eplb0_mtp0_256.yaml + srt-recipe: disagg-2p1d-dp8-b32-eplb0-mtp0-c256.yaml prefill: num-worker: 2 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx2_gen1_dp8_batch32_eplb0_mtp0_256.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-2p1d-dp8-b32-eplb0-mtp0-c256.yaml" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: true - conc-list: [512] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen1_dp8_batch64_eplb0_mtp0_512.yaml + srt-recipe: disagg-3p1d-dp8-b64-eplb0-mtp0-c512.yaml prefill: num-worker: 3 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen1_dp8_batch64_eplb0_mtp0_512.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-3p1d-dp8-b64-eplb0-mtp0-c512.yaml" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: true - conc-list: [256] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen5_tp8_batch64_eplb0_mtp0_256.yaml + srt-recipe: disagg-3p5d-tp8-b64-eplb0-mtp0-c256.yaml prefill: num-worker: 3 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen5_tp8_batch64_eplb0_mtp0_256.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-3p5d-tp8-b64-eplb0-mtp0-c256.yaml" decode: num-worker: 5 tp: 8 ep: 1 dp-attn: false - conc-list: [1075] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx5_gen1_dp8_batch128_eplb0_mtp0_1075.yaml + srt-recipe: disagg-5p1d-dp8-b128-eplb0-mtp0-c1075.yaml prefill: num-worker: 5 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx5_gen1_dp8_batch128_eplb0_mtp0_1075.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-5p1d-dp8-b128-eplb0-mtp0-c1075.yaml" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: true - conc-list: [3072] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx7_gen1_dep8_batch384_eplb0_mtp0_3072.yaml + srt-recipe: disagg-7p1d-dep8-b384-eplb0-mtp0-c3072.yaml prefill: num-worker: 7 tp: 4 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx7_gen1_dep8_batch384_eplb0_mtp0_3072.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-7p1d-dep8-b384-eplb0-mtp0-c3072.yaml" decode: num-worker: 1 tp: 8 @@ -867,6 +817,7 @@ dsr1-fp4-b200-sglang: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/b200-fp4 scenarios: fixed-seq-len: - isl: 8192 @@ -876,13 +827,13 @@ dsr1-fp4-b200-sglang: ep: 1 conc-start: 1 conc-end: 32 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 4 dp-attn: true conc-start: 64 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml # agentic-coding: temporarily disabled — blocked by e2e-tests.yml artifact # name mismatch (downloads `agentic_*` but benchmark-tmpl.yml uploads as # `bmk_agentic_*`). Re-enable once that workflow is aligned. @@ -900,6 +851,7 @@ dsr1-fp4-b200-sglang-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/b200-fp4-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -910,14 +862,14 @@ dsr1-fp4-b200-sglang-mtp: conc-start: 1 conc-end: 32 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 4 dp-attn: true conc-start: 64 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml dsv4-fp4-b200-sglang-agentic-hicache-mtp: image: lmsysorg/sglang:v0.5.20-cu130 @@ -927,13 +879,14 @@ dsv4-fp4-b200-sglang-agentic-hicache-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv4/sglang/b200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 3, 4, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 10, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 3, 4, 5], srt-recipe: agentic.yaml } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 10, 16], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } dsv4-fp4-b200-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-591bb95 @@ -943,14 +896,15 @@ dsv4-fp4-b200-vllm-agentic-mtp: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv4/vllm/b200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.90 search-space: # TP8 resident KV + DSpark6 - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 6, 10, 14, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 6, 10, 14, 16], srt-recipe: agentic.yaml } # DEP8 SimpleCPU + DSpark6 - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [32, 64, 96, 128, 160, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [32, 64, 96, 128, 160, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 # does not have a B300-specific recipe, so this config reuses the existing DSR1 FP4 @@ -963,6 +917,7 @@ dsr1-fp4-b300-sglang: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/b300-fp4 scenarios: fixed-seq-len: - isl: 8192 @@ -972,12 +927,12 @@ dsr1-fp4-b300-sglang: ep: 4 conc-start: 1 conc-end: 128 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b300-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 8 conc-start: 1 conc-end: 16 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b300-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp4-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 @@ -987,6 +942,7 @@ dsr1-fp4-b200-trt: precision: fp4 framework: trt multinode: false + srt-recipe-dir: dsr1/trtllm/b200-fp4 scenarios: fixed-seq-len: - isl: 8192 @@ -998,28 +954,28 @@ dsr1-fp4-b200-trt: - tp: 4 conc-start: 4 conc-end: 32 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 4 conc-start: 32 conc-end: 32 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 4 dp-attn: true conc-start: 256 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 conc-start: 4 conc-end: 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 8 dp-attn: true conc-start: 128 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp4-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 @@ -1029,6 +985,7 @@ dsr1-fp4-b200-trt-mtp: precision: fp4 framework: trt multinode: false + srt-recipe-dir: dsr1/trtllm/b200-fp4-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -1039,33 +996,33 @@ dsr1-fp4-b200-trt-mtp: conc-start: 4 conc-end: 16 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 4 conc-start: 32 conc-end: 32 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 4 dp-attn: true conc-start: 256 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml # TP=8 configurations - tp: 8 conc-start: 4 conc-end: 4 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 8 dp-attn: true conc-start: 64 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-b200-sglang: image: lmsysorg/sglang:v0.5.12-cu130 @@ -1075,6 +1032,7 @@ dsr1-fp8-b200-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/b200-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -1084,12 +1042,12 @@ dsr1-fp8-b200-sglang: ep: 1 conc-start: 1 conc-end: 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 1 conc-start: 1 conc-end: 32 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 # does not have a B300-specific recipe, so this config reuses the existing DSR1 FP8 @@ -1102,6 +1060,7 @@ dsr1-fp8-b300-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/b300-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -1111,12 +1070,12 @@ dsr1-fp8-b300-sglang: ep: 1 conc-start: 1 conc-end: 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b300-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 1 conc-start: 1 conc-end: 32 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b300-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml dsv4-fp4-b300-sglang-agentic-hicache-mtp: image: lmsysorg/sglang:nightly-dev-20260901-07c8f729 @@ -1126,12 +1085,13 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv4/sglang/b300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.95 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } # DeepSeek-V4-Pro-0813 on B300 with DSpark speculative decoding. Recipe is # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by @@ -1153,13 +1113,14 @@ dsv4-fp4-b300-sglang-agentic-hicache-eagle: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv4/sglang/b300-fp4-eagle scenarios: agentic-coding: - dram-utilization: 0.95 search-space: - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-eagle/agentic.yaml } - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-eagle/agentic.yaml } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 128, 256, 384, 512], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-eagle/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 128, 256, 384, 512], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } qwen3.5-fp8-b200-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 @@ -1169,6 +1130,7 @@ qwen3.5-fp8-b200-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b200-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -1177,12 +1139,12 @@ qwen3.5-fp8-b200-sglang: - tp: 8 conc-start: 1 conc-end: 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 1 conc-start: 2 conc-end: 512 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 1 conc-list: @@ -1190,7 +1152,7 @@ qwen3.5-fp8-b200-sglang: - 384 - 448 - 640 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-b200-sglang-agentic-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 @@ -1200,14 +1162,15 @@ qwen3.5-fp8-b200-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b200-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32], srt-recipe: agentic.yaml } qwen3.5-fp4-b200-sglang: image: lmsysorg/sglang:v0.5.19-cu130 @@ -1217,6 +1180,7 @@ qwen3.5-fp4-b200-sglang: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b200-fp4 scenarios: fixed-seq-len: - isl: 8192 @@ -1226,12 +1190,12 @@ qwen3.5-fp4-b200-sglang: ep: 1 conc-start: 4 conc-end: 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 2 ep: 1 conc-start: 4 conc-end: 128 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp4-b200-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130 @@ -1241,6 +1205,7 @@ qwen3.5-fp4-b200-sglang-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b200-fp4-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -1251,13 +1216,13 @@ qwen3.5-fp4-b200-sglang-mtp: conc-start: 4 conc-end: 4 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 2 ep: 1 conc-start: 4 conc-end: 64 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 2 ep: 2 conc-list: @@ -1265,7 +1230,7 @@ qwen3.5-fp4-b200-sglang-mtp: - 32 - 64 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-b200-sglang-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 @@ -1275,6 +1240,7 @@ qwen3.5-fp8-b200-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b200-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -1285,13 +1251,13 @@ qwen3.5-fp8-b200-sglang-mtp: conc-start: 4 conc-end: 4 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 1 conc-start: 4 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-b300-sglang-mtp: image: lmsysorg/sglang:v0.5.12-cu130 @@ -1301,6 +1267,7 @@ qwen3.5-fp8-b300-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b300-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -1311,7 +1278,7 @@ qwen3.5-fp8-b300-sglang-mtp: conc-start: 4 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-b300-sglang: image: lmsysorg/sglang:v0.5.12-cu130 @@ -1321,6 +1288,7 @@ qwen3.5-fp8-b300-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b300-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -1330,7 +1298,7 @@ qwen3.5-fp8-b300-sglang: ep: 1 conc-start: 4 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp4-b300-sglang: image: lmsysorg/sglang:v0.5.14-cu130 @@ -1340,6 +1308,7 @@ qwen3.5-fp4-b300-sglang: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b300-fp4 scenarios: fixed-seq-len: - isl: 8192 @@ -1349,12 +1318,12 @@ qwen3.5-fp4-b300-sglang: ep: 1 conc-start: 4 conc-end: 128 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 2 ep: 2 conc-start: 4 conc-end: 128 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp4-b300-sglang-mtp: image: lmsysorg/sglang:v0.5.19-cu130 @@ -1364,6 +1333,7 @@ qwen3.5-fp4-b300-sglang-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b300-fp4-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -1374,13 +1344,13 @@ qwen3.5-fp4-b300-sglang-mtp: conc-start: 4 conc-end: 128 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 2 ep: 2 conc-start: 4 conc-end: 128 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml # Kimi K3 is a 2.8T MXFP4 MoE served on H200 nodes. # These are the three aggregated strategy classes from the official H200 @@ -1402,6 +1372,7 @@ kimik3-fp4-h200-vllm-agentic-latency: framework: vllm multinode: true disagg: false + srt-recipe-dir: kimik3/vllm/h200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -1411,13 +1382,12 @@ kimik3-fp4-h200-vllm-agentic-latency: kv-offloading: none conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12] num-nodes: 4 + srt-recipe: agg-tp16dp2ep32-latency.yaml worker: num-worker: 2 tp: 16 ep: 32 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml" kimik3-fp4-h200-vllm-agentic-balanced: image: vllm/vllm-openai:kimi-k3 @@ -1428,6 +1398,7 @@ kimik3-fp4-h200-vllm-agentic-balanced: framework: vllm multinode: true disagg: false + srt-recipe-dir: kimik3/vllm/h200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -1437,13 +1408,12 @@ kimik3-fp4-h200-vllm-agentic-balanced: kv-offloading: none conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16] num-nodes: 4 + srt-recipe: agg-tp8dp4ep32-balanced.yaml worker: num-worker: 4 tp: 8 ep: 32 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml" kimik3-fp4-h200-vllm-agentic-simple: image: vllm/vllm-openai:kimi-k3 @@ -1454,6 +1424,7 @@ kimik3-fp4-h200-vllm-agentic-simple: framework: vllm multinode: true disagg: false + srt-recipe-dir: kimik3/vllm/h200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -1464,13 +1435,12 @@ kimik3-fp4-h200-vllm-agentic-simple: kv-offload-backend: { name: vllm-simple } conc-list: [8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] num-nodes: 4 + srt-recipe: agg-tp8dp4ep32-vllm-simple.yaml worker: num-worker: 4 tp: 8 ep: 32 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml" dsr1-fp8-b200-sglang-mtp: image: lmsysorg/sglang:v0.5.12-cu130 @@ -1480,6 +1450,7 @@ dsr1-fp8-b200-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/b200-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -1490,7 +1461,7 @@ dsr1-fp8-b200-sglang-mtp: conc-start: 1 conc-end: 512 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b200-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 # does not have a B300-specific recipe, so this config reuses the existing DSR1 FP8 @@ -1503,6 +1474,7 @@ dsr1-fp8-b300-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/b300-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -1513,7 +1485,7 @@ dsr1-fp8-b300-sglang-mtp: conc-start: 1 conc-end: 512 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b300-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml kimik3-fp4-b300-vllm-agentic-dspark: # TP8 x DCP8 with Mooncake as the external KV tier. The recipe drafts with @@ -1531,6 +1503,7 @@ kimik3-fp4-b300-vllm-agentic-dspark: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: kimik3/vllm/b300-fp4-mtp scenarios: # Agentic-coding only: no fixed-seq-len (1k1k / 8k1k) arms for this recipe. agentic-coding: @@ -1542,7 +1515,7 @@ kimik3-fp4-b300-vllm-agentic-dspark: # One entry for every arm: the recipe drafts at DSpark 7 up to conc 8, # DSpark 3 at conc 16, and not at all above. Keep the concurrencies # disjoint across arms so exp-names stay unique. - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: agentic.yaml } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 @@ -1552,6 +1525,7 @@ dsr1-fp8-b200-trt: precision: fp8 framework: trt multinode: false + srt-recipe-dir: dsr1/trtllm/b200-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -1561,17 +1535,17 @@ dsr1-fp8-b200-trt: ep: 1 conc-start: 64 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 1 conc-start: 8 conc-end: 32 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 1 conc-start: 4 conc-end: 8 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 @@ -1581,6 +1555,7 @@ dsr1-fp8-b200-trt-mtp: precision: fp8 framework: trt multinode: false + srt-recipe-dir: dsr1/trtllm/b200-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -1592,7 +1567,7 @@ dsr1-fp8-b200-trt-mtp: conc-start: 4 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-h200-sglang: image: lmsysorg/sglang:v0.5.12-cu130 @@ -1602,6 +1577,7 @@ dsr1-fp8-h200-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/h200-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -1610,7 +1586,7 @@ dsr1-fp8-h200-sglang: - tp: 8 conc-start: 4 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-h200-sglang-mtp: image: lmsysorg/sglang:v0.5.19-cu130 @@ -1620,6 +1596,7 @@ dsr1-fp8-h200-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: dsr1/sglang/h200-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -1630,7 +1607,7 @@ dsr1-fp8-h200-sglang-mtp: conc-start: 4 conc-end: 64 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/h200-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml # DeepSeek-V4-Pro AgentX on one aggregated TP8 H200 worker. Keep the serving # topology fixed and sweep only concurrency to produce the Pareto curve. @@ -1644,6 +1621,7 @@ dsv4-fp8-h200-dynamo-sglang-agentic-agg: router: { name: dynamo-router, version: "1.5.0.dev20260910" } multinode: true disagg: false + srt-recipe-dir: dsv4/sglang/h200-fp8/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -1653,13 +1631,12 @@ dsv4-fp8-h200-dynamo-sglang-agentic-agg: kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-tp8-mtp-kvoffload.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml" dsv4-fp4-b300-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-20ebb28 model: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -1668,16 +1645,17 @@ dsv4-fp4-b300-vllm-agentic-mtp: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv4/vllm/b300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.95 search-space: # TP8 GPU-resident + DSpark6 - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: agentic.yaml } # TP4 GPU-resident + DSpark6 - - { tp: 4, kv-offloading: none, spec-decoding: draft_model, conc-list: [8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: draft_model, conc-list: [8], srt-recipe: agentic.yaml } # DEP8 SimpleCPU + DSpark6 - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [64, 96, 128, 192, 256, 288], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [64, 96, 128, 192, 256, 288], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } qwen3.5-fp8-h200-sglang: image: lmsysorg/sglang:v0.5.14-cu130 @@ -1687,6 +1665,7 @@ qwen3.5-fp8-h200-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/h200-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -1696,7 +1675,7 @@ qwen3.5-fp8-h200-sglang: ep: 8 conc-start: 4 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-h200-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130 @@ -1706,6 +1685,7 @@ qwen3.5-fp8-h200-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/h200-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -1716,7 +1696,7 @@ qwen3.5-fp8-h200-sglang-mtp: conc-start: 4 conc-end: 128 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-h200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 @@ -1726,6 +1706,7 @@ dsr1-fp8-h200-trt: precision: fp8 framework: trt multinode: false + srt-recipe-dir: dsr1/trtllm/h200-fp8 # For all sequence lengths, EP=TP scenarios: fixed-seq-len: @@ -1737,13 +1718,13 @@ dsr1-fp8-h200-trt: ep: 8 conc-start: 4 conc-end: 32 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 8 dp-attn: true conc-start: 64 conc-end: 64 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-h200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 @@ -1753,6 +1734,7 @@ dsr1-fp8-h200-trt-mtp: precision: fp8 framework: trt multinode: false + srt-recipe-dir: dsr1/trtllm/h200-fp8-mtp # For all sequence lengths, EP=TP, MOE_BACKEND=CUTLASS, MTP=3 (or MTP=1 when DP_ATTN=true) scenarios: fixed-seq-len: @@ -1765,14 +1747,14 @@ dsr1-fp8-h200-trt-mtp: conc-start: 4 conc-end: 32 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 8 dp-attn: true conc-start: 64 conc-end: 256 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml dsr1-fp8-h200-dynamo-trt: image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1 @@ -1785,6 +1767,7 @@ dsr1-fp8-h200-dynamo-trt: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/trtllm/h200-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -1793,14 +1776,13 @@ dsr1-fp8-h200-dynamo-trt: # MTP configurations - spec-decoding: "mtp" conc-list: [1] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp3.yaml + srt-recipe: disagg-1p7d-tep8-b1-eplb0-mtp3-c9.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b1-eplb0-mtp3-c9.yaml" decode: num-worker: 7 tp: 8 @@ -1808,14 +1790,13 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [4] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp3.yaml + srt-recipe: disagg-1p7d-tep8-b32-eplb0-mtp3-c28.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b32-eplb0-mtp3-c28.yaml" decode: num-worker: 7 tp: 8 @@ -1823,14 +1804,13 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [8] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c8_ctx1_gen6_tep8_batch32_eplb0_mtp3.yaml + srt-recipe: disagg-1p6d-tep8-b32-eplb0-mtp3-c48.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c8_ctx1_gen6_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p6d-tep8-b32-eplb0-mtp3-c48.yaml" decode: num-worker: 6 tp: 8 @@ -1838,14 +1818,13 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [16] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp2.yaml + srt-recipe: disagg-1p3d-tep8-b32-eplb0-mtp2-c48.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p3d-tep8-b32-eplb0-mtp2-c48.yaml" decode: num-worker: 3 tp: 8 @@ -1853,14 +1832,13 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [32] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c32_ctx3_gen5_tep8_batch32_eplb0_mtp3.yaml + srt-recipe: disagg-3p5d-tep8-b32-eplb0-mtp3-c160.yaml prefill: num-worker: 3 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c32_ctx3_gen5_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p5d-tep8-b32-eplb0-mtp3-c160.yaml" decode: num-worker: 5 tp: 8 @@ -1868,14 +1846,13 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [64] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c64_ctx1_gen1_dep8_batch32_eplb0_mtp2.yaml + srt-recipe: disagg-1p1d-dep8-b32-eplb0-mtp2-c64.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c64_ctx1_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p1d-dep8-b32-eplb0-mtp2-c64.yaml" decode: num-worker: 1 tp: 8 @@ -1883,14 +1860,13 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c128_ctx2_gen1_dep8_batch32_eplb0_mtp2.yaml + srt-recipe: disagg-2p1d-dep8-b32-eplb0-mtp2-c128.yaml prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c128_ctx2_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p1d-dep8-b32-eplb0-mtp2-c128.yaml" decode: num-worker: 1 tp: 8 @@ -1898,14 +1874,13 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [256] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c256_ctx3_gen1_dep8_batch32_eplb0_mtp2.yaml + srt-recipe: disagg-3p1d-dep8-b32-eplb0-mtp2-c256.yaml prefill: num-worker: 3 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c256_ctx3_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b32-eplb0-mtp2-c256.yaml" decode: num-worker: 1 tp: 8 @@ -1913,14 +1888,13 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [512] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c512_ctx3_gen1_dep8_batch64_eplb0_mtp1.yaml + srt-recipe: disagg-3p1d-dep8-b64-eplb0-mtp1-c512.yaml prefill: num-worker: 3 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c512_ctx3_gen1_dep8_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp1-c512.yaml" decode: num-worker: 1 tp: 8 @@ -1928,126 +1902,117 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: true # Non-MTP configurations (STP) - conc-list: [1] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp0.yaml + srt-recipe: disagg-1p7d-tep8-b1-eplb0-mtp0-c9.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b1-eplb0-mtp0-c9.yaml" decode: num-worker: 7 tp: 8 ep: 8 dp-attn: false - conc-list: [4] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp0.yaml + srt-recipe: disagg-1p7d-tep8-b32-eplb0-mtp0-c28.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b32-eplb0-mtp0-c28.yaml" decode: num-worker: 7 tp: 8 ep: 8 dp-attn: false - conc-list: [8] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c8_ctx1_gen6_tep8_batch16_eplb0_mtp0.yaml + srt-recipe: disagg-1p6d-tep8-b16-eplb0-mtp0-c48.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c8_ctx1_gen6_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p6d-tep8-b16-eplb0-mtp0-c48.yaml" decode: num-worker: 6 tp: 8 ep: 8 dp-attn: false - conc-list: [16] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp0.yaml + srt-recipe: disagg-1p3d-tep8-b32-eplb0-mtp0-c48.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p3d-tep8-b32-eplb0-mtp0-c48.yaml" decode: num-worker: 3 tp: 8 ep: 8 dp-attn: false - conc-list: [32] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c32_ctx2_gen5_tep8_batch128_eplb0_mtp0.yaml + srt-recipe: disagg-2p5d-tep8-b128-eplb0-mtp0-c160.yaml prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c32_ctx2_gen5_tep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p5d-tep8-b128-eplb0-mtp0-c160.yaml" decode: num-worker: 5 tp: 8 ep: 8 dp-attn: false - conc-list: [64] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c64_ctx2_gen3_dep8_batch128_eplb0_mtp0.yaml + srt-recipe: disagg-2p3d-dep8-b128-eplb0-mtp0-c192.yaml prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c64_ctx2_gen3_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p3d-dep8-b128-eplb0-mtp0-c192.yaml" decode: num-worker: 3 tp: 8 ep: 8 dp-attn: true - conc-list: [128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c128_ctx1_gen1_dep8_batch256_eplb0_mtp0.yaml + srt-recipe: disagg-1p1d-dep8-b256-eplb0-mtp0-c128.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c128_ctx1_gen1_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p1d-dep8-b256-eplb0-mtp0-c128.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - conc-list: [256] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c256_ctx5_gen3_dep8_batch256_eplb0_mtp0.yaml + srt-recipe: disagg-5p3d-dep8-b256-eplb0-mtp0-c768.yaml prefill: num-worker: 5 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c256_ctx5_gen3_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-5p3d-dep8-b256-eplb0-mtp0-c768.yaml" decode: num-worker: 3 tp: 8 ep: 8 dp-attn: true - conc-list: [512] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c512_ctx3_gen1_dep8_batch512_eplb0_mtp0.yaml + srt-recipe: disagg-3p1d-dep8-b512-eplb0-mtp0-c512.yaml prefill: num-worker: 3 tp: 8 ep: 8 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c512_ctx3_gen1_dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b512-eplb0-mtp0-c512.yaml" decode: num-worker: 1 tp: 8 @@ -2065,6 +2030,7 @@ dsr1-fp8-h100-dynamo-trt: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/trtllm/h100-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -2073,14 +2039,13 @@ dsr1-fp8-h100-dynamo-trt: # MTP configurations (6 points) - spec-decoding: "mtp" conc-list: [6] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml + srt-recipe: disagg-1p3d-tep16-b1-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 16 ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b1-eplb0-mtp3.yaml" decode: num-worker: 3 tp: 16 @@ -2088,14 +2053,13 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [9] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml + srt-recipe: disagg-1p3d-tep16-b2-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 16 ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b2-eplb0-mtp3.yaml" decode: num-worker: 3 tp: 16 @@ -2103,14 +2067,13 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [30] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml + srt-recipe: disagg-1p3d-tep16-b8-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 16 ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b8-eplb0-mtp3.yaml" decode: num-worker: 3 tp: 16 @@ -2118,14 +2081,13 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [77] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen1_dep16_batch4_eplb0_mtp3.yaml + srt-recipe: disagg-1p1d-dep16-b4-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 16 ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen1_dep16_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p1d-dep16-b4-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -2135,14 +2097,13 @@ dsr1-fp8-h100-dynamo-trt: # https://github.com/InferenceMAX/InferenceMAX/actions/runs/21769314582/job/62813105509 # - spec-decoding: "mtp" # conc-list: [78] + # # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen2_tep16_batch32_eplb0_mtp3.yaml + # srt-recipe: disagg-1p2d-tep16-b32-eplb0-mtp3.yaml # prefill: # num-worker: 1 # tp: 16 # ep: 16 # dp-attn: true - # additional-settings: - # # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen2_tep16_batch32_eplb0_mtp3.yaml - # - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p2d-tep16-b32-eplb0-mtp3.yaml" # decode: # num-worker: 2 # tp: 16 @@ -2150,14 +2111,13 @@ dsr1-fp8-h100-dynamo-trt: # dp-attn: false - spec-decoding: "mtp" conc-list: [154] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx2_gen1_dep16_batch8_eplb0_mtp3.yaml + srt-recipe: disagg-2p1d-dep16-b8-eplb0-mtp3.yaml prefill: num-worker: 2 tp: 16 ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx2_gen1_dep16_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-2p1d-dep16-b8-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -2165,70 +2125,65 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true # STP configurations (5 points) - conc-list: [6] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml + srt-recipe: disagg-1p3d-tep16-b1-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 16 ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b1-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 16 ep: 16 dp-attn: false - conc-list: [9] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml + srt-recipe: disagg-1p3d-tep16-b2-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 16 ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b2-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 16 ep: 16 dp-attn: false - conc-list: [30] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml + srt-recipe: disagg-1p3d-tep16-b8-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 16 ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b8-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 16 ep: 16 dp-attn: false - conc-list: [154] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen2_tep16_batch64_eplb0_mtp0.yaml + srt-recipe: disagg-1p2d-tep16-b64-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 16 ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen2_tep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p2d-tep16-b64-eplb0-mtp0.yaml" decode: num-worker: 2 tp: 16 ep: 16 dp-attn: false - conc-list: [308] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx2_gen1_dep16_batch16_eplb0_mtp0.yaml + srt-recipe: disagg-2p1d-dep16-b16-eplb0-mtp0.yaml prefill: num-worker: 2 tp: 16 ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx2_gen1_dep16_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-2p1d-dep16-b16-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -2246,6 +2201,7 @@ dsr1-fp8-h100-dynamo-sglang: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/h100-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -2253,13 +2209,12 @@ dsr1-fp8-h100-dynamo-sglang: search-space: # # STP: Max throughput TEP (1 prefill, 1 decode) # - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + # srt-recipe: disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-stp.yaml # prefill: # num-worker: 1 # tp: 16 # ep: 1 # dp-attn: false - # additional-settings: - # - "CONFIG_FILE=recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-stp.yaml" # decode: # num-worker: 1 # tp: 16 @@ -2267,13 +2222,12 @@ dsr1-fp8-h100-dynamo-sglang: # dp-attn: false # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) # - conc-list: [1, 2, 4, 8, 16, 32, 64] + # srt-recipe: disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-stp.yaml # prefill: # num-worker: 1 # tp: 16 # ep: 1 # dp-attn: false - # additional-settings: - # - "CONFIG_FILE=recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-stp.yaml" # decode: # num-worker: 1 # tp: 16 @@ -2282,13 +2236,12 @@ dsr1-fp8-h100-dynamo-sglang: # MTP: Max throughput TEP (1 prefill, 1 decode) - spec-decoding: "mtp" conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + srt-recipe: disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-mtp.yaml prefill: num-worker: 1 tp: 16 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -2297,13 +2250,12 @@ dsr1-fp8-h100-dynamo-sglang: # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) - spec-decoding: "mtp" conc-list: [1, 2, 4, 8, 16, 32, 64] + srt-recipe: disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-mtp.yaml prefill: num-worker: 1 tp: 16 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -2321,6 +2273,7 @@ dsr1-fp4-gb200-dynamo-trt: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/trtllm/gb200-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -2329,14 +2282,13 @@ dsr1-fp4-gb200-dynamo-trt: # MTP configurations (spec_decoding="mtp") - spec-decoding: "mtp" conc-list: [ 4, 8, 12, 24, 48 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml + srt-recipe: disagg-1p4d-tep8-b8-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -2344,14 +2296,13 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [ 180 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx3_gen1_dep32_batch4_eplb0_mtp3.yaml + srt-recipe: disagg-3p1d-dep32-b4-eplb0-mtp3.yaml prefill: num-worker: 3 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx3_gen1_dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-3p1d-dep32-b4-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2359,14 +2310,13 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [ 1229 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx7_gen1_dep16_batch64_eplb256_mtp1.yaml + srt-recipe: disagg-7p1d-dep16-b64-eplb256-mtp1.yaml prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx7_gen1_dep16_batch64_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep16-b64-eplb256-mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -2374,14 +2324,13 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [ 666 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx8_gen1_dep32_batch16_eplb0_mtp3.yaml + srt-recipe: disagg-8p1d-dep32-b16-eplb0-mtp3.yaml prefill: num-worker: 8 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx8_gen1_dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep32-b16-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2389,14 +2338,13 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [ 4301 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx11_gen1_dep16_batch256_eplb256_mtp1.yaml + srt-recipe: disagg-11p1d-dep16-b256-eplb256-mtp1.yaml prefill: num-worker: 11 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx11_gen1_dep16_batch256_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-11p1d-dep16-b256-eplb256-mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -2405,84 +2353,78 @@ dsr1-fp4-gb200-dynamo-trt: # Non-MTP configurations (default spec_decoding="none") - conc-list: [ 12, 44, 76 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0.yaml + srt-recipe: disagg-1p4d-tep8-b16-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b16-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - conc-list: [ 5 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml + srt-recipe: disagg-1p4d-tep8-b1-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - conc-list: [ 333 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0.yaml + srt-recipe: disagg-2p1d-dep32-b8-eplb0-mtp0.yaml prefill: num-worker: 2 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - conc-list: [ 1229 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx7_gen1_dep32_batch32_eplb0_mtp0.yaml + srt-recipe: disagg-7p1d-dep32-b32-eplb0-mtp0.yaml prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx7_gen1_dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep32-b32-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - conc-list: [ 2253 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx8_gen1_dep16_batch128_eplb0_mtp0.yaml + srt-recipe: disagg-8p1d-dep16-b128-eplb0-mtp0.yaml prefill: num-worker: 8 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx8_gen1_dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep16-b128-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - conc-list: [ 4096 ] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx10_gen1_dep16_batch256_eplb256_mtp0.yaml + srt-recipe: disagg-10p1d-dep16-b256-eplb256-mtp0.yaml prefill: num-worker: 10 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx10_gen1_dep16_batch256_eplb256_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-10p1d-dep16-b256-eplb256-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -2500,6 +2442,7 @@ dsr1-fp8-gb200-dynamo-trt: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/trtllm/gb200-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -2507,14 +2450,13 @@ dsr1-fp8-gb200-dynamo-trt: search-space: - spec-decoding: "mtp" conc-list: [666] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3_666.yaml + srt-recipe: disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml prefill: num-worker: 3 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml" decode: num-worker: 1 tp: 8 @@ -2522,14 +2464,13 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [666] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx5_gen1_dep16_batch32_eplb0_mtp3_666.yaml + srt-recipe: disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml prefill: num-worker: 5 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx5_gen1_dep16_batch32_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml" decode: num-worker: 1 tp: 16 @@ -2537,14 +2478,13 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [333] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep16_batch16_eplb0_mtp3_333.yaml + srt-recipe: disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml prefill: num-worker: 3 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep16_batch16_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml" decode: num-worker: 1 tp: 16 @@ -2552,14 +2492,13 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [333] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx4_gen1_dep32_batch8_eplb0_mtp3_333.yaml + srt-recipe: disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml prefill: num-worker: 4 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx4_gen1_dep32_batch8_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml" decode: num-worker: 1 tp: 32 @@ -2567,14 +2506,13 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [90] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx2_gen1_dep32_batch2_eplb0_mtp3_90.yaml + srt-recipe: disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx2_gen1_dep32_batch2_eplb0_mtp3_90.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml" decode: num-worker: 1 tp: 32 @@ -2582,14 +2520,13 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [15] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_15.yaml + srt-recipe: disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_15.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml" decode: num-worker: 3 tp: 8 @@ -2597,14 +2534,13 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [6] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_6.yaml + srt-recipe: disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_6.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml" decode: num-worker: 3 tp: 8 @@ -2612,98 +2548,91 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: false # 8k1k STP configs - conc-list: [1229] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx5_gen1_dep16_batch64_eplb0_mtp0_1229.yaml + srt-recipe: disagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml prefill: num-worker: 5 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx5_gen1_dep16_batch64_eplb0_mtp0_1229.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - conc-list: [666] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx4_gen1_dep32_batch16_eplb0_mtp0_666.yaml + srt-recipe: disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml prefill: num-worker: 4 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx4_gen1_dep32_batch16_eplb0_mtp0_666.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - conc-list: [615] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx3_gen1_dep16_batch32_eplb0_mtp0_615.yaml + srt-recipe: disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml prefill: num-worker: 3 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx3_gen1_dep16_batch32_eplb0_mtp0_615.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - conc-list: [333] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0_333.yaml + srt-recipe: disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0_333.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - conc-list: [63] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0_63.yaml + srt-recipe: disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0_63.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml" decode: num-worker: 3 tp: 8 ep: 8 dp-attn: false - conc-list: [18] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch4_eplb0_mtp0_18.yaml + srt-recipe: disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch4_eplb0_mtp0_18.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml" decode: num-worker: 3 tp: 8 ep: 8 dp-attn: false - conc-list: [6] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_6.yaml + srt-recipe: disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_6.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml" decode: num-worker: 3 tp: 8 @@ -2721,37 +2650,48 @@ dsr1-fp8-gb200-dynamo-sglang: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/gb200-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 osl: 1024 search-space: - # "Low latency" (1 prefill worker at TP8 and 1 decode worker at TP8) - - conc-list: [4, 8, 16] + # "Low latency" (1 prefill worker at TP8 and 7 decode workers at TP8) + - conc-list: [4, 8] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/low-latency.yaml + srt-recipe: disagg-tp8-stp-variants.yaml:zip_override_lowlat[0] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/low-latency.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-tp8-stp-variants.yaml" decode: + num-worker: 7 + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [16] + srt-recipe: disagg-tp8-stp-variants.yaml:zip_override_lowlat[1] + prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false + decode: + num-worker: 7 + tp: 8 + ep: 1 + dp-attn: false # "Mid curve" (5 prefill workers at DEP8 and 1 decode worker at DEP32) - conc-list: [512, 1024, 2048, 6144] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/mid-curve.yaml + srt-recipe: disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b8192-c512x1024x2048x6144-stp.yaml prefill: num-worker: 5 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/mid-curve.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b8192-c512x1024x2048x6144-stp.yaml" decode: num-worker: 1 tp: 32 @@ -2760,14 +2700,13 @@ dsr1-fp8-gb200-dynamo-sglang: # "Max throughput" (6 prefill workers at DEP8 and 1 decode worker at DEP24) - conc-list: [2048, 4096, 6144] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/max_tpt.yaml + srt-recipe: disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b8192-c2048x4096x6144-stp.yaml prefill: num-worker: 6 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/max_tpt.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b8192-c2048x4096x6144-stp.yaml" decode: num-worker: 1 tp: 24 @@ -2785,6 +2724,7 @@ dsr1-fp8-gb300-dynamo-sglang: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/gb300-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -2792,14 +2732,13 @@ dsr1-fp8-gb300-dynamo-sglang: search-space: # "Low latency" (1 prefill worker at TP4 and 1 decode worker at TP4) - conc-list: [4, 8] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/low-latency.yaml + srt-recipe: disagg-1p1d-p-tp4-d-tp4-b128-c4x8-stp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/low-latency.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c4x8-stp.yaml" decode: num-worker: 1 tp: 4 @@ -2808,14 +2747,13 @@ dsr1-fp8-gb300-dynamo-sglang: # "Mid curve" (5 prefill workers at DEP8 and 1 decode worker at DEP32) - conc-list: [128, 256, 512, 1024] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/mid.yaml + srt-recipe: disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b45000-c128x256x512x1024-stp.yaml prefill: num-worker: 5 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/mid.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b45000-c128x256x512x1024-stp.yaml" decode: num-worker: 1 tp: 32 @@ -2824,14 +2762,13 @@ dsr1-fp8-gb300-dynamo-sglang: # "Max throughput" (6 prefill workers at DEP8 and 1 decode worker at DEP24) - conc-list: [2048, 4096] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/max.yaml + srt-recipe: disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b45000-c2048x4096-stp.yaml prefill: num-worker: 6 tp: 8 ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/max.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b45000-c2048x4096-stp.yaml" decode: num-worker: 1 tp: 24 @@ -2849,6 +2786,7 @@ dsr1-fp4-gb200-dynamo-sglang: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/gb200-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -2857,13 +2795,12 @@ dsr1-fp4-gb200-dynamo-sglang: # Low latency (1 prefill node, 4 decode nodes) - spec-decoding: "none" conc-list: [ 4, 8 ] + srt-recipe: disagg-1p4d-p-tp4-d-tp4-c4x8-stp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-1p4d-p-tp4-d-tp4-c4x8-stp.yaml" decode: num-worker: 4 tp: 4 @@ -2873,13 +2810,12 @@ dsr1-fp4-gb200-dynamo-sglang: # Mid curve (6 prefill nodes, 12 decode nodes) - spec-decoding: "none" conc-list: [ 512, 2048, 4096 ] + srt-recipe: disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml prefill: num-worker: 6 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml" decode: num-worker: 1 tp: 48 @@ -2889,13 +2825,12 @@ dsr1-fp4-gb200-dynamo-sglang: # Max throughput (10 prefill nodes, 8 decode nodes) - spec-decoding: "none" conc-list: [ 2048 ] + srt-recipe: disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml prefill: num-worker: 10 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml" decode: num-worker: 1 tp: 32 @@ -2913,6 +2848,7 @@ dsr1-fp4-gb300-dynamo-trt: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/trtllm/gb300-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -2921,14 +2857,13 @@ dsr1-fp4-gb300-dynamo-trt: # MTP configurations (spec_decoding="mtp") - spec-decoding: "mtp" conc-list: [33] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3.yaml + srt-recipe: disagg-1p3d-tep8-b8-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b8-eplb0-mtp3.yaml" decode: num-worker: 3 tp: 8 @@ -2936,14 +2871,13 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [5] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml + srt-recipe: disagg-1p4d-tep8-b1-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -2951,14 +2885,13 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [12, 24] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml + srt-recipe: disagg-1p4d-tep8-b4-eplb0-mtp3.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -2966,14 +2899,13 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [180] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx4_gen1_dep32_batch4_eplb0_mtp3.yaml + srt-recipe: disagg-4p1d-dep32-b4-eplb0-mtp3.yaml prefill: num-worker: 4 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx4_gen1_dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep32-b4-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2981,14 +2913,13 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [308] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx8_gen1_dep32_batch8_eplb0_mtp3.yaml + srt-recipe: disagg-8p1d-dep32-b8-eplb0-mtp3.yaml prefill: num-worker: 8 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx8_gen1_dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep32-b8-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2996,14 +2927,13 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [2253] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml + srt-recipe: disagg-10p1d-dep8-b256-eplb0-mtp1.yaml prefill: num-worker: 10 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -3011,14 +2941,13 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [666] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep16_batch32_eplb0_mtp3.yaml + srt-recipe: disagg-10p1d-dep16-b32-eplb0-mtp3.yaml prefill: num-worker: 10 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep16-b32-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -3026,14 +2955,13 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [1127] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx13_gen1_dep16_batch64_eplb256_mtp3.yaml + srt-recipe: disagg-13p1d-dep16-b64-eplb256-mtp3.yaml prefill: num-worker: 13 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx13_gen1_dep16_batch64_eplb256_mtp3.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-13p1d-dep16-b64-eplb256-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -3041,112 +2969,104 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true # Non-MTP configurations (default spec_decoding="none") - conc-list: [72] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml + srt-recipe: disagg-1p3d-tep8-b16-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 8 ep: 8 dp-attn: false - conc-list: [5] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml + srt-recipe: disagg-1p4d-tep8-b1-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - conc-list: [12] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch2_eplb0_mtp0.yaml + srt-recipe: disagg-1p4d-tep8-b2-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - conc-list: [5, 15, 30] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml + srt-recipe: disagg-1p5d-tep4-b4-eplb0-mtp0.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p5d-tep4-b4-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 ep: 4 dp-attn: false - conc-list: [666] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx7_gen1_dep32_batch16_eplb0_mtp0.yaml + srt-recipe: disagg-7p1d-dep32-b16-eplb0-mtp0.yaml prefill: num-worker: 7 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx7_gen1_dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep32-b16-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - conc-list: [1229] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx9_gen1_dep16_batch64_eplb0_mtp0.yaml + srt-recipe: disagg-9p1d-dep16-b64-eplb0-mtp0.yaml prefill: num-worker: 9 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx9_gen1_dep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep16-b64-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - conc-list: [3228] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx11_gen3_dep4_batch256_eplb0_mtp0.yaml + srt-recipe: disagg-11p3d-dep4-b256-eplb0-mtp0.yaml prefill: num-worker: 11 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx11_gen3_dep4_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-11p3d-dep4-b256-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 4 ep: 4 dp-attn: true - conc-list: [2253] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx14_gen1_dep16_batch128_eplb0_mtp0.yaml + srt-recipe: disagg-14p1d-dep16-b128-eplb0-mtp0.yaml prefill: num-worker: 14 tp: 2 ep: 2 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx14_gen1_dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-14p1d-dep16-b128-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -3163,6 +3083,7 @@ dsr1-fp4-gb300-dynamo-sglang: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/gb300-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -3171,13 +3092,12 @@ dsr1-fp4-gb300-dynamo-sglang: # Low latency (1 prefill node, 4 decode nodes) - spec-decoding: "none" conc-list: [ 4, 8, 32, 64 ] + srt-recipe: disagg-1p4d-p-tp4-d-tp4-c4x8x32x64-stp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-1p4d-p-tp4-d-tp4-c4x8x32x64-stp.yaml" decode: num-worker: 4 tp: 4 @@ -3187,13 +3107,12 @@ dsr1-fp4-gb300-dynamo-sglang: # Mid curve (6 prefill nodes, 12 decode nodes) - spec-decoding: "none" conc-list: [ 512, 2048, 4096 ] + srt-recipe: disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml prefill: num-worker: 6 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml" decode: num-worker: 1 tp: 48 @@ -3203,13 +3122,12 @@ dsr1-fp4-gb300-dynamo-sglang: # Max throughput (10 prefill nodes, 8 decode nodes) - spec-decoding: "none" conc-list: [ 2048 ] + srt-recipe: disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml prefill: num-worker: 10 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml" decode: num-worker: 1 tp: 32 @@ -3227,6 +3145,7 @@ dsr1-fp8-gb300-dynamo-trt: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/trtllm/gb300-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -3235,14 +3154,13 @@ dsr1-fp8-gb300-dynamo-trt: # MTP configurations (spec_decoding="mtp") - spec-decoding: "mtp" conc-list: [8] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml + srt-recipe: disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml" decode: num-worker: 4 tp: 8 @@ -3250,14 +3168,13 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [24] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml + srt-recipe: disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml" decode: num-worker: 4 tp: 8 @@ -3265,14 +3182,13 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: false - spec-decoding: "mtp" conc-list: [333] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx6_gen1_dep32_batch8_eplb0_mtp3_333.yaml + srt-recipe: disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml prefill: num-worker: 6 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx6_gen1_dep32_batch8_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml" decode: num-worker: 1 tp: 32 @@ -3280,14 +3196,13 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [666] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx8_gen1_dep16_batch32_eplb0_mtp3_666.yaml + srt-recipe: disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml prefill: num-worker: 8 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx8_gen1_dep16_batch32_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml" decode: num-worker: 1 tp: 16 @@ -3295,14 +3210,13 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [1229] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx10_gen1_dep16_batch64_eplb0_mtp1_1229.yaml + srt-recipe: disagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml prefill: num-worker: 10 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx10_gen1_dep16_batch64_eplb0_mtp1_1229.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml" decode: num-worker: 1 tp: 16 @@ -3310,14 +3224,13 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true - spec-decoding: "mtp" conc-list: [1229] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx7_gen1_dep8_batch128_eplb0_mtp1_1229.yaml + srt-recipe: disagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx7_gen1_dep8_batch128_eplb0_mtp1_1229.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml" decode: num-worker: 1 tp: 8 @@ -3325,98 +3238,91 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true # STP configurations (no spec_decoding) - conc-list: [4] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml + srt-recipe: disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - conc-list: [24] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml + srt-recipe: disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - conc-list: [36] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch8_eplb0_mtp0_36.yaml + srt-recipe: disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch8_eplb0_mtp0_36.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - conc-list: [512] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx6_gen1_dep32_batch16_eplb0_mtp0_512.yaml + srt-recipe: disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml prefill: num-worker: 6 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx6_gen1_dep32_batch16_eplb0_mtp0_512.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - conc-list: [666] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx4_gen1_dep16_batch32_eplb0_mtp0_666.yaml + srt-recipe: disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx4_gen1_dep16_batch32_eplb0_mtp0_666.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - conc-list: [1229] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep16_batch64_eplb0_mtp0_1229.yaml + srt-recipe: disagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep16_batch64_eplb0_mtp0_1229.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - conc-list: [2151] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep8_batch256_eplb0_mtp0_2151.yaml + srt-recipe: disagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep8_batch256_eplb0_mtp0_2151.yaml - - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml" decode: num-worker: 1 tp: 8 @@ -3433,6 +3339,7 @@ dsr1-fp8-h200-dynamo-sglang: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/h200-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -3441,13 +3348,12 @@ dsr1-fp8-h200-dynamo-sglang: # STP: Low latency TEP (1 prefill, 7 decode) - spec-decoding: "none" conc-list: [1, 4, 8] + srt-recipe: disagg-bs4-1p7d-stp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs4-1p7d-stp.yaml" decode: num-worker: 7 tp: 8 @@ -3456,13 +3362,12 @@ dsr1-fp8-h200-dynamo-sglang: # STP: TEP (1 prefill, 6 decode) - spec-decoding: "none" conc-list: [4, 8, 16] + srt-recipe: disagg-bs8-1p6d-stp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs8-1p6d-stp.yaml" decode: num-worker: 6 tp: 8 @@ -3471,13 +3376,12 @@ dsr1-fp8-h200-dynamo-sglang: # STP: TEP (1 prefill, 3 decode) - spec-decoding: "none" conc-list: [8, 16, 32] + srt-recipe: disagg-bs16-1p3d-stp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs16-1p3d-stp.yaml" decode: num-worker: 3 tp: 8 @@ -3486,13 +3390,12 @@ dsr1-fp8-h200-dynamo-sglang: # STP: TEP (2 prefill, 3 decode) - spec-decoding: "none" conc-list: [32, 64, 128] + srt-recipe: disagg-bs64-2p3d-stp.yaml prefill: num-worker: 2 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs64-2p3d-stp.yaml" decode: num-worker: 3 tp: 8 @@ -3501,13 +3404,12 @@ dsr1-fp8-h200-dynamo-sglang: # STP: High throughput DEP (1 prefill, 1 decode, dp-attention) - spec-decoding: "none" conc-list: [64, 128, 256] + srt-recipe: disagg-bs128-1p1d-dep-stp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs128-1p1d-dep-stp.yaml" decode: num-worker: 1 tp: 8 @@ -3516,13 +3418,12 @@ dsr1-fp8-h200-dynamo-sglang: # MTP: Low latency TEP (1 prefill, 7 decode) - spec-decoding: "mtp" conc-list: [1, 4, 8] + srt-recipe: disagg-bs4-1p7d-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs4-1p7d-mtp.yaml" decode: num-worker: 7 tp: 8 @@ -3531,13 +3432,12 @@ dsr1-fp8-h200-dynamo-sglang: # MTP: TEP (1 prefill, 6 decode) - spec-decoding: "mtp" conc-list: [2, 4, 8, 16, 32] + srt-recipe: disagg-bs8-1p6d-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs8-1p6d-mtp.yaml" decode: num-worker: 6 tp: 8 @@ -3546,13 +3446,12 @@ dsr1-fp8-h200-dynamo-sglang: # MTP: TEP (1 prefill, 3 decode) - spec-decoding: "mtp" conc-list: [4, 8, 16, 32, 64] + srt-recipe: disagg-bs16-1p3d-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs16-1p3d-mtp.yaml" decode: num-worker: 3 tp: 8 @@ -3561,13 +3460,12 @@ dsr1-fp8-h200-dynamo-sglang: # MTP: TEP (2 prefill, 3 decode) - spec-decoding: "mtp" conc-list: [32, 64, 128] + srt-recipe: disagg-bs64-2p3d-mtp.yaml prefill: num-worker: 2 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs64-2p3d-mtp.yaml" decode: num-worker: 3 tp: 8 @@ -3576,13 +3474,12 @@ dsr1-fp8-h200-dynamo-sglang: # MTP: High throughput DEP (1 prefill, 1 decode, dp-attention) - spec-decoding: "mtp" conc-list: [32, 64, 128, 256, 512] + srt-recipe: disagg-bs128-1p1d-dep-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs128-1p1d-dep-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -3599,6 +3496,7 @@ dsr1-fp4-b200-dynamo-sglang: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/b200-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -3606,65 +3504,60 @@ dsr1-fp4-b200-dynamo-sglang: search-space: # Non-MTP configurations - conc-list: [64, 128] + srt-recipe: disagg-stp-mtp-variants.yaml:zip_override_stp_lowlat[0] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:zip_override_stp_lowlat[0]" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: false - conc-list: [8] + srt-recipe: disagg-stp-mtp-variants.yaml:zip_override_stp_lowlat[1] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:zip_override_stp_lowlat[1]" decode: num-worker: 5 tp: 8 ep: 8 dp-attn: false - conc-list: [4, 128] + srt-recipe: disagg-stp-mtp-variants.yaml:zip_override_stp_lowlat[2] prefill: num-worker: 2 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:zip_override_stp_lowlat[2]" decode: num-worker: 5 tp: 8 ep: 8 dp-attn: false - conc-list: [4, 8, 16, 64] + srt-recipe: disagg-stp-mtp-variants.yaml:override_stp_tp4 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:override_stp_tp4" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - conc-list: [1024, 2048] + srt-recipe: disagg-stp-mtp-variants.yaml:override_stp_maxtpt_7p2d prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:override_stp_maxtpt_7p2d" decode: num-worker: 2 tp: 8 @@ -3680,6 +3573,7 @@ dsr1-fp8-b200-dynamo-sglang: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/b200-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -3687,14 +3581,13 @@ dsr1-fp8-b200-dynamo-sglang: search-space: # STP low-latency: resolved from 8k1k.yaml zip_override_stp_lowlat - conc-list: [128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_0.yaml + srt-recipe: disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_0.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 3 @@ -3702,14 +3595,13 @@ dsr1-fp8-b200-dynamo-sglang: ep: 1 dp-attn: false - conc-list: [128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_1.yaml + srt-recipe: disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_1.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 4 @@ -3717,14 +3609,13 @@ dsr1-fp8-b200-dynamo-sglang: ep: 1 dp-attn: false - conc-list: [8, 16, 32, 64, 128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_2.yaml + srt-recipe: disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-stp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_2.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 6 @@ -3733,14 +3624,13 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: false # STP max-throughput: resolved from 8k1k.yaml zip_override_stp_maxtpt - conc-list: [288] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml + srt-recipe: disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-stp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 2 @@ -3748,14 +3638,13 @@ dsr1-fp8-b200-dynamo-sglang: ep: 8 dp-attn: true - conc-list: [160, 288] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml + srt-recipe: disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-stp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -3763,14 +3652,13 @@ dsr1-fp8-b200-dynamo-sglang: ep: 8 dp-attn: true - conc-list: [512] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml + srt-recipe: disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-stp.yaml prefill: num-worker: 2 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -3778,14 +3666,13 @@ dsr1-fp8-b200-dynamo-sglang: ep: 8 dp-attn: true - conc-list: [1024] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml + srt-recipe: disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-stp.yaml prefill: num-worker: 3 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -3803,6 +3690,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/b200-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -3811,14 +3699,13 @@ dsr1-fp8-b200-dynamo-sglang-mtp: # MTP low-latency: resolved from 8k1k.yaml zip_override_mtp_lowlat - spec-decoding: "mtp" conc-list: [128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml + srt-recipe: disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 3 @@ -3827,14 +3714,13 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: false - spec-decoding: "mtp" conc-list: [128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml + srt-recipe: disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 4 @@ -3843,14 +3729,13 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: false - spec-decoding: "mtp" conc-list: [8, 16, 32, 64, 128] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml + srt-recipe: disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 6 @@ -3860,14 +3745,13 @@ dsr1-fp8-b200-dynamo-sglang-mtp: # MTP max-throughput: resolved from 8k1k.yaml zip_override_mtp_maxtpt - spec-decoding: "mtp" conc-list: [288] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml + srt-recipe: disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 2 @@ -3876,14 +3760,13 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true - spec-decoding: "mtp" conc-list: [160, 288] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml + srt-recipe: disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -3892,14 +3775,13 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true - spec-decoding: "mtp" conc-list: [512] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml + srt-recipe: disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-mtp.yaml prefill: num-worker: 2 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -3908,14 +3790,13 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true - spec-decoding: "mtp" conc-list: [1024] + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml + srt-recipe: disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-mtp.yaml prefill: num-worker: 3 tp: 8 ep: 1 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -3934,6 +3815,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsr1/sglang/b200-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -3942,13 +3824,12 @@ dsr1-fp4-b200-dynamo-sglang-mtp: # 1p5d low-latency (decode-heavy). - spec-decoding: "mtp" conc-list: [4, 8, 16, 32] + srt-recipe: disagg-1p5d-p-tp4-d-tp8-b512-c4x8x16x32-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p5d-p-tp4-d-tp8-b512-c4x8x16x32-mtp.yaml" decode: num-worker: 5 tp: 8 @@ -3957,13 +3838,12 @@ dsr1-fp4-b200-dynamo-sglang-mtp: # 1p3d low-latency. - spec-decoding: "mtp" conc-list: [32, 64] + srt-recipe: disagg-1p3d-p-tp4-d-tp8-b512-c32x64-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p3d-p-tp4-d-tp8-b512-c32x64-mtp.yaml" decode: num-worker: 3 tp: 8 @@ -3972,13 +3852,12 @@ dsr1-fp4-b200-dynamo-sglang-mtp: # 1p1d low-latency. - spec-decoding: "mtp" conc-list: [32] + srt-recipe: disagg-1p1d-p-tp4-d-tp8-b512-c32-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp4-d-tp8-b512-c32-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -3988,13 +3867,12 @@ dsr1-fp4-b200-dynamo-sglang-mtp: # 1p1d throughput. - spec-decoding: "mtp" conc-list: [512] + srt-recipe: disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4003,13 +3881,12 @@ dsr1-fp4-b200-dynamo-sglang-mtp: # 2p1d throughput. - spec-decoding: "mtp" conc-list: [768] + srt-recipe: disagg-2p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c768-mtp.yaml prefill: num-worker: 2 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c768-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4018,13 +3895,12 @@ dsr1-fp4-b200-dynamo-sglang-mtp: # 3p1d throughput. - spec-decoding: "mtp" conc-list: [1024] + srt-recipe: disagg-3p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c1024-mtp.yaml prefill: num-worker: 3 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c1024-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4033,13 +3909,12 @@ dsr1-fp4-b200-dynamo-sglang-mtp: # 4p1d throughput. - spec-decoding: "mtp" conc-list: [512] + srt-recipe: disagg-4p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4048,13 +3923,12 @@ dsr1-fp4-b200-dynamo-sglang-mtp: # 5p1d throughput. - spec-decoding: "mtp" conc-list: [2048] + srt-recipe: disagg-5p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c2048-mtp.yaml prefill: num-worker: 5 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c2048-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4063,13 +3937,12 @@ dsr1-fp4-b200-dynamo-sglang-mtp: # 4p1d high-throughput (dedicated c2048 tune). - spec-decoding: "mtp" conc-list: [2048] + srt-recipe: disagg-mtp-4p1d-c2048.yaml prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-mtp-4p1d-c2048.yaml" decode: num-worker: 1 tp: 8 @@ -4087,6 +3960,7 @@ qwen3.5-fp8-gb200-dynamo-sglang: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/gb200-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -4095,13 +3969,12 @@ qwen3.5-fp8-gb200-dynamo-sglang: # 1P1D STP: TP4 prefill + TP4 decode (pure tensor parallel). 2 nodes (1+1). - spec-decoding: "none" conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + srt-recipe: disagg-1p1d-tp4-tp4-stp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -4110,13 +3983,12 @@ qwen3.5-fp8-gb200-dynamo-sglang: # 4P1D wide-EP: 4 prefill DEP4 + decode DEP16. 8 nodes (4+4). - spec-decoding: "none" conc-list: [1024] + srt-recipe: disagg-4p1d-dep4-dep16-stp.yaml prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -4125,13 +3997,12 @@ qwen3.5-fp8-gb200-dynamo-sglang: # 8P1D wide-EP: 8 prefill DEP4 + decode DEP16. 12 nodes (8+4). - spec-decoding: "none" conc-list: [2048, 4096] + srt-recipe: disagg-8p1d-dep4-dep16-stp.yaml prefill: num-worker: 8 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -4146,6 +4017,7 @@ qwen3.5-fp8-h100-sglang: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/h100-fp8 scenarios: fixed-seq-len: - isl: 8192 @@ -4155,12 +4027,12 @@ qwen3.5-fp8-h100-sglang: ep: 1 conc-start: 1 conc-end: 8 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 8 conc-start: 16 conc-end: 256 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp8-h100-sglang-mtp: image: lmsysorg/sglang:v0.5.19-cu130 @@ -4170,6 +4042,7 @@ qwen3.5-fp8-h100-sglang-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/h100-fp8-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -4180,7 +4053,7 @@ qwen3.5-fp8-h100-sglang-mtp: conc-start: 4 conc-end: 32 spec-decoding: mtp - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp4-gb300-dynamo-sglang: image: lmsysorg/sglang:v0.5.14-cu130 @@ -4192,6 +4065,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: router: { name: dynamo-router, version: "1.1.0" } multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/gb300-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -4203,13 +4077,12 @@ qwen3.5-fp4-gb300-dynamo-sglang: - spec-decoding: "none" kv-p2p-transfer: mooncake conc-list: [1, 4, 8, 16, 32, 64, 256] + srt-recipe: disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml" decode: num-worker: 1 tp: 4 @@ -4220,13 +4093,12 @@ qwen3.5-fp4-gb300-dynamo-sglang: - spec-decoding: "none" kv-p2p-transfer: nixl conc-list: [2048] + srt-recipe: disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml prefill: num-worker: 5 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml" decode: num-worker: 1 tp: 16 @@ -4237,13 +4109,12 @@ qwen3.5-fp4-gb300-dynamo-sglang: - spec-decoding: "none" kv-p2p-transfer: mooncake conc-list: [5120] + srt-recipe: disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml prefill: num-worker: 6 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml" decode: num-worker: 1 tp: 16 @@ -4254,13 +4125,12 @@ qwen3.5-fp4-gb300-dynamo-sglang: - spec-decoding: "none" kv-p2p-transfer: mooncake conc-list: [5120] + srt-recipe: disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml" decode: num-worker: 1 tp: 16 @@ -4277,6 +4147,7 @@ qwen3.5-fp4-gb300-dynamo-trt: multinode: true disagg: true kv-p2p-transfer: nixl + srt-recipe-dir: qwen3.5/trtllm/gb300-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -4284,13 +4155,12 @@ qwen3.5-fp4-gb300-dynamo-trt: search-space: # 1P2D: 1 prefill (TP1/EP1/dp-attn), 2 decode (TP8/EP8/no-dp-attn), conc=42 - conc-list: [42] + srt-recipe: disagg-1p2d-tep8-b16-eplb0-mtp0-c42.yaml prefill: num-worker: 1 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p2d-tep8-b16-eplb0-mtp0-c42.yaml" decode: num-worker: 2 tp: 8 @@ -4298,13 +4168,12 @@ qwen3.5-fp4-gb300-dynamo-trt: dp-attn: false # 1P4D: 1 prefill (TP2/EP2/dp-attn), 4 decode (TP8/EP8/no-dp-attn), conc=8 - conc-list: [8] + srt-recipe: disagg-1p4d-tep8-b1-eplb0-mtp0-c8.yaml prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c8.yaml" decode: num-worker: 4 tp: 8 @@ -4312,13 +4181,12 @@ qwen3.5-fp4-gb300-dynamo-trt: dp-attn: false # 1P4D: 1 prefill (TP1/EP1/dp-attn), 4 decode (TP8/EP8/no-dp-attn), conc=24 - conc-list: [24] + srt-recipe: disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml prefill: num-worker: 1 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml" decode: num-worker: 4 tp: 8 @@ -4326,13 +4194,12 @@ qwen3.5-fp4-gb300-dynamo-trt: dp-attn: false # 4P1D: 4 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=282 - conc-list: [282] + srt-recipe: disagg-4p1d-dep16-b16-eplb0-mtp0-c282.yaml prefill: num-worker: 4 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep16-b16-eplb0-mtp0-c282.yaml" decode: num-worker: 1 tp: 16 @@ -4340,13 +4207,12 @@ qwen3.5-fp4-gb300-dynamo-trt: dp-attn: true # 4P3D: 4 prefill (TP1/EP1/dp-attn), 3 decode (TP8/EP8/no-dp-attn), conc=210 - conc-list: [210] + srt-recipe: disagg-4p3d-tep8-b64-eplb0-mtp0-c210.yaml prefill: num-worker: 4 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p3d-tep8-b64-eplb0-mtp0-c210.yaml" decode: num-worker: 3 tp: 8 @@ -4354,13 +4220,12 @@ qwen3.5-fp4-gb300-dynamo-trt: dp-attn: false # 7P1D: 7 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=563 - conc-list: [563] + srt-recipe: disagg-7p1d-dep16-b32-eplb0-mtp0-c563.yaml prefill: num-worker: 7 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep16-b32-eplb0-mtp0-c563.yaml" decode: num-worker: 1 tp: 16 @@ -4368,13 +4233,12 @@ qwen3.5-fp4-gb300-dynamo-trt: dp-attn: true # 10P1D: 10 prefill (TP1/EP1/dp-attn), 1 decode (TP8/EP8/dp-attn), conc=2150 - conc-list: [2150] + srt-recipe: disagg-10p1d-dep8-b256-eplb0-mtp0-c2150.yaml prefill: num-worker: 10 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp0-c2150.yaml" decode: num-worker: 1 tp: 8 @@ -4382,13 +4246,12 @@ qwen3.5-fp4-gb300-dynamo-trt: dp-attn: true # 11P1D: 11 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=1076 - conc-list: [1076] + srt-recipe: disagg-11p1d-dep16-b64-eplb0-mtp0-c1076.yaml prefill: num-worker: 11 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-11p1d-dep16-b64-eplb0-mtp0-c1076.yaml" decode: num-worker: 1 tp: 16 @@ -4396,13 +4259,12 @@ qwen3.5-fp4-gb300-dynamo-trt: dp-attn: true # 16P1D: 16 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=2253 - conc-list: [2253] + srt-recipe: disagg-16p1d-dep16-b128-eplb0-mtp0-c2253.yaml prefill: num-worker: 16 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-16p1d-dep16-b128-eplb0-mtp0-c2253.yaml" decode: num-worker: 1 tp: 16 @@ -4410,13 +4272,12 @@ qwen3.5-fp4-gb300-dynamo-trt: dp-attn: true # 24P1D: 24 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=8192 - conc-list: [8192] + srt-recipe: disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-stp-sweep.yaml prefill: num-worker: 24 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-stp-sweep.yaml" decode: num-worker: 1 tp: 16 @@ -4425,7 +4286,7 @@ qwen3.5-fp4-gb300-dynamo-trt: # GB300 SGLang AgentX, re-swept across low/mid/high concurrency after aiperf # restored real inter-tool-call pacing. The seven aggregate frontier points use -# one TP2 worker with MTP/NEXTN + hierarchical KV cache. CONFIG_FILE carries the +# one TP2 worker with MTP/NEXTN + hierarchical KV cache. srt-recipe carries the # full srtctl recipe; the matrix topology stays in sync for naming/GPU accounting. qwen3.5-fp4-gb300-dynamo-trt-mtp: image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 @@ -4437,6 +4298,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: multinode: true disagg: true kv-p2p-transfer: nixl + srt-recipe-dir: qwen3.5/trtllm/gb300-fp4/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -4445,13 +4307,12 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: # 1P2D: 1 prefill (TP1/EP1/dp-attn), 2 decode (TP8/EP8/no-dp-attn), conc=20 - spec-decoding: "mtp" conc-list: [20] + srt-recipe: disagg-1p2d-tep8-b8-eplb0-mtp3-c20.yaml prefill: num-worker: 1 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p2d-tep8-b8-eplb0-mtp3-c20.yaml" decode: num-worker: 2 tp: 8 @@ -4460,13 +4321,12 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: # 1P4D: 1 prefill (TP1/EP1/dp-attn), 4 decode (TP8/EP8/no-dp-attn), conc=12 - spec-decoding: "mtp" conc-list: [12] + srt-recipe: disagg-1p4d-tep8-b1-eplb0-mtp3-c12.yaml prefill: num-worker: 1 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c12.yaml" decode: num-worker: 4 tp: 8 @@ -4475,13 +4335,12 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: # 1P4D: 1 prefill (TP1/EP1/dp-attn), 4 decode (TP8/EP8/no-dp-attn), conc=8 - spec-decoding: "mtp" conc-list: [8] + srt-recipe: disagg-1p4d-tep8-b2-eplb0-mtp3-c8.yaml prefill: num-worker: 1 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp3-c8.yaml" decode: num-worker: 4 tp: 8 @@ -4490,13 +4349,12 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: # 4P3D: 4 prefill (TP1/EP1/dp-attn), 3 decode (TP8/EP8/no-dp-attn), conc=126 - spec-decoding: "mtp" conc-list: [126] + srt-recipe: disagg-4p3d-tep8-b32-eplb0-mtp3-c126.yaml prefill: num-worker: 4 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p3d-tep8-b32-eplb0-mtp3-c126.yaml" decode: num-worker: 3 tp: 8 @@ -4505,13 +4363,12 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: # 5P1D: 5 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=154 - spec-decoding: "mtp" conc-list: [154] + srt-recipe: disagg-5p1d-dep16-b8-eplb0-mtp3-c154.yaml prefill: num-worker: 5 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-5p1d-dep16-b8-eplb0-mtp3-c154.yaml" decode: num-worker: 1 tp: 16 @@ -4520,13 +4377,12 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: # 7P1D: 7 prefill (TP2/EP2/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=666 - spec-decoding: "mtp" conc-list: [666] + srt-recipe: disagg-7p1d-dep16-b32-eplb0-mtp3-c666.yaml prefill: num-worker: 7 tp: 2 ep: 2 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep16-b32-eplb0-mtp3-c666.yaml" decode: num-worker: 1 tp: 16 @@ -4535,13 +4391,12 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: # 11P1D: 11 prefill (TP1/EP1/dp-attn), 1 decode (TP8/EP8/dp-attn), conc=1229 - spec-decoding: "mtp" conc-list: [1229] + srt-recipe: disagg-11p1d-dep8-b128-eplb0-mtp3-c1229.yaml prefill: num-worker: 11 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-11p1d-dep8-b128-eplb0-mtp3-c1229.yaml" decode: num-worker: 1 tp: 8 @@ -4550,13 +4405,12 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: # 17P2D: 17 prefill (TP1/EP1/dp-attn), 2 decode (TP8/EP8/dp-attn), conc=1126 - spec-decoding: "mtp" conc-list: [1126] + srt-recipe: disagg-17p2d-dep8-b64-eplb0-mtp3-c1126.yaml prefill: num-worker: 17 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-17p2d-dep8-b64-eplb0-mtp3-c1126.yaml" decode: num-worker: 2 tp: 8 @@ -4565,13 +4419,12 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: # 26P1D: 26 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=4301, MTP draft-len 2 - spec-decoding: "mtp" conc-list: [4301] + srt-recipe: disagg-26p1d-dep16-b256-eplb0-mtp2-c4301.yaml prefill: num-worker: 26 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-26p1d-dep16-b256-eplb0-mtp2-c4301.yaml" decode: num-worker: 1 tp: 16 @@ -4580,13 +4433,12 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: # 24P1D: 24 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=8192, MTP curve tail (spec decoding disabled) - spec-decoding: "none" conc-list: [8192] + srt-recipe: disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-mtp-sweep.yaml prefill: num-worker: 24 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-mtp-sweep.yaml" decode: num-worker: 1 tp: 16 @@ -4604,6 +4456,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: qwen3.5/trtllm/gb300-fp4/agentx scenarios: agentic-coding: # Existing recipes reserve 128 GiB per rank; each node has 840 GiB and 4 GPUs. @@ -4614,13 +4467,12 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [44] + srt-recipe: disagg-variants.yaml:override_1p1d_dep1_tep2_c44_b8 prefill: num-worker: 1 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_dep1_tep2_c44_b8" decode: num-worker: 1 tp: 2 @@ -4631,13 +4483,12 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [7] + srt-recipe: disagg-variants.yaml:override_1p7d_dep4_tep8_c7_b1 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p7d_dep4_tep8_c7_b1" decode: num-worker: 7 tp: 8 @@ -4648,13 +4499,12 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [52] + srt-recipe: disagg-variants.yaml:override_2p2d_dep1_tep2_c52_b4 prefill: num-worker: 2 tp: 1 ep: 1 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p2d_dep1_tep2_c52_b4" decode: num-worker: 2 tp: 2 @@ -4665,13 +4515,12 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [96] + srt-recipe: disagg-variants.yaml:override_2p3d_tep2_tep8_c96_b128 prefill: num-worker: 2 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p3d_tep2_tep8_c96_b128" decode: num-worker: 3 tp: 8 @@ -4682,13 +4531,12 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [565] + srt-recipe: disagg-variants.yaml:override_3p1d_dep4_dep16_c565_b8 prefill: num-worker: 3 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p1d_dep4_dep16_c565_b8" decode: num-worker: 1 tp: 16 @@ -4699,13 +4547,12 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [704] + srt-recipe: disagg-variants.yaml:override_3p2d_dep4_dep4_c704_b32 prefill: num-worker: 3 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p2d_dep4_dep4_c704_b32" decode: num-worker: 2 tp: 4 @@ -4721,6 +4568,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: framework: dynamo-sglang multinode: true disagg: false + srt-recipe-dir: qwen3.5/sglang/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.90 @@ -4730,85 +4578,78 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: kv-offload-backend: { name: hicache } conc-list: [1] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c1 worker: num-worker: 1 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c1" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [24] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c24 worker: num-worker: 1 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c24" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [32] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c32 worker: num-worker: 1 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c32" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [40] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c40 worker: num-worker: 1 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c40" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [48] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c48 worker: num-worker: 1 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c48" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [52] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c52 worker: num-worker: 1 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c52" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [64] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c64 worker: num-worker: 1 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c64" # The seven disaggregated frontier points use six TP4 shapes plus one TP2 # shape. Stable X-Dynamo-Session-ID affinity replaces the removed conv-aware # routing message path. @@ -4823,6 +4664,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.90 @@ -4831,13 +4673,12 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [8] + srt-recipe: disagg-variants.yaml:override_tp4_tp4_c8 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c8" decode: num-worker: 1 tp: 4 @@ -4847,13 +4688,12 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [16] + srt-recipe: disagg-variants.yaml:override_tp4_tp4_c16 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c16" decode: num-worker: 1 tp: 4 @@ -4863,13 +4703,12 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [32] + srt-recipe: disagg-variants.yaml:override_tp4_tp4_c32 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c32" decode: num-worker: 1 tp: 4 @@ -4879,13 +4718,12 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [64] + srt-recipe: disagg-variants.yaml:override_tp4_tp4_c64 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c64" decode: num-worker: 1 tp: 4 @@ -4895,13 +4733,12 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [96] + srt-recipe: disagg-variants.yaml:override_tp4_tp4_c96 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c96" decode: num-worker: 1 tp: 4 @@ -4911,13 +4748,12 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [128] + srt-recipe: disagg-variants.yaml:override_tp4_tp4_c128 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c128" decode: num-worker: 1 tp: 4 @@ -4927,13 +4763,12 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [72] + srt-recipe: disagg-variants.yaml:override_tp2_tp2_c72 prefill: num-worker: 1 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp2_tp2_c72" decode: num-worker: 1 tp: 2 @@ -4955,6 +4790,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.90 @@ -4963,14 +4799,13 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [704] + srt-recipe: disagg-pp-pareto-variants.yaml:override_3p2d_c704 prefill: num-worker: 3 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml:override_3p2d_c704" decode: num-worker: 2 tp: 4 @@ -4981,14 +4816,13 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [565] + srt-recipe: disagg-pp-pareto-variants.yaml:override_3p4d_c565 prefill: num-worker: 3 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml:override_3p4d_c565" decode: num-worker: 4 tp: 4 @@ -5005,6 +4839,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: framework: dynamo-sglang multinode: true disagg: false + srt-recipe-dir: qwen3.5/sglang/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.90 @@ -5014,40 +4849,37 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: kv-offload-backend: { name: hicache } conc-list: [44] num-nodes: 1 + srt-recipe: agg-pareto-variants.yaml:override_tp2_c44_b2 worker: num-worker: 1 tp: 2 pp: 1 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_tp2_c44_b2" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [44] num-nodes: 1 + srt-recipe: agg-pareto-variants.yaml:override_tp2_c44_b1 worker: num-worker: 1 tp: 2 pp: 1 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_tp2_c44_b1" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [7] num-nodes: 2 + srt-recipe: agg-pareto-variants.yaml:override_tp8_c7_b1 worker: num-worker: 1 tp: 8 pp: 1 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_tp8_c7_b1" qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto: image: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 @@ -5060,6 +4892,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/gb300-fp8/agentx scenarios: agentic-coding: - dram-utilization: 0.90 @@ -5068,14 +4901,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [1000] + srt-recipe: disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913.yaml prefill: num-worker: 5 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913.yaml" decode: num-worker: 3 tp: 4 @@ -5086,14 +4918,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [600] + srt-recipe: disagg-gb300-3p2d-pp4-dep4-c600-mtp-hicache-nightly-20260913.yaml prefill: num-worker: 3 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-3p2d-pp4-dep4-c600-mtp-hicache-nightly-20260913.yaml" decode: num-worker: 2 tp: 4 @@ -5104,14 +4935,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [200] + srt-recipe: disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913.yaml prefill: num-worker: 1 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913.yaml" decode: num-worker: 1 tp: 4 @@ -5122,14 +4952,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [100] + srt-recipe: disagg-gb300-1p1d-pp4-dep4-c100-mtp-hicache-nightly-20260913.yaml prefill: num-worker: 1 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c100-mtp-hicache-nightly-20260913.yaml" decode: num-worker: 1 tp: 4 @@ -5146,6 +4975,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto: framework: dynamo-sglang multinode: true disagg: false + srt-recipe-dir: qwen3.5/sglang/gb300-fp8/agentx scenarios: agentic-coding: - dram-utilization: 0.90 @@ -5155,66 +4985,61 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto: kv-offload-backend: { name: hicache } conc-list: [44] num-nodes: 1 + srt-recipe: agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml worker: num-worker: 1 tp: 4 pp: 1 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [44] num-nodes: 1 + srt-recipe: agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml worker: num-worker: 1 tp: 4 pp: 1 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [44] num-nodes: 1 + srt-recipe: agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml worker: num-worker: 1 tp: 4 pp: 1 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [1] num-nodes: 1 + srt-recipe: agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml worker: num-worker: 1 tp: 4 pp: 1 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [1] num-nodes: 2 + srt-recipe: agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml worker: num-worker: 1 tp: 8 pp: 1 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml" qwen3.5-fp8-b300-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 @@ -5224,13 +5049,14 @@ qwen3.5-fp8-b300-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b300-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 34, 36, 38, 40, 44, 48, 52, 56], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72], srt-recipe: agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32], srt-recipe: agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 34, 36, 38, 40, 44, 48, 52, 56], srt-recipe: agentic.yaml } qwen3.5-fp4-b300-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 @@ -5240,13 +5066,14 @@ qwen3.5-fp4-b300-sglang-agentic-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [36, 44, 52], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72], srt-recipe: agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32], srt-recipe: agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [36, 44, 52], srt-recipe: agentic.yaml } # Qwen3.8-Flash-Next NVFP4 AgentX on B300 via SGLang with native NEXTN MTP. # Day-zero recipe; mirrors the B200 arm. TP1 per the cookbook's verified @@ -5259,11 +5086,12 @@ qwen3.8next-fp4-b300-sglang-agentic-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.8next/sglang/b300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: agentic.yaml } # CONC range conservative for H100's 80 GB HBM3 under the long-ISL with- # subagents corpus. hicache arm capped at conc 16 since high-conc + hicache @@ -5280,6 +5108,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsv4/vllm/gb300-fp4/agentx scenarios: agentic-coding: - search-space: @@ -5289,13 +5118,12 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: # at conc=192. Reuses the 1p6d recipe; no separate recipe file needed. - spec-decoding: none conc-list: [32] + srt-recipe: disagg-variants.yaml:override_1p6d_dep4_tp4 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p6d_dep4_tp4" decode: num-worker: 6 tp: 4 @@ -5305,13 +5133,12 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: # Mirrors fixed-seq-len conc=192 entry. - spec-decoding: none conc-list: [192] + srt-recipe: disagg-variants.yaml:override_1p6d_dep4_tp4 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p6d_dep4_tp4" decode: num-worker: 6 tp: 4 @@ -5322,13 +5149,12 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: # on both sides. Mirrors fixed-seq-len conc=4096 entry (4p1d variant). - spec-decoding: none conc-list: [4096] + srt-recipe: disagg-variants.yaml:override_4p1d_dep4_dep8_24_c4096 prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_4p1d_dep4_dep8_24_c4096" decode: num-worker: 1 tp: 8 @@ -5343,12 +5169,13 @@ qwen3.5-fp8-h100-sglang-agentic: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/h100-fp8 scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [12, 14, 16, 20, 24, 28, 32, 42], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [12, 14, 16, 20, 24, 28, 32, 42], srt-recipe: agentic.yaml } # MTP speculative-decoding (spec-decoding: mtp) variant of # qwen3.5-fp8-h100-sglang-agentic: same TP8/EP8 GPU-resident and HiCache arms, @@ -5378,12 +5205,13 @@ qwen3.5-fp8-h200-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/h200-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 8, 12, 16], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16], srt-recipe: agentic.yaml } # Qwen3.8-Flash-Next FP8 AgentX on H200 via SGLang with native NEXTN MTP. # Day-zero recipe. H200 is Hopper, so this arm is FP8: NVFP4 is greyed out for @@ -5398,11 +5226,12 @@ qwen3.8next-fp8-h200-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.8next/sglang/h200-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: agentic.yaml } # H200 AgentX MTP frontier with DRAM HiCache. This is intentionally an MTP-only # submission; the model's non-speculative AgentX arm is not included. qwen3.5-fp8-h200-sglang-agentic-hicache-mtp: @@ -5413,11 +5242,12 @@ qwen3.5-fp8-h200-sglang-agentic-hicache-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/h200-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [2, 4, 8, 10, 12, 16, 20, 24], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [2, 4, 8, 10, 12, 16, 20, 24], srt-recipe: agentic.yaml } qwen3.5-fp8-h100-sglang-agentic-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 @@ -5427,12 +5257,13 @@ qwen3.5-fp8-h100-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/h100-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16], srt-recipe: agentic.yaml } qwen3.5-fp4-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 model: nvidia/Qwen3.5-397B-A17B-NVFP4 @@ -5441,6 +5272,7 @@ qwen3.5-fp4-b200-trt: precision: fp4 framework: trt multinode: false + srt-recipe-dir: qwen3.5/trtllm/b200-fp4 scenarios: fixed-seq-len: - isl: 8192 @@ -5451,29 +5283,29 @@ qwen3.5-fp4-b200-trt: conc-list: - 4 - 16 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 1 conc-list: - 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 2 ep: 2 conc-list: - 8 - 32 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 8 conc-list: - 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 4 dp-attn: true conc-list: - 1024 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 8 dp-attn: true @@ -5481,7 +5313,7 @@ qwen3.5-fp4-b200-trt: - 256 - 512 - 1024 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4/8k1k.yaml + srt-recipe: 8k1k.yaml qwen3.5-fp4-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 @@ -5491,6 +5323,7 @@ qwen3.5-fp4-b200-trt-mtp: precision: fp4 framework: trt multinode: false + srt-recipe-dir: qwen3.5/trtllm/b200-fp4-mtp scenarios: fixed-seq-len: - isl: 8192 @@ -5501,26 +5334,26 @@ qwen3.5-fp4-b200-trt-mtp: spec-decoding: mtp conc-list: - 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 2 ep: 2 spec-decoding: mtp conc-list: - 8 - 16 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 4 ep: 4 spec-decoding: mtp conc-list: - 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 8 spec-decoding: mtp conc-list: - 4 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml - tp: 8 ep: 8 dp-attn: true @@ -5529,7 +5362,7 @@ qwen3.5-fp4-b200-trt-mtp: - 128 - 256 - 1024 - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/trtllm/b200-fp4-mtp/8k1k.yaml + srt-recipe: 8k1k.yaml minimaxm3-fp8-h100-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-ddd6fbca148a867aad1fcab7ec72f582b9977db4 @@ -5539,13 +5372,14 @@ minimaxm3-fp8-h100-vllm-agentic-mtp: precision: fp8 framework: vllm multinode: false + srt-recipe-dir: minimaxm3/vllm/h100-fp8-mtp scenarios: agentic-coding: # The fast sweep places the resident HBM cliff between c5 and c6. - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5], srt-recipe: agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8], srt-recipe: agentic.yaml } minimaxm3-fp8-h200-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-1a001d584244bd21829a5e5bd4a376ea141c0ce3 @@ -5555,12 +5389,13 @@ minimaxm3-fp8-h200-vllm-agentic-mtp: precision: fp8 framework: vllm multinode: false + srt-recipe-dir: minimaxm3/vllm/h200-fp8-mtp scenarios: agentic-coding: # Use the FP8 MSA indexer cache for all H200 concurrency variants. - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12, 14], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12, 14], srt-recipe: agentic.yaml } qwen3.5-fp4-b200-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 @@ -5570,14 +5405,15 @@ qwen3.5-fp4-b200-sglang-agentic-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.5/sglang/b200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32], srt-recipe: agentic.yaml } # Qwen3.8-Flash-Next NVFP4 AgentX on B200 via SGLang with native NEXTN MTP. @@ -5590,11 +5426,12 @@ qwen3.8next-fp4-b200-sglang-agentic-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.8next/sglang/b200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml } qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 @@ -5604,6 +5441,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: framework: dynamo-sglang multinode: true disagg: false + srt-recipe-dir: qwen3.5/sglang/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.90 @@ -5614,35 +5452,32 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: kv-offloading: none conc-list: [1, 4, 8, 12, 20] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp4" - spec-decoding: mtp kv-offloading: none conc-list: [2] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_no_symm worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp4_no_symm" - spec-decoding: mtp kv-offloading: none conc-list: [24] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_parity worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp4_parity" # Keep only the reproducible TP2/EP2 HiCache K5 transition point; C28 # was dominated by the published C28 result in the official PR sweep. - spec-decoding: mtp @@ -5650,13 +5485,12 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: kv-offload-backend: { name: hicache } conc-list: [22] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp2ep2_hicache worker: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp2ep2_hicache" # Keep the published C32 interactivity anchor. The tighter capacity # layout becomes Pareto-relevant at C40 and remains deployable at C48. - spec-decoding: mtp @@ -5664,25 +5498,23 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: kv-offload-backend: { name: hicache } conc-list: [32] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp2ep2_hicache_k3_baseline worker: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp2ep2_hicache_k3_baseline" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [40, 48] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp2ep2_hicache_cap48 worker: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp2ep2_hicache_cap48" minimaxm3-fp4-b300-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-af7f9488c2210d67e1033ecdc845b087ee7fe92b model: nvidia/MiniMax-M3-NVFP4 @@ -5691,16 +5523,17 @@ minimaxm3-fp4-b300-vllm-agentic-mtp: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: minimaxm3/vllm/b300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 1.0 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30, 40, 48, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30, 40, 48, 64], srt-recipe: agentic.yaml } - dram-utilization: 1.0 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24, 32, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24, 32, 40], srt-recipe: agentic.yaml } minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 @@ -5713,6 +5546,7 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: minimaxm3/vllm/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.887 @@ -5721,66 +5555,61 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } conc-list: [1] + srt-recipe: disagg-variants.yaml:override_1p1d_tep4_tp4_c1 + eval-srt-recipe: disagg-variants.yaml:override_1p1d_tep4_tp4_c1_eval prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tep4_tp4_c1" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tep4_tp4_c1_eval" decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } conc-list: [24] + srt-recipe: disagg-variants.yaml:override_1p3d_dep4_tp4_c24 + eval-srt-recipe: disagg-variants.yaml:override_1p3d_dep4_tp4_c24_eval prefill: num-worker: 1 tp: 1 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_dep4_tp4_c24" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_dep4_tp4_c24_eval" decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } conc-list: [20, 24] + srt-recipe: disagg-variants.yaml:override_1p1d_tp2_tp4_c20_c24 + eval-srt-recipe: disagg-variants.yaml:override_1p1d_tp2_tp4_c20_c24_eval prefill: num-worker: 1 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tp2_tp4_c20_c24" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tp2_tp4_c20_c24_eval" decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } conc-list: [48] + srt-recipe: disagg-variants.yaml:override_1p3d_tp2_tp2_c48 + eval-srt-recipe: disagg-variants.yaml:override_1p3d_tp2_tp2_c48_eval prefill: num-worker: 1 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_tp2_tp2_c48" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_tp2_tp2_c48_eval" decode: { num-worker: 3, tp: 2, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } conc-list: [120] + srt-recipe: disagg-variants.yaml:override_2p5d_tp2_tp2_c120 + eval-srt-recipe: disagg-variants.yaml:override_2p5d_tp2_tp2_c120_eval prefill: num-worker: 2 tp: 2 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p5d_tp2_tp2_c120" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p5d_tp2_tp2_c120_eval" decode: { num-worker: 5, tp: 2, ep: 1, dp-attn: false } minimaxm3-fp4-b300-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 @@ -5790,13 +5619,14 @@ minimaxm3-fp4-b300-trtllm-agentic-mtp: precision: fp4 framework: trt multinode: false + srt-recipe-dir: minimaxm3/trtllm/b300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [20, 25, 30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [20, 25, 30], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30], srt-recipe: agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: agentic.yaml } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP @@ -5845,15 +5675,16 @@ minimaxm3-fp4-b200-vllm-agentic-mtp: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: minimaxm3/vllm/b200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.683 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: agentic.yaml } - dram-utilization: 1.0 search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [15, 20, 25, 30, 32, 34, 36, 38, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [15, 20, 25, 30, 32, 34, 36, 38, 40], srt-recipe: agentic.yaml } minimaxm3-fp4-b200-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 model: nvidia/MiniMax-M3-NVFP4 @@ -5862,6 +5693,7 @@ minimaxm3-fp4-b200-trtllm-agentic-mtp: precision: fp4 framework: trt multinode: false + srt-recipe-dir: minimaxm3/trtllm/b200-fp4-mtp scenarios: agentic-coding: # The benchmark script pins TRT-LLM kv_cache_config.host_cache_size for @@ -5869,8 +5701,8 @@ minimaxm3-fp4-b200-trtllm-agentic-mtp: # does not size the pool. - dram-utilization: 0.8 search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30, 35, 40, 45], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30, 35, 40, 45], srt-recipe: agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: agentic.yaml } # Run the GB200 aggregate configuration with TP4 resident, TP8 resident across # two NVL4 nodes, and TP4 lazy SimpleCPU KV offload variants. The pinned # runtime uses Dynamo-native MiniMax parsing on the worker without the vLLM @@ -5885,6 +5717,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: router: { name: dynamo-router, version: "1.5.0.dev20260908" } multinode: true disagg: false + srt-recipe-dir: minimaxm3/vllm/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.861904761904762 @@ -5893,36 +5726,33 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: kv-offloading: none conc-list: [1, 10] num-nodes: 1 + srt-recipe: agg-nightly-native-variants.yaml:override_tp4 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_tp4" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple } conc-list: [15, 25, 30] num-nodes: 1 + srt-recipe: agg-nightly-native-variants.yaml:override_tp4_vllm_simple worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_tp4_vllm_simple" - spec-decoding: mtp kv-offloading: none conc-list: [1] num-nodes: 2 + srt-recipe: agg-nightly-native-variants.yaml:override_tp8 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_tp8" minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: image: vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6 @@ -5935,6 +5765,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: minimaxm3/vllm/gb200-fp4/agentx scenarios: agentic-coding: # Each active four-rank prefill node reserves 150 GB/rank in Mooncake. @@ -5944,37 +5775,34 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } conc-list: [1] + srt-recipe: disagg-variants.yaml:override_1p1d_tp8_c1 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p1d_tp8_c1" decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: false } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } conc-list: [8, 16] + srt-recipe: disagg-variants.yaml:override_1p2d_c8_c16 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p2d_c8_c16" decode: { num-worker: 2, tp: 4, ep: 4, dp-attn: false } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } conc-list: [24] + srt-recipe: disagg-variants.yaml:override_1p1d_c24 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p1d_c24" decode: { num-worker: 1, tp: 4, ep: 4, dp-attn: false } minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: @@ -5987,6 +5815,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: router: { name: dynamo-router, version: "1.4.0.dev20260807" } multinode: true disagg: false + srt-recipe-dir: minimaxm3/trtllm/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.61 @@ -5996,85 +5825,78 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: kv-offload-backend: { name: native } conc-list: [5] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c5_b5 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c5_b5" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } conc-list: [10] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c10_b10 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c10_b10" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } conc-list: [15] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c15_b15 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c15_b15" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } conc-list: [20] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c20_b20 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c20_b20" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } conc-list: [25] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c25_b25 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c25_b25" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } conc-list: [30] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c30_b30 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c30_b30" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } conc-list: [40] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c40_b40 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c40_b40" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 @@ -6086,6 +5908,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: router: { name: dynamo-router, version: "1.3.1" } multinode: true disagg: false + srt-recipe-dir: dsv4/vllm/gb200-fp4/agentx scenarios: agentic-coding: - search-space: @@ -6094,23 +5917,21 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: - spec-decoding: mtp conc-list: [1, 2, 4, 6, 8] num-nodes: 2 + srt-recipe: agg-mtp-variants.yaml:override_tp8 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml:override_tp8" - spec-decoding: mtp conc-list: [8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80] num-nodes: 2 + srt-recipe: agg-mtp-variants.yaml:override_dep8 worker: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml:override_dep8" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 model: deepseek-ai/DeepSeek-V4-Pro @@ -6122,19 +5943,19 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsv4/vllm/gb200-fp4/agentx scenarios: agentic-coding: - search-space: # One two-tray DEP8 prefill and decode worker covers the balanced region. - spec-decoding: mtp conc-list: [32, 64, 96, 128, 192, 256] + srt-recipe: disagg-mtp-variants.yaml:override_1p1d prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d" decode: num-worker: 1 tp: 8 @@ -6143,13 +5964,12 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: # A second DEP8 prefill worker extends the high-concurrency region. - spec-decoding: mtp conc-list: [192, 256, 384, 512] + srt-recipe: disagg-mtp-variants.yaml:override_2p1d prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml:override_2p1d" decode: num-worker: 1 tp: 8 @@ -6166,6 +5986,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: router: { name: dynamo-router, version: "1.2.1" } multinode: true disagg: false + srt-recipe-dir: dsv4/vllm/gb300-fp4/agentx scenarios: agentic-coding: - search-space: @@ -6174,33 +5995,30 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - spec-decoding: mtp conc-list: [1] num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp8 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_tp8" - spec-decoding: mtp conc-list: [4] num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp8 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_tp8" - spec-decoding: mtp conc-list: [1, 2, 4, 6, 8] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_tp4" dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro @@ -6211,6 +6029,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsv4/vllm/gb300-fp4/agentx scenarios: agentic-coding: # Four DP ranks each reserve 180 GiB on an 840 GiB node. @@ -6223,13 +6042,12 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } router: { name: dynamo-router, version: "1.3.0.dev20260720" } + srt-recipe: disagg-mtp-variants.yaml:override_2p1d_dep8_dep12_c1152 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_2p1d_dep8_dep12_c1152" decode: num-worker: 1 tp: 12 @@ -6240,13 +6058,12 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } router: { name: dynamo-router, version: "1.3.0.dev20260720" } + srt-recipe: disagg-mtp-variants.yaml:override_2p1d_dep8_dep16_c1024 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_2p1d_dep8_dep16_c1024" decode: num-worker: 1 tp: 16 @@ -6257,13 +6074,12 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } router: { name: dynamo-router, version: "1.3.0.dev20260720" } + srt-recipe: disagg-mtp-variants.yaml:override_1p1d_dep4_dep8_c256 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep4_dep8_c256" decode: num-worker: 1 tp: 8 @@ -6274,13 +6090,12 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } router: { name: dynamo-router, version: "1.4.0" } + srt-recipe: disagg-mtp-variants.yaml:override_1p1d_dep8_dep16_c512 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep8_dep16_c512" decode: num-worker: 1 tp: 16 @@ -6289,13 +6104,12 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - spec-decoding: mtp conc-list: [4] router: { name: dynamo-router, version: "1.4.0" } + srt-recipe: disagg-1p4d-dep4-tp8-c4-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml" decode: num-worker: 4 tp: 8 @@ -6306,13 +6120,12 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } router: { name: dynamo-router, version: "1.4.0" } + srt-recipe: disagg-mtp-variants.yaml:override_1p1d_dep4_dep16_c128 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep4_dep16_c128" decode: num-worker: 1 tp: 16 @@ -6323,13 +6136,12 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } router: { name: dynamo-router, version: "1.4.0" } + srt-recipe: disagg-mtp-variants.yaml:override_1p1d_dep4_dep16_c256 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep4_dep16_c256" decode: num-worker: 1 tp: 16 @@ -6351,6 +6163,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: router: { name: dynamo-router, version: "1.2.1" } multinode: true disagg: false + srt-recipe-dir: dsv4/vllm/gb200-fp4/agentx scenarios: agentic-coding: - search-space: @@ -6359,33 +6172,30 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: - spec-decoding: mtp conc-list: [1] num-nodes: 2 + srt-recipe: agg-mtp3-variants.yaml:override_c4 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [4] num-nodes: 2 + srt-recipe: agg-mtp3-variants.yaml:override_c4 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [8] num-nodes: 2 + srt-recipe: agg-mtp3-variants.yaml:override_c8 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_c8" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro @@ -6396,6 +6206,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsv4/vllm/gb200-fp4/agentx scenarios: agentic-coding: # Four DP ranks each reserve 140 GiB on an 840 GiB node. @@ -6408,13 +6219,12 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } router: { name: dynamo-router, version: "1.3.0.dev20260720" } + srt-recipe: disagg-mtp3-variants.yaml:override_2p1d_dep12_c576 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_2p1d_dep12_c576" decode: num-worker: 1 tp: 12 @@ -6425,13 +6235,12 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } router: { name: dynamo-router, version: "1.3.0.dev20260720" } + srt-recipe: disagg-mtp3-variants.yaml:override_2p1d_dep16_c512 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_2p1d_dep16_c512" decode: num-worker: 1 tp: 16 @@ -6442,13 +6251,12 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } router: { name: dynamo-router, version: "1.3.0.dev20260720" } + srt-recipe: disagg-mtp3-variants.yaml:override_1p1d_c256 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_1p1d_c256" decode: num-worker: 1 tp: 8 @@ -6459,13 +6267,12 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } router: { name: dynamo-router, version: "1.3.0.dev20260720" } + srt-recipe: disagg-mtp3-variants.yaml:override_1p1d_c128 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_1p1d_c128" decode: num-worker: 1 tp: 8 @@ -6491,6 +6298,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: router: { name: dynamo-router, version: "1.3.0" } multinode: true disagg: false + srt-recipe-dir: kimik3/vllm/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.61 @@ -6501,25 +6309,23 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: - spec-decoding: mtp conc-list: [1, 2, 4, 8] num-nodes: 4 + srt-recipe: agg-variants.yaml:override_tp16_latency worker: num-worker: 1 tp: 16 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_tp16_latency" # Balanced: multi_node_tep strategy, TEP16 across four GB200 nodes. # https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep - spec-decoding: mtp conc-list: [8, 16, 24, 32] num-nodes: 4 + srt-recipe: agg-variants.yaml:override_tep16_balanced worker: num-worker: 1 tp: 16 ep: 16 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_tep16_balanced" # Throughput oriented: official multi_node_dep strategy, DEP16 across # four GB200 nodes (TP4 x DP4 = EP16, one local DP rank per node). # The recipe allows up to 3600s for full-context saturation warmup to drain. @@ -6527,13 +6333,12 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: - spec-decoding: mtp conc-list: [32, 64, 96, 128, 192, 256] num-nodes: 4 + srt-recipe: agg-variants.yaml:override_dep16 worker: num-worker: 1 tp: 4 ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_dep16" # High-concurrency DEP16 with vLLM Simple CPU KV offloading. c384 # exercises 384 of the 393 AgentX trajectories and remains below the # aggregate max-num-seqs capacity of 512 (128 per DP rank). @@ -6542,13 +6347,12 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: kv-offload-backend: { name: vllm-simple, version: "13c59a3" } conc-list: [128, 192, 256, 384] num-nodes: 4 + srt-recipe: agg-variants.yaml:override_dep16_vllm_simple_offload worker: num-worker: 1 tp: 4 ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_dep16_vllm_simple_offload" # Kimi-K3 GB200 TP16/DCP16 profiles using Mooncake DRAM offload. kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef @@ -6559,6 +6363,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg: framework: dynamo-vllm multinode: true disagg: false + srt-recipe-dir: kimik3/vllm/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.1065 @@ -6568,14 +6373,13 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg: kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } conc-list: [1, 2, 4, 8, 16] num-nodes: 4 + srt-recipe: agg-dcp16-dspark4-maxseq2-mooncake.yaml worker: num-worker: 1 tp: 16 dcp-size: 16 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml" kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef @@ -6586,6 +6390,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg: framework: dynamo-vllm multinode: true disagg: false + srt-recipe-dir: kimik3/vllm/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.1065 @@ -6594,14 +6399,13 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg: kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } conc-list: [8, 40, 48] num-nodes: 4 + srt-recipe: agg-dcp16-nospec-mooncake.yaml worker: num-worker: 1 tp: 16 dcp-size: 16 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml" dsv4-fp4-gb300-dynamo-trt-agentx: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 @@ -6614,6 +6418,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsv4/trtllm/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.21429 @@ -6622,14 +6427,12 @@ dsv4-fp4-gb300-dynamo-trt-agentx: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [4] + srt-recipe: disagg-variants.yaml:override_1p4d_dep4_tep8_c4_b1 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p4d_dep4_tep8_c4_b1" - - "SLURM_PARTITION=batch_1" decode: num-worker: 4 tp: 8 @@ -6639,14 +6442,12 @@ dsv4-fp4-gb300-dynamo-trt-agentx: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [24] + srt-recipe: disagg-variants.yaml:override_1p6d_dep4_tep4_c24_b4 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p6d_dep4_tep4_c24_b4" - - "SLURM_PARTITION=batch_1" decode: num-worker: 6 tp: 4 @@ -6656,14 +6457,12 @@ dsv4-fp4-gb300-dynamo-trt-agentx: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [388] + srt-recipe: disagg-variants.yaml:override_1p1d_dep8_dep32_c388_b4 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_dep8_dep32_c388_b4" - - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 32 @@ -6673,14 +6472,12 @@ dsv4-fp4-gb300-dynamo-trt-agentx: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [736] + srt-recipe: disagg-variants.yaml:override_2p1d_dep8_dep32_c736_b8 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p1d_dep8_dep32_c736_b8" - - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 32 @@ -6690,14 +6487,12 @@ dsv4-fp4-gb300-dynamo-trt-agentx: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [1152] + srt-recipe: disagg-variants.yaml:override_3p1d_dep8_dep16_c1152_b32 prefill: num-worker: 3 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p1d_dep8_dep16_c1152_b32" - - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -6707,14 +6502,12 @@ dsv4-fp4-gb300-dynamo-trt-agentx: kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc24" } conc-list: [2626] + srt-recipe: disagg-variants.yaml:override_5p1d_dep8_dep16_c2626_b96 prefill: num-worker: 5 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_5p1d_dep8_dep16_c2626_b96" - - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -6731,6 +6524,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: framework: dynamo-vllm multinode: true disagg: false + srt-recipe-dir: kimik3/vllm/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.8 @@ -6740,6 +6534,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } conc-list: [16] num-nodes: 4 + srt-recipe: agg-tp8pp2-mooncake-c16.yaml worker: num-worker: 1 tp: 8 @@ -6747,13 +6542,12 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } conc-list: [32] num-nodes: 4 + srt-recipe: agg-tp8pp2-mooncake-c32.yaml worker: num-worker: 1 tp: 8 @@ -6761,12 +6555,11 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml" - kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } conc-list: [48] num-nodes: 4 + srt-recipe: agg-tp8pp2-mooncake-c48.yaml worker: num-worker: 1 tp: 8 @@ -6774,12 +6567,11 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml" - kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } conc-list: [72] num-nodes: 4 + srt-recipe: agg-tp8pp2-mooncake-c72.yaml worker: num-worker: 1 tp: 8 @@ -6787,12 +6579,11 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml" - kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } conc-list: [96] num-nodes: 4 + srt-recipe: agg-tp8pp2-mooncake-c96.yaml worker: num-worker: 1 tp: 8 @@ -6800,8 +6591,6 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml" dsv4-fp4-gb200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 model: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -6812,6 +6601,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-agg: router: { name: dynamo-router, version: "1.5.0.dev20260910" } multinode: true disagg: false + srt-recipe-dir: dsv4/sglang/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -6821,25 +6611,23 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-agg: kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 2 + srt-recipe: agg-gb200-tp8-c1-mtp-hicache.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml" - spec-decoding: draft_model conc-list: [4] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 2 + srt-recipe: agg-gb200-tp8-c4-mtp-hicache.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml" dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 model: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -6851,6 +6639,7 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: dsv4/sglang/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -6859,13 +6648,12 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: conc-list: [64] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 @@ -6875,13 +6663,12 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: conc-list: [128] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 @@ -6891,13 +6678,12 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: conc-list: [256] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml prefill: num-worker: 1 tp: 16 ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 32 @@ -6907,13 +6693,12 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: conc-list: [768] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml prefill: num-worker: 2 tp: 16 ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 32 @@ -6923,13 +6708,12 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: conc-list: [1024] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml prefill: num-worker: 2 tp: 16 ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 32 @@ -6939,13 +6723,12 @@ dsv4-fp4-gb200-dynamo-sglang-agentic-disagg: conc-list: [1280] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml prefill: num-worker: 2 tp: 16 ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 32 @@ -6961,19 +6744,19 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: router: { name: dynamo-router, version: "1.5.0.dev20260910" } multinode: true disagg: false + srt-recipe-dir: dsv4/sglang/gb300-fp4/agentx scenarios: agentic-coding: - search-space: - spec-decoding: draft_model conc-list: [1, 4] num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp8 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml:override_tp8" dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9 model: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -6985,6 +6768,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: dsv4/sglang/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -6993,13 +6777,12 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: conc-list: [120, 480] kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p1d_c480 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_c480" decode: num-worker: 1 tp: 16 @@ -7009,13 +6792,12 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: conc-list: [960] kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_2p1d_c960 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_2p1d_c960" decode: num-worker: 1 tp: 16 @@ -7025,13 +6807,12 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: conc-list: [1440] kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_3p1d_c1440 prefill: num-worker: 3 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_3p1d_c1440" decode: num-worker: 1 tp: 16 @@ -7041,13 +6822,12 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: conc-list: [2400] kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_4p1d_c2400 prefill: num-worker: 4 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_4p1d_c2400" decode: num-worker: 1 tp: 16 @@ -7064,6 +6844,7 @@ qwen3.5-fp8-gb300-dynamo-sglang: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/gb300-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -7072,13 +6853,12 @@ qwen3.5-fp8-gb300-dynamo-sglang: # 1P1D STP: TP4 prefill + TP4 decode (pure tensor parallel). 2 nodes (1+1). - spec-decoding: "none" conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + srt-recipe: disagg-1p1d-tp4-tp4-stp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -7087,13 +6867,12 @@ qwen3.5-fp8-gb300-dynamo-sglang: # 4P1D wide-EP: 4 prefill DEP4 + decode DEP16. 8 nodes (4+4). - spec-decoding: "none" conc-list: [1024] + srt-recipe: disagg-4p1d-dep4-dep16-stp.yaml prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -7102,13 +6881,12 @@ qwen3.5-fp8-gb300-dynamo-sglang: # 8P1D wide-EP: 8 prefill DEP4 + decode DEP16. 12 nodes (8+4). - spec-decoding: "none" conc-list: [2048, 4096] + srt-recipe: disagg-8p1d-dep4-dep16-stp.yaml prefill: num-worker: 8 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -7129,6 +6907,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: glm5.2/sglang/h200-fp8/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -7136,14 +6915,13 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: - spec-decoding: mtp conc-list: [8] kv-offloading: none + srt-recipe: disagg-2p2d-pcp8-tp8-dp8-mtp.yaml prefill: num-worker: 2 tp: 1 pcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml" decode: num-worker: 2 tp: 8 @@ -7152,14 +6930,13 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: - spec-decoding: mtp conc-list: [12] kv-offloading: none + srt-recipe: disagg-2p2d-pcp8-tp8-dp8-mtp.yaml prefill: num-worker: 2 tp: 1 pcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml" decode: num-worker: 2 tp: 8 @@ -7168,14 +6945,13 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: - spec-decoding: mtp conc-list: [16] kv-offloading: none + srt-recipe: disagg-2p2d-pcp8-tp8-dp8-mtp.yaml prefill: num-worker: 2 tp: 1 pcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml" decode: num-worker: 2 tp: 8 @@ -7196,6 +6972,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: glm5.2/sglang/h200-fp8/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -7204,14 +6981,13 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: conc-list: [8] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml prefill: num-worker: 1 tp: 1 pcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml" decode: num-worker: 1 tp: 8 @@ -7221,14 +6997,13 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: conc-list: [12] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml prefill: num-worker: 1 tp: 1 pcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml" decode: num-worker: 1 tp: 8 @@ -7238,14 +7013,13 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: conc-list: [16] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml prefill: num-worker: 1 tp: 1 pcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml" decode: num-worker: 1 tp: 8 @@ -7265,6 +7039,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg: router: { name: dynamo-router, version: "1.5.0.dev20260913" } multinode: true disagg: false + srt-recipe-dir: glm5.2/sglang/h200-fp8/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -7274,61 +7049,56 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg: kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-h200-tp8-mtp-c2.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml" - spec-decoding: mtp conc-list: [4] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-h200-tp8-mtp-c4.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml" - spec-decoding: mtp conc-list: [8] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-h200-tp8-mtp-c8.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml" - spec-decoding: mtp conc-list: [12] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-h200-tp8-mtp-c12.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml" - spec-decoding: mtp conc-list: [16] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-h200-tp8-mtp-c16.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml" # GLM-5.2 B300 NVFP4 AgentX with EAGLE/MTP speculative decoding, following the # AgentX speculative-decoding policy in docs/MODELS.md. SGLang EAGLE runs off @@ -7351,11 +7121,12 @@ glm5.2-fp4-b300-sglang-agentic-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: glm5.2/sglang/b300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48], srt-recipe: agentic.yaml } # First GLM-5.2 FP8 AgentX recipe on B300, the FP8 precision sibling of # glm5.2-fp4-b300-sglang-agentic-mtp and the B300 counterpart of @@ -7382,11 +7153,12 @@ glm5.2-fp8-b300-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: glm5.2/sglang/b300-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: agentic.yaml } # First GLM-5.2 AgentX recipe on B200, shipped spec-decode-only per the AgentX # policy that agentic arms enable speculative decoding rather than running a @@ -7411,11 +7183,12 @@ glm5.2-fp4-b200-sglang-agentic-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: glm5.2/sglang/b200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: agentic.yaml } # First GLM-5.2 FP8 AgentX recipe on B200, the FP8 precision sibling of # glm5.2-fp4-b200-sglang-agentic-mtp. Same spec-decode-only shape per the AgentX @@ -7441,11 +7214,12 @@ glm5.2-fp8-b200-sglang-agentic-mtp: precision: fp8 framework: sglang multinode: false + srt-recipe-dir: glm5.2/sglang/b200-fp8-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: agentic.yaml } # GLM-5.2 NVFP4 B200 AgentX on Dynamo + SGLang. EAGLE uses the model's # built-in nextn head, with acceptance pinned to the golden thinking-on AL in @@ -7461,6 +7235,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: router: { name: dynamo-router, version: "1.5.0.dev20260909" } multinode: true disagg: false + srt-recipe-dir: glm5.2/sglang/b200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -7470,37 +7245,34 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c1 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_c1" - spec-decoding: mtp conc-list: [4] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c4 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [8] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c8 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_c8" glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260910-00840301 @@ -7513,6 +7285,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: glm5.2/sglang/b200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -7522,13 +7295,12 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: conc-list: [48] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p4d_tp4_c48 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p4d_tp4_c48" decode: num-worker: 4 tp: 4 @@ -7539,13 +7311,12 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: conc-list: [64] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p1d_c64 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p1d_c64" decode: num-worker: 1 tp: 8 @@ -7562,6 +7333,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } multinode: true disagg: false + srt-recipe-dir: glm5.2/sglang/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -7571,14 +7343,13 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 2 + # Shared deployment GPU power telemetry. + srt-recipe: agg.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - # Shared deployment GPU power telemetry. - - "CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml" glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:v0.5.18-cu130@sha256:db37df8aded8fa169f7264404f3c893672f8046609a4a43d15bedcb4f63d692a model: nvidia/GLM-5.2-NVFP4 @@ -7590,6 +7361,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: glm5.2/sglang/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -7598,13 +7370,12 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg: conc-list: [10, 12] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-mtp-variants.yaml:zip_override_mtp_agentx_frontier[0] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml:zip_override_mtp_agentx_frontier[0]" decode: num-worker: 1 tp: 4 @@ -7626,6 +7397,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: glm5.2/sglang/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.8 @@ -7636,13 +7408,12 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: kv-offloading: dram kv-offload-backend: name: hicache + srt-recipe: disagg-dep8-mtp-variants.yaml:override_1p6d_tp4_c45 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_1p6d_tp4_c45 decode: num-worker: 6 tp: 4 @@ -7654,13 +7425,12 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: kv-offloading: dram kv-offload-backend: name: hicache + srt-recipe: disagg-dep8-mtp-variants.yaml:override_1p4d_tp4_c48 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_1p4d_tp4_c48 decode: num-worker: 4 tp: 4 @@ -7672,13 +7442,12 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: kv-offloading: dram kv-offload-backend: name: hicache + srt-recipe: disagg-dep8-mtp-variants.yaml:override_2p1d_dep16_c128 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_2p1d_dep16_c128 decode: num-worker: 1 tp: 16 @@ -7699,6 +7468,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: version: 1.2.1 multinode: true disagg: false + srt-recipe-dir: glm5.2/sglang/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.8 @@ -7710,13 +7480,12 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: conc-list: - 2 num-nodes: 2 + srt-recipe: agg-mtp-variants.yaml:override_c2 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_c2 - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -7724,13 +7493,12 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: conc-list: - 4 num-nodes: 2 + srt-recipe: agg-mtp-variants.yaml:override_c4 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_c4 - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -7738,13 +7506,12 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: conc-list: - 8 num-nodes: 2 + srt-recipe: agg-mtp-variants.yaml:override_c8 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_c8 glm5.2-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-20260914-4358a161 model: nvidia/GLM-5.2-NVFP4 @@ -7755,6 +7522,7 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-agg: router: { name: dynamo-router, version: "1.5.0.dev20260913" } multinode: true disagg: false + srt-recipe-dir: glm5.2/sglang/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -7765,39 +7533,36 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-agg: kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 2 + srt-recipe: agg-tp8-c1-mtp.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml" # aggregated 1-worker TP8 (prefill+decode on the same GPUs), concurrency 2 - spec-decoding: mtp conc-list: [2] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 2 + srt-recipe: agg-tp8-c2-mtp.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c2-mtp.yaml" # aggregated 1-worker TP8 (prefill+decode on the same GPUs), concurrency 4 - spec-decoding: mtp conc-list: [4] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 2 + srt-recipe: agg-tp8-c4-mtp.yaml worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c4-mtp.yaml" glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260914-4358a161 model: nvidia/GLM-5.2-NVFP4 @@ -7809,6 +7574,7 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: glm5.2/sglang/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -7818,13 +7584,12 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: conc-list: [48] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-1p2d-p-dep4-d-tp4-c48-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p2d-p-dep4-d-tp4-c48-mtp.yaml" decode: num-worker: 2 tp: 4 @@ -7835,13 +7600,12 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: conc-list: [45] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-1p4d-p-dep4-d-tp4-c45-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p4d-p-dep4-d-tp4-c45-mtp.yaml" decode: num-worker: 4 tp: 4 @@ -7852,13 +7616,12 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: conc-list: [45] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-1p6d-p-dep4-d-tp4-c45-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p6d-p-dep4-d-tp4-c45-mtp.yaml" decode: num-worker: 6 tp: 4 @@ -7869,13 +7632,12 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: conc-list: [192] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-2p1d-p-dep8-d-dep16-c192-mtp.yaml prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-2p1d-p-dep8-d-dep16-c192-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -7895,6 +7657,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: glm5.2/trtllm/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.15239 @@ -7903,14 +7666,12 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: conc-list: [1] kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc26.dev202609040000" } + srt-recipe: disagg-variants.yaml:override_1p1d_tp8_c1_b1_mtp5 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tp8_c1_b1_mtp5" - - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 8 @@ -7920,14 +7681,12 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: conc-list: [30] kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc26.dev202609040000" } + srt-recipe: disagg-variants.yaml:override_1p4d_tep4_c30_b2_mtp5 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p4d_tep4_c30_b2_mtp5" - - "SLURM_PARTITION=batch_1" decode: num-worker: 4 tp: 4 @@ -7937,14 +7696,12 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: conc-list: [20] kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc26.dev202609040000" } + srt-recipe: disagg-variants.yaml:override_1p1d_tep8_c20_b5_mtp5 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tep8_c20_b5_mtp5" - - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 8 @@ -7954,14 +7711,12 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: conc-list: [60] kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc26.dev202609040000" } + srt-recipe: disagg-variants.yaml:override_3p4d_tep4_c60_b5_mtp5 prefill: num-worker: 3 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p4d_tep4_c60_b5_mtp5" - - "SLURM_PARTITION=batch_1" decode: num-worker: 4 tp: 4 @@ -7971,14 +7726,12 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: conc-list: [260] kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc26.dev202609040000" } + srt-recipe: disagg-variants.yaml:override_5p1d_dep16_c260_b16_mtp3 prefill: num-worker: 5 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_5p1d_dep16_c260_b16_mtp3" - - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -7988,14 +7741,12 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: conc-list: [227] kv-offloading: dram kv-offload-backend: { name: native, version: "1.3.0rc26.dev202609040000" } + srt-recipe: disagg-variants.yaml:override_4p1d_dep8_c227_b16_mtp3 prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_4p1d_dep8_c227_b16_mtp3" - - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 8 @@ -8013,6 +7764,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/gb200-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -8021,13 +7773,12 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: # 1P1D TP4/TP4. 8 worker GPUs. - spec-decoding: "mtp" conc-list: [1, 2, 8] + srt-recipe: disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -8036,13 +7787,12 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: # 1P1D TEP8/TEP8. 16 worker GPUs. - spec-decoding: "mtp" conc-list: [32, 48, 80] + srt-recipe: disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -8051,13 +7801,12 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: # 3P1D DEP4/DEP16. 28 worker GPUs. - spec-decoding: "mtp" conc-list: [480] + srt-recipe: disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml prefill: num-worker: 3 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8066,13 +7815,12 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: # 4P1D DEP4/DEP16. 32 worker GPUs. - spec-decoding: "mtp" conc-list: [768] + srt-recipe: disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8081,13 +7829,12 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: # 6P1D DEP4/DEP16. 40 worker GPUs. - spec-decoding: "mtp" conc-list: [1280] + srt-recipe: disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml prefill: num-worker: 6 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8096,13 +7843,12 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: # 7P1D DEP4/DEP16. 44 worker GPUs. - spec-decoding: "mtp" conc-list: [1344] + srt-recipe: disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8111,13 +7857,12 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: # 8P1D DEP4/DEP16. 48 worker GPUs. - spec-decoding: "mtp" conc-list: [1920, 2304] + srt-recipe: disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml prefill: num-worker: 8 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8137,6 +7882,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: kimik3/vllm/gb300-fp4/agentx scenarios: agentic-coding: # 0.1775 resolves to 160 GB of host DRAM per 4-GPU server node on @@ -8147,53 +7893,49 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } conc-list: [48, 56] + srt-recipe: disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml prefill: num-worker: 1 tp: 8 dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml" decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } conc-list: [48, 64] + srt-recipe: disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml prefill: num-worker: 1 tp: 8 dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml" decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } conc-list: [1] + srt-recipe: disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml prefill: num-worker: 1 tp: 8 dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml" decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } conc-list: [48] + srt-recipe: disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml prefill: num-worker: 1 tp: 8 dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml" decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: @@ -8205,6 +7947,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: framework: dynamo-vllm multinode: true disagg: false + srt-recipe-dir: kimik3/vllm/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.1775 @@ -8214,27 +7957,25 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } conc-list: [1, 4] num-nodes: 2 + srt-recipe: agg-dcp8-dspark7-maxseq2-mooncake.yaml worker: num-worker: 1 tp: 8 dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } conc-list: [48, 52, 56] num-nodes: 2 + srt-recipe: agg-dcp8-dspark4-mooncake.yaml worker: num-worker: 1 tp: 8 dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml" # Kimi-K3 MXFP4 B200 aggregated vLLM (TP8 x PP2, 2 nodes / 16 GPUs), agentic # coding. The native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does @@ -8259,6 +8000,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: framework: dynamo-vllm multinode: true disagg: false + srt-recipe-dir: kimik3/vllm/b200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.8 @@ -8267,6 +8009,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: kv-offloading: none conc-list: [1] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c1.yaml worker: num-worker: 1 tp: 8 @@ -8274,12 +8017,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [4] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c4.yaml worker: num-worker: 1 tp: 8 @@ -8287,12 +8029,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [8] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c8.yaml worker: num-worker: 1 tp: 8 @@ -8300,12 +8041,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [14] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c14.yaml worker: num-worker: 1 tp: 8 @@ -8313,12 +8053,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [24] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c24.yaml worker: num-worker: 1 tp: 8 @@ -8326,12 +8065,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [48] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c48.yaml worker: num-worker: 1 tp: 8 @@ -8339,11 +8077,10 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml" - kv-offloading: none conc-list: [96] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c96.yaml worker: num-worker: 1 tp: 8 @@ -8351,12 +8088,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [32] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c32.yaml worker: num-worker: 1 tp: 8 @@ -8364,12 +8100,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [56] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c56.yaml worker: num-worker: 1 tp: 8 @@ -8377,12 +8112,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c56.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [72] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c72.yaml worker: num-worker: 1 tp: 8 @@ -8390,12 +8124,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [80] num-nodes: 2 + srt-recipe: agg-tp8pp2-mooncake-c80.yaml worker: num-worker: 1 tp: 8 @@ -8403,8 +8136,6 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: dcp-size: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c80.yaml" qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -8416,6 +8147,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/gb300-fp8/8k1k scenarios: fixed-seq-len: - isl: 8192 @@ -8424,13 +8156,12 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: # 1P1D TP4/TP4. 8 worker GPUs. - spec-decoding: "mtp" conc-list: [1, 2, 8] + srt-recipe: disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -8439,13 +8170,12 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: # 1P1D TEP8/TEP8. 16 worker GPUs. - spec-decoding: "mtp" conc-list: [32, 48, 80] + srt-recipe: disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -8454,13 +8184,12 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: # 3P1D DEP4/DEP16. 28 worker GPUs. - spec-decoding: "mtp" conc-list: [480] + srt-recipe: disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml prefill: num-worker: 3 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8469,13 +8198,12 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: # 4P1D DEP4/DEP16. 32 worker GPUs. - spec-decoding: "mtp" conc-list: [768] + srt-recipe: disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8484,13 +8212,12 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: # 6P1D DEP4/DEP16. 40 worker GPUs. - spec-decoding: "mtp" conc-list: [1280] + srt-recipe: disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml prefill: num-worker: 6 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8499,13 +8226,12 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: # 7P1D DEP4/DEP16. 44 worker GPUs. - spec-decoding: "mtp" conc-list: [1344] + srt-recipe: disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8514,13 +8240,12 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: # 8P1D DEP4/DEP16. 48 worker GPUs. - spec-decoding: "mtp" conc-list: [1920, 2304] + srt-recipe: disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml prefill: num-worker: 8 tp: 4 ep: 4 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8542,6 +8267,7 @@ dsv41flash-fp4-h100-vllm-agentic-dspark: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv41flash/vllm/h100-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 @@ -8549,7 +8275,7 @@ dsv41flash-fp4-h100-vllm-agentic-dspark: # Extend beyond the c1 full-context capacity estimate to measure the # AgentX workload's practical saturation curve; c20, c24 and c28 fill in # the knee between c16 and the point where 1M-token trajectories preempt. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20, 24, 28], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20, 24, 28], srt-recipe: agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on H100, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). @@ -8570,11 +8296,12 @@ dsv41flash-fp4-h100-sglang-agentic-dspark: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv41flash/sglang/h100-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20], srt-recipe: agentic.yaml } # Sweep supported DP8 attention alongside the retained plain TP8 curve. dsv41flash-fp4-h100-sglang-agentic-dspark-dpa: @@ -8585,11 +8312,12 @@ dsv41flash-fp4-h100-sglang-agentic-dspark-dpa: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv41flash/sglang/h100-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 20], srt-recipe: agentic.yaml } dsv41flash-fp4-gb200-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-arm64-cu130-ac9126e58aa7 @@ -8599,17 +8327,18 @@ dsv41flash-fp4-gb200-vllm-agentic-dspark: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv41flash/vllm/gb200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. # TP4 covers low concurrency with FlashInfer attention and THP Engram tables. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 32], srt-recipe: agentic.yaml } # DEP2 is native TP1 x DP2 + EP2 with MegaMoE and adaptive verification behind a consistent-hash vLLM Router. - - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } # DEP4 is native TP1 x DP4 + EP4 with the same settings. - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on GB200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). @@ -8629,12 +8358,13 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv41flash/sglang/gb200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: agentic.yaml } dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 @@ -8644,16 +8374,17 @@ dsv41flash-fp4-b300-vllm-agentic-dspark: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv41flash/vllm/b300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. # TP4 covers low concurrency with FlashInfer attention and THP Engram tables. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml } # DEP2 is native TP1 x DP2 + EP2 with MegaMoE behind a consistent-hash vLLM # Router; FlashInfer attention up to conc 64, MegaAttention above. - - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32, 64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32, 64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 @@ -8663,17 +8394,18 @@ dsv41flash-fp4-b200-vllm-agentic-dspark: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv41flash/vllm/b200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. # TP4 covers concurrency 1-128 with FlashInfer attention and THP Engram tables. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 64, 128], srt-recipe: agentic.yaml } # DEP2 is native TP1 x DP2 + EP2 with MegaMoE behind a consistent-hash vLLM Router. - - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } # DEP4 is native TP1 x DP4 + EP4 with MegaMoE behind the same router. - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on B200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). @@ -8692,12 +8424,13 @@ dsv41flash-fp4-b200-sglang-agentic-dspark: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv41flash/sglang/b200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 24, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48, 128], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 24, 32], srt-recipe: agentic.yaml } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48, 128], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } dsv41flash-fp4-gb300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-arm64-cu130-ac9126e58aa7 @@ -8707,16 +8440,17 @@ dsv41flash-fp4-gb300-vllm-agentic-dspark: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv41flash/vllm/gb300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. # TP4 covers low concurrency with FlashInfer attention and THP Engram tables. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml } # DEP2 is native TP1 x DP2 + EP2 with MegaMoE behind a consistent-hash vLLM # Router; FlashInfer attention up to conc 64, MegaAttention above. - - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32, 64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32, 64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } # H200 AgentX arm for DeepSeek-V4.1-Flash. Upstream marks h200 verified and says # the GB200 NVL4 TP4 layout becomes TP8 on 8-GPU nodes, so this is TP8. @@ -8734,19 +8468,20 @@ dsv41flash-fp4-h200-vllm-agentic-dspark: precision: fp4 framework: vllm multinode: false + srt-recipe-dir: dsv41flash/vllm/h200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: # 8x141 GB holds the 511 GB checkpoint minus the UVA-offloaded Engram # tables, so the KV cache stays GPU-resident across the full range. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: agentic.yaml } # TP4 doubles the per-GPU share to ~99 GiB of weights on each 141 GB H200 # with the Engram tables offloaded, leaving ~15 GiB of KV per GPU (~17M # tokens across the four ranks, about 3x the pool that served c1-c128 on # B200 TP2). The script caps batched tokens and the scheduler batch on # this arm so the sparse-attention indexer buffer does not consume it. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on H200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). @@ -8765,12 +8500,13 @@ dsv41flash-fp4-h200-sglang-agentic-dspark: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv41flash/sglang/h200-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: agentic.yaml } dsv4-fp4-b200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 @@ -8782,6 +8518,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-agg: router: { name: dynamo-router, version: "1.5.0.dev20260914" } multinode: true disagg: false + srt-recipe-dir: dsv4/sglang/b200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -8791,43 +8528,34 @@ dsv4-fp4-b200-dynamo-sglang-agentic-agg: kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c1 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_c1" - spec-decoding: mtp conc-list: [4] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c4 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [8] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c8 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_c8" dsv4-fp4-b200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 @@ -8840,6 +8568,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-disagg: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: dsv4/sglang/b200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -8848,15 +8577,12 @@ dsv4-fp4-b200-dynamo-sglang-agentic-disagg: conc-list: [64] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p1d_c64 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p1d_c64" decode: num-worker: 1 tp: 8 @@ -8866,15 +8592,12 @@ dsv4-fp4-b200-dynamo-sglang-agentic-disagg: conc-list: [128] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p1d_c128 prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p1d_c128" decode: num-worker: 1 tp: 8 @@ -8884,15 +8607,12 @@ dsv4-fp4-b200-dynamo-sglang-agentic-disagg: conc-list: [256] kv-offloading: dram kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_2p1d_c256 prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_2p1d_c256" decode: num-worker: 1 tp: 8 @@ -8912,6 +8632,7 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/b200-fp8/agentx scenarios: agentic-coding: - dram-utilization: 0.8 @@ -8920,13 +8641,12 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: kv-offloading: dram conc-list: - 8 + srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml decode: num-worker: 1 tp: 4 @@ -8938,13 +8658,12 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: kv-offloading: dram conc-list: - 16 + srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml decode: num-worker: 1 tp: 4 @@ -8956,13 +8675,12 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: kv-offloading: dram conc-list: - 24 + srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml decode: num-worker: 1 tp: 4 @@ -8974,13 +8692,12 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: kv-offloading: dram conc-list: - 32 + srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml decode: num-worker: 1 tp: 4 @@ -8992,13 +8709,12 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: kv-offloading: dram conc-list: - 48 + srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml decode: num-worker: 1 tp: 4 @@ -9010,13 +8726,12 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: kv-offloading: dram conc-list: - 64 + srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml decode: num-worker: 1 tp: 4 @@ -9040,6 +8755,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/b300-fp8/agentx scenarios: agentic-coding: - dram-utilization: 0.8 @@ -9050,13 +8766,12 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: name: hicache conc-list: - 4 + srt-recipe: disagg-variants.yaml:override_tp4_tp4_c4 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4_tp4_c4 decode: num-worker: 1 tp: 4 @@ -9068,13 +8783,12 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: name: hicache conc-list: - 12 + srt-recipe: disagg-variants.yaml:override_tp4_tp4_c12 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4_tp4_c12 decode: num-worker: 1 tp: 4 @@ -9086,13 +8800,12 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: name: hicache conc-list: - 24 + srt-recipe: disagg-variants.yaml:override_tp4_tp4_c24 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4_tp4_c24 decode: num-worker: 1 tp: 4 @@ -9104,13 +8817,12 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: name: hicache conc-list: - 32 + srt-recipe: disagg-variants.yaml:override_tp4ep4_tp4_c32 prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4ep4_tp4_c32 decode: num-worker: 1 tp: 4 @@ -9122,13 +8834,12 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: name: hicache conc-list: - 32 + srt-recipe: disagg-variants.yaml:override_tp2ep2_tp2ep2_c32 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c32 decode: num-worker: 1 tp: 2 @@ -9140,13 +8851,12 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: name: hicache conc-list: - 40 + srt-recipe: disagg-variants.yaml:override_tp2ep2_tp2ep2_c40 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c40 decode: num-worker: 1 tp: 2 @@ -9158,13 +8868,12 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: name: hicache conc-list: - 44 + srt-recipe: disagg-variants.yaml:override_tp2ep2_tp2ep2_c44 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c44 decode: num-worker: 1 tp: 2 @@ -9176,13 +8885,12 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: name: hicache conc-list: - 48 + srt-recipe: disagg-variants.yaml:override_tp2ep2_tp2ep2_c48 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c48 decode: num-worker: 1 tp: 2 @@ -9194,13 +8902,12 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: name: hicache conc-list: - 56 + srt-recipe: disagg-variants.yaml:override_tp2ep2_tp2ep2_c56_replayssm prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c56_replayssm decode: num-worker: 1 tp: 2 @@ -9216,13 +8923,14 @@ dsv41flash-fp4-b300-sglang-agentic-dspark: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv41flash/sglang/b300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 16, 24, 32, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [192], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 16, 24, 32, 48], srt-recipe: agentic.yaml } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [192], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } # Official SGLang nightly; TP4 C1/C2 use GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: @@ -9233,13 +8941,14 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: dsv41flash/sglang/gb300-fp4-mtp scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } - - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2], srt-recipe: agentic.yaml } + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 32, 64, 128], srt-recipe: agentic.yaml } dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 model: deepseek-ai/DeepSeek-V4.1-Flash @@ -9250,6 +8959,7 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: router: { name: dynamo-router, version: "1.5.0.dev20260914" } multinode: true disagg: false + srt-recipe-dir: dsv41flash/sglang/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 @@ -9257,23 +8967,21 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: - spec-decoding: mtp conc-list: [1] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c1 worker: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml:override_tp4_c1" - spec-decoding: mtp conc-list: [1] num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp2_c1 worker: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml:override_tp2_c1" dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 model: deepseek-ai/DeepSeek-V4.1-Flash @@ -9285,19 +8993,19 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: dsv41flash/sglang/gb300-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.80 search-space: - spec-decoding: mtp conc-list: [16] + srt-recipe: disagg-variants.yaml:override_1p2d_spread_c16 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_1p2d_spread_c16" decode: num-worker: 2 tp: 2 @@ -9305,13 +9013,12 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: dp-attn: false - spec-decoding: mtp conc-list: [48] + srt-recipe: disagg-variants.yaml:override_1p2d_c48 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_1p2d_c48" decode: num-worker: 2 tp: 2 @@ -9319,13 +9026,12 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: dp-attn: false - spec-decoding: mtp conc-list: [64] + srt-recipe: disagg-variants.yaml:override_1p2d_c64 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_1p2d_c64" decode: num-worker: 2 tp: 2 @@ -9333,13 +9039,12 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: dp-attn: false - spec-decoding: mtp conc-list: [96] + srt-recipe: disagg-variants.yaml:override_1p1d_c96 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_c96" decode: num-worker: 1 tp: 2 @@ -9347,13 +9052,12 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: dp-attn: false - spec-decoding: mtp conc-list: [160] + srt-recipe: disagg-variants.yaml:override_1p1d_hicache_c160 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_hicache_c160" decode: num-worker: 1 tp: 2 @@ -9361,13 +9065,12 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: dp-attn: false - spec-decoding: mtp conc-list: [256] + srt-recipe: disagg-variants.yaml:override_2p1d_hicache_c256 prefill: num-worker: 2 tp: 2 ep: 2 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_2p1d_hicache_c256" decode: num-worker: 1 tp: 2 @@ -9383,6 +9086,7 @@ qwen3.8next-fp4-gb300-sglang-agentic-mtp: precision: fp4 framework: sglang multinode: false + srt-recipe-dir: qwen3.8next/sglang/gb300-fp4-mtp scenarios: agentic-coding: # Use the GB300 profile's proportional per-worker host DRAM allowance. @@ -9396,7 +9100,7 @@ qwen3.8next-fp4-gb300-sglang-agentic-mtp: kv-offloading: dram kv-offload-backend: { name: hicache } conc-list: [1, 2, 4, 8, 16, 32, 48, 64] - srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml + srt-recipe: agentic.yaml qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: image: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 @@ -9411,6 +9115,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: kv-p2p-transfer: mooncake multinode: true disagg: true + srt-recipe-dir: qwen3.5/sglang/gb300-fp8/agentx scenarios: agentic-coding: - dram-utilization: 0.9 @@ -9421,14 +9126,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: name: hicache conc-list: - 1000 + srt-recipe: disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913.yaml prefill: num-worker: 5 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913.yaml decode: num-worker: 3 tp: 4 @@ -9441,14 +9145,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: name: hicache conc-list: - 200 + srt-recipe: disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913.yaml prefill: num-worker: 1 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913.yaml decode: num-worker: 1 tp: 4 @@ -9461,14 +9164,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: name: hicache conc-list: - 400 + srt-recipe: disagg-dep16-mtp6-variants.yaml:override_3p1d prefill: num-worker: 3 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-dep16-mtp6-variants.yaml:override_3p1d decode: num-worker: 1 tp: 16 @@ -9481,14 +9183,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: name: hicache conc-list: - 300 + srt-recipe: disagg-dep16-mtp6-variants.yaml:override_2p1d prefill: num-worker: 2 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-dep16-mtp6-variants.yaml:override_2p1d decode: num-worker: 1 tp: 16 @@ -9501,14 +9202,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: name: hicache conc-list: - 180 + srt-recipe: disagg-dep16-mtp6-variants.yaml:override_1p1d prefill: num-worker: 1 tp: 1 pp: 4 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-dep16-mtp6-variants.yaml:override_1p1d decode: num-worker: 1 tp: 16 @@ -9524,6 +9224,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve: framework: dynamo-sglang multinode: true disagg: false + srt-recipe-dir: qwen3.5/sglang/gb300-fp8/agentx scenarios: agentic-coding: - dram-utilization: 0.9 @@ -9535,14 +9236,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve: conc-list: - 44 num-nodes: 1 + srt-recipe: agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml worker: num-worker: 1 tp: 4 pp: 1 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -9550,14 +9250,13 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve: conc-list: - 1 num-nodes: 1 + srt-recipe: agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml worker: num-worker: 1 tp: 4 pp: 1 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -9565,11 +9264,10 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve: conc-list: - 1 num-nodes: 2 + srt-recipe: agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml worker: num-worker: 1 tp: 8 pp: 1 ep: 1 dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml diff --git a/inferencex-e2e/docs/architecture.md b/inferencex-e2e/docs/architecture.md index 8e39bf2bf8..59cee4372b 100644 --- a/inferencex-e2e/docs/architecture.md +++ b/inferencex-e2e/docs/architecture.md @@ -158,7 +158,7 @@ Workflows using current tooling call `infx` modules directly, and tests import c `expand_full_sweep(master_config, runner_data, options=FullSweepOptions(...))` exposes full-sweep expansion without an `argparse` namespace. Both commands share config/scenario traversal and row builders, while command-specific selection stays explicit. Full-sweep filters concrete runner nodes; selected-key expansion also accepts matching scheduling labels and deduplicates nodes. Fixed-sequence single-node ranges are capped before expansion, while multi-node ranges and explicit lists are filtered afterward. Agentic bounds only filter existing points. Expansion returns rows before eval selection; use `select_matrix_evals` to apply eval or trim policy. Existing CLI commands and namespace-based Python adapters remain compatible. -For append-only bases and Klaud baseline producers, `infx.matrix.revision.snapshot` extracts configs (under `configs/`, or `.github/configs/` before the move to the project root), multi-node recipes, legacy entrypoints, and the `infx` package (when present) from the same Git revision. Revisions before the package migration run their standalone generator; newer revisions use their own package code. Working-tree source and configs never replace committed inputs, and the planner reads base entries only as raw YAML to bound append-only scope. Snapshot subprocesses remain isolated, and extracted inputs are removed when the operation ends, including failure paths. +For append-only bases and Klaud baseline producers, `infx.matrix.revision.snapshot` extracts configs (under `configs/`, or `.github/configs/` before the move to the project root), srt-slurm recipes, legacy entrypoints, and the `infx` package (when present) from the same Git revision. Revisions before the package migration run their standalone generator; newer revisions use their own package code. Working-tree source and configs never replace committed inputs, and the planner reads base entries only as raw YAML to bound append-only scope. Snapshot subprocesses remain isolated, and extracted inputs are removed when the operation ends, including failure paths. [`validation.py`](../infx/matrix/validation.py) validates master files and runner data before generation. Its strict models own accepted aliases and cross-field rules. Examples include mutually exclusive concurrency forms, single-node versus multi-node shapes, component metadata scope, prefill and decode hardware pairing, and cluster-label requirements for agentic scenarios. @@ -212,7 +212,7 @@ The first cleanup step, before checkout, cancels the runner's Slurm jobs with pl | Driver | Runs | | --- | --- | -| [`drivers/srt/`](../infx/launch/drivers/srt) | Single-node and multi-node srt-slurm recipes (`SRT_RECIPE`, `CONFIG_FILE`), including the cluster-maintained B200 Nscale lanes. Slurm only | +| [`drivers/srt/`](../infx/launch/drivers/srt) | Single-node and multi-node srt-slurm recipes (`SRT_RECIPE`, plus `EVAL_SRT_RECIPE` for multi-node eval-only runs), including the cluster-maintained B200 Nscale lanes. Slurm only | | [`drivers/script.py`](../infx/launch/drivers/script.py) | Single-node runs with an explicit `BENCH_SCRIPT_OVERRIDE`, such as SPEED-Bench collectors: one container through the backend interface, on any backend. The only driver clusters on other schedulers run | Depending on the driver, the launcher may: diff --git a/inferencex-e2e/docs/eval-agentx-procedures.md b/inferencex-e2e/docs/eval-agentx-procedures.md index 965536af3a..6e2adaf08b 100644 --- a/inferencex-e2e/docs/eval-agentx-procedures.md +++ b/inferencex-e2e/docs/eval-agentx-procedures.md @@ -110,7 +110,7 @@ Run these with Python 3.10 or newer, normally inside the serving container, beca Set `EVAL_ONLY=true` **before server launch**. It is not merely a switch inside the eval command: -1. For a single-node fixed-sequence job, the srt binder sets the server context to the matrix `MAX_MODEL_LEN` (`isl + osl + 256`) through `context-length` for SGLang, `max_seq_len` and `max_num_tokens` for TRT-LLM, or `max-model-len` for vLLM and ATOM. AgentX points and multi-node jobs keep their recipe's own context, and multi-node jobs can select a real-verification `EVAL_CONFIG_FILE`. +1. For a single-node fixed-sequence job, the srt binder sets the server context to the matrix `MAX_MODEL_LEN` (`isl + osl + 256`) through `context-length` for SGLang, `max_seq_len` and `max_num_tokens` for TRT-LLM, or `max-model-len` for vLLM and ATOM. AgentX points and multi-node jobs keep their recipe's own context, and multi-node jobs can select a real-verification `eval-srt-recipe`. 2. The health check still runs. In eval-only jobs, vendor frameworks also wait for the served model on the OpenAI chat route, within `EVAL_ENDPOINT_READY_TIMEOUT_SECONDS`. 3. Throughput is skipped. 4. `python3 -m infx.bench eval` sizes each lm-eval request from `EVAL_MAX_MODEL_LEN`, else from `MAX_MODEL_LEN` capped at the model's native maximum. diff --git a/inferencex-e2e/infx/klaud/__main__.py b/inferencex-e2e/infx/klaud/__main__.py index f2eb845e4b..56540d7738 100644 --- a/inferencex-e2e/infx/klaud/__main__.py +++ b/inferencex-e2e/infx/klaud/__main__.py @@ -70,7 +70,7 @@ def live_families(root: Path) -> dict[tuple, set[str]]: family = f"{path.relative_to(root).as_posix()}:{key}" try: entries = generate_test_config_sweep( - SimpleNamespace(config_keys=[key]), configs, runners + SimpleNamespace(config_keys=[key]), configs, runners, root ) except ValueError: # One unrenderable family must not hide unrelated working recipes. diff --git a/inferencex-e2e/infx/klaud/validation.py b/inferencex-e2e/infx/klaud/validation.py index 87aa61eac8..ff0ebca374 100644 --- a/inferencex-e2e/infx/klaud/validation.py +++ b/inferencex-e2e/infx/klaud/validation.py @@ -49,7 +49,11 @@ def canonical_matrix(repository: str, head: str, family: str) -> dict: from infx.matrix.generate import generate_test_config_sweep, mark_eval_entries from infx.matrix.plan import recipe_fingerprint - from infx.matrix.validation import validate_master_config, validate_runner_config + from infx.matrix.validation import ( + srt_recipe_references, + validate_master_config, + validate_runner_config, + ) OwnedCandidate(id="0" * 16 + "-" + "0" * 16, family=family, base=head) source, key = family.split(":", 1) @@ -57,11 +61,19 @@ def canonical_matrix(repository: str, head: str, family: str) -> dict: master = yaml.safe_load(github.file_at(repository, head, prefix + source)) runners = yaml.safe_load(github.file_at(repository, head, prefix + "configs/runners.yaml")) # Only the selected family is relevant; retired sibling schemas may have changed. - entries = generate_test_config_sweep( - SimpleNamespace(config_keys=[key]), - validate_master_config({key: master[key]}), - validate_runner_config(runners), - ) + family_config = {key: master[key]} + with tempfile.TemporaryDirectory(prefix="klaud-recipes-") as temp: + # Recipes are head data too: resolve them from the candidate, not this checkout. + root = Path(temp) + for path in {ref.partition(":")[0] for ref in srt_recipe_references(family_config)[key]}: + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_bytes(github.file_at(repository, head, prefix + path)) + entries = generate_test_config_sweep( + SimpleNamespace(config_keys=[key]), + validate_master_config(family_config, root), + validate_runner_config(runners), + root, + ) evals = [ dict(row, **{"eval-only": True}) for row in mark_eval_entries(deepcopy(entries)) diff --git a/inferencex-e2e/infx/launch/drivers/srt/__init__.py b/inferencex-e2e/infx/launch/drivers/srt/__init__.py index 4f7099498e..7ba96f215b 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/__init__.py +++ b/inferencex-e2e/infx/launch/drivers/srt/__init__.py @@ -28,9 +28,14 @@ prepare_checkout, run_setup, ) -from infx.launch.drivers.srt.recipe import eval_overrides, prepare_recipe +from infx.launch.drivers.srt.recipe import eval_overrides, prepare_recipe, staged_recipe from infx.launch.drivers.srt.run import SrtRun, require, slurm_backend -from infx.launch.request import BATCH_REENTRY_ENV, RequestError, SingleNodeRequest, SrtRequest +from infx.launch.request import ( + BATCH_REENTRY_ENV, + MultiNodeRequest, + RequestError, + SingleNodeRequest, +) if TYPE_CHECKING: from infx.clusters import Cluster @@ -120,13 +125,14 @@ def run_batch(launch: Launch) -> int: def run_multinode(launch: Launch) -> int: """One job on the cluster's lane for the launch path: prepare, submit, follow, collect.""" lane = lanes.srt_lane(launch.cluster.id, launch.path) - request = SrtRequest.from_env(launch.request.env) + request = MultiNodeRequest.from_env(launch.request.env) lanes.check_request(lane, request) - config_file = lanes.config_file(request) + srt_recipe = lanes.srt_recipe(request) + staged = staged_recipe(srt_recipe) decision = power.resolve_power(launch.cluster.id, launch.path, request) model = models.checkpoint(launch.cluster, request) served = models.served_path(launch.cluster, request, model) - model_paths = models.model_paths(launch.cluster, request, config_file, served) + model_paths = models.model_paths(launch.cluster, request, srt_recipe, served) run = SrtRun.create(launch, request, models.job_env(launch.cluster, request, served)) preflight = run.srt.preflight and not (model_paths and model and model.node_local) if request.framework == "tilert": @@ -147,12 +153,12 @@ def run_multinode(launch: Launch) -> int: conc_list = request.env.get("CONC_LIST", "") if decision.dcgm else None job_name = srtctl_job_name(request.runner_name) - prepare_recipe(checkout.root, config_file, job_name, run.srt.dist_timeout_s, conc_list) - arguments = submit.multinode_arguments(run, lane, config_file, overrides, preflight=preflight) + prepare_recipe(checkout.root, staged, job_name, run.srt.dist_timeout_s, conc_list) + arguments = submit.multinode_arguments(run, lane, staged, overrides, preflight=preflight) manifest = run.workspace / submit.MULTINODE_SUBMISSION submitted = submit.Submitted(manifest=manifest) run.life.callback(submitted.cancel, run.backend) - if rc := submit.submit_lane(run, submitted, checkout, config_file, arguments): + if rc := submit.submit_lane(run, submitted, checkout, staged, arguments): return rc return collect.collect(run, lane, checkout, submitted.adopted(), decision, infmax) diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index 57d31ce99d..1f042563f5 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -12,7 +12,7 @@ if TYPE_CHECKING: from infx.clusters.slurm import SrtSlurmSettings - from infx.launch.request import LaunchRequest, SrtRequest + from infx.launch.request import LaunchRequest, MultiNodeRequest, SrtRequest @dataclass(frozen=True) @@ -134,21 +134,15 @@ def check_request(lane: SrtLane, request: SrtRequest) -> None: raise LaunchError(f"{message} (FRAMEWORK={framework})") -def config_file(request: SrtRequest) -> str: - """CONFIG_FILE, or on an eval-only run its real-verification EVAL_CONFIG_FILE.""" - if request.eval_only and request.eval_config_file: +def srt_recipe(request: MultiNodeRequest) -> str: + """SRT_RECIPE, or on an eval-only run its real-verification EVAL_SRT_RECIPE.""" + if request.eval_only and request.eval_srt_recipe: print( - f"EVAL_ONLY=true: selecting real-verification recipe {request.eval_config_file}", + f"EVAL_ONLY=true: selecting real-verification recipe {request.eval_srt_recipe}", flush=True, ) - return request.eval_config_file - if not request.config_file: - raise LaunchError( - "CONFIG_FILE is not set. The srt-slurm path requires a CONFIG_FILE in additional-settings " - f"(MODEL_PREFIX={request.model_prefix} PRECISION={request.precision} " - f"FRAMEWORK={request.framework})" - ) - return request.config_file + return request.eval_srt_recipe + return request.srt_recipe def srt_time_limit( diff --git a/inferencex-e2e/infx/launch/drivers/srt/models.py b/inferencex-e2e/infx/launch/drivers/srt/models.py index 28ac33998c..03e41b30a4 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/models.py +++ b/inferencex-e2e/infx/launch/drivers/srt/models.py @@ -175,13 +175,13 @@ def recipe_aliases(recipe: Path) -> set[str]: def model_paths( cluster: Cluster, request: LaunchRequest, - config_file: str, + srt_recipe: str, served: str | None, ) -> dict[str, str]: """srtslurm.yaml ``model_paths``: each recipe alias mapped to ``served``.""" - recipe = recipe_mirror_path(request.workspace, config_file) + recipe = recipe_mirror_path(request.workspace, srt_recipe) if not recipe.is_file(): - raise LaunchError(f"CONFIG_FILE {config_file} is not in the recipe mirror: {recipe}") + raise LaunchError(f"{srt_recipe} is not in the recipe mirror: {recipe}") aliases = recipe_aliases(recipe) if aliases and served is None: raise LaunchError( diff --git a/inferencex-e2e/infx/launch/drivers/srt/power.py b/inferencex-e2e/infx/launch/drivers/srt/power.py index 29eb324dbb..e8e372a481 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/power.py +++ b/inferencex-e2e/infx/launch/drivers/srt/power.py @@ -13,11 +13,13 @@ import yaml from infx.launch.context import LaunchError -from infx.launch.drivers.srt.recipe import recipe_mirror_path, recipe_relpath +from infx.launch.drivers.srt.recipe import RECIPES_MIRROR, recipe_mirror_path, recipe_relpath from infx.launch.policy import LaunchPath, Match, any_of if TYPE_CHECKING: - from infx.launch.request import LaunchRequest + from infx.launch.request import LaunchRequest, MultiNodeRequest + +MIRROR = f"{RECIPES_MIRROR.as_posix()}/" def recipe_enables_dcgm_power(text: str) -> bool: @@ -64,12 +66,12 @@ class PowerLane: PowerRule( Match(any_of("glm5.2"), any_of("fp4"), any_of("dynamo-sglang"), agentic=True), agentx=True, - recipe_glob="recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml", + recipe_glob=f"{MIRROR}glm5.2/sglang/gb200-fp4/agentx/agg.yaml", ), PowerRule( Match(any_of("kimik3"), any_of("fp4"), any_of("dynamo-vllm"), agentic=True), agentx=True, - recipe_glob="recipes/kimik3/vllm/gb200-fp4/agentx/*", + recipe_glob=f"{MIRROR}kimik3/vllm/gb200-fp4/agentx/*", ), PowerRule(Match(frameworks=any_of("dynamo-sglang"), agentic=False), agentx=False), ), @@ -81,7 +83,7 @@ class PowerLane: PowerRule( Match(any_of("kimik3"), any_of("fp4"), any_of("dynamo-vllm"), agentic=True), agentx=True, - recipe_glob="recipes/kimik3/vllm/*/agentx/*", + recipe_glob=f"{MIRROR}kimik3/vllm/*/agentx/*", ), PowerRule(Match(frameworks=any_of("dynamo-sglang")), agentx=False), ), @@ -167,7 +169,7 @@ class PowerDecision: def decide_power( cluster_id: str, path: LaunchPath, *, dcgm: bool, request: LaunchRequest, recipe: str ) -> PowerDecision: - """Apply the lane's rules to the inspected ``recipe`` (``recipes/...``) when it enables dcgm. + """Apply the lane's rules to the inspected ``recipe`` path when it enables dcgm. Raises ``PowerPolicyError`` for a combination the lane does not allow. """ @@ -183,21 +185,19 @@ def decide_power( raise PowerPolicyError(message) -def resolve_power(cluster_id: str, path: LaunchPath, request: LaunchRequest) -> PowerDecision: - """Detect dcgm in the workspace mirror of the recipe the lane inspects, and decide. +def resolve_power(cluster_id: str, path: LaunchPath, request: MultiNodeRequest) -> PowerDecision: + """Detect dcgm in the workspace copy of the recipe the lane inspects, and decide. - A recipe only upstream (no mirror) stays non-power. + A recipe missing from the workspace stays non-power. """ lane = POWER_LANES.get((cluster_id, path)) if lane is None: return NO_POWER - config_file = request.config_file - if lane.eval_recipe_when_eval_only and request.eval_only and request.eval_config_file: - config_file = request.eval_config_file - if not config_file: - return NO_POWER - mirror = recipe_mirror_path(request.workspace, config_file) + srt_recipe = request.srt_recipe + if lane.eval_recipe_when_eval_only and request.eval_only and request.eval_srt_recipe: + srt_recipe = request.eval_srt_recipe + mirror = recipe_mirror_path(request.workspace, srt_recipe) dcgm = mirror.is_file() and recipe_enables_dcgm_power(mirror.read_text()) return decide_power( - cluster_id, path, dcgm=dcgm, request=request, recipe=recipe_relpath(config_file) + cluster_id, path, dcgm=dcgm, request=request, recipe=recipe_relpath(srt_recipe) ) diff --git a/inferencex-e2e/infx/launch/drivers/srt/recipe.py b/inferencex-e2e/infx/launch/drivers/srt/recipe.py index c03eda1734..9ad5eb7ee0 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/recipe.py +++ b/inferencex-e2e/infx/launch/drivers/srt/recipe.py @@ -27,14 +27,21 @@ FORCED_ACCEPTANCE_MARKER = "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS" -def recipe_relpath(config_file: str) -> str: - """The recipe path of ``CONFIG_FILE``, without its ``:`` selector.""" - return config_file.split(":", 1)[0] +def recipe_relpath(srt_recipe: str) -> str: + """``srt_recipe`` without its ``:``.""" + return srt_recipe.split(":", 1)[0] -def recipe_mirror_path(workspace: Path, config_file: str) -> Path: - """The workspace mirror of ``CONFIG_FILE``'s recipe.""" - return workspace / RECIPES_MIRROR / recipe_relpath(config_file).removeprefix("recipes/") +def recipe_mirror_path(workspace: Path, srt_recipe: str) -> Path: + return workspace / recipe_relpath(srt_recipe) + + +def staged_recipe(srt_recipe: str) -> str: + """The srtctl file argument: the recipe's copy in the checkout's ``recipes/``, selector kept.""" + relative = srt_recipe.removeprefix(f"{RECIPES_MIRROR.as_posix()}/") + if relative == srt_recipe: + raise LaunchError(f"{srt_recipe} is not under {RECIPES_MIRROR}") + return f"recipes/{relative}" def rename_job(text: str, name: str) -> str: @@ -74,15 +81,15 @@ def add_dist_timeout(text: str, seconds: int) -> str: def prepare_recipe( checkout: Path, - config_file: str, + staged: str, job_name: str, dist_timeout_s: int | None, conc_list: str | None, ) -> None: - """Edit the checkout's staged copy of ``config_file`` for this job.""" - config_path = checkout / recipe_relpath(config_file) + """Edit the checkout's staged copy of the recipe for this job.""" + config_path = checkout / recipe_relpath(staged) if not config_path.is_file(): - raise LaunchError(f"CONFIG_FILE does not exist after srt-slurm setup: {config_path}") + raise LaunchError(f"{staged} does not exist after srt-slurm setup: {config_path}") text = raise_health_attempts(rename_job(config_path.read_text(), job_name)) if dist_timeout_s is not None: text = add_dist_timeout(text, dist_timeout_s) diff --git a/inferencex-e2e/infx/launch/drivers/srt/submit.py b/inferencex-e2e/infx/launch/drivers/srt/submit.py index 6ccf728f51..98be347be6 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/submit.py +++ b/inferencex-e2e/infx/launch/drivers/srt/submit.py @@ -145,11 +145,11 @@ def adopted(self) -> SlurmJob: def submit_lane( - run: SrtRun, submitted: Submitted, checkout: Checkout, config_file: str, arguments: list[str] + run: SrtRun, submitted: Submitted, checkout: Checkout, recipe: str, arguments: list[str] ) -> int: """Submit a multi-node lane job, record it in ``submitted``, and return srtctl's exit code.""" applied = apply( - run, checkout, config_file, [*arguments, "--json", "--yes"], stdout=submitted.manifest + run, checkout, recipe, [*arguments, "--json", "--yes"], stdout=submitted.manifest ) print(applied.stdout, end="", flush=True) if applied.returncode: @@ -162,7 +162,7 @@ def submit_lane( def multinode_arguments( run: SrtRun, lane: SrtLane, - config_file: str, + recipe: str, overrides: list[str], *, preflight: bool, @@ -175,7 +175,7 @@ def multinode_arguments( "benchmark.stream_output=true", *overrides, "-f", - config_file, + recipe, ] if not preflight: arguments.append("--no-preflight") diff --git a/inferencex-e2e/infx/launch/request.py b/inferencex-e2e/infx/launch/request.py index f756af87bc..b41747cca1 100644 --- a/inferencex-e2e/infx/launch/request.py +++ b/inferencex-e2e/infx/launch/request.py @@ -70,8 +70,6 @@ class LaunchRequest(BaseModel): spec_decoding: str | None = Field(None, alias="SPEC_DECODING") is_multinode: TrueFlag = Field(False, alias="IS_MULTINODE") is_agentic: OneFlag = Field(False, alias="IS_AGENTIC") - config_file: str | None = Field(None, alias="CONFIG_FILE") - eval_config_file: str | None = Field(None, alias="EVAL_CONFIG_FILE") bench_script_override: str | None = Field(None, alias="BENCH_SCRIPT_OVERRIDE") batch_reentry: OneFlag = Field(False, alias=BATCH_REENTRY_ENV) conc: int | None = Field(None, alias="CONC") @@ -121,7 +119,7 @@ def workspace(self) -> Path: class SrtRequest(LaunchRequest): - """What every srt-slurm submission reads; a multi-node srt-slurm job needs exactly this.""" + """What every srt-slurm submission reads.""" github_workspace: Path = Field(alias="GITHUB_WORKSPACE") image: str = Field(alias="IMAGE") @@ -146,6 +144,13 @@ def _golden_curve_key(self) -> Self: return self +class MultiNodeRequest(SrtRequest): + """One multi-node srt-slurm job; eval-only runs may swap in a real-verification recipe.""" + + srt_recipe: str = Field(alias="SRT_RECIPE") + eval_srt_recipe: str | None = Field(None, alias="EVAL_SRT_RECIPE") + + class SingleNodeRequest(SrtRequest): """One native single-node srt-slurm point, bound to one recipe variant.""" diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 08dde7d4f7..6164a5f38d 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -6,18 +6,20 @@ from collections.abc import Iterable from dataclasses import dataclass from decimal import Decimal +from pathlib import Path from typing import Any, Literal import yaml from infx.clusters import CLUSTER_LABEL_PREFIX -from infx.config import repository_root from .validation import ( DEFAULT_AGENTIC_DURATION_SECONDS, Fields, + config_root, load_config_files, load_runner_file, + srt_recipe_path, validate_agentic_matrix_entry, validate_matrix_entry, ) @@ -277,41 +279,18 @@ def recipe_auxiliary_node_count(recipe: dict) -> int: return pool_nodes + dedicated_roles -def recipe_node_count(prefill: dict, decode: dict) -> int | None: - """Read the authoritative node count from a checked-in srt-slurm recipe.""" - config_files = { - setting.split("=", 1)[1] - for worker in (prefill, decode) - for setting in (worker.get(Fields.ADDITIONAL_SETTINGS.value, []) or []) - if setting.startswith("CONFIG_FILE=") - } - if not config_files: - return None - if len(config_files) != 1: - raise ValueError(f"Conflicting CONFIG_FILE settings: {sorted(config_files)}") - - config_file, _, selector = config_files.pop().partition(":") - repo_root = repository_root() - recipe_root = repo_root / "benchmarks" / "multi_node" / "srt-slurm-recipes" - if config_file.startswith("benchmarks/multi_node/srt-slurm-recipes/"): - recipe_path = repo_root / config_file - else: - recipe_path = recipe_root / config_file.removeprefix("recipes/") - if not recipe_path.exists(): - # Some srt-slurm recipes live only in the runtime image. Their master - # config topology remains the best available scheduling estimate. - return None - +def recipe_node_count(root: Path, srt_recipe: str) -> int | None: + """Read the authoritative node count from the row's checked-in srt-slurm recipe.""" + path, _, selector = srt_recipe.partition(":") + recipe_path = root / path recipe = yaml.safe_load(recipe_path.read_text()) if "base" in recipe: - # srtctl merges a named variant over base (null deletes a key) and - # carries a top-level schema into it. Zip groups and non-schema-2 - # variant files have no authoritative count here; the selected master - # topology supplies the estimate. - if not (selector == "base" or (selector.startswith("override_") and selector in recipe)): + # srtctl merges the variant over base (null deletes a key). Zip groups, + # whole bundles and non-schema-2 variants leave the estimate to the topology. + if not (selector == "base" or selector.startswith("override_")): return None schema = recipe.get("schema") - recipe = _merge_recipe(recipe["base"], recipe.get(selector) or {}) + recipe = _merge_recipe(recipe["base"], recipe[selector] or {}) recipe.setdefault("schema", schema) if recipe.get("schema") != 2: return None @@ -319,8 +298,7 @@ def recipe_node_count(prefill: dict, decode: dict) -> int | None: raise ValueError(f"srt-slurm recipes must declare schema: 2: {recipe_path}") roles = recipe.get("roles") if roles: - # Schema 2 groups node allocations by role. A colocated decode role - # shares prefill nodes and does not reserve another allocation. + # A colocated decode role shares prefill nodes. for name, role in roles.items(): if "nodes" not in role: raise ValueError(f"Recipe role {name!r} must specify nodes: {recipe_path}") @@ -358,9 +336,13 @@ def multinode_node_count( decode: dict, runner: str, runner_data: dict, + srt_recipe: str | None, + root: Path | None, ) -> int: """Return the total Slurm node request represented by a matrix row.""" - recipe_count = recipe_node_count(prefill, decode) + if srt_recipe is not None and root is None: + raise ValueError(f"Reading {srt_recipe} needs the master config's project root") + recipe_count = recipe_node_count(root, srt_recipe) if srt_recipe is not None else None if recipe_count is not None: return recipe_count return worker_node_count(prefill, "prefill", runner, runner_data) + worker_node_count( @@ -372,6 +354,7 @@ def add_multinode_node_count( entry: dict, runner_data: dict, num_nodes: int | None, + root: Path | None, ) -> dict: """Annotate a multi-node row with its scheduling node count.""" if not entry[Fields.DISAGG.value] and num_nodes is not None: @@ -384,6 +367,8 @@ def add_multinode_node_count( entry[Fields.DECODE.value], entry[Fields.RUNNER.value], runner_data, + entry.get(Fields.SRT_RECIPE.value), + root, ) return entry @@ -559,6 +544,19 @@ def component_metadata(benchmark: dict, config: dict) -> dict: return metadata +def srt_recipe_fields(config: dict, benchmark: dict) -> dict: + """The row's srt-recipe and eval-srt-recipe, relative to the project root.""" + return { + field.value: srt_recipe_path( + config.get(Fields.MULTINODE.value, False), + config[Fields.SRT_RECIPE_DIR.value], + benchmark[field.value], + ) + for field in (Fields.SRT_RECIPE, Fields.EVAL_SRT_RECIPE) + if benchmark.get(field.value) is not None + } + + def _multinode_parallelism_key(entry: dict) -> tuple: """Identify a multi-node config independently of eval/concurrency fields. @@ -1007,6 +1005,7 @@ def _fixed_sequence_entries( conc_values: list[int], runners: list[str], runner_data: dict, + root: Path | None, ) -> list[dict]: """Build fixed-sequence rows after the command has selected runners and points. @@ -1068,8 +1067,7 @@ def _fixed_sequence_entries( Fields.SPEC_DECODING.value: spec_decoding, } ) - if benchmark.get(Fields.SRT_RECIPE.value) is not None: - entry[Fields.SRT_RECIPE.value] = benchmark[Fields.SRT_RECIPE.value] + entry.update(srt_recipe_fields(config, benchmark)) entry.update( { Fields.EXP_NAME.value: f"{model_code}_{seq_len_to_str(isl, osl)}", @@ -1079,7 +1077,9 @@ def _fixed_sequence_entries( ) entry.update(component_metadata(benchmark, config)) if is_multinode: - add_multinode_node_count(entry, runner_data, benchmark.get(Fields.NUM_NODES.value)) + add_multinode_node_count( + entry, runner_data, benchmark.get(Fields.NUM_NODES.value), root + ) entries.append(validate_matrix_entry(entry, is_multinode)) return entries @@ -1095,6 +1095,7 @@ def _agentic_entries( min_conc: int | None = None, max_conc: int | None = None, conc_filter: list[int] | None = None, + root: Path | None = None, ) -> list[dict]: """Expand one AgentX deployment for either generator command. @@ -1183,13 +1184,12 @@ def _agentic_entries( Fields.CONC.value: conc, } ) - if benchmark.get(Fields.SRT_RECIPE.value) is not None: - entry[Fields.SRT_RECIPE.value] = benchmark[Fields.SRT_RECIPE.value] exp_name = ( f"{model_code}_tp{tp}_conc{conc}_" f"{agentic_kv_offload_suffix(kv_offloading, kv_offload_backend)}" + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ) + entry.update(srt_recipe_fields(config, benchmark)) entry.update( { Fields.KV_OFFLOADING.value: kv_offloading, @@ -1205,7 +1205,9 @@ def _agentic_entries( entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend entry.update(component_metadata(benchmark, config)) if is_multinode: - add_multinode_node_count(entry, runner_data, benchmark.get(Fields.NUM_NODES.value)) + add_multinode_node_count( + entry, runner_data, benchmark.get(Fields.NUM_NODES.value), root + ) entries.append(validate_agentic_matrix_entry(entry)) return entries @@ -1234,6 +1236,7 @@ def generate_full_sweep( args: argparse.Namespace, all_config_data: dict, runner_data: dict, + root: Path | None = None, ) -> list[dict]: """Compatibility adapter for callers passing the full-sweep CLI namespace.""" return expand_full_sweep( @@ -1255,6 +1258,7 @@ def generate_full_sweep( max_tp=args.max_tp, max_ep=args.max_ep, ), + root=root, ) @@ -1263,6 +1267,7 @@ def expand_full_sweep( runner_data: dict, *, options: FullSweepOptions = FullSweepOptions(), + root: Path | None = None, ) -> list[dict]: """Expand validated configs in declaration order, before eval selection.""" if options.step_size <= 1: @@ -1299,6 +1304,7 @@ def expand_full_sweep( scenario_types=options.scenario_types, runner_node_filter=options.runner_node_filter, full_sweep=options, + root=root, ) @@ -1379,6 +1385,7 @@ def generate_test_config_sweep( args: argparse.Namespace, all_config_data: dict, runner_data: dict | None = None, + root: Path | None = None, ) -> list[dict]: """Compatibility API for selected-key expansion without eval selection.""" return _expand_selected_configs( @@ -1389,6 +1396,7 @@ def generate_test_config_sweep( seq_lens=getattr(args, "seq_lens", None), scenario_types=getattr(args, "scenario_type", None), concurrencies=getattr(args, "conc", None), + root=root, ) @@ -1401,6 +1409,7 @@ def _expand_selected_configs( seq_lens: list[str] | None = None, scenario_types: tuple[str, ...] | list[str] | None = None, concurrencies: list[int] | None = None, + root: Path | None = None, ) -> list[dict]: resolved_keys = expand_config_keys(config_keys, all_config_data.keys()) return _expand_configs( @@ -1410,6 +1419,7 @@ def _expand_selected_configs( seq_lens=seq_lens, scenario_types=scenario_types, concurrencies=concurrencies, + root=root, ) @@ -1423,6 +1433,7 @@ def _expand_configs( scenario_types: tuple[str, ...] | list[str] | None = None, concurrencies: list[int] | None = None, full_sweep: FullSweepOptions | None = None, + root: Path | None = None, ) -> list[dict]: """Traverse configs and scenarios once for both generation commands. @@ -1489,7 +1500,7 @@ def _expand_configs( continue rows.extend( _fixed_sequence_entries( - config, benchmark, sequence, values, runners, runner_data + config, benchmark, sequence, values, runners, runner_data, root ) ) @@ -1513,6 +1524,7 @@ def _expand_configs( min_conc=full_sweep.min_conc if full_sweep is not None else None, max_conc=full_sweep.max_conc if full_sweep is not None else None, conc_filter=concurrencies, + root=root, ) ) return rows @@ -1620,18 +1632,20 @@ def generate_config_matrix( *, scenario_types: tuple[str, ...] | list[str] | None = None, eval_mode: EvalMode = "default", + root: Path | None = None, ) -> list[dict]: """Build selected configs and evals without a generator subprocess. - Every call builds independent rows. The caller loads master/runner inputs; - node-count resolution still reads checked-in recipes. Default, - throughput-only, subset-only, all-eval, and smoke modes use the same policy as the CLI. + Every call builds independent rows. Multi-node node counts read recipes under + ``root``. Default, throughput-only, subset-only, all-eval, and smoke modes use + the same policy as the CLI. """ rows = _expand_selected_configs( config_keys, master_config, runner_data, scenario_types=scenario_types, + root=root, ) rows = select_matrix_evals(rows, mode=eval_mode) # Retain the former JSON boundary: values and nested objects cannot leak @@ -1839,11 +1853,12 @@ def main() -> list[dict]: all_config_data = load_config_files(args.config_files) runner_data = load_runner_file(args.runner_config) + root = config_root(args.config_files) if args.command == "full-sweep": - matrix_values = generate_full_sweep(args, all_config_data, runner_data) + matrix_values = generate_full_sweep(args, all_config_data, runner_data, root) elif args.command == "test-config": - matrix_values = generate_test_config_sweep(args, all_config_data, runner_data) + matrix_values = generate_test_config_sweep(args, all_config_data, runner_data, root) else: parser.error(f"Unknown command: {args.command}") diff --git a/inferencex-e2e/infx/matrix/plan.py b/inferencex-e2e/infx/matrix/plan.py index 3d6ee34e97..03a91ad07f 100644 --- a/inferencex-e2e/infx/matrix/plan.py +++ b/inferencex-e2e/infx/matrix/plan.py @@ -26,6 +26,7 @@ from .validation import ( ChangelogEntry, ChangelogMatrixEntry, + config_root, load_config_files, load_runner_file, ) @@ -366,6 +367,7 @@ def build_plan( config_files = MASTER_CONFIGS if config_files is None else config_files master_config = load_config_files(config_files) + root = config_root(config_files) runner_data = None def generate_current( @@ -383,6 +385,7 @@ def generate_current( runner_data, scenario_types=scenarios, eval_mode=mode, + root=root, ) except Exception as error: command = [ diff --git a/inferencex-e2e/infx/matrix/revision.py b/inferencex-e2e/infx/matrix/revision.py index 4d86ee88b0..08ee0d6c58 100644 --- a/inferencex-e2e/infx/matrix/revision.py +++ b/inferencex-e2e/infx/matrix/revision.py @@ -44,6 +44,7 @@ def module_path(self) -> str: "infx", "utils/matrix_logic", *CONFIG_DIRS, + "benchmarks/single_node/srt-slurm-recipes", "benchmarks/multi_node/srt-slurm-recipes", ) INHERITED_ENV = ("PATH", "HOME", "LANG", "TMPDIR") diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index 63dad22ba5..0aead9091d 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -1,9 +1,12 @@ import pprint +import re from enum import Enum -from typing import Any, Literal, Self +from pathlib import Path, PurePosixPath +from typing import Annotated, Any, Literal, Self import yaml from pydantic import ( + AfterValidator, BaseModel, ConfigDict, Field, @@ -15,6 +18,11 @@ from infx.clusters import CLUSTER_LABEL_PREFIX, RunnerInventory DEFAULT_AGENTIC_DURATION_SECONDS = 3600 +SINGLE_NODE_RECIPES = "benchmarks/single_node/srt-slurm-recipes" +MULTI_NODE_RECIPES = "benchmarks/multi_node/srt-slurm-recipes" +# llm-d jobs read CONFIG_FILE (benchmarks/multi_node/llm-d-recipes) instead of srt-recipe. +LLMD_FRAMEWORK = "llmd-vllm" +RECIPE_SELECTOR = re.compile(r"base|override_[\w-]+|zip_override_[\w-]+\[\d+\]") """ The below class defines the field names expected to be present in the JSON entries @@ -46,6 +54,8 @@ class Fields(Enum): # Search-space/benchmark fields TP = "tp" SRT_RECIPE = "srt-recipe" + SRT_RECIPE_DIR = "srt-recipe-dir" + EVAL_SRT_RECIPE = "eval-srt-recipe" PP = "pp" DCP_SIZE = "dcp-size" PCP_SIZE = "pcp-size" @@ -265,6 +275,10 @@ class MultiNodeMatrixEntry(BaseModel): framework: str spec_decoding: Literal["mtp", "draft_model", "none"] = Field(alias=Fields.SPEC_DECODING.value) runner: str + srt_recipe: str | None = Field(default=None, alias=Fields.SRT_RECIPE.value, min_length=1) + eval_srt_recipe: str | None = Field( + default=None, alias=Fields.EVAL_SRT_RECIPE.value, min_length=1 + ) node_count: int = Field(alias=Fields.NODE_COUNT.value, gt=0, strict=True) isl: int osl: int @@ -366,6 +380,10 @@ class MultiNodeAgenticMatrixEntry(BaseModel): framework: str spec_decoding: Literal["mtp", "draft_model", "none"] = Field(alias=Fields.SPEC_DECODING.value) runner: str + srt_recipe: str | None = Field(default=None, alias=Fields.SRT_RECIPE.value, min_length=1) + eval_srt_recipe: str | None = Field( + default=None, alias=Fields.EVAL_SRT_RECIPE.value, min_length=1 + ) node_count: int = Field(alias=Fields.NODE_COUNT.value, gt=0, strict=True) prefill: WorkerConfig decode: WorkerConfig @@ -530,13 +548,41 @@ def _validate_kv_offload_fields(self: Any) -> Any: return self +def _recipe_path(value: str) -> str: + path = PurePosixPath(value) + if ( + not path.parts + or path.is_absolute() + or ".." in path.parts + or ":" in value + or path.as_posix() != value + ): + raise ValueError(f"{value!r} must be a normalized relative path") + return value + + +def _recipe_reference(value: str) -> str: + path, separator, selector = value.partition(":") + _recipe_path(path) + if separator and not RECIPE_SELECTOR.fullmatch(selector): + raise ValueError( + f"{value!r} must select base, override_ or zip_override_[]" + ) + return value + + +RecipeDir = Annotated[str, AfterValidator(_recipe_path)] +# [:] +RecipeReference = Annotated[str, AfterValidator(_recipe_reference)] + + class SingleNodeSearchSpaceEntry(BaseModel): """Single node search space configuration.""" model_config = ConfigDict(extra="forbid", populate_by_name=True) tp: int - srt_recipe: str | None = Field(default=None, alias=Fields.SRT_RECIPE.value, min_length=1) + srt_recipe: RecipeReference | None = Field(default=None, alias=Fields.SRT_RECIPE.value) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) pcp_size: int = Field(default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) @@ -570,6 +616,10 @@ class MultiNodeSearchSpaceEntry(BaseModel): worker: AggregateWorkerConfig | None = None prefill: WorkerConfig | None = None decode: WorkerConfig | None = None + srt_recipe: RecipeReference | None = Field(default=None, alias=Fields.SRT_RECIPE.value) + eval_srt_recipe: RecipeReference | None = Field( + default=None, alias=Fields.EVAL_SRT_RECIPE.value + ) num_nodes: int | None = Field(default=None, alias=Fields.NUM_NODES.value, gt=0, strict=True) router: ComponentMetadata | None = None kv_p2p_transfer: str | None = Field( @@ -627,7 +677,10 @@ class AgenticCodingSearchSpaceEntry(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) tp: int | None = None - srt_recipe: str | None = Field(default=None, alias=Fields.SRT_RECIPE.value, min_length=1) + srt_recipe: RecipeReference | None = Field(default=None, alias=Fields.SRT_RECIPE.value) + eval_srt_recipe: RecipeReference | None = Field( + default=None, alias=Fields.EVAL_SRT_RECIPE.value + ) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) pcp_size: int = Field(default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) @@ -872,6 +925,36 @@ def _validate_multinode_entry_scope(self: BaseModel) -> BaseModel: return self +_RECIPE_SETTINGS = {"CONFIG_FILE": Fields.SRT_RECIPE, "EVAL_CONFIG_FILE": Fields.EVAL_SRT_RECIPE} + + +def _validate_srt_recipes(self: Any) -> Any: + """Every srt-slurm row names its recipe; llm-d rows keep CONFIG_FILE.""" + entries = _master_search_space_entries(self) + if self.multinode and self.framework == LLMD_FRAMEWORK: + if self.srt_recipe_dir is not None or any( + entry.srt_recipe or entry.eval_srt_recipe for entry in entries + ): + raise ValueError(f"{LLMD_FRAMEWORK} selects recipes with CONFIG_FILE, not srt-recipe") + return self + if self.srt_recipe_dir is None: + raise ValueError(f"srt-slurm configs require {Fields.SRT_RECIPE_DIR.value}") + for entry in entries: + if entry.srt_recipe is None: + raise ValueError(f"Every search-space entry requires {Fields.SRT_RECIPE.value}") + if not self.multinode and getattr(entry, "eval_srt_recipe", None) is not None: + raise ValueError(f"{Fields.EVAL_SRT_RECIPE.value} is only valid for multinode configs") + workers = (getattr(entry, role, None) for role in ("worker", "prefill", "decode")) + for worker in filter(None, workers): + for setting in worker.additional_settings or []: + name = setting.partition("=")[0] + if name in _RECIPE_SETTINGS: + raise ValueError( + f"{name} is not an additional-setting; use {_RECIPE_SETTINGS[name].value}" + ) + return self + + class SingleNodeMasterConfigEntry(BaseModel): """Top-level single node master configuration entry.""" @@ -886,6 +969,7 @@ class SingleNodeMasterConfigEntry(BaseModel): multinode: Literal[False] disagg: Literal[False] = Field(default=False) router: ComponentMetadata | None = None + srt_recipe_dir: RecipeDir = Field(alias=Fields.SRT_RECIPE_DIR.value) scenarios: SingleNodeScenarios @model_validator(mode="after") @@ -901,6 +985,10 @@ def validate_component_metadata_scope(self) -> Self: def validate_multinode_entry_scope(self) -> Self: return _validate_multinode_entry_scope(self) + @model_validator(mode="after") + def validate_srt_recipes(self) -> Self: + return _validate_srt_recipes(self) + class MultiNodeMasterConfigEntry(BaseModel): """Top-level multinode master configuration entry.""" @@ -919,6 +1007,7 @@ class MultiNodeMasterConfigEntry(BaseModel): kv_p2p_transfer: str | None = Field( default=None, alias=Fields.KV_P2P_TRANSFER.value, min_length=1 ) + srt_recipe_dir: RecipeDir | None = Field(default=None, alias=Fields.SRT_RECIPE_DIR.value) scenarios: MultiNodeScenarios @model_validator(mode="after") @@ -934,19 +1023,80 @@ def validate_component_metadata_scope(self) -> Self: def validate_multinode_entry_scope(self) -> Self: return _validate_multinode_entry_scope(self) - -def validate_master_config(master_configs: dict) -> list[dict]: - """Validate input master configuration structure.""" + @model_validator(mode="after") + def validate_srt_recipes(self) -> Self: + return _validate_srt_recipes(self) + + +def srt_recipe_path(multinode: bool, directory: str, reference: str) -> str: + """A row's recipe reference relative to the project root, selector kept.""" + return f"{MULTI_NODE_RECIPES if multinode else SINGLE_NODE_RECIPES}/{directory}/{reference}" + + +def _zip_length(group: Any) -> int: + # srtctl broadcasts length-1 lists, so the longest list is the variant count. + if isinstance(group, list): + return len(group) + if isinstance(group, dict): + return max(map(_zip_length, group.values()), default=0) + return 0 + + +def _check_srt_recipe(root: Path, reference: str, recipes: dict[str, Any]) -> None: + path, _, selector = reference.partition(":") + if path not in recipes: + if not (root / path).is_file(): + raise ValueError(f"{reference} does not exist") + recipes[path] = yaml.safe_load((root / path).read_text()) + recipe = recipes[path] + if not selector: + return + if not isinstance(recipe, dict) or "base" not in recipe: + raise ValueError(f"{reference}: only an override bundle with a base takes a selector") + group, _, index = selector.removesuffix("]").partition("[") + if group not in recipe or (index and int(index) >= _zip_length(recipe[group])): + raise ValueError(f"{reference}: {path} has no variant {selector}") + + +def config_root(config_files: list[str]) -> Path: + """The project directory whose ``configs/`` holds ``config_files``; recipes resolve under it.""" + roots = {Path(config_file).resolve().parent.parent for config_file in config_files} + if len(roots) != 1: + raise ValueError(f"Master configs must share one project directory: {sorted(roots)}") + return roots.pop() + + +def srt_recipe_references(master_configs: dict) -> dict[str, list[str]]: + """Each validated config's srt-recipe and eval-srt-recipe, relative to its project root.""" + references = {} for key, entry in master_configs.items(): - is_multinode = entry.get("multinode", False) - + model = ( + MultiNodeMasterConfigEntry + if entry.get("multinode", False) + else SingleNodeMasterConfigEntry + ) try: - if is_multinode: - MultiNodeMasterConfigEntry(**entry) - else: - SingleNodeMasterConfigEntry(**entry) + config = model(**entry) except ValidationError as e: raise ValueError(f"Master config entry '{key}' failed validation:\n{e}") from e + references[key] = [ + srt_recipe_path(config.multinode, config.srt_recipe_dir, reference) + for row in _master_search_space_entries(config) + for reference in (row.srt_recipe, getattr(row, "eval_srt_recipe", None)) + if reference is not None + ] + return references + + +def validate_master_config(master_configs: dict, root: Path) -> dict: + """Validate master configs and the recipe variant each srt-recipe selects under ``root``.""" + recipes: dict[str, Any] = {} + for key, references in srt_recipe_references(master_configs).items(): + for reference in references: + try: + _check_srt_recipe(root, reference, recipes) + except ValueError as e: + raise ValueError(f"Master config entry '{key}' failed validation:\n{e}") from e return master_configs @@ -1105,7 +1255,7 @@ def load_config_files(config_files: list[str], validate: bool = True) -> dict: raise ValueError(f"Input file '{config_file}' does not exist.") from e if validate: - validate_master_config(all_config_data) + validate_master_config(all_config_data, config_root(config_files)) return all_config_data diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 1f20cfc6c7..0d4c217365 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -3,6 +3,7 @@ import shlex import subprocess import sys +from pathlib import Path from types import SimpleNamespace import pytest @@ -512,10 +513,11 @@ def family_configs(): "framework": "sglang", "runner": "fixture", "multinode": False, + "srt-recipe-dir": "fixture", "scenarios": { "fixed-seq-len": [{ "isl": 1024, "osl": 1024, - "search-space": [{"tp": 1, "conc-list": [2, 6]}], + "search-space": [{"tp": 1, "conc-list": [2, 6], "srt-recipe": "recipe.yaml"}], }], }, }, @@ -530,12 +532,20 @@ def family_configs(): return master, runners +def write_fixture_recipe(root: Path) -> None: + recipe = root / "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml" + recipe.parent.mkdir(parents=True, exist_ok=True) + recipe.write_text("{}\n") + + @pytest.mark.parametrize("prefix", ["", "inferencex-e2e/"]) def test_canonical_family_matrix_reads_revision_layout(monkeypatch, family_configs, prefix): master, runners = family_configs + # The candidate's recipe exists only at the candidate head, not in this checkout. files = { prefix + "configs/nvidia-master.yaml": yaml.safe_dump(master), prefix + "configs/runners.yaml": yaml.safe_dump(runners), + prefix + "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml": b"{}\n", } monkeypatch.setattr( github, "read", @@ -1139,6 +1149,7 @@ def test_live_families_preserve_public_identity_after_move(tmp_path, family_conf configs.mkdir(parents=True) (configs / "nvidia-master.yaml").write_text(yaml.safe_dump(master)) (configs / "runners.yaml").write_text(yaml.safe_dump(runners)) + write_fixture_recipe(project) families = klaud.live_families(tmp_path if from_repository_root else project) @@ -1154,6 +1165,7 @@ def test_historical_families_ignore_leftover_nested_results(tmp_path, family_con configs.mkdir() (configs / "nvidia-master.yaml").write_text(yaml.safe_dump(master)) (configs / "runners.yaml").write_text(yaml.safe_dump(runners)) + write_fixture_recipe(tmp_path) def git(*args): return subprocess.check_output( diff --git a/inferencex-e2e/infx/tests/launch/fake_slurm.py b/inferencex-e2e/infx/tests/launch/fake_slurm.py index 004e91e4d6..6b6852863d 100644 --- a/inferencex-e2e/infx/tests/launch/fake_slurm.py +++ b/inferencex-e2e/infx/tests/launch/fake_slurm.py @@ -234,7 +234,7 @@ def base_env(*, fakes: Path, logs: Path, workspace: Path, sandbox: Path) -> dict key: value for key, value in os.environ.items() if not key.startswith(("SLURM_", "FAKE_", "SRT_", "INFERENCEX_", "INFX_", "UV_")) - and key not in {"VIRTUAL_ENV", "MODEL_PATH", "CONFIG_FILE", "EVAL_CONFIG_FILE", "BENCH_SCRIPT_OVERRIDE"} + and key not in {"VIRTUAL_ENV", "MODEL_PATH", "CONFIG_FILE", "EVAL_SRT_RECIPE", "BENCH_SCRIPT_OVERRIDE"} } env.update( PATH=f"{fakes}{os.pathsep}{Path(sys.executable).parent}{os.pathsep}/usr/bin{os.pathsep}/bin", diff --git a/inferencex-e2e/infx/tests/launch/test_launch_policy.py b/inferencex-e2e/infx/tests/launch/test_launch_policy.py index a3d8d128d3..6f36a58d96 100644 --- a/inferencex-e2e/infx/tests/launch/test_launch_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_launch_policy.py @@ -19,7 +19,7 @@ ) from infx.launch.lifecycle import Lifecycle from infx.launch.policy import LaunchPath, Match, salloc_time_limit -from infx.launch.request import LaunchRequest +from infx.launch.request import LaunchRequest, MultiNodeRequest DCGM, AGENTX, ADAPTER, ERR = "dcgm", "agentx", "adapter", "error" MULTI, NATIVE = LaunchPath.SRT_MULTI, LaunchPath.SRT_NATIVE @@ -29,10 +29,11 @@ def _request(**env): return LaunchRequest.from_env({"RUNNER_NAME": "r_0", **env}) -GB200_GLM = "recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml" -KIMI_GB200 = "recipes/kimik3/vllm/gb200-fp4/agentx/k.yaml" -KIMI_GB300 = "recipes/kimik3/vllm/gb300-fp4/agentx/deep/k.yaml" -OTHER = "recipes/other.yaml" +MIRROR = "benchmarks/multi_node/srt-slurm-recipes" +GB200_GLM = f"{MIRROR}/glm5.2/sglang/gb200-fp4/agentx/agg.yaml" +KIMI_GB200 = f"{MIRROR}/kimik3/vllm/gb200-fp4/agentx/k.yaml" +KIMI_GB300 = f"{MIRROR}/kimik3/vllm/gb300-fp4/agentx/deep/k.yaml" +OTHER = f"{MIRROR}/other.yaml" CASES = [ ("gb200-nv", MULTI, "1", "glm5.2", "fp4", "dynamo-sglang", GB200_GLM, AGENTX), @@ -111,22 +112,30 @@ def _mirror(tmp_path, rel, text): POWER_RECIPE = "telemetry:\n dcgm_exporter:\n enabled: true\n" +def _multinode(**env): + return MultiNodeRequest.from_env({ + "RUNNER_NAME": "r_0", "IMAGE": "i", "SPEC_DECODING": "none", "RESULT_FILENAME": "r", + "RUN_EVAL": "false", **env, + }) + + def test_nscale_eval_only_inspects_eval_recipe(tmp_path): _mirror(tmp_path, "dsv4/eval.yaml", POWER_RECIPE) _mirror(tmp_path, "dsv4/bench.yaml", "model: {}\n") env = dict( GITHUB_WORKSPACE=str(tmp_path), IS_AGENTIC="0", MODEL_PREFIX="dsv4", PRECISION="fp4", - FRAMEWORK="dynamo-vllm", CONFIG_FILE="recipes/dsv4/bench.yaml:zip", - EVAL_CONFIG_FILE="recipes/dsv4/eval.yaml", + FRAMEWORK="dynamo-vllm", SRT_RECIPE=f"{MIRROR}/dsv4/bench.yaml:zip", + EVAL_SRT_RECIPE=f"{MIRROR}/dsv4/eval.yaml", ) - assert resolve_power("b200-nscale", MULTI, _request(**env, EVAL_ONLY="true")).dcgm - assert not resolve_power("b200-nscale", MULTI, _request(**env, EVAL_ONLY="false")).dcgm - assert not resolve_power("b300-dsxe", MULTI, _request(**env, EVAL_ONLY="true")).dcgm + assert resolve_power("b200-nscale", MULTI, _multinode(**env, EVAL_ONLY="true")).dcgm + assert not resolve_power("b200-nscale", MULTI, _multinode(**env, EVAL_ONLY="false")).dcgm + assert not resolve_power("b300-dsxe", MULTI, _multinode(**env, EVAL_ONLY="true")).dcgm -def test_upstream_only_recipe_stays_non_power(tmp_path): - request = _request(GITHUB_WORKSPACE=str(tmp_path), CONFIG_FILE="recipes/missing.yaml", - IS_AGENTIC="0", FRAMEWORK="dynamo-trt") +def test_recipe_missing_from_the_workspace_stays_non_power(tmp_path): + request = _multinode(GITHUB_WORKSPACE=str(tmp_path), SRT_RECIPE=f"{MIRROR}/missing.yaml", + IS_AGENTIC="0", FRAMEWORK="dynamo-trt", MODEL_PREFIX="m", + PRECISION="fp8", EVAL_ONLY="false") assert not resolve_power("gb300-nv", MULTI, request).dcgm diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index 0f3f1b307d..86b1727647 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -77,6 +77,7 @@ concurrencies: [4] """ POWER_TELEMETRY = "telemetry:\n dcgm_exporter:\n image: dcgm\n enabled: true\n" +MIRROR = "benchmarks/multi_node/srt-slurm-recipes" @pytest.fixture @@ -110,7 +111,7 @@ def lane_env(harness, cluster_id: str, recipe: str = LANE_RECIPE, **overrides: s mirror.write_text(recipe) env = { **harness.env, "RUNNER_NAME": overrides.pop("RUNNER_NAME", None) or runner_for(cluster_id), - "IS_MULTINODE": "true", "CONFIG_FILE": "recipes/test/lane.yaml", "IMAGE": "test:tag", + "IS_MULTINODE": "true", "SRT_RECIPE": f"{MIRROR}/test/lane.yaml", "IMAGE": "test:tag", "CONC_LIST": "4", "SPEC_DECODING": "none", "IS_AGENTIC": "0", "ISL": "1024", "OSL": "1024", "FAKE_RESULTS": "fixed", } # fmt: skip @@ -427,7 +428,7 @@ def test_eval_only_runs_the_eval_recipe_with_real_verification(harness): env = lane_env( harness, "gb300-nv", MODEL_PREFIX="dsv4", PRECISION="fp4", FRAMEWORK="dynamo-trt", MODEL="deepseek-ai/DeepSeek-V4-Pro", IS_AGENTIC="1", SPEC_DECODING="mtp", ISL="0", OSL="0", - EVAL_ONLY="true", EVAL_CONFIG_FILE="recipes/test/eval.yaml", FAKE_RESULTS="eval", + EVAL_ONLY="true", EVAL_SRT_RECIPE=f"{MIRROR}/test/eval.yaml", FAKE_RESULTS="eval", EVAL_CONC="4 8", RECIPE_FINGERPRINT="recipe-fixture", FAKE_EVAL_META=json.dumps(staged), ) # fmt: skip assert_ok(launch(env, harness.config, harness.workspace)) @@ -456,6 +457,27 @@ def test_eval_only_runs_the_eval_recipe_with_real_verification(harness): assert list(workspace.glob("point-identity_*.json")) == [] +def test_srtctl_runs_the_bundle_variant_staged_from_srt_recipe(harness): + bundle = "base:\n" + "".join(f" {line}\n" for line in LANE_RECIPE.splitlines()) + env = lane_env( + harness, "h200-dgxc", bundle + "override_x:\n benchmark:\n concurrencies: [8]\n", + MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-sglang", + MODEL="deepseek-ai/DeepSeek-R1-0528", SRT_RECIPE=f"{MIRROR}/test/lane.yaml:override_x", + ) # fmt: skip + assert_ok(launch(env, harness.config, harness.workspace)) + [call] = srtctl_calls(harness.logs) + argv = call["argv"] + assert argv[argv.index("--file") + 1] == "recipes/test/lane.yaml:override_x" + + +def test_srt_recipe_outside_the_recipe_mirror_fails_before_any_setup(harness): + env = lane_env(harness, "h200-dgxc", MODEL_PREFIX="dsr1", PRECISION="fp8", + FRAMEWORK="dynamo-sglang", MODEL="m", SRT_RECIPE="recipes/test/lane.yaml") # fmt: skip + result = launch(env, harness.config, harness.workspace) + assert result.returncode == 1 + assert "recipes/test/lane.yaml is not under benchmarks/multi_node/srt-slurm-recipes" in result.stderr + assert lines(harness.logs, "git") == [] + def test_setup_retries_only_after_discarding_a_truncated_archive(harness): env = lane_env( harness, "h200-dgxc", MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-sglang", @@ -482,8 +504,6 @@ def test_a_setup_failure_without_a_bad_archive_is_not_retried(harness): ("h100-dgxc", dict(MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-vllm"), "Unsupported framework"), ("b200-nscale", dict(MODEL_PREFIX="dsv4", PRECISION="fp4", FRAMEWORK="dynamo-trt"), "only dynamo-vllm"), ("gb300-nv", dict(MODEL_PREFIX="llama", PRECISION="fp8", FRAMEWORK="dynamo-sglang"), "stages no checkpoint"), - ("h200-dgxc", dict(MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-sglang", CONFIG_FILE=""), - "CONFIG_FILE is not set"), ]) # fmt: skip def test_unsupported_multinode_requests_fail_before_any_setup(harness, cluster_id, env, message): result = launch(lane_env(harness, cluster_id, MODEL="m", **env), harness.config, harness.workspace) @@ -495,6 +515,7 @@ def test_unsupported_multinode_requests_fail_before_any_setup(harness, cluster_i @pytest.mark.parametrize(("shape", "overrides", "missing"), [ ("single", dict(IS_AGENTIC="1", SPEC_DECODING="mtp", THINKING_MODE=""), "THINKING_MODE"), ("multi", dict(SPEC_DECODING=None), "SPEC_DECODING"), + ("multi", dict(SRT_RECIPE=None), "SRT_RECIPE"), ("batch", dict(SRT_RECIPE=None), "SRT_RECIPE"), ]) # fmt: skip def test_a_missing_input_is_named_before_any_setup(harness, shape, overrides, missing): diff --git a/inferencex-e2e/infx/tests/launch/test_srt_policy.py b/inferencex-e2e/infx/tests/launch/test_srt_policy.py index dc5a883fed..9e46580de1 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_policy.py @@ -45,8 +45,9 @@ def cluster(tmp_path, single_node_models: str = "staged") -> Cluster: return load_inventory({"labels": {"cluster:c": ["c_0"]}, "clusters": {"c": record}}).clusters["c"] -MULTI = dict(IS_MULTINODE="true", CONFIG_FILE="recipes/x.yaml") +MULTI = dict(IS_MULTINODE="true") SINGLE = dict(IS_MULTINODE="false") +MIRROR = "benchmarks/multi_node/srt-slurm-recipes" @pytest.mark.parametrize(("cluster_id", "env", "path"), [ @@ -139,16 +140,16 @@ def test_every_recipe_alias_maps_to_the_checkpoint_and_literals_pass_through(tmp point = request(MODEL=model, GITHUB_WORKSPACE=str(tmp_path / "ws")) if paths is LaunchError: with pytest.raises(LaunchError, match="stages no checkpoint"): - model_paths(c, point, "recipes/r.yaml:override_x", served_path(c, point, checkpoint(c, point))) + model_paths(c, point, f"{MIRROR}/r.yaml:override_x", served_path(c, point, checkpoint(c, point))) return - resolved = model_paths(c, point, "recipes/r.yaml:override_x", served_path(c, point, checkpoint(c, point))) + resolved = model_paths(c, point, f"{MIRROR}/r.yaml:override_x", served_path(c, point, checkpoint(c, point))) assert resolved == {alias: str(tmp_path / path) for alias, path in paths.items()} def test_missing_recipe_fails_before_model_resolution(tmp_path): c, point = cluster(tmp_path), request(MODEL="org/M", GITHUB_WORKSPACE=str(tmp_path)) with pytest.raises(LaunchError, match="not in the recipe mirror"): - model_paths(c, point, "recipes/missing.yaml", "/m") + model_paths(c, point, f"{MIRROR}/missing.yaml", "/m") def test_matching_single_node_points_read_the_shared_hub_cache(tmp_path, monkeypatch): diff --git a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py index ec26ec7fd8..2bfd43a8ed 100644 --- a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py +++ b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py @@ -37,7 +37,7 @@ def test_aggregated_multinode_node_count_uses_explicit_num_nodes(): "decode": {}, } - add_multinode_node_count(entry, {}, num_nodes=3) + add_multinode_node_count(entry, {}, num_nodes=3, root=None) assert entry["node-count"] == 3 @@ -51,7 +51,7 @@ def test_disaggregated_multinode_node_count_rejects_num_nodes(): } with pytest.raises(ValueError, match="num-nodes.*disaggregated"): - add_multinode_node_count(entry, {}, num_nodes=3) + add_multinode_node_count(entry, {}, num_nodes=3, root=None) def test_disaggregated_multinode_node_count_requires_hardware_inventory(): @@ -63,7 +63,7 @@ def test_disaggregated_multinode_node_count_requires_hardware_inventory(): } with pytest.raises(ValueError, match="Cannot resolve gpus-per-node"): - add_multinode_node_count(entry, {}, num_nodes=None) + add_multinode_node_count(entry, {}, num_nodes=None, root=None) def test_aggregated_worker_expands_to_legacy_matrix_pair(): @@ -74,7 +74,7 @@ def test_aggregated_worker_expands_to_legacy_matrix_pair(): "pp": 2, "ep": 1, "dp-attn": False, - "additional-settings": ["CONFIG_FILE=recipes/aggregate.yaml"], + "additional-settings": ["PREFILL_IMAGE=prefill:tag"], } } @@ -88,7 +88,7 @@ def test_aggregated_worker_expands_to_legacy_matrix_pair(): "pcp-size": 1, "ep": 1, "dp-attn": False, - "additional-settings": ["CONFIG_FILE=recipes/aggregate.yaml"], + "additional-settings": ["PREFILL_IMAGE=prefill:tag"], } assert decode == { "num-worker": 0, @@ -101,26 +101,33 @@ def test_aggregated_worker_expands_to_legacy_matrix_pair(): } +@pytest.fixture +def multinode_recipe(tmp_path): + """Write a multi-node recipe under the scratch project ``tmp_path``; return its matrix path.""" + + def write(recipe: dict, name: str = "test.yaml") -> str: + path = tmp_path / "benchmarks/multi_node/srt-slurm-recipes" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(recipe)) + return f"benchmarks/multi_node/srt-slurm-recipes/{name}" + + return write + + @pytest.mark.parametrize("roles, expected", [ ({"agg": {"nodes": 3, "workers": 6}}, 3), ({"prefill": {"nodes": 2}, "decode": {"nodes": 4}}, 6), ({"prefill": {"nodes": 2}, "decode": {"nodes": "colocate"}}, 2), ({"prefill": {"nodes": 2}, "decode": {"workers": 1}}, None), ]) -def test_multinode_node_count_reads_schema_two_roles(tmp_path, monkeypatch, roles, expected): - recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes/test.yaml" - recipe.parent.mkdir(parents=True) - recipe.write_text(yaml.safe_dump({"schema": 2, "roles": roles})) +def test_multinode_node_count_reads_schema_two_roles(tmp_path, multinode_recipe, roles, expected): import infx.matrix.generate as generate - import infx.config - (tmp_path / "configs").mkdir() - monkeypatch.setattr(infx.config, "__file__", str(tmp_path / "infx/config.py")) - prefill = {"additional-settings": ["CONFIG_FILE=recipes/test.yaml"]} + srt_recipe = multinode_recipe({"schema": 2, "roles": roles}) if expected is None: with pytest.raises(ValueError, match="role 'decode' must specify nodes"): - generate.recipe_node_count(prefill, {}) + generate.recipe_node_count(tmp_path, srt_recipe) else: - assert generate.recipe_node_count(prefill, {}) == expected + assert generate.recipe_node_count(tmp_path, srt_recipe) == expected @pytest.mark.parametrize("selector, expected", [ @@ -129,22 +136,16 @@ def test_multinode_node_count_reads_schema_two_roles(tmp_path, monkeypatch, role ("override_colocated", 2), ("zip_override_sweep[0]", None), ]) -def test_recipe_node_count_resolves_override_selectors(tmp_path, monkeypatch, selector, expected): - recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes/variants.yaml" - recipe.parent.mkdir(parents=True) - recipe.write_text(yaml.safe_dump({ +def test_recipe_node_count_resolves_override_selectors(tmp_path, multinode_recipe, selector, expected): + srt_recipe = multinode_recipe({ "schema": 2, "base": {"roles": {"prefill": {"nodes": 1}, "decode": {"nodes": 2}}}, "override_wide": {"roles": {"decode": {"nodes": 4}}}, "override_colocated": {"roles": {"prefill": {"nodes": 2}, "decode": {"nodes": "colocate"}}}, "zip_override_sweep": {"roles": {"decode": {"nodes": [1, 2]}}}, - })) + }, "variants.yaml") import infx.matrix.generate as generate - import infx.config - (tmp_path / "configs").mkdir() - monkeypatch.setattr(infx.config, "__file__", str(tmp_path / "infx/config.py")) - prefill = {"additional-settings": [f"CONFIG_FILE=recipes/variants.yaml:{selector}"]} - assert generate.recipe_node_count(prefill, {}) == expected + assert generate.recipe_node_count(tmp_path, f"{srt_recipe}:{selector}") == expected @pytest.mark.parametrize("auxiliary, expected", [ @@ -165,22 +166,16 @@ def test_recipe_node_count_resolves_override_selectors(tmp_path, monkeypatch, se "services": [{"type": "etcd", "placement": {"node": "dedicated"}, "enabled": False}]}, 3), ]) -def test_recipe_node_count_includes_auxiliary_nodes(tmp_path, monkeypatch, auxiliary, expected): +def test_recipe_node_count_includes_auxiliary_nodes(tmp_path, multinode_recipe, auxiliary, expected): import infx.matrix.generate as generate - import infx.config - recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes/test.yaml" - recipe.parent.mkdir(parents=True) - recipe.write_text(yaml.safe_dump({ + srt_recipe = multinode_recipe({ "schema": 2, "base": {"roles": {"prefill": {"nodes": 1}, "decode": {"nodes": 2}}}, "override_auxiliary": auxiliary, - })) - (tmp_path / "configs").mkdir() - monkeypatch.setattr(infx.config, "__file__", str(tmp_path / "infx/config.py")) - prefill = {"additional-settings": ["CONFIG_FILE=recipes/test.yaml:override_auxiliary"]} + }) - assert generate.recipe_node_count(prefill, {}) == expected + assert generate.recipe_node_count(tmp_path, f"{srt_recipe}:override_auxiliary") == expected def test_multinode_node_count_uses_role_gpu_footprints(sample_runner_config): @@ -188,7 +183,7 @@ def test_multinode_node_count_uses_role_gpu_footprints(sample_runner_config): decode = {"num-worker": 2, "tp": 8, "pp": 1, "pcp-size": 1} assert multinode_node_count( - prefill, decode, "cluster:b300-nv", sample_runner_config + prefill, decode, "cluster:b300-nv", sample_runner_config, None, None ) == 3 @@ -204,7 +199,7 @@ def test_multinode_node_count_honors_explicit_role_node_settings(): "additional-settings": ["DECODE_NODES=1"], } - assert multinode_node_count(prefill, decode, "unknown", {}) == 3 + assert multinode_node_count(prefill, decode, "unknown", {}, None, None) == 3 def test_multinode_node_count_resolves_heterogeneous_worker_hardware( @@ -214,7 +209,7 @@ def test_multinode_node_count_resolves_heterogeneous_worker_hardware( decode = {"hardware": "h100", "num-worker": 1, "tp": 8} assert multinode_node_count( - prefill, decode, "gb200", sample_runner_config + prefill, decode, "gb200", sample_runner_config, None, None ) == 6 @@ -229,36 +224,42 @@ def cluster_record(gpus_per_node, **facts): } -@pytest.mark.parametrize("config_file", [ - "recipes/test.yaml", - "benchmarks/multi_node/srt-slurm-recipes/test.yaml", -]) @pytest.mark.parametrize(("roles", "expected_nodes"), [ ({"agg": {"nodes": 3}}, 3), ({"prefill": {"nodes": 2}, "decode": {"nodes": 3}}, 5), ]) -def test_multinode_node_count_prefers_recipe_roles( - tmp_path, monkeypatch, config_file, roles, expected_nodes, -): - recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes/test.yaml" - recipe.parent.mkdir(parents=True) - recipe.write_text(yaml.safe_dump({"schema": 2, "roles": roles})) - import infx.config - (tmp_path / "configs").mkdir() - monkeypatch.setattr( - infx.config, "__file__", - str(tmp_path / "infx/config.py"), - ) - prefill = { - "num-worker": 1, "tp": 8, - "additional-settings": [f"CONFIG_FILE={config_file}", "PREFILL_NODES=7"], - } +def test_multinode_node_count_prefers_recipe_roles(tmp_path, multinode_recipe, roles, expected_nodes): + srt_recipe = multinode_recipe({"schema": 2, "roles": roles}) + prefill = {"num-worker": 1, "tp": 8, "additional-settings": ["PREFILL_NODES=7"]} decode = {"num-worker": 1, "tp": 8, "additional-settings": ["DECODE_NODES=9"]} # Recipe allocation wins over role overrides, even without an inventory. - assert multinode_node_count(prefill, decode, "unknown", {}) == expected_nodes + assert multinode_node_count(prefill, decode, "unknown", {}, srt_recipe, tmp_path) == expected_nodes +def test_multinode_rows_resolve_recipes_and_count_nodes_from_them( + tmp_path, multinode_recipe, sample_multinode_config, sample_runner_config, +): + multinode_recipe({ + "schema": 2, + "base": {"roles": {"prefill": {"nodes": 2}, "decode": {"nodes": 1}}}, + "override_wide": {"roles": {"decode": {"nodes": 3}}}, + }, "dsr1/bundle.yaml") + key, config = next(iter(sample_multinode_config.items())) + config["srt-recipe-dir"] = "dsr1" + config["scenarios"]["fixed-seq-len"][0]["search-space"][0].update({ + "srt-recipe": "bundle.yaml:override_wide", "eval-srt-recipe": "bundle.yaml:base", + }) + + [row] = generate_test_config_sweep( + argparse.Namespace(config_keys=[key]), sample_multinode_config, sample_runner_config, + root=tmp_path, + ) + + root = "benchmarks/multi_node/srt-slurm-recipes/dsr1" + assert (row["srt-recipe"], row["eval-srt-recipe"], row["node-count"]) == ( + f"{root}/bundle.yaml:override_wide", f"{root}/bundle.yaml:base", 5, + ) @pytest.mark.parametrize("command", ["full-sweep", "test-config"]) @@ -266,14 +267,31 @@ def test_srt_recipe_selection_stays_with_its_scenario( sample_single_node_config, sample_runner_config, full_sweep_args_both, command, ): key, config = next(iter(sample_single_node_config.items())) + config["srt-recipe-dir"] = "dsr1/sglang/mi300x-fp8" config["scenarios"]["fixed-seq-len"][0]["search-space"][0]["srt-recipe"] = "pilot.yaml:base" vars(full_sweep_args_both).update(config_keys=[key], no_evals=True) generate = generate_full_sweep if command == "full-sweep" else generate_test_config_sweep rows = generate(full_sweep_args_both, sample_single_node_config, sample_runner_config) assert rows - assert {row.get("srt-recipe") for row in rows if row["isl"] == 1024} == {"pilot.yaml:base"} + assert {row.get("srt-recipe") for row in rows if row["isl"] == 1024} == { + "benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/mi300x-fp8/pilot.yaml:base" + } assert {row.get("srt-recipe") for row in rows if row["isl"] == 8192} == {None} + +def name_fixture_recipe(config: dict, root: Path) -> None: + """Point every single-node row of ``config`` at one recipe written under ``root``.""" + recipe = root / "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml" + recipe.parent.mkdir(parents=True, exist_ok=True) + recipe.write_text("{}\n") + for entry in config.values(): + entry["srt-recipe-dir"] = "fixture" + for scenario in entry["scenarios"].values(): + for sequence in scenario: + for row in sequence["search-space"]: + row["srt-recipe"] = "recipe.yaml" + + @pytest.fixture def sample_single_node_config(): """Single node config based on dsr1-fp8-mi300x-sglang.""" @@ -2031,7 +2049,9 @@ def test_script_from_another_directory( command, invalid, ): """The module entrypoint resolves caller-relative inputs from another directory.""" - (tmp_path / "master config.yaml").write_text(yaml.safe_dump(sample_single_node_config)) + name_fixture_recipe(sample_single_node_config, tmp_path) + (tmp_path / "configs").mkdir() + (tmp_path / "configs/master config.yaml").write_text(yaml.safe_dump(sample_single_node_config)) nodes = sample_runner_config["labels"]["mi300x"] (tmp_path / "runners.yaml").write_text(yaml.safe_dump({ "labels": {"mi300x": nodes, "cluster:mi300x-amd": nodes}, @@ -2039,7 +2059,7 @@ def test_script_from_another_directory( })) repo_root = Path(__file__).resolve().parents[3] args = [ - command, "--config-files", "master config.yaml", + command, "--config-files", "configs/master config.yaml", "--runner-config", "runners.yaml", "--seq-lens", "1k1k", "--no-evals", ] if command == "test-config": @@ -2072,8 +2092,9 @@ def test_cli_uses_selected_runner_file( sample_runner_config, runner_file, ): monkeypatch.chdir(tmp_path) - (tmp_path / "master.yaml").write_text(yaml.safe_dump(sample_single_node_config)) + name_fixture_recipe(sample_single_node_config, tmp_path) (tmp_path / "configs").mkdir() + (tmp_path / "configs/master.yaml").write_text(yaml.safe_dump(sample_single_node_config)) # An explicit override must not fall back to the default inventory. (tmp_path / "configs/runners.yaml").write_text("invalid: default inventory") selected_file = tmp_path / (runner_file or "configs/runners.yaml") @@ -2084,7 +2105,7 @@ def test_cli_uses_selected_runner_file( })) argv = [ "generate_sweep_configs.py", "full-sweep", - "--config-files", "master.yaml", "--single-node", "--no-evals", + "--config-files", "configs/master.yaml", "--single-node", "--no-evals", "--runner-node-filter", "fixture-node", "--seq-lens", "1k1k", "--max-conc", "4", ] @@ -2098,6 +2119,42 @@ def test_cli_uses_selected_runner_file( ("fixture-node-0", 4), ("fixture-node-1", 4), ] + def test_recipes_resolve_under_the_loaded_config_project( + self, tmp_path, sample_multinode_config, sample_runner_config, + ): + """A recipe that exists only beside the loaded configs validates and sizes the job.""" + project = tmp_path / "project" + recipe = project / "benchmarks/multi_node/srt-slurm-recipes/only-here/recipe.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text(yaml.safe_dump( + {"schema": 2, "roles": {"prefill": {"nodes": 2}, "decode": {"nodes": 1}}} + )) + key, config = next(iter(sample_multinode_config.items())) + config["srt-recipe-dir"] = "only-here" + config["scenarios"]["fixed-seq-len"][0]["search-space"][0]["srt-recipe"] = "recipe.yaml" + (project / "configs").mkdir() + (project / "configs/master.yaml").write_text(yaml.safe_dump(sample_multinode_config)) + (project / "configs/runners.yaml").write_text(yaml.safe_dump({ + "labels": {"gb200": ["gb200-nv_0"], "cluster:gb200-nv": ["gb200-nv_0"]}, + "clusters": {"gb200-nv": cluster_record(4)}, + })) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + + result = subprocess.run( + [sys.executable, "-m", "infx.matrix.generate", "test-config", "--config-keys", key, + "--config-files", str(project / "configs/master.yaml"), + "--runner-config", str(project / "configs/runners.yaml"), "--no-evals"], + cwd=elsewhere, capture_output=True, text=True, check=False, + env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[3])}, + ) # fmt: skip + + assert result.returncode == 0, result.stderr + [row] = json.loads(result.stdout) + assert (row["srt-recipe"], row["node-count"]) == ( + "benchmarks/multi_node/srt-slurm-recipes/only-here/recipe.yaml", 3, + ) + def test_all_evals_cli_marks_every_fixed_sequence_entry( self, monkeypatch, @@ -3255,4 +3312,4 @@ def test_single_node_fixed_sequence_rejects_require_power(sample_single_node_con entry = next(iter(sample_single_node_config.values())) entry["scenarios"]["fixed-seq-len"][-1]["require-power"] = True with pytest.raises(ValueError, match="require-power"): - validate_master_config(sample_single_node_config) + validate_master_config(sample_single_node_config, Path()) diff --git a/inferencex-e2e/infx/tests/matrix/test_process_changelog.py b/inferencex-e2e/infx/tests/matrix/test_process_changelog.py index eef49ddacd..8a863af5e8 100644 --- a/inferencex-e2e/infx/tests/matrix/test_process_changelog.py +++ b/inferencex-e2e/infx/tests/matrix/test_process_changelog.py @@ -266,7 +266,10 @@ def test_append_only_delta_rejects_removed_existing_point(): raise AssertionError("removing an existing point should reject append-only mode") -def test_append_only_scope_allows_additive_top_level_restructuring(): +def test_append_only_scope_allows_additive_top_level_restructuring(tmp_path): + recipe = tmp_path / "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text("{}\n") router_a = {"name": "router-a", "version": "1"} router_b = {"name": "router-b", "version": "2"} base = { @@ -278,13 +281,16 @@ def test_append_only_scope_allows_additive_top_level_restructuring(): "framework": "vllm", "runner": "b200", "multinode": False, + "srt-recipe-dir": "fixture", "router": router_a, "scenarios": { "fixed-seq-len": [ { "isl": 8192, "osl": 1024, - "search-space": [{"tp": 4, "conc-list": [1, 4, 8]}], + "search-space": [ + {"tp": 4, "conc-list": [1, 4, 8], "srt-recipe": "recipe.yaml"} + ], } ] }, @@ -297,11 +303,11 @@ def test_append_only_scope_allows_additive_top_level_restructuring(): ] search_space[0]["router"] = router_a search_space.append( - {"tp": 8, "conc-list": [12, 16], "router": router_b} + {"tp": 8, "conc-list": [12, 16], "router": router_b, "srt-recipe": "recipe.yaml"} ) - validate_master_config(base) - validate_master_config(head) + validate_master_config(base, tmp_path) + validate_master_config(head, tmp_path) args = SimpleNamespace( config_keys=["test-config"], seq_lens=None, @@ -394,12 +400,14 @@ def planning_inputs() -> tuple[dict, dict]: "precision": "fp8", "framework": "sglang", "runner": "cluster:fixture", "multinode": multinode, "disagg": multinode, **({"kv-p2p-transfer": "nixl"} if multinode else {}), + "srt-recipe-dir": "fixture", "scenarios": { "fixed-seq-len": [{"isl": 8192, "osl": 1024, "search-space": [ - {**shape, "conc-list": [16, 32, 64]}, + {**shape, "conc-list": [16, 32, 64], "srt-recipe": "recipe.yaml"}, ]}], "agentic-coding": [{"search-space": [ - {**shape, "conc-list": [16, 32], **({} if multinode else {"kv-offloading": "none"})}, + {**shape, "conc-list": [16, 32], "srt-recipe": "recipe.yaml", + **({} if multinode else {"kv-offloading": "none"})}, ]}], }, } @@ -416,6 +424,10 @@ def planning_repo(tmp_path, monkeypatch): master, runners = planning_inputs() (tmp_path / "configs/runners.yaml").write_text(yaml.safe_dump(runners)) (tmp_path / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master, sort_keys=False)) + for node in ("single_node", "multi_node"): + recipe = tmp_path / f"benchmarks/{node}/srt-slurm-recipes/fixture/recipe.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text("schema: 2\nroles:\n prefill: {nodes: 1}\n decode: {nodes: 1}\n") monkeypatch.chdir(tmp_path) return tmp_path, master, runners @@ -451,12 +463,11 @@ def test_recovery_plans_with_the_checkouts_own_planner_and_recipes( root, base, head = committed_planning_repo master_path = root / "configs/nvidia-master.yaml" master = yaml.safe_load(master_path.read_text()) - master["multi"]["scenarios"]["fixed-seq-len"][0]["search-space"][0]["prefill"][ - "additional-settings" - ] = ["CONFIG_FILE=recipes/recovery-fixture.yaml"] + master["multi"]["scenarios"]["fixed-seq-len"][0]["search-space"][0][ + "srt-recipe" + ] = "recovery-fixture.yaml" master_path.write_text(yaml.safe_dump(master, sort_keys=False)) - recipe = root / "benchmarks/multi_node/srt-slurm-recipes/recovery-fixture.yaml" - recipe.parent.mkdir(parents=True) + recipe = root / "benchmarks/multi_node/srt-slurm-recipes/fixture/recovery-fixture.yaml" recipe.write_text("schema: 2\nroles:\n prefill: {nodes: 3}\n decode: {nodes: 4}\n") changelog = root / "perf-changelog.yaml" entries = yaml.safe_load(changelog.read_text()) @@ -494,19 +505,18 @@ def test_historical_generator_uses_snapshot_recipes_not_inherited_recovery_root( planning_repo, monkeypatch, nested_layout ): root, master, _ = planning_repo - master["multi"]["scenarios"]["fixed-seq-len"][0]["search-space"][0]["prefill"][ - "additional-settings" - ] = ["CONFIG_FILE=recipes/snapshot.yaml"] + master["multi"]["scenarios"]["fixed-seq-len"][0]["search-space"][0][ + "srt-recipe" + ] = "snapshot.yaml" (root / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master, sort_keys=False)) - recipe = root / "benchmarks/multi_node/srt-slurm-recipes/snapshot.yaml" - recipe.parent.mkdir(parents=True) + recipe = root / "benchmarks/multi_node/srt-slurm-recipes/fixture/snapshot.yaml" recipe.write_text("schema: 2\nroles:\n prefill: {nodes: 3}\n decode: {nodes: 4}\n") if nested_layout: project = root / "inferencex-e2e" project.mkdir() for name in ("infx", "configs", "benchmarks"): shutil.move(root / name, project / name) - recipe = project / "benchmarks/multi_node/srt-slurm-recipes/snapshot.yaml" + recipe = project / "benchmarks/multi_node/srt-slurm-recipes/fixture/snapshot.yaml" for command in ( ["init", "-q"], ["config", "user.name", "Test"], @@ -682,11 +692,13 @@ def test_append_only_main_runs_only_added_points_and_skips_evals(planning_repo, def test_generation_failure_does_not_publish_a_partial_matrix(planning_repo, changelog_run, capsys): # Single-node generation succeeds before multinode scheduling fails. - (planning_repo[0] / "configs/runners.yaml").write_text("labels: {}\nclusters: {}\n") + (planning_repo[0] / "benchmarks/multi_node/srt-slurm-recipes/fixture/recipe.yaml").write_text( + "schema: 2\n" + ) with pytest.raises(subprocess.CalledProcessError): changelog_run([{"config-keys": ["single", "multi"]}]) captured = capsys.readouterr() - assert "Cannot resolve gpus-per-node" in captured.out + assert "Recipe has no worker roles" in captured.out assert '\"single_node\":' not in captured.out @@ -736,16 +748,16 @@ def test_generation_api_preserves_inputs_and_returns_independent_nested_rows(pla import copy from infx.matrix.generate import generate_config_matrix - _, master, runners = planning_repo + root, master, runners = planning_repo master["multi"]["router"] = {"name": "fixture-router", "version": "1.0"} original = copy.deepcopy((master, runners)) - rows = generate_config_matrix(["multi"], master, runners, eval_mode="all") + rows = generate_config_matrix(["multi"], master, runners, eval_mode="all", root=root) assert [(r["conc"], r.get("eval-conc")) for r in rows] == [([16, 32, 64], None), ([16, 32], 32)] rows[0]["router"]["name"] = "mutated" rows[0]["prefill"]["tp"] = 999 rows[0]["conc"].append(999) assert (master, runners) == original - repeated = generate_config_matrix(["multi"], master, runners, eval_mode="none") + repeated = generate_config_matrix(["multi"], master, runners, eval_mode="none", root=root) assert repeated[0]["prefill"]["tp"] == 8 assert repeated[0]["conc"] == [16, 32, 64] @@ -782,14 +794,14 @@ def test_plan_rejects_empty_changelog_before_reading_inputs(): def test_generation_api_preserves_json_rejection_for_yaml_sets(planning_repo): from infx.matrix.generate import generate_config_matrix - _, master, runners = planning_repo + root, master, runners = planning_repo # Pydantic accepts this YAML set as a list, but validation returns the raw # config. The generator CLI has always rejected it at JSON serialization. worker = master["multi"]["scenarios"]["fixed-seq-len"][0]["search-space"][0]["prefill"] worker["additional-settings"] = {"A=1"} - validate_master_config(master) + validate_master_config(master, root) with pytest.raises(TypeError, match="set is not JSON serializable"): - generate_config_matrix(["multi"], master, runners, eval_mode="none") + generate_config_matrix(["multi"], master, runners, eval_mode="none", root=root) def test_changelog_move_preserves_history_and_selects_only_additions(tmp_path, monkeypatch): diff --git a/inferencex-e2e/infx/tests/matrix/test_revision.py b/inferencex-e2e/infx/tests/matrix/test_revision.py index 100e932d44..776faab792 100644 --- a/inferencex-e2e/infx/tests/matrix/test_revision.py +++ b/inferencex-e2e/infx/tests/matrix/test_revision.py @@ -38,6 +38,9 @@ def generation_repo(tmp_path, monkeypatch): (tmp_path / "infx/data 中文\t\r\n.bin").write_bytes(b"named asset") (tmp_path / "infx/data-link").symlink_to("data.bin") (tmp_path / ".gitattributes").write_text("infx/data.bin export-ignore\n") + recipe = tmp_path / "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text("{}\n") git = git_in(tmp_path) git("init", "-q") @@ -47,9 +50,10 @@ def generation_repo(tmp_path, monkeypatch): master = {"fixture": { "image": "example/image:stable", "model": name, "model-prefix": "dsr1", "precision": "fp8", "framework": "sglang", "runner": "fixture", - "multinode": False, + "multinode": False, "srt-recipe-dir": "fixture", "scenarios": {"fixed-seq-len": [{ - "isl": 1024, "osl": 1024, "search-space": [{"tp": 1, "conc-list": [conc]}], + "isl": 1024, "osl": 1024, + "search-space": [{"tp": 1, "conc-list": [conc], "srt-recipe": "recipe.yaml"}], }]}, }} (tmp_path / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master)) @@ -134,7 +138,7 @@ def test_historical_generation_from_nested_project(generation_repo, monkeypatch, root, git = generation_repo git("restore", ".") (root / "inferencex-e2e").mkdir() - git("mv", "infx", "configs", "inferencex-e2e/") + git("mv", "infx", "configs", "benchmarks", "inferencex-e2e/") (root / "infx/matrix").mkdir(parents=True) (root / "infx/matrix/generate.py").write_text( 'raise RuntimeError("root-level decoy source was used")\n' diff --git a/inferencex-e2e/infx/tests/matrix/test_validation.py b/inferencex-e2e/infx/tests/matrix/test_validation.py index 9d52b65446..3aeb8510fc 100644 --- a/inferencex-e2e/infx/tests/matrix/test_validation.py +++ b/inferencex-e2e/infx/tests/matrix/test_validation.py @@ -121,6 +121,7 @@ def valid_single_node_master_config(): "framework": "sglang", "runner": "mi300x", "multinode": False, + "srt-recipe-dir": "fixture", "scenarios": { "fixed-seq-len": [ @@ -128,7 +129,7 @@ def valid_single_node_master_config(): "isl": 1024, "osl": 1024, "search-space": [ - {"tp": 8, "conc-start": 4, "conc-end": 64} + {"tp": 8, "conc-start": 4, "conc-end": 64, "srt-recipe": "recipe.yaml"} ] } ] @@ -149,6 +150,7 @@ def valid_multinode_master_config(): "multinode": True, "disagg": True, "kv-p2p-transfer": "nixl", + "srt-recipe-dir": "fixture", "scenarios": { "fixed-seq-len": [ @@ -157,6 +159,7 @@ def valid_multinode_master_config(): "osl": 1024, "search-space": [ { + "srt-recipe": "recipe.yaml", "prefill": { "hardware": "gb200", "num-worker": 5, @@ -188,6 +191,19 @@ def valid_multinode_master_config(): } +@pytest.fixture +def recipe_root(tmp_path): + """Project root holding the fixture configs' recipes.""" + for node in ("single_node", "multi_node"): + recipes = tmp_path / f"benchmarks/{node}/srt-slurm-recipes/fixture" + recipes.mkdir(parents=True) + (recipes / "recipe.yaml").write_text("schema: 2\n") + (recipes / "bundle.yaml").write_text(yaml.safe_dump({ + "base": {}, "override_a": {}, "zip_override_z": {"benchmark": {"c": [1, 2], "d": [3]}}, + })) + return tmp_path + + class TestWorkerConfig: @@ -917,11 +933,13 @@ def test_single_node_agentic_master_config_requires_cluster_runner(self): "framework": "vllm", "runner": "b200", "multinode": False, + "srt-recipe-dir": "fixture", "scenarios": { "agentic-coding": [ { "search-space": [ - {"tp": 8, "conc-list": [1], "kv-offloading": "none"} + {"tp": 8, "conc-list": [1], "kv-offloading": "none", + "srt-recipe": "recipe.yaml"} ], } ] @@ -946,6 +964,7 @@ def test_multinode_agentic_master_config_requires_cluster_runner(self): "multinode": True, "disagg": True, "kv-p2p-transfer": "nixl", + "srt-recipe-dir": "fixture", "scenarios": { "agentic-coding": [ { @@ -953,6 +972,7 @@ def test_multinode_agentic_master_config_requires_cluster_runner(self): { "spec-decoding": "none", "conc-list": [1], + "srt-recipe": "recipe.yaml", "prefill": { "hardware": "b200", "num-worker": 1, @@ -989,10 +1009,92 @@ def test_invalid_config_raises_valueerror(self, valid_single_node_master_config) del valid_single_node_master_config["model"] configs = {"broken-config": valid_single_node_master_config} with pytest.raises(ValueError) as exc_info: - validate_master_config(configs) + validate_master_config(configs, Path()) assert "broken-config" in str(exc_info.value) assert "failed validation" in str(exc_info.value) + @pytest.mark.parametrize("multinode", [False, True]) + @pytest.mark.parametrize(("missing", "error"), [ + ("srt-recipe-dir", "srt-recipe-dir"), + ("srt-recipe", "requires srt-recipe"), + ]) + def test_srt_configs_name_a_recipe_on_every_row( + self, valid_single_node_master_config, valid_multinode_master_config, + multinode, missing, error, + ): + config = valid_multinode_master_config if multinode else valid_single_node_master_config + if missing == "srt-recipe-dir": + del config[missing] + else: + del config["scenarios"]["fixed-seq-len"][0]["search-space"][0][missing] + with pytest.raises(ValueError, match=error): + validate_master_config({"fixture": config}, Path()) + + @pytest.mark.parametrize(("setting", "replacement"), [ + ("CONFIG_FILE=recipes/fixture/recipe.yaml", "use srt-recipe"), + ("EVAL_CONFIG_FILE=recipes/fixture/recipe.yaml", "use eval-srt-recipe"), + ]) + def test_recipe_settings_are_not_additional_settings( + self, valid_multinode_master_config, setting, replacement, + ): + row = valid_multinode_master_config["scenarios"]["fixed-seq-len"][0]["search-space"][0] + row["prefill"]["additional-settings"].append(setting) + with pytest.raises(ValidationError, match=replacement): + MultiNodeMasterConfigEntry(**valid_multinode_master_config) + + def test_llmd_configs_keep_config_file(self, valid_multinode_master_config): + config = valid_multinode_master_config + config["framework"] = "llmd-vllm" + del config["srt-recipe-dir"] + row = config["scenarios"]["fixed-seq-len"][0]["search-space"][0] + del row["srt-recipe"] + row["prefill"]["additional-settings"].append("CONFIG_FILE=llmd.yaml") + assert validate_master_config({"llmd": config}, Path()) == {"llmd": config} + row["srt-recipe"] = "recipe.yaml" + with pytest.raises(ValidationError, match="llmd-vllm selects recipes with CONFIG_FILE"): + MultiNodeMasterConfigEntry(**config) + + def test_eval_srt_recipe_is_multinode_only(self, valid_single_node_master_config): + config = valid_single_node_master_config + config["runner"] = "cluster:b200-nscale" + config["scenarios"] = {"agentic-coding": [{"search-space": [{ + "tp": 8, "conc-list": [1], "kv-offloading": "none", + "srt-recipe": "recipe.yaml", "eval-srt-recipe": "recipe.yaml", + }]}]} + with pytest.raises(ValidationError, match="eval-srt-recipe is only valid for multinode"): + SingleNodeMasterConfigEntry(**config) + + @pytest.mark.parametrize("reference", [ + "/fixture/recipe.yaml", "../recipe.yaml", "sub//recipe.yaml", "./recipe.yaml", "", + "recipe.yaml:", "recipe.yaml:override_*", "recipe.yaml:zip_override_z", "recipe.yaml:z", + ]) + def test_recipe_references_are_relative_paths_with_one_variant(self, reference): + with pytest.raises(ValidationError, match="srt-recipe"): + SingleNodeSearchSpaceEntry(**{"tp": 8, "conc-list": [1], "srt-recipe": reference}) + + @pytest.mark.parametrize(("field", "reference", "error"), [ + ("srt-recipe", "bundle.yaml", None), + ("srt-recipe", "bundle.yaml:base", None), + ("srt-recipe", "bundle.yaml:override_a", None), + ("eval-srt-recipe", "bundle.yaml:zip_override_z[1]", None), + ("srt-recipe", "missing.yaml", "does not exist"), + ("eval-srt-recipe", "missing.yaml:base", "does not exist"), + ("srt-recipe", "bundle.yaml:override_b", "has no variant override_b"), + ("srt-recipe", "bundle.yaml:zip_override_z[2]", r"has no variant zip_override_z\[2\]"), + ("srt-recipe", "recipe.yaml:base", "only an override bundle"), + ]) + def test_load_time_validation_resolves_the_selected_variant( + self, recipe_root, valid_multinode_master_config, field, reference, error, + ): + row = valid_multinode_master_config["scenarios"]["fixed-seq-len"][0]["search-space"][0] + row[field] = reference + configs = {"fixture": valid_multinode_master_config} + if error is None: + assert validate_master_config(configs, recipe_root) is configs + return + with pytest.raises(ValueError, match=error): + validate_master_config(configs, recipe_root) + class TestValidateRunnerConfig: @@ -1234,9 +1336,10 @@ def test_validate_agentic_matrix_entry_dispatches_on_prefill_key(self): class TestLoadConfigFiles: - def test_load_single_file_with_validation(self, tmp_path, valid_single_node_master_config): - config_file = tmp_path / "config.yaml" - import yaml + def test_load_single_file_with_validation(self, recipe_root, valid_single_node_master_config): + # Recipes resolve under the project that holds configs/, not the tooling checkout. + config_file = recipe_root / "configs/config.yaml" + config_file.parent.mkdir() config_file.write_text(yaml.dump({"test-config": valid_single_node_master_config})) result = load_config_files([str(config_file)]) assert "test-config" in result diff --git a/inferencex-e2e/infx/tests/test_installed_package.py b/inferencex-e2e/infx/tests/test_installed_package.py index 714049b46c..582e914ff7 100644 --- a/inferencex-e2e/infx/tests/test_installed_package.py +++ b/inferencex-e2e/infx/tests/test_installed_package.py @@ -60,27 +60,18 @@ def test_installed_tools_use_callers_repository(tmp_path, run_installed, layout) "clusters:\n fixture-gpu:\n gpus-per-node: 8\n arch: x86_64\n" " scheduler: slurm\n slurm: {partition: batch, exclusive: true}\n" ) - recipes = project / "benchmarks/multi_node/srt-slurm-recipes" - recipes.mkdir(parents=True) - (recipes / "fixture.yaml").write_text( - "schema: 2\nroles:\n prefill: {nodes: 2}\n decode: {nodes: 4}\n" - ) result = run_installed( "-c", """ import json -from infx.matrix.generate import recipe_node_count from infx.workflows.calc_success_rate import load_hardware_labels -print(json.dumps({ - "nodes": recipe_node_count({"additional-settings": ["CONFIG_FILE=recipes/fixture.yaml"]}, {}), - "hardware": load_hardware_labels(), -})) +print(json.dumps({"hardware": load_hardware_labels()})) """, ) assert result.returncode == 0, result.stderr - assert json.loads(result.stdout) == {"nodes": 6, "hardware": ["fixture-gpu"]} + assert json.loads(result.stdout) == {"hardware": ["fixture-gpu"]} def test_installed_matrix_validation_rejects_zero_concurrency(run_installed): diff --git a/inferencex-e2e/infx/tests/workflows/test_recover_failed_ingest.py b/inferencex-e2e/infx/tests/workflows/test_recover_failed_ingest.py index 6e1d4ef4cc..34014eca14 100644 --- a/inferencex-e2e/infx/tests/workflows/test_recover_failed_ingest.py +++ b/inferencex-e2e/infx/tests/workflows/test_recover_failed_ingest.py @@ -407,11 +407,16 @@ def git(*args): master = {"fixture": { "image": "example/image:stable", "model": "example/model", "model-prefix": "dsr1", "precision": "fp8", "framework": "sglang", "runner": "fixture", "multinode": False, + "srt-recipe-dir": "fixture", "scenarios": {"fixed-seq-len": [{ - "isl": 8192, "osl": 1024, "search-space": [{"tp": 1, "conc-list": [2]}], + "isl": 8192, "osl": 1024, + "search-space": [{"tp": 1, "conc-list": [2], "srt-recipe": "recipe.yaml"}], }]}, }} (configs / "nvidia-master.yaml").write_text(yaml.safe_dump(master)) + recipe = project / "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text("{}\n") base_bytes = block("fixture", "https://github.com/SemiAnalysisAI/InferenceX/pull/1") (project / "perf-changelog.yaml").write_bytes(base_bytes) git("add", ".") @@ -420,7 +425,7 @@ def git(*args): if head_layout != base_layout: project = repo / head_layout project.mkdir() - git("mv", "configs", "infx", "perf-changelog.yaml", head_layout + "/") + git("mv", "configs", "infx", "benchmarks", "perf-changelog.yaml", head_layout + "/") (project / "perf-changelog.yaml").write_bytes( base_bytes + b"\n" + block("fixture", "https://github.com/SemiAnalysisAI/InferenceX/pull/42") ) diff --git a/operatorx/tests/test_inferencex_generator.py b/operatorx/tests/test_inferencex_generator.py index f03efc1b09..10aa7c0008 100644 --- a/operatorx/tests/test_inferencex_generator.py +++ b/operatorx/tests/test_inferencex_generator.py @@ -30,7 +30,7 @@ def test_generator_uses_requested_checkout_with_safe_path(tmp_path, monkeypatch, script.parent.mkdir(parents=True) script.write_text( "import json, sys\nfrom validation import model\n" - "assert sys.argv[1:] == ['full-sweep', '--config-files', 'master.yaml', '--no-evals']\n" + "assert sys.argv[1:] == ['full-sweep', '--config-files', 'configs/master.yaml', '--no-evals']\n" "print(json.dumps([{'model': model, 'conc': 2}]))\n" ) script.with_name("validation.py").write_text("model = 'legacy revision'\n") @@ -45,22 +45,25 @@ def test_generator_uses_requested_checkout_with_safe_path(tmp_path, monkeypatch, "clusters: {fixture: {gpus-per-node: 8, arch: x86_64, scheduler: slurm," " slurm: {partition: p, exclusive: false}}}\n" ) - (root / "master.yaml").write_text(yaml.safe_dump({"fixture": { + (root / "configs/master.yaml").write_text(yaml.safe_dump({"fixture": { "image": "example/image:stable", "model": "selected revision", "model-prefix": "dsr1", "precision": "fp8", "framework": "sglang", - "runner": "fixture", "multinode": False, + "runner": "fixture", "multinode": False, "srt-recipe-dir": "fixture", "scenarios": {"fixed-seq-len": [{ "isl": 1024, "osl": 1024, - "search-space": [{"tp": 1, "conc-list": [2]}], + "search-space": [{"tp": 1, "conc-list": [2], "srt-recipe": "recipe.yaml"}], }]}, }})) + recipe = root / "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text("{}\n") expected = "selected revision" if layout == "nested": nested = root / "inferencex-e2e" nested.mkdir() - for name in ("infx", "configs", "master.yaml"): + for name in ("infx", "configs", "benchmarks"): (root / name).rename(nested / name) - rows = _run_inferencex_generator(os.path.relpath(root), ["master.yaml"], ["--no-evals"]) + rows = _run_inferencex_generator(os.path.relpath(root), ["configs/master.yaml"], ["--no-evals"]) assert [(row["model"], row["conc"]) for row in rows] == [(expected, 2)]