cquil11 · Oct 13, 2026 · Oct 13, 2026 · Oct 13, 2026 · Oct 13, 2026 · Oct 13, 2026
@@ -1,3 +1,3 @@
 [submodule "utils/srt-slurm"]
 	path = inferencex-e2e/utils/srt-slurm
-	url = https://github.com/NVIDIA/srt-slurm.git
+	url = https://github.com/SemiAnalysisAI/srt-slurm.git
@@ -6,7 +6,7 @@ base:
   name: dsv41flash-fp4-h200-sglang-agentic
   model:
     path: hf:deepseek-ai/DeepSeek-V4.1-Flash
-    container: lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17
+    container: lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34
     precision: fp4
   resources:
     gpu_type: h200
@@ -140,7 +140,16 @@ override_tp4_c16:
       CONC: '16'
       AGENTIC_WARMUP_GRACE_PERIOD: '1800'
 
+# DNM: c32 runs behind a separate sglang-router frontend container.
 override_tp4_c32:
+  frontend:
+    type: sglang-router
+    args:
+      policy: cache_aware
+      request-id-headers: x-correlation-id
+      connect-timeout-secs: 900
+      request-timeout-secs: 14400
+      disable-retries: true
   roles:
     agg:
       gpus: 4

@@ -6,7 +6,7 @@ base:
   name: dsv41flash-fp4-h200-vllm-agentic
   model:
     path: hf:deepseek-ai/DeepSeek-V4.1-Flash
-    container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3
+    container: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7
     precision: fp4
   resources:
     gpu_type: h200
@@ -217,7 +217,15 @@ override_tp4_c16:
     env:
       CONC: '16'
 
+# DNM: c32 runs behind a separate vllm-router frontend container.
 override_tp4_c32:
+  frontend:
+    type: vllm-router
+    args:
+      policy: consistent_hash
+      request-timeout-secs: 14400
+      disable-retries: true
+  setup_script: vllm-router-0.1.14.sh
   roles:
     agg:
       gpus: 4

@@ -5382,8 +5382,8 @@ qwen3.5-fp8-h200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml }
-      - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml }
+      - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml }
+      - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml }
 
 # Qwen3.8-Flash-Next FP8 AgentX on H200 via SGLang with native NEXTN MTP.
 # Day-zero recipe. H200 is Hopper, so this arm is FP8: NVFP4 is greyed out for
@@ -5402,7 +5402,7 @@ qwen3.8next-fp8-h200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.8
       search-space:
-      - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml }
+      - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml }
 # H200 AgentX MTP frontier with DRAM HiCache. This is intentionally an MTP-only
 # submission; the model's non-speculative AgentX arm is not included.
 qwen3.5-fp8-h200-sglang-agentic-hicache-mtp:
@@ -5560,7 +5560,7 @@ minimaxm3-fp8-h200-vllm-agentic-mtp:
     # Use the FP8 MSA indexer cache for all H200 concurrency variants.
     - dram-utilization: 0.80
       search-space:
-      - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12, 14], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml }
 
 qwen3.5-fp4-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:v0.5.16-cu130
@@ -8535,7 +8535,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp:
 # caps batched tokens at 4096. Concurrency is bounded by the measured KV
 # ceiling rather than a guess; see the search-space comment.
 dsv41flash-fp4-h100-vllm-agentic-dspark:
-  image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3
+  image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7
   model: deepseek-ai/DeepSeek-V4.1-Flash
   model-prefix: dsv41flash
   runner: cluster:h100-dgxc
@@ -8727,7 +8727,7 @@ dsv41flash-fp4-gb300-vllm-agentic-dspark:
 # buffer at startup, which is 16 GiB at 1M context and does not fit 80 GB
 # alongside the weights. Tracked separately.
 dsv41flash-fp4-h200-vllm-agentic-dspark:
-  image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3
+  image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7
   model: deepseek-ai/DeepSeek-V4.1-Flash
   model-prefix: dsv41flash
   runner: cluster:h200-dgxc
@@ -8740,13 +8740,12 @@ dsv41flash-fp4-h200-vllm-agentic-dspark:
       search-space:
       # 8x141 GB holds the 511 GB checkpoint minus the UVA-offloaded Engram
       # tables, so the KV cache stays GPU-resident across the full range.
-      - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml }
       # TP4 doubles the per-GPU share to ~99 GiB of weights on each 141 GB H200
       # with the Engram tables offloaded, leaving ~15 GiB of KV per GPU (~17M
       # tokens across the four ranks, about 3x the pool that served c1-c128 on
       # B200 TP2). The script caps batched tokens and the scheduler batch on
       # this arm so the sparse-attention indexer buffer does not consume it.
-      - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml }
+      - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml }
 
 # SGLang arm for DeepSeek-V4.1-Flash AgentX on H200, from the SGLang cookbook
 # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1).
@@ -8758,7 +8757,7 @@ dsv41flash-fp4-h200-vllm-agentic-dspark:
 # Match published vLLM TP4/EP1 and TP8/EP1 using dsv4 attention and the nightly's
 # native MXFP4 x BF16 Marlin MoE runner.
 dsv41flash-fp4-h200-sglang-agentic-dspark:
-  image: lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17
+  image: lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34
   model: deepseek-ai/DeepSeek-V4.1-Flash
   model-prefix: dsv41flash
   runner: cluster:h200-dgxc
@@ -8769,8 +8768,7 @@ dsv41flash-fp4-h200-sglang-agentic-dspark:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml }
-      - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml }
+      - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml }
 
 dsv4-fp4-b200-dynamo-sglang-agentic-agg:
   image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9

@@ -511,6 +511,7 @@ clusters:
             command: "dcgm-exporter --collect-interval=1000 --address :{port} -f /configs/dcgm-counters-noprof.csv"
         network-interface: ""
         job-tag: h200
+        single-node-launcher: docker
         gpus-per-node-directive: true
         segment-directive: false
         container-aliases: [dynamo-trtllm, dynamo-sglang, dynamo-vllm, latest]

@@ -160,6 +160,10 @@ class SrtSlurmSettings(Record):
     )
     single_node_time_limit: int | None = Field(default=None, alias="single-node-time-limit", gt=0)
     single_node_models: Literal["staged", "hub"] = Field(default="hub", alias="single-node-models")
+    # docker: single-node points run srtctl's launcher: docker inside an exclusive salloc.
+    single_node_launcher: Literal["slurm", "docker"] = Field(
+        default="slurm", alias="single-node-launcher"
+    )
     preflight: bool = True
     dist_timeout_s: int | None = Field(default=None, alias="dist-timeout-s", gt=0)
     gpus_per_node_directive: bool | None = Field(default=None, alias="gpus-per-node-directive")

@@ -329,6 +329,13 @@ def _allocate(self, gpus: int, time_min: int) -> Job:
         self.life.callback(cli.cancel, job)
         return job
 
+    def allocate_node(self, time_min: int) -> Job:
+        """``salloc`` one whole node, exclusively, and cancel it when the launch ends."""
+        resources = self._resources(self.cluster.gpus_per_node, time_min)
+        job = cli.salloc(resources, extra=["--exclusive", *self.settings.salloc_args])
+        self.life.callback(cli.cancel, job)
+        return job
+
     def _ensure(self, image: str, policy: SquashPolicy) -> str:
         """Stage ``image`` before any allocation exists (a ``compute`` import runs its own step)."""
         alloc_args = self._import_step_args() if policy.import_mode == "compute" else ()

@@ -8,20 +8,23 @@
 from __future__ import annotations
 
 import os
+import re
 import shlex
+import subprocess
 import sys
 import tempfile
 from collections.abc import Mapping
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 from infx.clusters.slurm import SlurmSettings
-from infx.launch import policy
+from infx.launch import policy, proc
 from infx.launch.backends.base import BackendError
-from infx.launch.backends.slurm import srtctl_job_name
+from infx.launch.backends.slurm import cli, srtctl_job_name
 from infx.launch.context import Launch
 from infx.launch.drivers.srt import collect, config, lanes, models, power, submit
 from infx.launch.drivers.srt.checkout import (
+    Checkout,
     checkout_dir,
     compute_workspace,
     install_srtctl,
@@ -37,6 +40,14 @@
     from infx.clusters.slurm import SrtSlurmSettings
 
 
+# Host-specific names; docker gets every other launch variable, as pyxis would.
+DOCKER_HOST_ENV = re.compile(
+    r"PATH|HOME|USER|LOGNAME|SHELL|PWD|OLDPWD|SHLVL|HOSTNAME|TERM|TMPDIR|MAIL|_|"
+    r"PYTHONPATH|PYTHONHOME|VIRTUAL_ENV|CUDA_VISIBLE_DEVICES|INVOCATION_ID|JOURNAL_STREAM|"
+    r"(LD|LC|XDG|SSH|DBUS|UV|SLURM|SBATCH|SALLOC|SRUN|GITHUB|RUNNER|ACTIONS|NVIDIA)_\w*"
+)
+
+
 def run_single_node(launch: Launch) -> int:
     """One native single-node point: bind its recipe variant, submit, follow, verify."""
     request = SingleNodeRequest.from_env(launch.request.env)
@@ -53,22 +64,35 @@ def run_single_node(launch: Launch) -> int:
     if rc := submit.bind_point(run, checkout, root / "arguments"):
         return rc
     selected, runtime_args = submit.bound_arguments(root / "arguments")
+    docker = run.srt.single_node_launcher == "docker"
     job_config = config.SrtJob(
         srtctl_root=checkout.root,
         workspace=run.workspace,
         time_limit=time_limit,
         image=request.image,
-        container=run.backend.stage_image(request.image, single_node=True).reference,
+        container=request.image
+        if docker
+        else run.backend.stage_image(request.image, single_node=True).reference,
         nginx=config.NGINX_IMAGE if run.srt.nginx_aliases else None,
         model_paths={f"hf:{request.model}": model_path},
         mounts=[(str(hf_cache), request.hf_hub_cache)],
         single_node=True,
         account=run.account,
     )
     config.create_volume_mounts(run)
-    config.write(checkout.root / "srtslurm.yaml", config.render(run.cluster, job_config))
+    rendered = config.render(run.cluster, job_config)
+    if docker:
+        # Pyxis hands a container the whole submit environment; docker only what it is told.
+        names = sorted(n for n in run.env if n.isidentifier() and not DOCKER_HOST_ENV.fullmatch(n))
+        rendered["launcher"] = "docker"
+        rendered["docker_args"] = [
+            "-w", "/infmax-workspace", *(arg for name in names for arg in ("-e", name)),
+        ]  # fmt: skip
+    config.write(checkout.root / "srtslurm.yaml", rendered)
     if rc := run_setup(run, checkout):
         return rc
+    if docker:
+        return _run_single_node_docker(run, checkout, root, selected, runtime_args, time_limit)
 
     fetched = root / "fetched-outputs"
     submitted = submit.Submitted(manifest=run.workspace / submit.SINGLE_NODE_SUBMISSION)
@@ -89,6 +113,45 @@ def run_single_node(launch: Launch) -> int:
     return collect.check_single_node(run, run.backend.fetch_outputs(job, fetched) / "logs")
 
 
+def _run_single_node_docker(
+    run: SrtRun,
+    checkout: Checkout,
+    root: Path,
+    selected: str,
+    runtime_args: list[str],
+    time_limit: str,
+) -> int:
+    """Hold one whole node with ``salloc`` and run ``srtctl apply`` (launcher: docker) on it.
+
+    srtctl runs the job in the foreground on the allocated node, its containers under the
+    node's docker daemon, so the step's exit code is the job's.
+    """
+    allocation = run.backend.allocate_node(int(time_limit))
+    image = run.request.image.replace("#", "/", 1)
+    if cli.srun(allocation, ["docker", "pull", "--quiet", image]):
+        return 1
+    outputs = root / "outputs"
+    # Containers run as root: hand their outputs back so the runner can clean its workspace.
+    owner = f"{os.getuid()}:{os.getgid()}"
+    chown = ["docker", "run", "--rm", "--entrypoint", "chown", "-v", f"{root}:{root}", image]
+    run.life.callback(cli.srun, allocation, [*chown, "-R", owner, str(root)])
+    manifest = run.workspace / submit.SINGLE_NODE_SUBMISSION
+    eval_args = submit.eval_args(run.env, submit.SINGLE_NODE_EVAL_COMMAND)
+    arguments = ["--json", "--yes", "--output", str(outputs), *runtime_args, *eval_args]
+    argv = submit.apply_argv(run, checkout, selected, arguments)
+    env = {**run.env, "RUNNER_NAME": srtctl_job_name(run.request.runner_name)}
+    step = cli.srun_argv(allocation, argv)
+    proc.echo(step, env)
+    with manifest.open("w") as handle:
+        rc = subprocess.run(step, env=env, cwd=checkout.root, stdout=handle, check=False).returncode
+    jobs = sorted(outputs.glob("docker-*"), key=lambda path: path.stat().st_mtime)
+    if not jobs:
+        print(f"ERROR: srtctl wrote no job under {outputs} (exit {rc})", file=sys.stderr)
+        return rc or 1
+    run.life.callback(collect.stage_single_node, run, jobs[-1])
+    return rc or collect.check_single_node(run, jobs[-1] / "logs")
+
+
 def run_batch(launch: Launch) -> int:
     """Re-run this launch's command line inside a batch allocation and follow its log."""
     request = SingleNodeRequest.from_env(launch.request.env)

@@ -54,7 +54,11 @@ def finish_single_node(run: SrtRun, submitted: Submitted, fetched: Path) -> int:
     if job is None:
         return 0
     run.backend.cancel(job)
-    output = run.backend.fetch_outputs(job, fetched)
+    return stage_single_node(run, run.backend.fetch_outputs(job, fetched))
+
+
+def stage_single_node(run: SrtRun, output: Path) -> int:
+    """Stage a single-node job's server logs, result, and AgentX artifacts from ``output``."""
     if not output.is_dir():
         return 0
     rc = 0

@@ -78,6 +78,15 @@ def bound_arguments(arguments: Path) -> tuple[str, list[str]]:
     return selected, runtime_args
 
 
+def apply_argv(run: SrtRun, checkout: Checkout, config: str, arguments: list[str]) -> list[str]:
+    """``srtctl apply`` for ``config`` through the golden AgentX acceptance planner."""
+    return [
+        str(checkout.venv / "bin/python"), "-m", "infx.srt_slurm.synthetic_acceptance",
+        config, run.request.framework, "--",
+        "--set", 'srun_options.container-workdir="/infmax-workspace"', *arguments,
+    ]  # fmt: skip
+
+
 def apply(
     run: SrtRun,
     checkout: Checkout,
@@ -92,11 +101,7 @@ def apply(
     PyTorch's generated module imports fail from / with PYTHONPYCACHEPREFIX set.
     ``arguments`` still win. ``stdout`` receives srtctl's JSON manifest.
     """
-    argv = [
-        str(checkout.venv / "bin/python"), "-m", "infx.srt_slurm.synthetic_acceptance",
-        config, run.request.framework, "--",
-        "--set", 'srun_options.container-workdir="/infmax-workspace"', *arguments,
-    ]  # fmt: skip
+    argv = apply_argv(run, checkout, config, arguments)
     env = {**run.env, "RUNNER_NAME": srtctl_job_name(run.request.runner_name)}
     proc.echo(argv, env)
     with stdout.open("w") as handle:

@@ -14,7 +14,7 @@
 
 from infx.clusters import CLUSTER_LABEL_PREFIX, RunnerInventory
 
-DEFAULT_AGENTIC_DURATION_SECONDS = 3600
+DEFAULT_AGENTIC_DURATION_SECONDS = 1200
 
 """
     The below class defines the field names expected to be present in the JSON entries

@@ -9632,3 +9632,12 @@
     - "Enable SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL and ROCM_QUICK_REDUCE_QUANTIZATION=INT8, previously NONE."
     - "Run TP2 concurrency 32 and 64 on DP2 attention with TP MoE instead of TP-only."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3926
+
+- config-keys:
+    - dsv41flash-fp4-h200-sglang-agentic-dspark
+  scenario-type:
+    - agentic-coding
+  no-evals: true
+  description:
+    - "DNM: 20-minute AgentX points through srt-slurm launcher: docker on h200-dgxc."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3942