From 9410c4e0d8f1a15dda08773b1914d979cee83ef7 Mon Sep 17 00:00:00 2001
From: mkuznet1 <Mikhail.Kuznetsov@amd.com>
Date: Fri, 16 Oct 2026 12:40:01 +0000
Subject: [PATCH 1/8] ci(rccl): add DeepSeek-R1 SGLang 2P+2D to the weekly
 madengine runs

A 4-node prefill/decode-disaggregated inference workload. The baseline is
MAD's sglang overlay recipe built on a digest-pinned base, the candidate is
the same recipe with the CI RCCL swapped in, so both phases differ only in
librccl. The gate compares output throughput at two concurrency-64 sweep
points.

- per-phase /run_logs mount; the RCCL banner is read from the server logs
- weights mounted read-only from the cluster's models_dir
- image rdma-core kept: host libibverbs is not mounted for this workload
- DNS preflight on every node, optional MADENGINE_SLURM_EXCLUDE hostlist
- weekly matrix entry, dispatch choice, unit tests

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .github/workflows/therock-rccl-ci-linux.yml   |   6 +
 .../workflows/therock-rccl-test-madengine.yml |   4 +
 projects/rccl/ci/scripts/test_madengine.py    | 366 ++++++++++++++++--
 .../ci/scripts/tests/test_sglang_disagg.py    | 213 ++++++++++
 4 files changed, 567 insertions(+), 22 deletions(-)
 create mode 100644 projects/rccl/ci/scripts/tests/test_sglang_disagg.py

diff --git a/.github/workflows/therock-rccl-ci-linux.yml b/.github/workflows/therock-rccl-ci-linux.yml
index 0f3e6a750a27..fdad3586a44c 100644
--- a/.github/workflows/therock-rccl-ci-linux.yml
+++ b/.github/workflows/therock-rccl-ci-linux.yml
@@ -316,6 +316,12 @@ jobs:
           - workload: llama-3.1-405b-maxtext
             nodes: '4'
             job_timeout_minutes: '450'
+          # 4 nodes: two prefill, two decode. 360 is its 3 h time_limit plus
+          # 120 minutes, because the head node also pulls the sglang base and
+          # builds both images before the allocation is requested.
+          - workload: deepseek-r1-sglang-disagg
+            nodes: '4'
+            job_timeout_minutes: '360'
     uses: ./.github/workflows/therock-rccl-test-madengine.yml
     secrets: inherit
     with:
diff --git a/.github/workflows/therock-rccl-test-madengine.yml b/.github/workflows/therock-rccl-test-madengine.yml
index 028ab2f80a35..84b1b897520e 100644
--- a/.github/workflows/therock-rccl-test-madengine.yml
+++ b/.github/workflows/therock-rccl-test-madengine.yml
@@ -64,6 +64,7 @@ on:
           - llama-3.1-70b-pp4-training
           - gpt-oss-120b-training
           - llama-3.1-405b-maxtext
+          - deepseek-r1-sglang-disagg
         default: 'llama-3.1-70b-training'
       nodes:
         description: 'Number of SLURM nodes to allocate'
@@ -245,6 +246,8 @@ jobs:
           NCCL_DEBUG: WARN
           MAD_SECRETS_HFTOKEN: ${{ secrets.HF_TOKEN || '' }}
           JOB_TIMEOUT_MINUTES: ${{ inputs.job_timeout_minutes }}
+          # Nodes to keep out of the allocation, as a SLURM hostlist.
+          MADENGINE_SLURM_EXCLUDE: ${{ vars.RCCL_MADENGINE_SLURM_EXCLUDE || '' }}
         run: |
           export PYTHONPATH="rocm-systems/projects/rccl/ci/scripts:${PYTHONPATH}"
 
@@ -302,6 +305,7 @@ jobs:
             ${{ env.WORK_DIR }}/ab/*/manifest.json
             ${{ env.WORK_DIR }}/ab/*/slurm_output/*.out
             ${{ env.WORK_DIR }}/ab/*/slurm_output/*.err
+            ${{ env.WORK_DIR }}/ab/*/slurm_output/run_logs/*/*.log
           retention-days: 30
 
       # What a rerun by hand needs once the cleanup below has deleted WORK_DIR:
diff --git a/projects/rccl/ci/scripts/test_madengine.py b/projects/rccl/ci/scripts/test_madengine.py
index c23edcab3385..536a5d8ba035 100644
--- a/projects/rccl/ci/scripts/test_madengine.py
+++ b/projects/rccl/ci/scripts/test_madengine.py
@@ -81,6 +81,14 @@
 # The maxtext family needs a second image: its A/B baseline cannot be the stock
 # registry tag, which carries no Primus. See build_maxtext_baseline_image().
 MAXTEXT_BASELINE_DOCKERFILE = "Dockerfile.maxtext-baseline"
+# Likewise for sglang-disagg, whose baseline is MAD's Dockerfile built here.
+SGLANG_BASELINE_DOCKERFILE = "Dockerfile.sglang-baseline"
+SGLANG_MAD_DOCKERFILE = "docker/sglang_disagg_inference_full_overlay.ubuntu.amd.Dockerfile"
+# Families whose baseline is built here rather than pulled as the stock tag.
+LOCAL_BASELINE_DOCKERFILES = {
+    "maxtext": MAXTEXT_BASELINE_DOCKERFILE,
+    "sglang-disagg": SGLANG_BASELINE_DOCKERFILE,
+}
 
 # The A/B below needs both runs in one allocation, which requires madengine to
 # execute a multi-node workload in place instead of submitting its own sbatch
@@ -277,6 +285,116 @@
             "--ulimit memlock=-1 --ulimit nofile=1048576 "
             "-v /sys:/sys:ro -v /run/udev:/run/udev:ro",
     },
+    # The first inference workload: DeepSeek-R1 served disaggregated, two
+    # prefill and two decode nodes, KV cache moved by nixl. MoE all-to-all is
+    # left to RCCL (MOE_A2A_BACKEND=none) rather than MoRI, so expert traffic
+    # is what the candidate library carries.
+    #
+    # Both A/B images are MAD's sglang Dockerfile built here; see
+    # build_sglang_baseline_image().
+    "deepseek-r1-sglang-disagg": {
+        "type": "inference",
+        "family": "sglang-disagg",
+        # madengine names each result row <this>_2p2d_isl<I>_osl<O>_con<C>.
+        "model_repo": "sglang-disagg-deepseek-r1",
+        "base_image": "rocm/sgl-dev:v0.5.16-rocm720-mi35x-20260807",
+        "gpu_target": "gfx950",
+        "metric_key": "output_token_throughput_tok_s",
+        "metric_unit": "output tok/s of the whole 2P+2D deployment",
+        "multiple_results": "perf_sglang-disagg-DeepSeek-R1.csv",
+        # The sweep points that are scored, in place of precisions; the rest of
+        # the sweep is run and ignored. Only the highest concurrency: the lower
+        # ones are latency-bound and say little about the collectives.
+        "precisions": ["isl1024_osl1024_con64", "isl8192_osl1024_con64"],
+        "tags": ["pyt", "sglang", "disagg", "inference", "deepseek-r1"],
+        "model_args": "--model-name DeepSeek-R1 --model-path /models/DeepSeek-R1-0528 "
+            "--run-mori 1 --dp-mode 1 --xp 2 --yd 2 --gpus-per-node 8 "
+            "--kv-transfer-backend nixl",
+        "disagg": {"prefill_nodes": 2, "decode_nodes": 2},
+        "env_vars": {
+            "MODEL_NAME": "DeepSeek-R1",
+            "MODEL_PATH": "/models/DeepSeek-R1-0528",
+            "RUN_MORI": "1",
+            "DP_MODE": "1",
+            "MOE_A2A_BACKEND": "none",
+            "xP": "2",
+            "yD": "2",
+            "GPUS_PER_NODE": "8",
+            "KV_TRANSFER_BACKEND": "nixl",
+            # INFO for the same reason as the MaxText entry: at WARN only a
+            # communicator's rank 0 prints the version banner, and a server
+            # spanning two nodes has it on one of them.
+            "NCCL_DEBUG": "INFO",
+            "NCCL_DEBUG_SUBSYS": "INIT,NET",
+            # Thor2 settings the sglang stack runs with, over the cluster's.
+            "NCCL_IB_QPS_PER_CONNECTION": "1",
+            "NCCL_IB_USE_INLINE": "0",
+            "NCCL_IB_MERGE_NICS": "0",
+            "NCCL_IB_SPLIT_DATA_ON_QPS": "0",
+            "NCCL_IB_ADAPTIVE_ROUTING": "0",
+            "NCCL_GDR_FLUSH_DISABLE": "1",
+            "NCCL_DMABUF_ENABLE": "0",
+            "NCCL_IB_ROCE_VERSION_NUM": "2",
+            "IB_DEVICES": "bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,"
+                "bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7",
+            "MORI_RDMA_DEVICES": "bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,"
+                "bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7",
+            "MORI_SOCKET_IFNAME": "fenic0",
+            "MORI_IB_GID_INDEX": "3",
+            "MORI_JIT_CACHE_DIR": "/tmp/mori_jit",
+            "MORI_SHMEM_HEAP_SIZE": "17179869184",
+            "RDMAV_DRIVERS": "bnxt_re",
+            "IBV_DRIVERS": "bnxt_re",
+            "GPU_MAX_HW_QUEUES": "2",
+            "HSA_ENABLE_SDMA": "0",
+            "HSA_FORCE_FINE_GRAIN_PCIE": "1",
+            # The common OMP/TORCH_NCCL settings reach only the deployment
+            # env, not the container, and without OMP_NUM_THREADS each of the
+            # eight server processes per node sizes its pool to every core.
+            "OMP_NUM_THREADS": "8",
+            "TORCH_NCCL_HIGH_PRIORITY": "1",
+            "TORCH_NCCL_ASYNC_ERROR_HANDLING": "1",
+            # Part of meta64 has less VRAM per GPU, and sglang refuses a TP
+            # group spanning both kinds ("The memory capacity is unbalanced").
+            # Off, it warns and sizes the KV pool by the smaller GPU; both A/B
+            # phases share the nodes, so the ratio is unaffected.
+            "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
+            # A hang otherwise holds four nodes for two hours before failing.
+            "BARRIER_TIMEOUT_S": "1800",
+            "IP_SYNC_TIMEOUT": "1800",
+            "SMOKE": "0",
+            # A point is only 2 x concurrency requests, and one pass at
+            # isl1024 con64 moved 6.8% between two A/A phases on the same
+            # nodes. The parser keeps each point's best iteration, so three
+            # passes of just the gated points cost about what the full sweep
+            # did.
+            "BENCHMARK_ITR": "3",
+            "BENCHMARK_COMBINATIONS": "1024/1024 8192/1024",
+            "BENCHMARK_CONCURRENCY_LEVELS": "64",
+            "BENCHMARK_PRECHECK": "0",
+            "BENCHMARK_FAIL_FAST": "1",
+        },
+        # The image builds its own rdma-core 63 for Thor2; the host's 61
+        # mounted over it would put nixl and MoRI on a stack never run here.
+        "mount_host_ib_libs": False,
+        "required_nodes": 4,
+        "slurm_partition": "meta64",
+        "gpus_per_node": 8,
+        # One phase took 31 min by hand with the image cached; each phase here
+        # also builds, saves and loads it.
+        "time_limit": "03:00:00",
+        "docker_mounts": {
+            "/dev/infiniband": "/dev/infiniband",
+            "/sys/class/infiniband": "/sys/class/infiniband",
+        },
+        # --init: the launcher polls its servers with `kill -0`, which succeeds
+        # on a zombie, and without a reaping PID 1 every node exits 1 after a
+        # complete sweep.
+        "docker_run_options": "--init --privileged --group-add render --shm-size 64G "
+            "--device=/dev/infiniband --cap-add IPC_LOCK --ulimit memlock=-1 "
+            "--ulimit nofile=1048576:1048576 "
+            "-v /sys:/sys:ro -v /run/udev:/run/udev:ro -e SLURM_JOB_ID",
+    },
 }
 
 CLUSTER_CONFIGS = {
@@ -297,6 +415,8 @@
             "NCCL_DEBUG": "WARN",
         },
         "results_base": "/apps/rccl-ci/madengine/perf",
+        # Mounted read-only at /models for the inference workloads.
+        "models_dir": "/apps/data/models",
     },
 }
 
@@ -424,6 +544,9 @@ def install_madengine(work_dir: Path, family: str = "megatron") -> Path:
     if family == "maxtext":
         scripts_src = mad_dir / "scripts" / "jax-maxtext"
         scripts_name = "jax-maxtext"
+    elif family == "sglang-disagg":
+        scripts_src = mad_dir / "scripts" / "sglang_disagg"
+        scripts_name = "sglang_disagg"
     else:
         scripts_src = mad_dir / "scripts" / "primus_megatron-lm"
         if not scripts_src.is_dir():
@@ -436,18 +559,28 @@ def install_madengine(work_dir: Path, family: str = "megatron") -> Path:
         subprocess.run(["cp", "-r", str(scripts_src), str(scripts_dst)], check=True)
         log.info("Copied MAD %s scripts to %s", scripts_name, scripts_dst)
     elif not scripts_src.is_dir():
-        # Fatal for maxtext: the manifest points `scripts` at run.sh, and a
-        # missing one surfaces as a container-side "no such file" on every
-        # node instead of here. The megatron path keeps its historical warning.
+        # Fatal for maxtext and sglang: the manifest points `scripts` at
+        # run.sh, and a missing one surfaces as a container-side "no such
+        # file" on every node instead of here. The megatron path keeps its
+        # historical warning.
         if family == "maxtext":
             raise RuntimeError(
                 f"MAD jax-maxtext scripts not found at {scripts_src} -- "
                 f"MAD pin {MAD_REF[:12]} predates ROCm/MAD#228"
             )
+        if family == "sglang-disagg":
+            raise RuntimeError(
+                f"MAD sglang_disagg scripts not found at {scripts_src} -- "
+                f"MAD pin {MAD_REF[:12]} does not carry them"
+            )
         log.error("MAD primus scripts not found — expected at %s", scripts_src)
 
     if family == "maxtext":
         fetch_primus(mad_dir)
+    if family == "sglang-disagg" and not (mad_dir / SGLANG_MAD_DOCKERFILE).is_file():
+        raise RuntimeError(
+            f"MAD pin {MAD_REF[:12]} has no {SGLANG_MAD_DOCKERFILE}"
+        )
 
     result = subprocess.run(
         ["madengine", "--version"],
@@ -610,11 +743,27 @@ def squash_log_spam(run_dir: Path) -> int:
     return saved
 
 
+def rccl_banner_logs(work_dir: Path) -> list[Path]:
+    """One log per node of a phase, the one its RCCL version banner lands in.
+
+    madengine's per-node logs, unless the phase has sglang server logs: that
+    launcher sends the servers' output, and with it the banner, to /run_logs
+    (one prefill or decode log per node) instead.
+    """
+    log_dir = work_dir / "slurm_output"
+    if not log_dir.is_dir():
+        return []
+    server_logs = sorted(
+        p for p in log_dir.glob("run_logs/*/*_NODE*.log")
+        if p.name.startswith(("prefill_", "decode_"))
+    )
+    return server_logs or sorted(log_dir.glob("*node_*.out"))
+
+
 def runtime_rccl_versions(work_dir: Path) -> list[str]:
     """Distinct ``RCCL version :`` banners across one phase's node logs."""
-    log_dir = work_dir / "slurm_output"
     versions: list[str] = []
-    for log_file in sorted(log_dir.glob("*node_*.out")) if log_dir.is_dir() else []:
+    for log_file in rccl_banner_logs(work_dir):
         for v in re.findall(r"RCCL version\s*:\s*(\S+)", log_file.read_text(errors="replace")):
             if v not in versions:
                 versions.append(v)
@@ -628,7 +777,7 @@ def verify_rccl_replacement(
 ) -> tuple[bool, str]:
     """Verify that every node used the CI-built RCCL, not its bundled copy.
 
-    Checks *all* ``*node_*.out`` logs for ``RCCL version :`` and compares
+    Checks every node's log (rccl_banner_logs) for ``RCCL version :`` and compares
     against the expected fingerprint.  A library that reached only one node
     still fails — this is the multi-node failure this job exists to catch.
 
@@ -638,8 +787,7 @@ def verify_rccl_replacement(
     if not expected.get("version"):
         return True, "No RCCL version to verify"
 
-    log_dir = work_dir / "slurm_output"
-    node_logs = sorted(log_dir.glob("*node_*.out")) if log_dir.is_dir() else []
+    node_logs = rccl_banner_logs(work_dir)
     if not node_logs:
         return False, "No node logs found — cannot verify RCCL"
 
@@ -1032,6 +1180,58 @@ def build_maxtext_baseline_image(
     return tag
 
 
+# MAD's Dockerfile ends with `dpkg -r --force-all` on the distro rdma-core,
+# which leaves apt unusable, so the swap's patchelf has to come from pip.
+SGLANG_SWAP_TOOLS = (
+    "RUN command -v patchelf >/dev/null || pip install --no-cache-dir patchelf\n"
+)
+
+
+def sglang_recipe(mad_dir: Path, from_ref: str) -> str:
+    """MAD's sglang Dockerfile on *from_ref*, plus what the RCCL swap needs.
+
+    The base becomes a literal FROM rather than the ARG default, so no build
+    argument can move it off the digest both A/B images were pinned to.
+    """
+    text = (mad_dir / SGLANG_MAD_DOCKERFILE).read_text()
+    pinned, count = re.subn(
+        r"(?m)^FROM \$\{?BASE_DOCKER\}?\s*$", f"FROM {from_ref}", text
+    )
+    if count != 1:
+        raise RuntimeError(
+            f"{SGLANG_MAD_DOCKERFILE} at MAD {MAD_REF[:12]} has {count} "
+            f"`FROM $BASE_DOCKER` lines, expected 1"
+        )
+    return f"{pinned}\n{SGLANG_SWAP_TOOLS}"
+
+
+def build_sglang_baseline_image(
+    base_image: str,
+    work_dir: Path,
+    run_token: str,
+    from_ref: str,
+) -> str:
+    """Build the A/B baseline for sglang-disagg: MAD's image, stock RCCL.
+
+    The base alone is not what MAD runs: its Dockerfile adds rdma-core 63,
+    without which queue-pair creation fails on Thor2. Built every run and
+    tagged like the MaxText baseline, for the same reasons.
+    """
+    ctx_dir = work_dir / OVERLAY_CTX
+    ctx_dir.mkdir(exist_ok=True)
+    dockerfile = ctx_dir / SGLANG_BASELINE_DOCKERFILE
+    dockerfile.write_text(sglang_recipe(work_dir / "MAD", from_ref))
+
+    tag = f"{base_image}-mad-{MAD_REF[:8]}-{run_token}-{base_identity(from_ref)}"
+    log.info("Building sglang baseline image (MAD recipe, stock RCCL): %s", tag)
+    subprocess.run(
+        ["docker", "build", "-t", tag, "-f", str(dockerfile), str(ctx_dir)],
+        check=True,
+    )
+    log.info("sglang baseline image built: %s", tag)
+    return tag
+
+
 def stage_rccl_overlay(
     rccl_lib: Path,
     base_image: str,
@@ -1054,7 +1254,7 @@ def stage_rccl_overlay(
     """
     rccl_commit = get_rccl_commit(rccl_lib)
     tag = f"{base_image}-rccl-{gpu_target}-{rccl_commit}"
-    if family == "maxtext" and from_ref:
+    if family in ("maxtext", "sglang-disagg") and from_ref:
         # Same identity the baseline carries, for the same reason: the run id
         # alone survives a re-run, so without this the candidate could hit the
         # previous attempt's image and skip its build while the baseline was
@@ -1129,6 +1329,18 @@ def stage_rccl_overlay(
             f"FROM {from_ref or base_image}\n\n{MAXTEXT_PRIMUS_PAYLOAD}\n"
             f"COPY rccl_libs/ /tmp/rccl_ci/\n{swap}\nENV NCCL_DEBUG=WARN\n"
         )
+    elif family == "sglang-disagg":
+        # The baseline's recipe plus the swap, as for MaxText: the sgl-dev
+        # base does not have the Primus image's layout, so the targets are
+        # discovered rather than assumed.
+        swap = _librccl_swap_snippet(
+            bridge=_soname_bridge_discovered(rccl_needed),
+            kpack=has_kpack_files,
+        )
+        dockerfile.write_text(
+            f"{sglang_recipe(work_dir / 'MAD', from_ref or base_image)}\n"
+            f"COPY rccl_libs/ /tmp/rccl_ci/\n{swap}\nENV NCCL_DEBUG=WARN\n"
+        )
     elif uses_kpack:
         log.info(
             "CI-built librccl.so uses kpack (%.1f MB .so). "
@@ -1232,6 +1444,8 @@ def generate_manifest(
     family = workload_config.get("family", "megatron")
     if family == "maxtext":
         scripts_dir = work_dir / "scripts" / "jax-maxtext"
+    elif family == "sglang-disagg":
+        scripts_dir = work_dir / "scripts" / "sglang_disagg"
     else:
         scripts_dir = work_dir / "scripts" / "primus_megatron-lm"
         if not scripts_dir.is_dir():
@@ -1245,6 +1459,8 @@ def generate_manifest(
         extra_args = workload_config.get("model_args_extra", "")
         if extra_args:
             model_args += f" {extra_args}"
+    elif family == "sglang-disagg":
+        model_args = workload_config["model_args"]
     else:
         model_args = f"--model_repo {model_repo}"
 
@@ -1271,7 +1487,9 @@ def generate_manifest(
     # /usr/lib64/libibverbs/ so the providers must appear there.
     # The library itself replaces the container's copy so the linker
     # picks it up from the standard search path.
-    if cluster_config.get("mount_host_ib_libs"):
+    if cluster_config.get("mount_host_ib_libs") and workload_config.get(
+        "mount_host_ib_libs", True
+    ):
         docker_run_opts += (
             " -v /usr/lib64/libibverbs.so.1"
             ":/usr/lib/x86_64-linux-gnu/libibverbs.so.1:ro"
@@ -1280,6 +1498,20 @@ def generate_manifest(
             ":/usr/lib/x86_64-linux-gnu/libibumad.so.3:ro"
         )
 
+    if family == "sglang-disagg":
+        # The launcher writes its server logs, and the RCCL banner the
+        # candidate is verified by, to /run_logs/$SLURM_JOB_ID. Both phases
+        # share one job id, so each gets its own directory. It must be shared
+        # storage and world-writable: Docker runs as root, which a
+        # root-squashed NFS maps to nobody.
+        run_logs = (run_dir or work_dir) / "slurm_output" / "run_logs"
+        run_logs.mkdir(parents=True, exist_ok=True)
+        run_logs.chmod(0o777)
+        docker_run_opts += f" -v {run_logs}:/run_logs"
+        # Read-only: run.sh downloads the weights into MODEL_PATH when they
+        # look incomplete, and that must fail rather than fill shared storage.
+        docker_run_opts += f" -v {cluster_config['models_dir']}:/models:ro"
+
     # When --skip-overlay-build is used the base image still has its
     # bundled RCCL.  Bind-mount the CI-built librccl.so (and kpack
     # files if present) over the container's copies so we actually
@@ -1289,9 +1521,9 @@ def generate_manifest(
         # do not exist, and `docker run -v` CREATES a missing source as an empty
         # root-owned directory rather than refusing -- so the run would look
         # healthy while measuring the image's own RCCL. Refuse instead.
-        if family == "maxtext":
+        if family in ("maxtext", "sglang-disagg"):
             raise RuntimeError(
-                "--skip-overlay-build is not supported for the maxtext family: "
+                f"--skip-overlay-build is not supported for the {family} family: "
                 "the bind-mount targets are Primus-image paths absent from "
                 + workload_config["base_image"] + ", and Docker would silently "
                 "mount empty directories over nothing. Build the overlay."
@@ -1397,7 +1629,7 @@ def generate_manifest(
             "target": "slurm",
             "slurm": slurm_config,
             "distributed": {
-                "launcher": "primus",
+                "launcher": "sglang-disagg" if family == "sglang-disagg" else "primus",
                 "backend": "nccl",
                 "port": 29500,
                 "nnodes": nodes,
@@ -1417,6 +1649,11 @@ def generate_manifest(
                     if family == "maxtext"
                     else {}
                 ),
+                **(
+                    {"sglang_disagg": workload_config["disagg"]}
+                    if family == "sglang-disagg"
+                    else {}
+                ),
             },
             "env_vars": {
                 **docker_env_vars,
@@ -1515,6 +1752,14 @@ def repro_manifest(manifest: dict, work_dir: Path) -> dict:
         model = out.get("built_models", {}).get(key)
         if model is not None:
             model["dockercontext"] = OVERLAY_CTX
+    # The CI phase directory is gone by then; a rerun salloc's from its copy.
+    for model in out.get("built_models", {}).values():
+        if "additional_docker_run_options" in model:
+            model["additional_docker_run_options"] = re.sub(
+                r"-v \S+/slurm_output/run_logs:/run_logs",
+                "-v ${SLURM_SUBMIT_DIR}/slurm_output/run_logs:/run_logs",
+                model["additional_docker_run_options"],
+            )
     return out
 
 
@@ -1624,6 +1869,9 @@ def repro_env(
 
 - The baseline phase is `manifest.baseline.repro.json`. Run it from a second copy
   of this directory: each phase writes `perf.csv` and `slurm_output/` to its cwd.
+- An sglang-disagg run also needs `mkdir -m 0777 -p slurm_output/run_logs` in
+  each copy before `salloc`, and the weights under `/models` as the manifest
+  mounts them.
 - To shorten a Megatron run, add `"PRIMUS_TRAIN_ITERS": "15"` to both
   `context.docker_env_vars` and `deployment_config.env_vars`.
 - `BASE_DIGEST` is the base the CI head node built from. The nodes build from
@@ -1680,6 +1928,21 @@ def write_repro_bundle(
     return bundle
 
 
+# The sglang launcher reaches every node at the address DNS gives its name and
+# binds its barrier to the node's own one, so a stale record leaves rank 0
+# waiting out BARRIER_TIMEOUT_S for a node that is up and never answers.
+PREFLIGHT_DNS = """\
+#!/bin/bash
+h=$(hostname -s)
+ip=$(getent ahostsv4 "$h" | awk 'NR==1 {print $1}')
+if [ -n "$ip" ] && hostname -I | tr ' ' '\\n' | grep -qxF "$ip"; then
+    echo "  ok   $h -> $ip"
+else
+    echo "  BAD  $h -> ${ip:-unresolved}, not an address of this node"
+    exit 1
+fi
+"""
+
 # Run on every node once the phases are done. The images are per-run tags that
 # no later run can reuse, and each is tens of GB on a node's local disk.
 CLEANUP_IMAGES = """\
@@ -1768,6 +2031,20 @@ def run_ab_in_one_allocation(
         "  exit 3",
         "}",
     ]
+    if workload_config.get("family") == "sglang-disagg":
+        dns_check = work_dir / "ab" / "preflight_dns.sh"
+        dns_check.parent.mkdir(parents=True, exist_ok=True)
+        dns_check.write_text(PREFLIGHT_DNS)
+        script_lines += [
+            'echo "===== preflight: every node resolves to an address it owns ====="',
+            'srun --ntasks="${SLURM_JOB_NUM_NODES}" --ntasks-per-node=1 '
+            f"--overlap bash {shlex.quote(str(dns_check))} || {{",
+            '  echo "::error::At least one allocated node resolves to an address '
+            'it does not have (see BAD above); the sglang barrier would wait for '
+            'it until timeout. Exclude it via MADENGINE_SLURM_EXCLUDE."',
+            "  exit 3",
+            "}",
+        ]
     for phase, run_dir in phases:
         script_lines += [
             "",
@@ -1806,6 +2083,15 @@ def run_ab_in_one_allocation(
     qos = cluster_config.get("slurm_qos", "")
     if qos:
         cmd += ["--qos", qos]
+    # Set from a repository variable by the workflow: the nodes to avoid are
+    # the cluster's business and their names do not belong in this file.
+    exclude = os.environ.get("MADENGINE_SLURM_EXCLUDE", "").strip()
+    if exclude:
+        if not re.fullmatch(r"[A-Za-z0-9_.,\[\]-]+", exclude):
+            raise RuntimeError(
+                f"MADENGINE_SLURM_EXCLUDE is not a SLURM hostlist: {exclude!r}"
+            )
+        cmd += ["--exclude", exclude]
     cmd += ["bash", str(script_path)]
 
     log.info("Running: %s", " ".join(cmd))
@@ -1979,17 +2265,38 @@ def parse_live_log_metrics(work_dir: Path) -> list[dict]:
     return runs
 
 
+_SWEEP_POINT_RE = re.compile(r"_(isl\d+_osl\d+_con\d+)$")
+
+
+def _row_precision(row: dict) -> str:
+    """The key a result row is paired on: its precision, or its sweep point.
+
+    An inference sweep has no precision; its rows are named
+    ``<model>_<xPyD>_isl<I>_osl<O>_con<C>``, and the point is what the two
+    phases have in common.
+    """
+    precision = (row.get("training_precision")
+                 or row.get("multi_results", {}).get("precision", ""))
+    if precision:
+        return precision
+    point = _SWEEP_POINT_RE.search(row.get("model", ""))
+    return point.group(1) if point else ""
+
+
 def collect_phase_results(
     run_dir: Path,
     metric_key: str,
     extra_aliases: set[str] | None = None,
     extra_tflops_aliases: set[str] | None = None,
+    only: list[str] | None = None,
 ) -> list[dict]:
     """Per-precision metrics for one phase.
 
     Prefers madengine's structured output and falls back to scraping the live
     log.  Each entry carries precision, metric value and a pass/fail status, so
     the A/B comparison and the datastore writes are driven from one list.
+
+    *only* keeps the rows of those precisions (sweep points) and drops the rest.
     """
     perf_results = parse_perf_results(run_dir)
     results: list[dict] = []
@@ -2012,8 +2319,9 @@ def collect_phase_results(
             if row.get("metric") not in accepted_metrics:
                 continue
             perf_val = row.get("performance", "")
-            precision = (row.get("training_precision")
-                         or row.get("multi_results", {}).get("precision", ""))
+            precision = _row_precision(row)
+            if only is not None and precision not in only:
+                continue
             row_status = row.get("status", "")
             if not perf_val:
                 continue
@@ -2041,8 +2349,7 @@ def collect_phase_results(
         for row in perf_results:
             if row.get("metric") not in accepted_tflops:
                 continue
-            prec = (row.get("training_precision")
-                    or row.get("multi_results", {}).get("precision", ""))
+            prec = _row_precision(row)
             try:
                 tflops_by_precision[prec] = float(row["performance"])
             except (KeyError, ValueError, TypeError):
@@ -2203,6 +2510,7 @@ def generate_summary_report(
     nodelist: str = "",
     job_id: str = "",
     phase_rccl: dict[str, str] | None = None,
+    unit: str = "tok/s/GPU",
 ) -> str:
     """Generate a plain-text summary report."""
     status = "PASSED" if exit_code == 0 else "FAILED"
@@ -2223,7 +2531,7 @@ def generate_summary_report(
         "Both numbers below were measured on those same nodes, back to back,",
         "inside one allocation. Baseline is the stock image; candidate is the",
         "same image with the CI-built RCCL laid over it. Throughput in",
-        "tok/s/GPU.",
+        f"{unit}.",
         "",
     ]
 
@@ -2369,7 +2677,7 @@ def main() -> None:
     # Resolved once and written into both A/B Dockerfiles, so the pair cannot
     # straddle a push to the mutable base tag between the two builds.
     base_ref = ""
-    if family == "maxtext" and not args.skip_overlay_build:
+    if family in ("maxtext", "sglang-disagg") and not args.skip_overlay_build:
         base_ref = resolve_base_digest(workload_config["base_image"])
 
     # Step 3: Stage the candidate image (or use pre-built). Images are built
@@ -2404,6 +2712,13 @@ def main() -> None:
             from_ref=base_ref or workload_config["base_image"],
         )
         baseline_pull_only = False
+    elif family == "sglang-disagg":
+        baseline_image = build_sglang_baseline_image(
+            workload_config["base_image"], work_dir,
+            run_token=rccl_commit,
+            from_ref=base_ref or workload_config["base_image"],
+        )
+        baseline_pull_only = False
     else:
         baseline_image = workload_config["base_image"]
         baseline_pull_only = True
@@ -2435,8 +2750,8 @@ def main() -> None:
             # and have it rebuild the swapped image under the baseline tag --
             # an A/B comparing the CI RCCL against itself.
             dockerfile_name=(
-                MAXTEXT_BASELINE_DOCKERFILE
-                if family == "maxtext" and phase == BASELINE
+                LOCAL_BASELINE_DOCKERFILES.get(family, OVERLAY_DOCKERFILE)
+                if phase == BASELINE
                 else OVERLAY_DOCKERFILE
             ),
         )
@@ -2505,6 +2820,9 @@ def main() -> None:
             run_dir, metric_key,
             extra_aliases=workload_config.get("metric_aliases"),
             extra_tflops_aliases=workload_config.get("tflops_aliases"),
+            # A sweep reports many points; only the configured ones are scored.
+            only=(workload_config["precisions"]
+                  if workload_config["type"] == "inference" else None),
         )
 
         # Matrix legs share GITHUB_RUN_ID, so the workload keeps them apart.
@@ -2604,7 +2922,10 @@ def main() -> None:
                     extra=phase_extra,
                     precision=c["precision"],
                     tflops=row.get("tflops_avg"),
-                    tokens_per_sec=row.get("metric_value"),
+                    # Per GPU for training only; inference scores the whole
+                    # deployment, and metric_value already carries that.
+                    tokens_per_sec=(row.get("metric_value")
+                                    if workload_config["type"] == "training" else None),
                 )
     else:
         append_result(
@@ -2629,6 +2950,7 @@ def main() -> None:
             phase: ", ".join(runtime_rccl_versions(run_dir))
             for phase, run_dir in phases
         },
+        unit=workload_config.get("metric_unit", "tok/s/GPU"),
     )
     log.info("\n%s", report)
     write_github_summary(report)
diff --git a/projects/rccl/ci/scripts/tests/test_sglang_disagg.py b/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
new file mode 100644
index 000000000000..0af9f4fe4d57
--- /dev/null
+++ b/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
@@ -0,0 +1,213 @@
+"""Tests for the sglang-disagg family in test_madengine.py.
+
+Run from projects/rccl/ci/scripts:  python3 -m unittest discover -s tests
+"""
+
+import json
+import os
+import re
+import stat
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, os.fspath(Path(__file__).resolve().parent.parent))
+import test_madengine as tm
+
+WORKLOAD = "deepseek-r1-sglang-disagg"
+CONFIG = tm.WORKLOAD_CONFIGS[WORKLOAD]
+RUBY = tm.CLUSTER_CONFIGS["ruby"]
+MAD_DOCKERFILE = """\
+# CONTEXT {'gpu_vendor': 'AMD', 'guest_os': 'UBUNTU'}
+ARG BASE_DOCKER=rocm/sgl-dev:v0.5.16-rocm720-mi35x-20260807
+FROM $BASE_DOCKER
+RUN echo payload
+"""
+
+
+def sweep_row(point: str, metric: str, value: float, status: str = "SUCCESS") -> dict:
+    return {"model": f"{CONFIG['model_repo']}_2p2d_{point}", "metric": metric,
+            "performance": value, "training_precision": "",
+            "multi_results": {"launcher": "sglang-disagg"}, "status": status}
+
+
+class TempWork(unittest.TestCase):
+    def setUp(self):
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.work = Path(tmp.name) / "work"
+        (self.work / "scripts" / "sglang_disagg").mkdir(parents=True)
+        (self.work / "MAD" / "docker").mkdir(parents=True)
+        (self.work / "MAD" / tm.SGLANG_MAD_DOCKERFILE).write_text(MAD_DOCKERFILE)
+
+    def manifest(self, phase: str) -> dict:
+        run_dir = tm.prepare_phase_dir(self.work, phase)
+        path = tm.generate_manifest(
+            WORKLOAD, CONFIG, RUBY, f"image-{phase}", CONFIG["required_nodes"],
+            self.work, run_dir=run_dir,
+            dockerfile_name=(tm.SGLANG_BASELINE_DOCKERFILE if phase == tm.BASELINE
+                             else tm.OVERLAY_DOCKERFILE),
+        )
+        return json.loads(path.read_text())
+
+
+class SweepTest(unittest.TestCase):
+    def test_every_gate_point_is_benchmarked(self):
+        env = CONFIG["env_vars"]
+        combos = set(env["BENCHMARK_COMBINATIONS"].split())
+        levels = set(env["BENCHMARK_CONCURRENCY_LEVELS"].split())
+        for point in CONFIG["precisions"]:
+            isl, osl, con = re.fullmatch(r"isl(\d+)_osl(\d+)_con(\d+)", point).groups()
+            self.assertIn(f"{isl}/{osl}", combos, point)
+            self.assertIn(con, levels, point)
+
+
+class GenerateManifestTest(TempWork):
+    def test_launcher_and_layout(self):
+        m = self.manifest(tm.CANDIDATE)
+        dist = m["deployment_config"]["distributed"]
+        self.assertEqual(dist["launcher"], "sglang-disagg")
+        self.assertEqual(dist["sglang_disagg"], {"prefill_nodes": 2, "decode_nodes": 2})
+        self.assertEqual(dist["nnodes"], 4)
+        model = m["built_models"]["overlay"]
+        self.assertEqual(model["scripts"], "scripts/sglang_disagg/run.sh")
+        self.assertIn("--xp 2 --yd 2", model["args"])
+        for env in (m["context"]["docker_env_vars"], m["deployment_config"]["env_vars"]):
+            self.assertEqual(env["MOE_A2A_BACKEND"], "none")
+            self.assertEqual(env["NCCL_DEBUG"], "INFO")
+            self.assertEqual(env["SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK"], "0")
+            self.assertEqual(env["OMP_NUM_THREADS"], "8")
+
+    def test_docker_options(self):
+        opts = self.manifest(tm.CANDIDATE)["built_models"]["overlay"][
+            "additional_docker_run_options"]
+        self.assertTrue(opts.startswith("--init "), opts)
+        self.assertIn(f"-v {RUBY['models_dir']}:/models:ro", opts)
+        self.assertNotIn("libibverbs", opts, "the image's own rdma-core must win")
+
+    def test_each_phase_gets_its_own_shared_run_logs(self):
+        mounts = {}
+        for phase in (tm.BASELINE, tm.CANDIDATE):
+            opts = self.manifest(phase)["built_models"]["overlay"][
+                "additional_docker_run_options"]
+            run_logs = self.work / "ab" / phase / "slurm_output" / "run_logs"
+            self.assertIn(f"-v {run_logs}:/run_logs", opts)
+            self.assertEqual(stat.S_IMODE(run_logs.stat().st_mode), 0o777)
+            mounts[phase] = run_logs
+        self.assertNotEqual(mounts[tm.BASELINE], mounts[tm.CANDIDATE])
+
+    def test_megatron_keeps_host_ib_libs(self):
+        config = tm.WORKLOAD_CONFIGS["llama-3.1-70b-training"]
+        (self.work / "scripts" / "primus_megatron-lm").mkdir()
+        path = tm.generate_manifest("llama-3.1-70b-training", config, RUBY, "img", 2,
+                                    self.work, pull_only=True)
+        opts = json.loads(path.read_text())["built_models"]["overlay"][
+            "additional_docker_run_options"]
+        self.assertIn("libibverbs", opts)
+        self.assertNotIn("/run_logs", opts)
+
+    def test_repro_bundle_points_run_logs_at_the_submit_dir(self):
+        m = tm.repro_manifest(self.manifest(tm.CANDIDATE), self.work)
+        opts = m["built_models"]["overlay"]["additional_docker_run_options"]
+        self.assertIn("-v ${SLURM_SUBMIT_DIR}/slurm_output/run_logs:/run_logs", opts)
+        self.assertNotIn(str(self.work), opts)
+
+
+class RecipeTest(TempWork):
+    def test_base_pinned_and_patchelf_added(self):
+        text = tm.sglang_recipe(self.work / "MAD", "rocm/sgl-dev@sha256:abc")
+        self.assertIn("\nFROM rocm/sgl-dev@sha256:abc\n", text)
+        self.assertNotIn("FROM $BASE_DOCKER", text)
+        self.assertTrue(text.rstrip().endswith(tm.SGLANG_SWAP_TOOLS.rstrip()))
+        self.assertLess(text.index("RUN echo payload"), text.index("patchelf"))
+
+    def test_unexpected_recipe_fails(self):
+        (self.work / "MAD" / tm.SGLANG_MAD_DOCKERFILE).write_text("FROM ubuntu\n")
+        with self.assertRaisesRegex(RuntimeError, "FROM \\$BASE_DOCKER"):
+            tm.sglang_recipe(self.work / "MAD", "x")
+
+
+class CollectResultsTest(unittest.TestCase):
+    def test_only_gate_points_keyed_by_point(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            run_dir = Path(tmp)
+            rows = [
+                sweep_row("isl1024_osl1024_con8", CONFIG["metric_key"], 217.2),
+                sweep_row("isl1024_osl1024_con64", CONFIG["metric_key"], 1638.6),
+                sweep_row("isl1024_osl1024_con64", "mean_itl_ms", 37.2),
+                sweep_row("isl8192_osl1024_con64", CONFIG["metric_key"], 1229.3, "FAILURE"),
+            ]
+            (run_dir / "perf_entry_super.json").write_text(json.dumps(rows))
+            results = tm.collect_phase_results(
+                run_dir, CONFIG["metric_key"], only=CONFIG["precisions"])
+        got = {(r["precision"], r["metric_value"], r["status"]) for r in results}
+        self.assertEqual(got, {("isl1024_osl1024_con64", 1638.6, "pass"),
+                               ("isl8192_osl1024_con64", 1229.3, "fail")})
+
+    def test_training_rows_unaffected(self):
+        row = {"model": "primus_x_Llama-3.1-70B", "metric": "tok_per_s_per_gpu_avg",
+               "performance": 1519.5, "training_precision": "",
+               "multi_results": {"precision": "BF16"}, "status": "SUCCESS"}
+        self.assertEqual(tm._row_precision(row), "BF16")
+        self.assertEqual(tm._row_precision({"model": "primus_x_Llama-3.1-70B"}), "")
+
+
+class BannerLogsTest(unittest.TestCase):
+    def test_server_logs_preferred_and_verified(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            run_dir = Path(tmp)
+            out = run_dir / "slurm_output"
+            logs = out / "run_logs" / "79405"
+            logs.mkdir(parents=True)
+            (out / "madengine-x_79405_node_0.out").write_text("no banner here\n")
+            banner = "h:1:1 [0] NCCL INFO RCCL version : 2.32.3-develop:2b88119\n"
+            for name in ("prefill_NODE0.log", "prefill_NODE1.log",
+                         "decode_NODE2.log", "decode_NODE3.log"):
+                (logs / name).write_text(banner)
+            (logs / "proxy_NODE0.log").write_text("router\n")
+            self.assertEqual([p.name for p in tm.rccl_banner_logs(run_dir)],
+                             ["decode_NODE2.log", "decode_NODE3.log",
+                              "prefill_NODE0.log", "prefill_NODE1.log"])
+            ok, msg = tm.verify_rccl_replacement(
+                run_dir, {"version": "2.32.3-develop:2b88119"})
+            self.assertTrue(ok, msg)
+            (logs / "decode_NODE3.log").write_text("died before init\n")
+            ok, _ = tm.verify_rccl_replacement(
+                run_dir, {"version": "2.32.3-develop:2b88119"})
+            self.assertFalse(ok)
+
+
+class AllocationScriptTest(TempWork):
+    def run_allocation(self, workload: str, env: dict) -> tuple[list[str], str]:
+        popen = mock.MagicMock()
+        popen.return_value.wait.return_value = 0
+        phases = [(tm.BASELINE, self.work / "ab" / tm.BASELINE)]
+        with mock.patch.dict(os.environ, env), \
+                mock.patch.object(tm.subprocess, "Popen", popen), \
+                mock.patch.object(tm.subprocess, "run"):
+            tm.run_ab_in_one_allocation(
+                phases, self.work, 4, workload, tm.WORKLOAD_CONFIGS[workload],
+                RUBY, timeout_minutes=1)
+        cmd = popen.call_args.args[0]
+        return cmd, (self.work / "ab" / "run_phases.sh").read_text()
+
+    def test_dns_preflight_for_sglang_only(self):
+        _, script = self.run_allocation(WORKLOAD, {})
+        self.assertIn("preflight_dns.sh", script)
+        self.assertTrue((self.work / "ab" / "preflight_dns.sh").is_file())
+        _, script = self.run_allocation("llama-3.1-405b-maxtext", {})
+        self.assertNotIn("preflight_dns.sh", script)
+
+    def test_exclude_from_environment(self):
+        cmd, _ = self.run_allocation(WORKLOAD, {"MADENGINE_SLURM_EXCLUDE": "n[01-02],n07"})
+        self.assertEqual(cmd[cmd.index("--exclude") + 1], "n[01-02],n07")
+        cmd, _ = self.run_allocation(WORKLOAD, {"MADENGINE_SLURM_EXCLUDE": ""})
+        self.assertNotIn("--exclude", cmd)
+        with self.assertRaises(RuntimeError):
+            self.run_allocation(WORKLOAD, {"MADENGINE_SLURM_EXCLUDE": "n01; rm -rf /"})
+
+
+if __name__ == "__main__":
+    unittest.main()

From 431a721074edf8b8c337317f1962aa38967def90 Mon Sep 17 00:00:00 2001
From: mkuznet1 <Mikhail.Kuznetsov@amd.com>
Date: Tue, 20 Oct 2026 10:40:10 +0000
Subject: [PATCH 2/8] ci(rccl): require artifact_run_id instead of guessing it

That run's RCCL build is the A/B candidate, so which build it is has to
be the dispatcher's choice. With the field empty, the job looked up the
latest TheRock run, which carries TheRock's pinned RCCL rather than this
repository's, and then fetched it from this repository, where that run
does not exist: dispatches 39279508468, 39279792903 and 39280034280 all
stopped at a 404 in "Fetch RCCL artifacts".

The input is now required on both triggers, the lookup step is gone, and
an empty ID fails before the fetch with a message naming the input. The
scheduled callers always pass inputs.artifact_run_id || github.run_id.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../workflows/therock-rccl-test-madengine.yml | 35 ++++++-------------
 1 file changed, 11 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/therock-rccl-test-madengine.yml b/.github/workflows/therock-rccl-test-madengine.yml
index 84b1b897520e..b22595c936d6 100644
--- a/.github/workflows/therock-rccl-test-madengine.yml
+++ b/.github/workflows/therock-rccl-test-madengine.yml
@@ -11,6 +11,7 @@ on:
         type: string
       artifact_run_id:
         type: string
+        required: true
       workload:
         # A key of WORKLOAD_CONFIGS in test_madengine.py, which validates it.
         # workflow_call inputs cannot be typed as choice, so the list of
@@ -53,9 +54,13 @@ on:
       test_runs_on:
         type: string
         default: 'ruby-linux-slurm-scale-runner'
+      # No default: its RCCL is the A/B candidate, so the dispatcher picks the
+      # build. The latest TheRock run would carry TheRock's pinned RCCL, not
+      # this repository's.
       artifact_run_id:
-        description: 'Reuse artifacts from a previous build run ID'
+        description: 'rocm-systems run ID whose RCCL build is the A/B candidate (e.g. a develop nightly of TheRock CI for rccl)'
         type: string
+        required: true
       workload:
         description: 'MADEngine workload to run'
         type: choice
@@ -207,33 +212,15 @@ jobs:
           python3 -m pip install --upgrade pip
           python3 -m pip install -r requirements.txt
 
-      - name: Resolve TheRock artifact run
-        if: ${{ inputs.artifact_run_id == '' }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-          GITHUB_TOKEN: ${{ github.token }}
-        run: |
-          run_id="$(python3 -c "
-          import sys, contextlib
-          sys.path.insert(0, 'build_tools')
-          from find_latest_artifacts import find_latest_artifacts
-          with contextlib.redirect_stdout(sys.stderr):
-              results = find_latest_artifacts(
-                  artifact_groups=['${{ inputs.artifact_group }}'],
-                  verbose=True,
-              )
-          if not results:
-              print('No artifacts found for ${{ inputs.artifact_group }}', file=sys.stderr)
-              sys.exit(1)
-          print(results[0].workflow_run_id)
-          ")"
-          echo "Resolved TheRock run_id=${run_id}"
-          echo "ARTIFACT_RUN_ID=${run_id}" >> $GITHUB_ENV
-
       - name: Fetch RCCL artifacts
         env:
           GITHUB_TOKEN: ${{ github.token }}
         run: |
+          # `required` does not stop an API dispatch from passing an empty string.
+          if [ -z "${ARTIFACT_RUN_ID}" ]; then
+            echo "::error::artifact_run_id is required: it names the RCCL build tested as the candidate"
+            exit 1
+          fi
           python3 build_tools/install_rocm_from_artifacts.py \
             --run-id="${ARTIFACT_RUN_ID}" \
             --artifact-group="${{ inputs.artifact_group }}" \

From 6afc9a11fc8318939ffe56f2ea3271ecca405551 Mon Sep 17 00:00:00 2001
From: mkuznet1 <Mikhail.Kuznetsov@amd.com>
Date: Tue, 20 Oct 2026 15:32:03 +0000
Subject: [PATCH 3/8] fix(rccl-ci): stop pulling and building the MaxText and
 sglang images on the runner host

The base PR moved the candidate overlay to the compute nodes; the MaxText
and sglang families still pulled the base on the runner host to resolve
its digest and built their baselines there. The nodes rebuild both from
the manifest's Dockerfile anyway, and pulling the 27 GB sglang base ran
that host's shared Docker disk out of space.

- Resolve the base digest with a HEAD request to the registry; no pull.
- stage_maxtext_baseline / stage_sglang_baseline only write the Dockerfile.
- The repro bundle's BASE_DIGEST comes from the registry as well.
- DSR1 job timeout 360 -> 330: the head-node pull no longer needs time.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .github/workflows/therock-rccl-ci-linux.yml   |   7 +-
 projects/rccl/ci/scripts/test_madengine.py    | 166 ++++++++++--------
 .../rccl/ci/scripts/tests/test_base_digest.py |  87 +++++++++
 .../ci/scripts/tests/test_sglang_disagg.py    |  18 +-
 4 files changed, 203 insertions(+), 75 deletions(-)
 create mode 100644 projects/rccl/ci/scripts/tests/test_base_digest.py

diff --git a/.github/workflows/therock-rccl-ci-linux.yml b/.github/workflows/therock-rccl-ci-linux.yml
index fdad3586a44c..6652d3df7d14 100644
--- a/.github/workflows/therock-rccl-ci-linux.yml
+++ b/.github/workflows/therock-rccl-ci-linux.yml
@@ -316,12 +316,11 @@ jobs:
           - workload: llama-3.1-405b-maxtext
             nodes: '4'
             job_timeout_minutes: '450'
-          # 4 nodes: two prefill, two decode. 360 is its 3 h time_limit plus
-          # 120 minutes, because the head node also pulls the sglang base and
-          # builds both images before the allocation is requested.
+          # 4 nodes: two prefill, two decode. 330 is its 3 h time_limit plus
+          # the same 90 minutes.
           - workload: deepseek-r1-sglang-disagg
             nodes: '4'
-            job_timeout_minutes: '360'
+            job_timeout_minutes: '330'
     uses: ./.github/workflows/therock-rccl-test-madengine.yml
     secrets: inherit
     with:
diff --git a/projects/rccl/ci/scripts/test_madengine.py b/projects/rccl/ci/scripts/test_madengine.py
index 536a5d8ba035..7a730ead24f5 100644
--- a/projects/rccl/ci/scripts/test_madengine.py
+++ b/projects/rccl/ci/scripts/test_madengine.py
@@ -38,6 +38,9 @@
 import subprocess
 import sys
 import tempfile
+import urllib.error
+import urllib.parse
+import urllib.request
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -79,9 +82,9 @@
 OVERLAY_CTX = "overlay_ctx"
 OVERLAY_DOCKERFILE = "Dockerfile.rccl-overlay"
 # The maxtext family needs a second image: its A/B baseline cannot be the stock
-# registry tag, which carries no Primus. See build_maxtext_baseline_image().
+# registry tag, which carries no Primus. See stage_maxtext_baseline().
 MAXTEXT_BASELINE_DOCKERFILE = "Dockerfile.maxtext-baseline"
-# Likewise for sglang-disagg, whose baseline is MAD's Dockerfile built here.
+# Likewise for sglang-disagg, whose baseline is MAD's Dockerfile.
 SGLANG_BASELINE_DOCKERFILE = "Dockerfile.sglang-baseline"
 SGLANG_MAD_DOCKERFILE = "docker/sglang_disagg_inference_full_overlay.ubuntu.amd.Dockerfile"
 # Families whose baseline is built here rather than pulled as the stock tag.
@@ -290,8 +293,8 @@
     # left to RCCL (MOE_A2A_BACKEND=none) rather than MoRI, so expert traffic
     # is what the candidate library carries.
     #
-    # Both A/B images are MAD's sglang Dockerfile built here; see
-    # build_sglang_baseline_image().
+    # Both A/B images are MAD's sglang Dockerfile; see
+    # stage_sglang_baseline().
     "deepseek-r1-sglang-disagg": {
         "type": "inference",
         "family": "sglang-disagg",
@@ -1092,30 +1095,78 @@ def _stage_primus_context(ctx_dir: Path, primus_dir: Path) -> None:
         raise RuntimeError(f"failed to stage Primus from {primus_dir}")
 
 
+# What `docker pull` accepts, so the digest is the one a pull by tag would
+# record in RepoDigests: the index for a multi-arch tag, else the manifest.
+_MANIFEST_MEDIA_TYPES = ", ".join((
+    "application/vnd.oci.image.index.v1+json",
+    "application/vnd.docker.distribution.manifest.list.v2+json",
+    "application/vnd.oci.image.manifest.v1+json",
+    "application/vnd.docker.distribution.manifest.v2+json",
+))
+
+
+def registry_digest(image: str, timeout: int = 60) -> str:
+    """*image* pinned by digest as its registry serves it now, or "" on failure.
+
+    Asks the registry for the manifest digest and downloads no layer: the
+    runner host's disk is shared by every runner on it and has no room for a
+    base image (the sgl-dev one is 27 GB compressed), and nothing there would
+    use it.
+    """
+    if "@sha256:" in image:
+        return image
+    name, _, tag = image.rpartition(":")
+    if not name or "/" in tag:
+        name, tag = image, "latest"
+    first, _, rest = name.partition("/")
+    if rest and ("." in first or ":" in first or first == "localhost"):
+        host, path = first, rest
+    else:
+        host, path = "registry-1.docker.io", name if rest else f"library/{name}"
+    url = f"https://{host}/v2/{path}/manifests/{tag}"
+    headers = {"Accept": _MANIFEST_MEDIA_TYPES}
+    try:
+        try:
+            req = urllib.request.Request(url, headers=headers, method="HEAD")
+            resp = urllib.request.urlopen(req, timeout=timeout)
+        except urllib.error.HTTPError as exc:
+            challenge = exc.headers.get("WWW-Authenticate", "")
+            if exc.code != 401 or not challenge.startswith("Bearer "):
+                raise
+            params = dict(re.findall(r'(\w+)="([^"]*)"', challenge))
+            realm = params.pop("realm")
+            params.setdefault("scope", f"repository:{path}:pull")
+            token_url = f"{realm}?{urllib.parse.urlencode(params)}"
+            with urllib.request.urlopen(token_url, timeout=timeout) as tok:
+                body = json.load(tok)
+            headers["Authorization"] = (
+                f"Bearer {body.get('token') or body.get('access_token')}"
+            )
+            req = urllib.request.Request(url, headers=headers, method="HEAD")
+            resp = urllib.request.urlopen(req, timeout=timeout)
+        with resp:
+            digest = resp.headers.get("Docker-Content-Digest", "")
+    except (OSError, ValueError, KeyError) as exc:
+        log.warning("Could not get the registry digest of %s: %s", image, exc)
+        return ""
+    return f"{name}@{digest}" if digest.startswith("sha256:") else ""
+
+
 def resolve_base_digest(base_image: str) -> str:
-    """Pull *base_image* once and return it pinned by digest.
+    """*base_image* pinned by digest, for both A/B Dockerfiles.
 
     Both A/B images must come from the same base, and a tag cannot promise
-    that: the baseline and the candidate are separate `docker build`
-    invocations, and `rocm/jax-training:maxtext-v26.6` is mutable. Adding
-    `--pull` to both would only narrow the window, not close it -- a push
-    between the two builds still splits the pair. A digest is immutable, so
+    that: each phase builds its own image on the compute nodes, and
+    `rocm/jax-training:maxtext-v26.6` is mutable. A digest is immutable, so
     resolving it once and writing it into both Dockerfiles closes it.
 
-    Falls back to the tag when the image has no RepoDigest, which is the case
-    for a locally built base; there is nothing to pin to and nothing remote
-    that could move underneath us either.
+    Falls back to the tag when the registry cannot tell.
     """
-    subprocess.run(["docker", "pull", base_image], check=True)
-    result = subprocess.run(
-        ["docker", "image", "inspect", "--format", "{{index .RepoDigests 0}}", base_image],
-        capture_output=True, text=True,
-    )
-    digest = result.stdout.strip()
-    if result.returncode != 0 or "@sha256:" not in digest:
+    digest = registry_digest(base_image)
+    if not digest:
         log.warning(
-            "No RepoDigest for %s; both A/B images will use the tag. They are "
-            "built minutes apart, so a push in between would split the pair.",
+            "No digest for %s; both A/B images will use the tag, so a push "
+            "between the two phases' builds would split the pair.",
             base_image,
         )
         return base_image
@@ -1138,14 +1189,14 @@ def base_identity(from_ref: str) -> str:
     return hashlib.sha256(from_ref.encode()).hexdigest()[:10]
 
 
-def build_maxtext_baseline_image(
+def stage_maxtext_baseline(
     base_image: str,
     work_dir: Path,
     primus_dir: Path,
     run_token: str,
     from_ref: str,
 ) -> str:
-    """Build the A/B baseline for the MaxText family: Primus, stock RCCL.
+    """Stage the A/B baseline for the MaxText family: Primus, stock RCCL.
 
     The Megatron workloads use the stock registry image as their baseline, but
     that cannot work here: ``rocm/jax-training:maxtext-*`` ships MaxText and no
@@ -1156,6 +1207,9 @@ def build_maxtext_baseline_image(
     That makes the pair *tighter* than the Megatron A/B rather than looser: the
     two images differ by the librccl swap alone, which is exactly the variable
     the ratio is supposed to isolate.
+
+    Only the Dockerfile and its context are written here; node 0 of the
+    allocation builds the image and the other nodes load it.
     """
     ctx_dir = work_dir / OVERLAY_CTX
     ctx_dir.mkdir(exist_ok=True)
@@ -1164,19 +1218,12 @@ def build_maxtext_baseline_image(
     dockerfile = ctx_dir / MAXTEXT_BASELINE_DOCKERFILE
     dockerfile.write_text(f"FROM {from_ref}\n\n{MAXTEXT_PRIMUS_PAYLOAD}")
 
-    # Rebuilt every run under a per-run tag, with the same token the candidate
-    # uses: a cached tag would skip the build and could pair a fresh candidate
-    # with a stale baseline, breaking the one invariant this pair exists to
-    # hold -- that the two differ by the RCCL swap alone. The payload layers
-    # are shared with the candidate, so a cached rebuild takes ~2 s.
+    # A per-run tag, with the same token the candidate uses: a node holding an
+    # image under a reused tag would skip the build and could pair a fresh
+    # candidate with a stale baseline, breaking the one invariant this pair
+    # exists to hold -- that the two differ by the RCCL swap alone.
     tag = f"{base_image}-primus-{PRIMUS_REF}-{run_token}-{base_identity(from_ref)}"
-    log.info("Building MaxText baseline image (Primus, stock RCCL): %s", tag)
-    # No --pull: from_ref is already a digest, resolved once for both images.
-    subprocess.run(
-        ["docker", "build", "-t", tag, "-f", str(dockerfile), str(ctx_dir)],
-        check=True,
-    )
-    log.info("MaxText baseline image built: %s", tag)
+    log.info("MaxText baseline (Primus, stock RCCL), built on the nodes: %s", tag)
     return tag
 
 
@@ -1205,16 +1252,16 @@ def sglang_recipe(mad_dir: Path, from_ref: str) -> str:
     return f"{pinned}\n{SGLANG_SWAP_TOOLS}"
 
 
-def build_sglang_baseline_image(
+def stage_sglang_baseline(
     base_image: str,
     work_dir: Path,
     run_token: str,
     from_ref: str,
 ) -> str:
-    """Build the A/B baseline for sglang-disagg: MAD's image, stock RCCL.
+    """Stage the A/B baseline for sglang-disagg: MAD's image, stock RCCL.
 
     The base alone is not what MAD runs: its Dockerfile adds rdma-core 63,
-    without which queue-pair creation fails on Thor2. Built every run and
+    without which queue-pair creation fails on Thor2. Built on the nodes and
     tagged like the MaxText baseline, for the same reasons.
     """
     ctx_dir = work_dir / OVERLAY_CTX
@@ -1223,12 +1270,7 @@ def build_sglang_baseline_image(
     dockerfile.write_text(sglang_recipe(work_dir / "MAD", from_ref))
 
     tag = f"{base_image}-mad-{MAD_REF[:8]}-{run_token}-{base_identity(from_ref)}"
-    log.info("Building sglang baseline image (MAD recipe, stock RCCL): %s", tag)
-    subprocess.run(
-        ["docker", "build", "-t", tag, "-f", str(dockerfile), str(ctx_dir)],
-        check=True,
-    )
-    log.info("sglang baseline image built: %s", tag)
+    log.info("sglang baseline (MAD recipe, stock RCCL), built on the nodes: %s", tag)
     return tag
 
 
@@ -1713,21 +1755,6 @@ def prepare_phase_dir(work_dir: Path, phase: str) -> Path:
 THEROCK_ARTIFACT_BUCKET = "therock-ci-artifacts-external"
 
 
-def local_repo_digest(image: str) -> str:
-    """*image* pinned by digest as the local daemon knows it, or "" if it cannot tell.
-
-    Never pulls: this only records what the head node built from.
-    """
-    if "@sha256:" in image:
-        return image
-    result = subprocess.run(
-        ["docker", "image", "inspect", "--format", "{{index .RepoDigests 0}}", image],
-        capture_output=True, text=True,
-    )
-    digest = result.stdout.strip()
-    return digest if result.returncode == 0 and "@sha256:" in digest else ""
-
-
 def repro_manifest(manifest: dict, work_dir: Path) -> dict:
     """*manifest* with its build paths made relative to the bundle root.
 
@@ -1874,8 +1901,9 @@ def repro_env(
   mounts them.
 - To shorten a Megatron run, add `"PRIMUS_TRAIN_ITERS": "15"` to both
   `context.docker_env_vars` and `deployment_config.env_vars`.
-- `BASE_DIGEST` is the base the CI head node built from. The nodes build from
-  `BASE_IMAGE` by tag; if the base has moved since, the image differs.
+- `BASE_DIGEST` is what `BASE_IMAGE` pointed to when CI ran. MaxText and
+  sglang-disagg images build from that digest; Megatron ones from `BASE_IMAGE`
+  by tag, so if the base has moved since, the image differs.
 """
 
 
@@ -2704,7 +2732,7 @@ def main() -> None:
     # phase would die before step 0. Build base + Primus instead, which also
     # makes the pair differ by the RCCL swap and nothing else.
     if family == "maxtext":
-        baseline_image = build_maxtext_baseline_image(
+        baseline_image = stage_maxtext_baseline(
             workload_config["base_image"], work_dir, primus_dir,
             # Same token the candidate tag carries, so the pair is unambiguously
             # from one run, and the same pinned base digest.
@@ -2713,7 +2741,7 @@ def main() -> None:
         )
         baseline_pull_only = False
     elif family == "sglang-disagg":
-        baseline_image = build_sglang_baseline_image(
+        baseline_image = stage_sglang_baseline(
             workload_config["base_image"], work_dir,
             run_token=rccl_commit,
             from_ref=base_ref or workload_config["base_image"],
@@ -2744,11 +2772,11 @@ def main() -> None:
             ),
             run_dir=run_dir,
             pull_only=pull_only,
-            # A maxtext baseline is a locally built image, so its manifest is
-            # on the build-capable path and must advertise ITS OWN dockerfile.
-            # Left at the default it would hand a node the candidate recipe
-            # and have it rebuild the swapped image under the baseline tag --
-            # an A/B comparing the CI RCCL against itself.
+            # A maxtext baseline is built on the nodes, so its manifest must
+            # advertise ITS OWN dockerfile. Left at the default it would hand
+            # a node the candidate recipe and have it build the swapped image
+            # under the baseline tag -- an A/B comparing the CI RCCL against
+            # itself.
             dockerfile_name=(
                 LOCAL_BASELINE_DOCKERFILES.get(family, OVERLAY_DOCKERFILE)
                 if phase == BASELINE
@@ -2767,7 +2795,7 @@ def main() -> None:
             work_dir, phase_dirs, args.workload, workload_config, cluster_config,
             args.nodes, rccl_fingerprint,
             images={BASELINE: baseline_image, CANDIDATE: overlay_image},
-            base_digest=base_ref or local_repo_digest(workload_config["base_image"]),
+            base_digest=base_ref or registry_digest(workload_config["base_image"]),
         )
     except Exception:
         log.exception("Could not write the repro bundle; continuing without it")
diff --git a/projects/rccl/ci/scripts/tests/test_base_digest.py b/projects/rccl/ci/scripts/tests/test_base_digest.py
new file mode 100644
index 000000000000..876875be72b4
--- /dev/null
+++ b/projects/rccl/ci/scripts/tests/test_base_digest.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""The base digest comes from the registry, without pulling the image."""
+
+import io
+import json
+import sys
+import unittest
+import urllib.error
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import test_madengine as tm  # noqa: E402
+
+DIGEST = "sha256:" + "a" * 64
+
+
+class Response(io.BytesIO):
+    def __init__(self, headers: dict | None = None, body: bytes = b""):
+        super().__init__(body)
+        self.headers = headers or {}
+
+
+def unauthorized(scope: str) -> urllib.error.HTTPError:
+    challenge = (f'Bearer realm="https://auth.docker.io/token",'
+                 f'service="registry.docker.io",scope="{scope}"')
+    return urllib.error.HTTPError(
+        "url", 401, "Unauthorized", {"WWW-Authenticate": challenge}, None)
+
+
+class RegistryDigestTest(unittest.TestCase):
+    def test_docker_hub_token_flow(self):
+        calls = []
+
+        def urlopen(req, timeout):
+            calls.append(req)
+            if isinstance(req, str):
+                return Response(body=json.dumps({"token": "t"}).encode())
+            if "Authorization" not in req.headers:
+                raise unauthorized("repository:rocm/primus:pull")
+            return Response({"Docker-Content-Digest": DIGEST})
+
+        with mock.patch.object(tm.urllib.request, "urlopen", urlopen):
+            self.assertEqual(tm.registry_digest("rocm/primus:v26.4"),
+                             f"rocm/primus@{DIGEST}")
+        head = calls[0]
+        self.assertEqual(head.get_method(), "HEAD")
+        self.assertEqual(head.full_url,
+                         "https://registry-1.docker.io/v2/rocm/primus/manifests/v26.4")
+        self.assertIn("manifest.list.v2+json", head.headers["Accept"])
+        self.assertIn("scope=repository%3Arocm%2Fprimus%3Apull", calls[1])
+        self.assertEqual(calls[2].headers["Authorization"], "Bearer t")
+
+    def test_reference_forms(self):
+        urls = []
+
+        def urlopen(req, timeout):
+            urls.append(req.full_url)
+            return Response({"Docker-Content-Digest": DIGEST})
+
+        with mock.patch.object(tm.urllib.request, "urlopen", urlopen):
+            self.assertEqual(tm.registry_digest("ubuntu"), f"ubuntu@{DIGEST}")
+            tm.registry_digest("ghcr.io/rocm/x:1")
+            tm.registry_digest("localhost:5000/x")
+        self.assertEqual(urls, [
+            "https://registry-1.docker.io/v2/library/ubuntu/manifests/latest",
+            "https://ghcr.io/v2/rocm/x/manifests/1",
+            "https://localhost:5000/v2/x/manifests/latest",
+        ])
+        self.assertEqual(tm.registry_digest(f"rocm/primus@{DIGEST}"),
+                         f"rocm/primus@{DIGEST}")
+
+    def test_failure_falls_back_to_the_tag(self):
+        def urlopen(req, timeout):
+            raise urllib.error.HTTPError("url", 404, "Not Found", {}, None)
+
+        with mock.patch.object(tm.urllib.request, "urlopen", urlopen), \
+                mock.patch.object(tm.subprocess, "run") as run:
+            self.assertEqual(tm.registry_digest("rocm/primus:gone"), "")
+            self.assertEqual(tm.resolve_base_digest("rocm/primus:gone"),
+                             "rocm/primus:gone")
+        run.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/projects/rccl/ci/scripts/tests/test_sglang_disagg.py b/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
index 0af9f4fe4d57..a1712cbdbedb 100644
--- a/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
+++ b/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
@@ -180,7 +180,8 @@ def test_server_logs_preferred_and_verified(self):
 
 
 class AllocationScriptTest(TempWork):
-    def run_allocation(self, workload: str, env: dict) -> tuple[list[str], str]:
+    def run_allocation(self, workload: str, env: dict,
+                       built_images: list[str] | None = None) -> tuple[list[str], str]:
         popen = mock.MagicMock()
         popen.return_value.wait.return_value = 0
         phases = [(tm.BASELINE, self.work / "ab" / tm.BASELINE)]
@@ -189,10 +190,23 @@ def run_allocation(self, workload: str, env: dict) -> tuple[list[str], str]:
                 mock.patch.object(tm.subprocess, "run"):
             tm.run_ab_in_one_allocation(
                 phases, self.work, 4, workload, tm.WORKLOAD_CONFIGS[workload],
-                RUBY, timeout_minutes=1)
+                RUBY, timeout_minutes=1, built_images=built_images)
         cmd = popen.call_args.args[0]
         return cmd, (self.work / "ab" / "run_phases.sh").read_text()
 
+    def test_built_images_removed_from_every_node_after_the_phases(self):
+        images = ["rocm/sgl-dev:t-mad-36339bd8-run1-abc", "rocm/sgl-dev:t-rccl-gfx950-run1-abc"]
+        _, script = self.run_allocation(WORKLOAD, {}, built_images=images)
+        lines = script.splitlines()
+        cleanup = next(i for i, line in enumerate(lines) if "cleanup_images.sh" in line)
+        self.assertGreater(cleanup, max(i for i, line in enumerate(lines)
+                                        if line.startswith("madengine run")))
+        self.assertIn('--ntasks="${SLURM_JOB_NUM_NODES}" --ntasks-per-node=1', lines[cleanup])
+        self.assertTrue(lines[cleanup].endswith(" ".join(images) + " || true"))
+        self.assertTrue((self.work / "ab" / "cleanup_images.sh").is_file())
+        _, script = self.run_allocation(WORKLOAD, {})
+        self.assertNotIn("cleanup_images.sh", script)
+
     def test_dns_preflight_for_sglang_only(self):
         _, script = self.run_allocation(WORKLOAD, {})
         self.assertIn("preflight_dns.sh", script)

From 05aad238044933affc6ccb2d3a889daa5bbe96b7 Mon Sep 17 00:00:00 2001
From: mkuznet1 <Mikhail.Kuznetsov@amd.com>
Date: Wed, 21 Oct 2026 11:08:32 +0000
Subject: [PATCH 4/8] fix(rccl-ci): run DSR1 sglang-disagg on a ROCm SDK base

The candidate librccl from TheRock is kpack-split: device code lives in
rccl_lib_<gfx>.kpack and .hip_fatbin is NOBITS. The rocm720 sgl-dev base
has a pre-kpack HIP runtime (7.2.26015), so every rank segfaulted in
ncclInitKernelsForDevice and the candidate phase produced no result
(run 39458139070).

Move to rocm/sgl-dev:v0.5.22-rocm10-mi35x-20261020: ROCm SDK 10 in the
venv, HIP 7.15, the same layout the MaxText image swaps into. MAD's
recipe builds on it unchanged.

That venv ships patchelf 0.19.1, which refuses to set a RUNPATH on the
kpack-split library ("section content is not naturally aligned"), so the
swap tools pin 0.18.0, the version the MaxText swap uses from apt.

Checked on one MI355X node: both images build, the swap gate passes on
all 3 librccl copies, and an 8-GPU all_reduce + all_to_all runs on each,
the candidate reporting RCCL 2.32.3-develop:4b7b5e1.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 projects/rccl/ci/scripts/test_madengine.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/projects/rccl/ci/scripts/test_madengine.py b/projects/rccl/ci/scripts/test_madengine.py
index 7a730ead24f5..055a4d2e67ab 100644
--- a/projects/rccl/ci/scripts/test_madengine.py
+++ b/projects/rccl/ci/scripts/test_madengine.py
@@ -300,7 +300,10 @@
         "family": "sglang-disagg",
         # madengine names each result row <this>_2p2d_isl<I>_osl<O>_con<C>.
         "model_repo": "sglang-disagg-deepseek-r1",
-        "base_image": "rocm/sgl-dev:v0.5.16-rocm720-mi35x-20260807",
+        # A ROCm SDK (TheRock) base, not a rocm7xx one: the candidate librccl
+        # is kpack-split, and a pre-kpack HIP runtime segfaults on it in
+        # ncclInitKernelsForDevice.
+        "base_image": "rocm/sgl-dev:v0.5.22-rocm10-mi35x-20261020",
         "gpu_target": "gfx950",
         "metric_key": "output_token_throughput_tok_s",
         "metric_unit": "output tok/s of the whole 2P+2D deployment",
@@ -1229,9 +1232,10 @@ def stage_maxtext_baseline(
 
 # MAD's Dockerfile ends with `dpkg -r --force-all` on the distro rdma-core,
 # which leaves apt unusable, so the swap's patchelf has to come from pip.
-SGLANG_SWAP_TOOLS = (
-    "RUN command -v patchelf >/dev/null || pip install --no-cache-dir patchelf\n"
-)
+# Pinned to the version the MaxText swap gets from apt: the 0.19.1 the
+# rocm10 sgl-dev venv ships refuses to set a RUNPATH on the kpack-split
+# librccl ("section content is not naturally aligned").
+SGLANG_SWAP_TOOLS = "RUN pip install --no-cache-dir patchelf==0.18.0.0\n"
 
 
 def sglang_recipe(mad_dir: Path, from_ref: str) -> str:

From 9884736e0930e4c600ae3767c493667a855a64ac Mon Sep 17 00:00:00 2001
From: mkuznet1 <Mikhail.Kuznetsov@amd.com>
Date: Wed, 21 Oct 2026 18:05:35 +0000
Subject: [PATCH 5/8] fix(rccl-ci): pin DSR1 sglang-disagg base to sglang
 0.5.18

sglang 0.5.19 split --cuda-graph-bs into --cuda-graph-bs-decode and
--cuda-graph-bs-prefill, so every server in the 0.5.22 base exited on
argparse's ambiguous-option error for the flag MAD's models.yaml passes,
and both phases reported nothing. v0.5.18-rocm10 keeps the ROCm SDK
runtime the kpack-split candidate librccl needs and parses all of MAD's
server command lines.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 projects/rccl/ci/scripts/test_madengine.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/projects/rccl/ci/scripts/test_madengine.py b/projects/rccl/ci/scripts/test_madengine.py
index 055a4d2e67ab..70e7f1573d34 100644
--- a/projects/rccl/ci/scripts/test_madengine.py
+++ b/projects/rccl/ci/scripts/test_madengine.py
@@ -302,8 +302,10 @@
         "model_repo": "sglang-disagg-deepseek-r1",
         # A ROCm SDK (TheRock) base, not a rocm7xx one: the candidate librccl
         # is kpack-split, and a pre-kpack HIP runtime segfaults on it in
-        # ncclInitKernelsForDevice.
-        "base_image": "rocm/sgl-dev:v0.5.22-rocm10-mi35x-20261020",
+        # ncclInitKernelsForDevice. No newer than sglang 0.5.18: 0.5.19 split
+        # --cuda-graph-bs into -decode/-prefill, and argparse rejects the
+        # plain flag MAD's models.yaml passes as ambiguous.
+        "base_image": "rocm/sgl-dev:v0.5.18-rocm10-mi35x-20260904",
         "gpu_target": "gfx950",
         "metric_key": "output_token_throughput_tok_s",
         "metric_unit": "output tok/s of the whole 2P+2D deployment",

From 2c03ade3ac4f23bab10b98acc64b27c9f3267edd Mon Sep 17 00:00:00 2001
From: mkuznet1 <Mikhail.Kuznetsov@amd.com>
Date: Wed, 21 Oct 2026 18:05:36 +0000
Subject: [PATCH 6/8] fix(rccl-ci): release host memory before each
 sglang-disagg phase

sglang's nixl backend registers every KV-cache buffer with every RDMA
NIC, and each bnxt_re registration needs an order-9 host allocation. On
nodes whose RAM is nearly all page cache that fails (dmesg: bnxt_re
Failed to register user MR - rc = -12) and the server dies at startup
with NIXL_ERR_BACKEND. The nodes are held exclusively, so drop the
clean page cache and compact memory on each of them before a phase;
best effort when passwordless sudo is unavailable.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 projects/rccl/ci/scripts/test_madengine.py    | 27 +++++++++++++++++--
 .../ci/scripts/tests/test_sglang_disagg.py    | 11 ++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/projects/rccl/ci/scripts/test_madengine.py b/projects/rccl/ci/scripts/test_madengine.py
index 70e7f1573d34..f82037947564 100644
--- a/projects/rccl/ci/scripts/test_madengine.py
+++ b/projects/rccl/ci/scripts/test_madengine.py
@@ -1977,6 +1977,23 @@ def write_repro_bundle(
 fi
 """
 
+# sglang's nixl backend registers every KV-cache buffer with every RDMA NIC,
+# and each bnxt_re registration needs physically contiguous host memory (an
+# order-9 allocation). On a node whose RAM is nearly all page cache those fail
+# -- "bnxt_re3: Failed to register user MR - rc = -12" in dmesg -- and the
+# server that hits one dies at startup with NIXL_ERR_BACKEND. The nodes are
+# held exclusively, so drop the clean page cache and compact before each phase.
+# Best effort: without passwordless sudo the phase runs as before.
+RELEASE_HOST_MEMORY = """\
+#!/bin/bash
+if sudo -n sh -c 'sync; echo 1 > /proc/sys/vm/drop_caches; echo 1 > /proc/sys/vm/compact_memory'; then
+    echo "  ok   $(hostname -s) free=$(free -g | awk '/^Mem:/ {print $4}')G"
+else
+    echo "  skip $(hostname -s): no passwordless sudo"
+fi
+exit 0
+"""
+
 # Run on every node once the phases are done. The images are per-run tags that
 # no later run can reuse, and each is tens of GB on a node's local disk.
 CLEANUP_IMAGES = """\
@@ -2079,10 +2096,16 @@ def run_ab_in_one_allocation(
             "  exit 3",
             "}",
         ]
+        release_memory = work_dir / "ab" / "release_host_memory.sh"
+        release_memory.write_text(RELEASE_HOST_MEMORY)
     for phase, run_dir in phases:
+        script_lines += ["", f'echo "===== phase: {phase} ====="']
+        if workload_config.get("family") == "sglang-disagg":
+            script_lines.append(
+                'srun --ntasks="${SLURM_JOB_NUM_NODES}" --ntasks-per-node=1 '
+                f"--overlap bash {shlex.quote(str(release_memory))}"
+            )
         script_lines += [
-            "",
-            f'echo "===== phase: {phase} ====="',
             f"cd {shlex.quote(str(run_dir))} || exit 1",
             "madengine run -m manifest.json -o perf.csv --live-output --verbose",
             "echo $? > " + shlex.quote(str(run_dir / "exit_code")),
diff --git a/projects/rccl/ci/scripts/tests/test_sglang_disagg.py b/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
index a1712cbdbedb..6d3ae0dca6bc 100644
--- a/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
+++ b/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
@@ -214,6 +214,17 @@ def test_dns_preflight_for_sglang_only(self):
         _, script = self.run_allocation("llama-3.1-405b-maxtext", {})
         self.assertNotIn("preflight_dns.sh", script)
 
+    def test_host_memory_released_before_each_sglang_phase(self):
+        _, script = self.run_allocation(WORKLOAD, {})
+        lines = script.splitlines()
+        release = next(i for i, line in enumerate(lines) if "release_host_memory.sh" in line)
+        self.assertLess(release, next(i for i, line in enumerate(lines)
+                                      if line.startswith("madengine run")))
+        self.assertIn('--ntasks="${SLURM_JOB_NUM_NODES}" --ntasks-per-node=1', lines[release])
+        self.assertTrue((self.work / "ab" / "release_host_memory.sh").is_file())
+        _, script = self.run_allocation("llama-3.1-405b-maxtext", {})
+        self.assertNotIn("release_host_memory.sh", script)
+
     def test_exclude_from_environment(self):
         cmd, _ = self.run_allocation(WORKLOAD, {"MADENGINE_SLURM_EXCLUDE": "n[01-02],n07"})
         self.assertEqual(cmd[cmd.index("--exclude") + 1], "n[01-02],n07")

From 98f11fc717214e8e9a4e98255755f7692bd54ed5 Mon Sep 17 00:00:00 2001
From: mkuznet1 <Mikhail.Kuznetsov@amd.com>
Date: Wed, 21 Oct 2026 19:21:48 +0000
Subject: [PATCH 7/8] fix(rccl-ci): keep a swapped librccl's DT_RPATH a
 DT_RPATH

patchelf --set-rpath writes DT_RUNPATH, which LD_LIBRARY_PATH outranks;
the SDK wheels' librccl carries DT_RPATH, which outranks it. In the
rocm10 sgl-dev image LD_LIBRARY_PATH names _rocm_sdk_devel/lib, so the
swapped library loaded that directory's copy of libamd_smi.so.27 next to
the _rocm_sdk_core one amdsmi uses. The copies interpose on each other,
amdsmi reports no GPUs, torch.cuda.device_count() returns 0, and every
candidate sglang server died on 'Invalid device id'. Pass --force-rpath
when the replaced library had a DT_RPATH.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 projects/rccl/ci/scripts/test_madengine.py          | 13 +++++++++++--
 .../rccl/ci/scripts/tests/test_sglang_disagg.py     |  5 +++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/projects/rccl/ci/scripts/test_madengine.py b/projects/rccl/ci/scripts/test_madengine.py
index f82037947564..42fcd2507c90 100644
--- a/projects/rccl/ci/scripts/test_madengine.py
+++ b/projects/rccl/ci/scripts/test_madengine.py
@@ -932,6 +932,14 @@ def _librccl_swap_snippet(bridge: str = "", kpack: bool = False) -> str:
       matters for the same reason in reverse: that destination resolves through
       the loader's search path, and leaving ours behind would silently change
       where its dependencies come from.
+
+      The tag is kept too: patchelf writes DT_RUNPATH unless told otherwise,
+      and DT_RPATH, which the SDK wheels' librccl carries, outranks
+      LD_LIBRARY_PATH while DT_RUNPATH does not. In the rocm10 sgl-dev image,
+      LD_LIBRARY_PATH names _rocm_sdk_devel/lib, which holds its own copy of
+      libamd_smi.so.27. As a RUNPATH the swapped library loads that copy next
+      to the _rocm_sdk_core one amdsmi uses. The two interpose on each other,
+      amdsmi then reports no GPUs, and so does torch.cuda.device_count().
     * **The gate compares unresolved sonames by LIBRARY, not by exact name**,
       per target, before and after. Counts alone are not enough -- dropping one
       missing soname while introducing another leaves the count unchanged. An
@@ -997,9 +1005,10 @@ def _librccl_swap_snippet(bridge: str = "", kpack: bool = False) -> str:
     while read -r t; do \\
       [ "$(readlink -f "$t")" = "$canon_src" ] && continue; \\
       rp="$(patchelf --print-rpath "$t" 2>/dev/null || true)"; \\
-      echo "  overwrite: $t (rpath: ${{rp:-none}})"; \\
+      if readelf -d "$t" 2>/dev/null | grep -qF '(RPATH)'; then tag=--force-rpath; else tag=; fi; \\
+      echo "  overwrite: $t (rpath: ${{rp:-none}}${{tag:+, kept as DT_RPATH}})"; \\
       cp -fL --remove-destination "$SRC" "$t"; \\
-      if [ -n "$rp" ]; then patchelf --set-rpath "$rp" "$t" || \\
+      if [ -n "$rp" ]; then patchelf $tag --set-rpath "$rp" "$t" || \\
         {{ echo "GATE FAIL: could not restore RUNPATH on $t"; exit 1; }}; \\
       else patchelf --remove-rpath "$t" || \\
         {{ echo "GATE FAIL: could not clear RUNPATH on $t"; exit 1; }}; fi; \\
diff --git a/projects/rccl/ci/scripts/tests/test_sglang_disagg.py b/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
index 6d3ae0dca6bc..1a8af7fa000b 100644
--- a/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
+++ b/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
@@ -123,6 +123,11 @@ def test_base_pinned_and_patchelf_added(self):
         self.assertTrue(text.rstrip().endswith(tm.SGLANG_SWAP_TOOLS.rstrip()))
         self.assertLess(text.index("RUN echo payload"), text.index("patchelf"))
 
+    def test_swap_keeps_a_dt_rpath_a_dt_rpath(self):
+        snippet = tm._librccl_swap_snippet(kpack=True)
+        self.assertIn("grep -qF '(RPATH)'; then tag=--force-rpath", snippet)
+        self.assertIn('patchelf $tag --set-rpath "$rp" "$t"', snippet)
+
     def test_unexpected_recipe_fails(self):
         (self.work / "MAD" / tm.SGLANG_MAD_DOCKERFILE).write_text("FROM ubuntu\n")
         with self.assertRaisesRegex(RuntimeError, "FROM \\$BASE_DOCKER"):

From daff1957de2c496dd0dfc99bea464de7bbdf34d0 Mon Sep 17 00:00:00 2001
From: mkuznet1 <Mikhail.Kuznetsov@amd.com>
Date: Thu, 22 Oct 2026 11:22:18 +0000
Subject: [PATCH 8/8] fix(rccl-ci): count sglang server logs as node logs in
 RCCL verification

verify_rccl_replacement now requires a banner log from every node, keyed by
the node index in the log name. sglang phases verify against the servers'
prefill_NODE<i>.log / decode_NODE<i>.log rather than madengine's
*_node_<i>.out, so read the index from either form; without this every
sglang phase would fail verification.

Tests: pass the node count, add a missing-node case, and expect the cleanup
step to run without built images too, with only the stock base as helper.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 projects/rccl/ci/scripts/test_madengine.py        |  2 +-
 .../rccl/ci/scripts/tests/test_sglang_disagg.py   | 15 ++++++++++++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/projects/rccl/ci/scripts/test_madengine.py b/projects/rccl/ci/scripts/test_madengine.py
index 1d5effb5cf10..b1f7e0cb81ba 100644
--- a/projects/rccl/ci/scripts/test_madengine.py
+++ b/projects/rccl/ci/scripts/test_madengine.py
@@ -807,7 +807,7 @@ def verify_rccl_replacement(
     logged = {
         int(m.group(1))
         for log_file in node_logs
-        if (m := re.search(r"_node_(\d+)\.out$", log_file.name))
+        if (m := re.search(r"_(?:node_|NODE)(\d+)\.(?:out|log)$", log_file.name))
     }
     missing = sorted(set(range(nodes)) - logged)
     if missing:
diff --git a/projects/rccl/ci/scripts/tests/test_sglang_disagg.py b/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
index 1a8af7fa000b..6da8b889f97f 100644
--- a/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
+++ b/projects/rccl/ci/scripts/tests/test_sglang_disagg.py
@@ -176,12 +176,17 @@ def test_server_logs_preferred_and_verified(self):
                              ["decode_NODE2.log", "decode_NODE3.log",
                               "prefill_NODE0.log", "prefill_NODE1.log"])
             ok, msg = tm.verify_rccl_replacement(
-                run_dir, {"version": "2.32.3-develop:2b88119"})
+                run_dir, {"version": "2.32.3-develop:2b88119"}, 4)
             self.assertTrue(ok, msg)
             (logs / "decode_NODE3.log").write_text("died before init\n")
             ok, _ = tm.verify_rccl_replacement(
-                run_dir, {"version": "2.32.3-develop:2b88119"})
+                run_dir, {"version": "2.32.3-develop:2b88119"}, 4)
             self.assertFalse(ok)
+            (logs / "decode_NODE3.log").unlink()
+            ok, msg = tm.verify_rccl_replacement(
+                run_dir, {"version": "2.32.3-develop:2b88119"}, 4)
+            self.assertFalse(ok)
+            self.assertIn("node(s) 3 of 4", msg)
 
 
 class AllocationScriptTest(TempWork):
@@ -209,8 +214,12 @@ def test_built_images_removed_from_every_node_after_the_phases(self):
         self.assertIn('--ntasks="${SLURM_JOB_NUM_NODES}" --ntasks-per-node=1', lines[cleanup])
         self.assertTrue(lines[cleanup].endswith(" ".join(images) + " || true"))
         self.assertTrue((self.work / "ab" / "cleanup_images.sh").is_file())
+        # Node workspaces are cleaned up even when no image was built; the
+        # stock base is only the helper, never removed.
         _, script = self.run_allocation(WORKLOAD, {})
-        self.assertNotIn("cleanup_images.sh", script)
+        cleanup = next(line for line in script.splitlines() if "cleanup_images.sh" in line)
+        self.assertTrue(cleanup.endswith(
+            "cleanup_images.sh " + tm.WORKLOAD_CONFIGS[WORKLOAD]["base_image"] + " || true"))
 
     def test_dns_preflight_for_sglang_only(self):
         _, script = self.run_allocation(WORKLOAD, {})