ci(rccl): add DeepSeek-R1 SGLang 2P+2D to the weekly madengine runs

.github/workflows/therock-rccl-ci-linux.yml

@@ -316,6 +316,12 @@ jobs:
           - workload: llama-3.1-405b-maxtext
             nodes: '4'
             job_timeout_minutes: '450'
+          # 4 nodes: two prefill, two decode. 360 is its 3 h time_limit plus
+          # 120 minutes, because the head node also pulls the sglang base and
+          # builds both images before the allocation is requested.
+          - workload: deepseek-r1-sglang-disagg
+            nodes: '4'
+            job_timeout_minutes: '360'
     uses: ./.github/workflows/therock-rccl-test-madengine.yml
     secrets: inherit
     with:

.github/workflows/therock-rccl-test-madengine.yml

@@ -11,6 +11,7 @@ on:
         type: string
       artifact_run_id:
         type: string
+        required: true
       workload:
         # A key of WORKLOAD_CONFIGS in test_madengine.py, which validates it.
         # workflow_call inputs cannot be typed as choice, so the list of
@@ -53,9 +54,13 @@ on:
       test_runs_on:
         type: string
         default: 'ruby-linux-slurm-scale-runner'
+      # No default: its RCCL is the A/B candidate, so the dispatcher picks the
+      # build. The latest TheRock run would carry TheRock's pinned RCCL, not
+      # this repository's.
       artifact_run_id:
-        description: 'Reuse artifacts from a previous build run ID'
+        description: 'rocm-systems run ID whose RCCL build is the A/B candidate (e.g. a develop nightly of TheRock CI for rccl)'
         type: string
+        required: true
       workload:
         description: 'MADEngine workload to run'
         type: choice
@@ -64,6 +69,7 @@ on:
           - llama-3.1-70b-pp4-training
           - gpt-oss-120b-training
           - llama-3.1-405b-maxtext
+          - deepseek-r1-sglang-disagg
         default: 'llama-3.1-70b-training'
       nodes:
         description: 'Number of SLURM nodes to allocate'
@@ -204,33 +210,15 @@ jobs:
           python3 -m pip install --upgrade pip
           python3 -m pip install -r requirements.txt
 
-      - name: Resolve TheRock artifact run
-        if: ${{ inputs.artifact_run_id == '' }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-          GITHUB_TOKEN: ${{ github.token }}
-        run: |
-          run_id="$(python3 -c "
-          import sys, contextlib
-          sys.path.insert(0, 'build_tools')
-          from find_latest_artifacts import find_latest_artifacts
-          with contextlib.redirect_stdout(sys.stderr):
-              results = find_latest_artifacts(
-                  artifact_groups=['${{ inputs.artifact_group }}'],
-                  verbose=True,
-              )
-          if not results:
-              print('No artifacts found for ${{ inputs.artifact_group }}', file=sys.stderr)
-              sys.exit(1)
-          print(results[0].workflow_run_id)
-          ")"
-          echo "Resolved TheRock run_id=${run_id}"
-          echo "ARTIFACT_RUN_ID=${run_id}" >> $GITHUB_ENV
-
       - name: Fetch RCCL artifacts
         env:
           GITHUB_TOKEN: ${{ github.token }}
         run: |
+          # `required` does not stop an API dispatch from passing an empty string.
+          if [ -z "${ARTIFACT_RUN_ID}" ]; then
+            echo "::error::artifact_run_id is required: it names the RCCL build tested as the candidate"
+            exit 1
+          fi
           python3 build_tools/install_rocm_from_artifacts.py \
             --run-id="${ARTIFACT_RUN_ID}" \
             --artifact-group="${{ inputs.artifact_group }}" \
@@ -243,6 +231,8 @@ jobs:
           NCCL_DEBUG: WARN
           MAD_SECRETS_HFTOKEN: ${{ secrets.HF_TOKEN || '' }}
           JOB_TIMEOUT_MINUTES: ${{ inputs.job_timeout_minutes }}
+          # Nodes to keep out of the allocation, as a SLURM hostlist.
+          MADENGINE_SLURM_EXCLUDE: ${{ vars.RCCL_MADENGINE_SLURM_EXCLUDE || '' }}
         run: |
           export PYTHONPATH="rocm-systems/projects/rccl/ci/scripts:${PYTHONPATH}"
 
@@ -300,6 +290,7 @@ jobs:
             ${{ env.WORK_DIR }}/ab/*/manifest.json
             ${{ env.WORK_DIR }}/ab/*/slurm_output/*.out
             ${{ env.WORK_DIR }}/ab/*/slurm_output/*.err
+            ${{ env.WORK_DIR }}/ab/*/slurm_output/run_logs/*/*.log
           retention-days: 30
 
       # What a rerun by hand needs once the cleanup below has deleted WORK_DIR: