Repository navigation
ROCm Slurm CI #24
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| # Offline PR checks and an opt-in GPU lane using the protected workflow commit. | |
| name: ROCm Slurm CI | |
| on: | |
| pull_request: | |
| paths: | |
| - '.github/workflows/rocm-ci.yaml' | |
| - 'container/**' | |
| - 'scripts/ci/**' | |
| - 'tests/conftest.py' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| offline: | |
| name: Validate ROCm CI helpers | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| persist-credentials: false | |
| - name: Run dependency-free controller tests | |
| shell: bash | |
| run: python3 -m unittest discover -s scripts/ci/tests -p 'test_*.py' -v | |
| - name: Validate ROCm Dockerfile rendering | |
| shell: bash | |
| run: | | |
| python3 -m venv "$RUNNER_TEMP/rocm-render-tests" | |
| "$RUNNER_TEMP/rocm-render-tests/bin/python" -m pip install Jinja2==3.1.6 PyYAML==6.0.3 | |
| "$RUNNER_TEMP/rocm-render-tests/bin/python" -m unittest discover -s container/tests -p 'test_*.py' -v | |
| - name: Check shell syntax | |
| shell: bash | |
| run: | | |
| for script in scripts/ci/slurm.sbatch scripts/ci/rocm/run-tests.sh; do | |
| bash -n "$script" | |
| done | |
| aggregate: | |
| needs: offline | |
| concurrency: | |
| group: ${{ format('{0}-ci', vars.ROCM_SLURM_ENVIRONMENT || 'rocm-slurm') }} | |
| cancel-in-progress: false | |
| if: >- | |
| github.event_name == 'workflow_dispatch' && | |
| github.ref_type == 'branch' && | |
| vars.ROCM_SLURM_ENABLED == 'true' | |
| runs-on: ubuntu-latest | |
| environment: ${{ vars.ROCM_SLURM_ENVIRONMENT || 'rocm-slurm' }} | |
| timeout-minutes: 330 | |
| env: | |
| SLURM_UID: ${{ vars.SLURM_UID }} | |
| PREFERRED_PARTITION: ${{ vars.SLURM_PREFERRED_PARTITION || 'compute-1' }} | |
| FALLBACK_PARTITION: ${{ vars.SLURM_FALLBACK_PARTITION || 'compute-0' }} | |
| steps: | |
| - name: Initialize evidence before checkout | |
| timeout-minutes: 1 | |
| shell: bash | |
| run: | | |
| umask 077 | |
| mkdir -p "$RUNNER_TEMP/dynamo-rocm/evidence" | |
| printf 'ROCM_SSH_DIR=%s/dynamo-rocm/ssh\n' "$RUNNER_TEMP" >> "$GITHUB_ENV" | |
| printf '%s\n' "workflow_sha=$GITHUB_SHA" "phase=before-checkout" \ | |
| > "$RUNNER_TEMP/dynamo-rocm/evidence/controller-start.txt" | |
| - name: Checkout the workflow commit and materialize LFS files | |
| id: controller_checkout | |
| timeout-minutes: 10 | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ github.sha }} | |
| lfs: true | |
| persist-credentials: false | |
| - name: Configure scoped SSH credentials and pinned host keys | |
| timeout-minutes: 1 | |
| shell: bash | |
| env: | |
| SLURM_SSH_CONFIG: ${{ vars.SLURM_SSH_CONFIG }} | |
| SLURM_SSH_KEY: ${{ secrets.SLURM_SSH_KEY }} | |
| SLURM_GATEWAY_SSH_KEY: ${{ secrets.SLURM_GATEWAY_SSH_KEY }} | |
| SLURM_KNOWN_HOSTS: ${{ vars.SLURM_KNOWN_HOSTS }} | |
| run: | | |
| set -euo pipefail | |
| : "${SLURM_SSH_CONFIG:?Missing SSH host configuration}" | |
| : "${SLURM_SSH_KEY:?Missing cluster SSH key}" | |
| : "${SLURM_KNOWN_HOSTS:?Missing pinned host keys}" | |
| umask 077 | |
| mkdir "$ROCM_SSH_DIR" | |
| printf '%s\n' "$SLURM_SSH_KEY" > "$ROCM_SSH_DIR/ssh_key" | |
| printf '%s\n' "$SLURM_KNOWN_HOSTS" > "$ROCM_SSH_DIR/known_hosts" | |
| cat > "$ROCM_SSH_DIR/config" <<EOF | |
| Host * | |
| BatchMode yes | |
| IdentitiesOnly yes | |
| IdentityAgent none | |
| ForwardAgent no | |
| StrictHostKeyChecking yes | |
| GlobalKnownHostsFile /dev/null | |
| UserKnownHostsFile "$ROCM_SSH_DIR/known_hosts" | |
| ConnectTimeout 20 | |
| ServerAliveInterval 30 | |
| ServerAliveCountMax 3 | |
| EOF | |
| printf '%s\n' "$SLURM_SSH_CONFIG" >> "$ROCM_SSH_DIR/config" | |
| ssh-keygen -y -P '' -f "$ROCM_SSH_DIR/ssh_key" > /dev/null | |
| ssh-keygen -l -f "$ROCM_SSH_DIR/known_hosts" > /dev/null | |
| if [[ -n "$SLURM_GATEWAY_SSH_KEY" ]]; then | |
| printf '%s\n' "$SLURM_GATEWAY_SSH_KEY" > "$ROCM_SSH_DIR/gateway_key" | |
| ssh-keygen -y -P '' -f "$ROCM_SSH_DIR/gateway_key" > /dev/null | |
| fi | |
| ssh -G -T -F "$ROCM_SSH_DIR/config" ci-slurm-login > /dev/null | |
| - name: Check cluster access before submitting a GPU job | |
| timeout-minutes: 2 | |
| run: python3 -B scripts/ci/slurm_client.py preflight | |
| - name: Build ROCm images and run MI300X tests through Slurm | |
| timeout-minutes: 290 | |
| run: python3 -B scripts/ci/slurm_client.py run | |
| - name: Reconcile this run and collect partial evidence | |
| if: always() && steps.controller_checkout.outcome == 'success' | |
| timeout-minutes: 5 | |
| run: python3 -B scripts/ci/slurm_client.py finalize | |
| - name: Record controller checkout failure | |
| if: always() && steps.controller_checkout.outcome != 'success' | |
| timeout-minutes: 1 | |
| shell: bash | |
| run: | | |
| printf '%s\n' 'controller checkout failed; no Slurm submission was attempted' \ | |
| > "$RUNNER_TEMP/dynamo-rocm/evidence/checkout-failure.txt" | |
| - name: Upload build logs and test evidence | |
| if: always() | |
| timeout-minutes: 5 | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: rocm-slurm-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: ${{ runner.temp }}/dynamo-rocm/evidence/ | |
| if-no-files-found: error | |
| retention-days: 14 | |
| - name: Remove temporary SSH material | |
| if: always() | |
| timeout-minutes: 1 | |
| shell: bash | |
| run: rm -rf -- "$RUNNER_TEMP/dynamo-rocm/ssh" |