diff --git a/.github/configs/runners.yaml b/.github/configs/runners.yaml index f613688d0b..c6e9b46149 100644 --- a/.github/configs/runners.yaml +++ b/.github/configs/runners.yaml @@ -80,6 +80,8 @@ gb200: - gb200-nv_2 b200-multinode-slurm: - 'b200-dgxc-slurm_0' +- 'b200-dgxc-slurm_1' +- 'b200-dgxc-slurm_2' b300: - 'b300-nv_0' gb300: diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 49919b3c96..7a3da89a02 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -120,11 +120,19 @@ jobs: - name: Slurm cleanup (pre-run) run: &slurm-cleanup | if command -v squeue >/dev/null 2>&1; then +<<<<<<< sahithi/b200-multi-runner + if [[ "${{ runner.name }}" == mi355x-amds* ]] || [[ "${{ runner.name }}" == b200-dgxc-slurm* ]]; then + echo "[Slurm] Cleaning up jobs with name: ${{ runner.name }} for user: $USER ..." + scancel -u "$USER" --name="${{ runner.name }}" || true + while [ -n "$(squeue -u "$USER" --name='${{ runner.name }}' --noheader --format='%i')" ]; do + squeue -u "$USER" --name="${{ runner.name }}" +======= if [[ "${{ runner.name }}" == mi355x-amds* || "${{ runner.name }}" == gb200-nv* || "${{ runner.name }}" == h100-dgxc-slurm* ]]; then echo "[Slurm] Cleaning up jobs with name: ${{ runner.name }} ..." scancel --name="${{ runner.name }}" || true while [ -n "$(squeue --name='${{ runner.name }}' --noheader --format='%i')" ]; do squeue --name="${{ runner.name }}" +>>>>>>> chore/add-more-nv-runners sleep 5 done else diff --git a/runners/launch_b200-dgxc-slurm.sh b/runners/launch_b200-dgxc-slurm.sh index 0aba09e81c..a8880fc5e3 100644 --- a/runners/launch_b200-dgxc-slurm.sh +++ b/runners/launch_b200-dgxc-slurm.sh @@ -19,7 +19,20 @@ source $HOME/.local/bin/env uv venv source .venv/bin/activate -uv pip install -e . + +# Retry uv pip install to handle "Text file busy" when multiple runners +# share the uv binary and one is still being written by the installer. +UV_INSTALL_RETRIES=5 +UV_INSTALL_DELAY=10 +for i in $(seq 1 $UV_INSTALL_RETRIES); do + uv pip install -e . && break + echo "uv pip install attempt $i/$UV_INSTALL_RETRIES failed, retrying in ${UV_INSTALL_DELAY}s..." + sleep $UV_INSTALL_DELAY + if [ $i -eq $UV_INSTALL_RETRIES ]; then + echo "Error: uv pip install failed after $UV_INSTALL_RETRIES attempts" + exit 1 + fi +done if ! command -v srtctl &> /dev/null; then echo "Error: Failed to install srtctl"