[NVIDIA] Enable multiple runners for b200-multinode-slurm

.github/configs/runners.yaml

@@ -80,6 +80,8 @@ gb200:
 - gb200-nv_2
 b200-multinode-slurm:
 - 'b200-dgxc-slurm_0'
+- 'b200-dgxc-slurm_1'
+- 'b200-dgxc-slurm_2'
 b300:
 - 'b300-nv_0'
 gb300:

.github/workflows/benchmark-multinode-tmpl.yml

@@ -120,11 +120,19 @@ jobs:
       - name: Slurm cleanup (pre-run)
         run: &slurm-cleanup |
           if command -v squeue >/dev/null 2>&1; then
+<<<<<<< sahithi/b200-multi-runner
+            if [[ "${{ runner.name }}" == mi355x-amds* ]] || [[ "${{ runner.name }}" == b200-dgxc-slurm* ]]; then
+              echo "[Slurm] Cleaning up jobs with name: ${{ runner.name }} for user: $USER ..."
+              scancel -u "$USER" --name="${{ runner.name }}" || true
+              while [ -n "$(squeue -u "$USER" --name='${{ runner.name }}' --noheader --format='%i')" ]; do
+                squeue -u "$USER" --name="${{ runner.name }}"
+=======
             if [[ "${{ runner.name }}" == mi355x-amds* || "${{ runner.name }}" == gb200-nv* || "${{ runner.name }}" == h100-dgxc-slurm* ]]; then
               echo "[Slurm] Cleaning up jobs with name: ${{ runner.name }} ..."
               scancel --name="${{ runner.name }}" || true
               while [ -n "$(squeue --name='${{ runner.name }}' --noheader --format='%i')" ]; do
                 squeue --name="${{ runner.name }}"
+>>>>>>> chore/add-more-nv-runners
                 sleep 5
               done
             else

runners/launch_b200-dgxc-slurm.sh

@@ -19,7 +19,20 @@ source $HOME/.local/bin/env
 
 uv venv
 source .venv/bin/activate
-uv pip install -e .
+
+# Retry uv pip install to handle "Text file busy" when multiple runners
+# share the uv binary and one is still being written by the installer.
+UV_INSTALL_RETRIES=5
+UV_INSTALL_DELAY=10
+for i in $(seq 1 $UV_INSTALL_RETRIES); do
+    uv pip install -e . && break
+    echo "uv pip install attempt $i/$UV_INSTALL_RETRIES failed, retrying in ${UV_INSTALL_DELAY}s..."
+    sleep $UV_INSTALL_DELAY
+    if [ $i -eq $UV_INSTALL_RETRIES ]; then
+        echo "Error: uv pip install failed after $UV_INSTALL_RETRIES attempts"
+        exit 1
+    fi
+done
 
 if ! command -v srtctl &> /dev/null; then
     echo "Error: Failed to install srtctl"