From bad0c0f6ac77e8c20ace40ce029446776fd8a34d Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Tue, 28 Jul 2026 01:02:30 -0700 Subject: [PATCH 1/3] ci: restore the image's cuDNN pin after reconciling requirements The GPU jobs install requirements.txt inside the test container. torchft-nightly pulls torch, torch pins nvidia-cudnn-cu13 exactly, and that resolve uninstalls the cuDNN the image deliberately installed last: Collecting nvidia-cudnn-cu13==9.19.0.56 (from torch>=2.7->torchft-nightly->-r requirements.txt) Uninstalling nvidia-cudnn-cu13-9.22.0.52 Successfully installed nvidia-cudnn-cu13-9.19.0.56 transformer_engine 2.17's fused attention backward then fails with CUDNN_STATUS_BAD_PARAM on CUDNN_ATTR_OPERATION_RESHAPE_MODE, which is what has been failing test_lora_qwen2.5_0.5B, test_gpt_oss_20b_moe_lora_ci and test_qwen3_5_35b_a3b_lora_ci on radixark/miles#1795. Record the version the image shipped and reinstall it after the resolve, rather than hardcoding it here: the Dockerfile stays the single place the version is chosen, and --no-deps keeps the rest of the resolved set untouched. Declaring the pin in requirements.txt instead does not work -- pip resolves the conflict against torch's exact pin by downgrading torch itself to 2.10.0 and swapping the whole CUDA stack to cu12. Verified on an H200 devbox running the exact CI image digest: the requirements resolve reproduces the downgrade (9.22.0.52 -> 9.19.0.56, torch untouched), the restore puts 9.22.0.52 back, and TE fused attention backward then passes 8/8 configs that fail at 9.19.0.56. --- .github/workflows/_run-ci.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_run-ci.yml b/.github/workflows/_run-ci.yml index 46d7dfd8ff..e730fc3be3 100644 --- a/.github/workflows/_run-ci.yml +++ b/.github/workflows/_run-ci.yml @@ -134,7 +134,27 @@ jobs: - name: Reconcile Miles dependencies shell: bash - run: python -m pip install -r "$GITHUB_WORKSPACE/requirements.txt" --break-system-packages + run: | + # The image pins cuDNN deliberately: transformer_engine's fused attention needs a + # newer build than torch's own exact pin, so the Dockerfile installs it last. This + # resolve drags it back down to whatever torch asks for, which makes fused attention + # backward fail with CUDNN_STATUS_BAD_PARAM. Record what the image shipped and put it + # back afterwards; --no-deps so nothing else in the resolved set moves. + pinned="" + for pkg in nvidia-cudnn-cu13 nvidia-cudnn-cu12; do + v=$(python -m pip show "$pkg" 2>/dev/null | awk '/^Version:/{print $2}') + [ -n "$v" ] && pinned="$pinned $pkg==$v" + done + python -m pip install -r "$GITHUB_WORKSPACE/requirements.txt" --break-system-packages + for spec in $pinned; do + pkg=${spec%%==*} + want=${spec##*==} + got=$(python -m pip show "$pkg" 2>/dev/null | awk '/^Version:/{print $2}') + if [ -n "$got" ] && [ "$got" != "$want" ]; then + echo "Restoring image pin: $pkg $got -> $want" + python -m pip install --no-deps --break-system-packages "$spec" + fi + done - name: Sync dependency sources if: ${{ !inputs.skip_dependency_install }} From 47ecd5dba4db65233691c2a55fa120ab90855548 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Tue, 28 Jul 2026 02:08:41 -0700 Subject: [PATCH 2/3] ci: do not let the empty-package probe trip errexit GitHub runs 'shell: bash' as 'bash -e {0}'. '[ -n "$v" ] && pinned=...' as the last statement of the loop body makes the loop exit non-zero whenever a package is absent (cu12 on a cu13 image), which aborted the whole step and every GPU job with it. Use an if-block so the loop always ends cleanly. Verified by extracting the step from the workflow and running it under 'bash -e' with a stub pip: exits 0 both when the version is untouched and when it has to be restored. --- .github/workflows/_run-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_run-ci.yml b/.github/workflows/_run-ci.yml index e730fc3be3..aa4ce50483 100644 --- a/.github/workflows/_run-ci.yml +++ b/.github/workflows/_run-ci.yml @@ -143,7 +143,7 @@ jobs: pinned="" for pkg in nvidia-cudnn-cu13 nvidia-cudnn-cu12; do v=$(python -m pip show "$pkg" 2>/dev/null | awk '/^Version:/{print $2}') - [ -n "$v" ] && pinned="$pinned $pkg==$v" + if [ -n "$v" ]; then pinned="$pinned $pkg==$v"; fi done python -m pip install -r "$GITHUB_WORKSPACE/requirements.txt" --break-system-packages for spec in $pinned; do From 20e30ced19029972a089d05f437baa202de1110b Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Tue, 28 Jul 2026 03:01:59 -0700 Subject: [PATCH 3/3] ci: guard the pip-show probe against pipefail GitHub runs steps as 'bash --noprofile --norc -e -o pipefail'. 'pip show' exits 1 for a package that is not installed (cu12 on a cu13 image), pipefail propagates that through the awk pipe, and -e then aborts the step -- taking every GPU job with it. Terminate both command substitutions with '|| true', matching how this file already reads optional values out of the PR body. Verified by extracting the step from the workflow and running it under the exact shell line GitHub logs, with a stub pip where cu12 is absent: exits 0, and still restores the pin when the resolve moves it. --- .github/workflows/_run-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_run-ci.yml b/.github/workflows/_run-ci.yml index aa4ce50483..dd420cc562 100644 --- a/.github/workflows/_run-ci.yml +++ b/.github/workflows/_run-ci.yml @@ -142,14 +142,14 @@ jobs: # back afterwards; --no-deps so nothing else in the resolved set moves. pinned="" for pkg in nvidia-cudnn-cu13 nvidia-cudnn-cu12; do - v=$(python -m pip show "$pkg" 2>/dev/null | awk '/^Version:/{print $2}') + v=$(python -m pip show "$pkg" 2>/dev/null | awk '/^Version:/{print $2}' || true) if [ -n "$v" ]; then pinned="$pinned $pkg==$v"; fi done python -m pip install -r "$GITHUB_WORKSPACE/requirements.txt" --break-system-packages for spec in $pinned; do pkg=${spec%%==*} want=${spec##*==} - got=$(python -m pip show "$pkg" 2>/dev/null | awk '/^Version:/{print $2}') + got=$(python -m pip show "$pkg" 2>/dev/null | awk '/^Version:/{print $2}' || true) if [ -n "$got" ] && [ "$got" != "$want" ]; then echo "Restoring image pin: $pkg $got -> $want" python -m pip install --no-deps --break-system-packages "$spec"