From b20b4b7ad9bdff41a1cf529d5bd7c78a86d253a1 Mon Sep 17 00:00:00 2001 From: Zhichenzzz Date: Sat, 25 Jul 2026 23:34:48 -0700 Subject: [PATCH 1/2] ci: stop the docker gate from skipping the whole test suite #1791 made resolve-ci-image depend on docker-build so that a docker PR runs its suites inside the freshly built image. docker-build only runs when docker files change, so on every other PR it is skipped -- and GitHub propagates a skip down the entire needs closure, including through resolve-ci-image's always() guard. stage-a-cpu carries no if: at all, so it was skipped, and stage-b/stage-c gate on stage-a-cpu succeeding, so the whole matrix went with it. The run still reports success, because a skipped job is not a failed one. Since #1791 merged, PRs 1792/1793/1794 each completed "green" with zero test jobs expanded; the last run that actually executed anything was #1790 at 07-24T21:17Z, before the merge. #1791's own CI was green and did run, because it changed docker/Dockerfile -- the regression could not appear in the PR that introduced it. Fix keeps the new behavior and restores the old graph: docker-paths and docker-build now always reach a conclusion instead of skipping, so nothing downstream is poisoned. With nothing to build, docker-build no-ops on ubuntu-latest rather than occupying a GPU runner, and reports built=false; resolve-ci-image reads that output instead of the job result. All seven stage-* jobs are byte-identical to their pre-#1791 definitions. docker-paths also loses its `if: github.event_name == 'pull_request'`, which had the same effect on the nightly cron: no PR context meant a skipped job and a silently empty scheduled run. It now reports changed=false for non-PR events. --- .github/workflows/pr-test.yml | 38 ++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 88036d7a70..9ace27fa29 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -71,7 +71,6 @@ jobs: run: python -m tests.ci.ci_policy docker-paths: - if: github.event_name == 'pull_request' runs-on: ubuntu-latest outputs: changed: ${{ steps.diff.outputs.changed }} @@ -81,10 +80,17 @@ jobs: persist-credentials: false - id: diff shell: bash + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} run: | - git fetch --depth 1 origin "${{ github.event.pull_request.base.sha }}" + # Cron and manual runs have no PR diff and never build a PR image. + if [ -z "$BASE_SHA" ]; then + echo "changed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + git fetch --depth 1 origin "$BASE_SHA" # Dockerfile.rocm doesn't feed the cu13 multi-arch build this pipeline produces. - if git diff --name-only "${{ github.event.pull_request.base.sha }}" HEAD \ + if git diff --name-only "$BASE_SHA" HEAD \ | grep -qE '^(docker/(Dockerfile|build\.py|verify_transformer_engine\.py)|requirements\.txt)$|^docker/patch/'; then echo "changed=true" >> "$GITHUB_OUTPUT" else @@ -92,37 +98,55 @@ jobs: fi # Docker PRs: build the multi-arch image first and run every suite inside it. + # Always runs: with nothing to build every step is a no-op and it stays off the + # GPU fleet. docker-build: needs: [docker-paths] - if: needs.docker-paths.outputs.changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository - runs-on: ["h200", "2gpu"] + if: always() && !cancelled() && needs.docker-paths.result == 'success' + runs-on: ${{ (needs.docker-paths.outputs.changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["h200", "2gpu"]') || 'ubuntu-latest' }} timeout-minutes: 180 + outputs: + built: ${{ steps.build.outputs.built || 'false' }} + env: + # Forks cannot push to the registry, so they keep the released image. + BUILD: ${{ needs.docker-paths.outputs.changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository }} steps: - name: Checkout repository + if: env.BUILD == 'true' uses: actions/checkout@v4 - name: Set up Docker Buildx + if: env.BUILD == 'true' uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:latest network=host - name: Install Python + dependencies + if: env.BUILD == 'true' run: | apt-get update && apt-get install -y python3 python3-pip pip3 install --break-system-packages typer - name: Login to Docker Hub + if: env.BUILD == 'true' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push PR tag + id: build + if: env.BUILD == 'true' run: | python3 docker/build.py --variant cu13 --image-tag custom \ --custom-tag pr-${{ github.event.pull_request.number }} --push + echo "built=true" >> "$GITHUB_OUTPUT" + - name: Nothing to build + if: env.BUILD != 'true' + run: echo "No docker-relevant changes; suites use the released image." + # No `if:` guard: a real build failure must stop the suites rather than run them + # against a stale image. resolve-ci-image: needs: [docker-paths, docker-build] - if: always() && !cancelled() && (needs.docker-build.result == 'success' || needs.docker-build.result == 'skipped') runs-on: ubuntu-latest outputs: container_image: ${{ steps.resolve.outputs.container_image }} @@ -133,7 +157,7 @@ jobs: env: PR_BODY: ${{ github.event.pull_request.body || '' }} INPUT_CI_IMAGE_TAG: ${{ github.event.inputs.ci_image_tag || '' }} - DOCKER_BUILT: ${{ needs.docker-build.result == 'success' }} + DOCKER_BUILT: ${{ needs.docker-build.outputs.built }} PR_NUMBER: ${{ github.event.pull_request.number || '' }} run: | CI_IMAGE_TAG="${INPUT_CI_IMAGE_TAG}" From edf9b7103ada7505a967f0df2eb7ef01974bc9ba Mon Sep 17 00:00:00 2001 From: Zhichenzzz Date: Sat, 25 Jul 2026 23:48:05 -0700 Subject: [PATCH 2/2] ci: keep the docker-build runner expression type-uniform The ternary mixed a JSON array with a bare string, and only the string branch is exercised by a PR that touches no docker file. Make both branches arrays so the untested branch cannot fail on a type mismatch. --- .github/workflows/pr-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 9ace27fa29..7804e814d9 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -102,8 +102,8 @@ jobs: # GPU fleet. docker-build: needs: [docker-paths] - if: always() && !cancelled() && needs.docker-paths.result == 'success' - runs-on: ${{ (needs.docker-paths.outputs.changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["h200", "2gpu"]') || 'ubuntu-latest' }} + if: always() && !cancelled() + runs-on: ${{ fromJSON((needs.docker-paths.outputs.changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository) && '["h200", "2gpu"]' || '["ubuntu-latest"]') }} timeout-minutes: 180 outputs: built: ${{ steps.build.outputs.built || 'false' }} @@ -146,7 +146,7 @@ jobs: # No `if:` guard: a real build failure must stop the suites rather than run them # against a stale image. resolve-ci-image: - needs: [docker-paths, docker-build] + needs: [docker-build] runs-on: ubuntu-latest outputs: container_image: ${{ steps.resolve.outputs.container_image }}