[release/1.6] Fix the Windows CUDA CI jobs (cherry-pick #24238) (#24266)

Cherry-pick of #24238 onto
`release/1.6`, preserving the original author and recording the merged
source commit `0db7a88ff808ecd39fc61e794952cbe033bb4619`.

This brings all five Windows CUDA fixes to the release: the distutils
logging call, CUDA 13.2 selection for the installed runner toolkits,
copying the required ExecuTorch DLLs beside the example runners, driver
installation and reliable model-test failure checks, and the test-infra
pin that supports final Python 3.15 wheel builds. Linux model exports
retain CUDA 13.0.

Validation: the cherry-pick applied without conflicts and its stable
patch ID matches the merged source commit exactly. Both changed
workflows parse, `setup.py` parses, and `git diff --check` passes. A
merge-tree check with the open release dependency PR #24199 is clean.
Windows GPU tests and wheel builds require CI.

Cherry-pick prepared with Codex assistance; original implementation by
Anthony Shoumikhin.

Co-authored-by: Anthony Shoumikhin <anthony@shoumikh.in>

Author: JacobSzwejbka

.ci/scripts/test_model_e2e_windows.ps1

@@ -133,6 +133,8 @@ function Download-IfNeeded {
 Push-Location $executorchRoot
 try {
     Write-Host "::group::Check CUDA toolchain"
+    # The GPU runners come without a driver, and every CUDA call fails without one.
+    & (Join-Path -Path $scriptDir -ChildPath "install_nvidia_driver_windows.ps1")
     $nvccOutput = nvcc --version | Out-String
     Write-Host $nvccOutput
     $nvidiaSmiCmd = Get-Command nvidia-smi -ErrorAction SilentlyContinue
@@ -299,7 +301,8 @@ try {
             -RedirectStandardOutput $stdoutFile `
             -RedirectStandardError $stderrFile
 
-        $stdout = if (Test-Path -Path $stdoutFile -PathType Leaf) { Get-Content -Path $stdoutFile -Raw } else { "" }
+        # [string] because Get-Content returns nothing for an empty file, and matching against nothing never fails.
+        $stdout = if (Test-Path -Path $stdoutFile -PathType Leaf) { [string](Get-Content -Path $stdoutFile -Raw) } else { "" }
         $stderr = if (Test-Path -Path $stderrFile -PathType Leaf) { Get-Content -Path $stderrFile -Raw } else { "" }
         $exitCode = $proc.ExitCode
     }
@@ -313,7 +316,7 @@ try {
     Write-Host $stderr
 
     if ($exitCode -ne 0) {
-        Write-Warning "Runner exited with code $exitCode (may be benign)"
+        throw "Runner exited with code $exitCode"
     }
 
     if ($expectedOutput -ne "") {

.github/workflows/build-wheels-cuda-windows.yml

@@ -94,9 +94,11 @@ jobs:
       repository: ${{ matrix.repository }}
       ref: ""
       test-infra-repository: pytorch/test-infra
-      # Avoid the unsatisfiable conda upgrade in the Windows runner's base env.
+      # Avoid the unsatisfiable conda upgrade in the Windows runner's base env, and the
+      # Python 3.15 release candidate download that fails on the current runner image.
       # https://github.com/pytorch/test-infra/pull/9179
-      test-infra-ref: e3d344d886a7281f6a919115d332df2a080da8ad
+      # https://github.com/pytorch/test-infra/pull/9265
+      test-infra-ref: b46669025788e540ed4ecc7c56cc52db343e284f
       build-matrix: ${{ needs.filter-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       env-script: ${{ matrix.env-script }}

.github/workflows/cuda-windows.yml

@@ -17,8 +17,8 @@ on:
       - backends/cuda/**
       - backends/aoti/**
       - extension/cuda/**
-      # Read only by unittest-cuda-runtime-windows; the other jobs filter on
-      # their own paths and stay skipped.
+      # Only unittest-cuda-runtime-windows runs for these alone; the other jobs
+      # filter on their own paths and stay skipped.
       - .ci/scripts/test_cuda_runtime_windows.ps1
       - .ci/scripts/install_nvidia_driver_windows.ps1
       - tools/cmake/Test.cmake
@@ -202,7 +202,10 @@ jobs:
       timeout: 240
       runner: windows.g5.4xlarge.nvidia.gpu
       gpu-arch-type: cuda
-      gpu-arch-version: "13.0"
+      # The Windows runner image carries only the toolkits the CUDA wheels are built for,
+      # 13.2 and 13.4, so this cannot match the export's 13.0. The exported program loads
+      # cudart64_13.dll, which every CUDA 13 toolkit provides.
+      gpu-arch-version: "13.2"
       download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       script: |
@@ -214,7 +217,7 @@ jobs:
           \$ErrorActionPreference = 'Stop'
           \$PSNativeCommandUseErrorActionPreference = \$true
 
-          \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.0'
+          \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2'
           \$env:CUDA_PATH = \$env:CUDA_HOME
           \$env:PATH = \"\$env:CUDA_HOME\bin;\$env:PATH\"
           nvcc --version
@@ -225,7 +228,7 @@ jobs:
             throw 'RUNNER_ARTIFACT_DIR is empty. Ensure download-artifact is configured for windows_job.yml.'
           }
 
-          .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '13.0'
+          .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '13.2'
         }"
 
   # The counterpart of cuda.yml's unittest-cuda-runtime on Windows. The Windows GPU runners
@@ -248,7 +251,8 @@ jobs:
       timeout: 120
       runner: windows.g5.4xlarge.nvidia.gpu
       gpu-arch-type: cuda
-      gpu-arch-version: "13.0"
+      # The same toolkit as test-model-cuda-windows-e2e, for the same reason.
+      gpu-arch-version: "13.2"
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       script: |
         git config --global http.sslBackend openssl
@@ -257,7 +261,7 @@ jobs:
         powershell -Command "& {
           Set-PSDebug -Trace 1
           \$ErrorActionPreference = 'Stop'
-          \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.0'
+          \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2'
           \$env:CUDA_PATH = \$env:CUDA_HOME
           .ci/scripts/setup-windows.ps1
           .ci/scripts/test_cuda_runtime_windows.ps1

examples/models/dinov2/CMakeLists.txt

@@ -90,8 +90,10 @@ if(MSVC)
   add_custom_command(
     TARGET dinov2_runner
     POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:aoti_cuda_shims>
-            $<TARGET_FILE_DIR:dinov2_runner>
-    COMMENT "Copying aoti_cuda_shims.dll to dinov2_runner directory"
+    COMMAND
+      ${CMAKE_COMMAND} -E copy_if_different $<TARGET_RUNTIME_DLLS:dinov2_runner>
+      $<TARGET_FILE_DIR:dinov2_runner>
+    COMMENT "Copying runtime DLLs to dinov2_runner directory"
+    COMMAND_EXPAND_LISTS
   )
 endif()

examples/models/nemotron3_diarization/CMakeLists.txt

@@ -80,8 +80,10 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA)
     TARGET nemotron3_diarization_runner
     POST_BUILD
     COMMAND
-      ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:aoti_cuda_shims>
+      ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_RUNTIME_DLLS:nemotron3_diarization_runner>
       $<TARGET_FILE_DIR:nemotron3_diarization_runner>
+    COMMAND_EXPAND_LISTS
   )
 endif()
 if(EXECUTORCH_BUILD_VULKAN)

examples/models/parakeet/CMakeLists.txt

@@ -151,9 +151,11 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA)
       TARGET ${parakeet_target}
       POST_BUILD
       COMMAND
-        ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:aoti_cuda_shims>
+        ${CMAKE_COMMAND} -E copy_if_different
+        $<TARGET_RUNTIME_DLLS:${parakeet_target}>
         $<TARGET_FILE_DIR:${parakeet_target}>
-      COMMENT "Copying aoti_cuda_shims.dll to ${parakeet_target} directory"
+      COMMENT "Copying runtime DLLs to ${parakeet_target} directory"
+      COMMAND_EXPAND_LISTS
     )
   endforeach()
 endif()

examples/models/sortformer/CMakeLists.txt

@@ -111,8 +111,11 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA)
   add_custom_command(
     TARGET sortformer_runner
     POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:aoti_cuda_shims>
-            $<TARGET_FILE_DIR:sortformer_runner>
-    COMMENT "Copying aoti_cuda_shims.dll to sortformer_runner directory"
+    COMMAND
+      ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_RUNTIME_DLLS:sortformer_runner>
+      $<TARGET_FILE_DIR:sortformer_runner>
+    COMMENT "Copying runtime DLLs to sortformer_runner directory"
+    COMMAND_EXPAND_LISTS
   )
 endif()

examples/models/voxtral/CMakeLists.txt

@@ -141,8 +141,10 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA)
   add_custom_command(
     TARGET voxtral_runner
     POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:aoti_cuda_shims>
-            $<TARGET_FILE_DIR:voxtral_runner>
-    COMMENT "Copying aoti_cuda_shims.dll to voxtral_runner directory"
+    COMMAND
+      ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_RUNTIME_DLLS:voxtral_runner> $<TARGET_FILE_DIR:voxtral_runner>
+    COMMENT "Copying runtime DLLs to voxtral_runner directory"
+    COMMAND_EXPAND_LISTS
   )
 endif()

examples/models/voxtral_realtime/CMakeLists.txt

@@ -129,8 +129,11 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA)
   add_custom_command(
     TARGET voxtral_realtime_runner
     POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:aoti_cuda_shims>
-            $<TARGET_FILE_DIR:voxtral_realtime_runner>
-    COMMENT "Copying aoti_cuda_shims.dll to voxtral_realtime_runner directory"
+    COMMAND
+      ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_RUNTIME_DLLS:voxtral_realtime_runner>
+      $<TARGET_FILE_DIR:voxtral_realtime_runner>
+    COMMENT "Copying runtime DLLs to voxtral_realtime_runner directory"
+    COMMAND_EXPAND_LISTS
   )
 endif()

examples/models/whisper/CMakeLists.txt

@@ -114,8 +114,10 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA)
   add_custom_command(
     TARGET whisper_runner
     POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:aoti_cuda_shims>
-            $<TARGET_FILE_DIR:whisper_runner>
-    COMMENT "Copying aoti_cuda_shims.dll to whisper_runner directory"
+    COMMAND
+      ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_RUNTIME_DLLS:whisper_runner> $<TARGET_FILE_DIR:whisper_runner>
+    COMMENT "Copying runtime DLLs to whisper_runner directory"
+    COMMAND_EXPAND_LISTS
   )
 endif()