diff --git a/.github/scripts/winx64test/bootstrap_buildtools.bat b/.github/scripts/winx64test/bootstrap_buildtools.bat new file mode 100644 index 0000000000..a7ced81f0d --- /dev/null +++ b/.github/scripts/winx64test/bootstrap_buildtools.bat @@ -0,0 +1,67 @@ +@echo off + +echo Dependency MSVC Build Tools with C++ with ARM64/ARM64EC components installation started. + +:: Pre-check for downloads and dependencies folders +if not exist "%DOWNLOADS_DIR%" mkdir "%DOWNLOADS_DIR%" +if not exist "%DEPENDENCIES_DIR%" mkdir "%DEPENDENCIES_DIR%" + +:: Set download URL for the Visual Studio Installer +set DOWNLOAD_URL=https://aka.ms/vs/17/release/vs_BuildTools.exe +set INSTALLER_FILE=%DOWNLOADS_DIR%\vs_BuildTools.exe + +:: Download installer +echo Downloading Visual Studio Build Tools with C++ installer... +curl -L -o "%INSTALLER_FILE%" %DOWNLOAD_URL% + +:: Install the Visual Studio Build Tools with C++ components +echo Installing Visual Studio Build Tools with C++ components... +echo Installing MSVC %MSVC_VERSION% +if "%MSVC_VERSION%" == "latest" ( + "%INSTALLER_FILE%" --norestart --quiet --wait --installPath "%DEPENDENCIES_DIR%\VSBuildTools" ^ + --add Microsoft.VisualStudio.Workload.VCTools ^ + --add Microsoft.VisualStudio.Component.Windows10SDK ^ + --add Microsoft.VisualStudio.Component.Windows11SDK.22621 ^ + --add Microsoft.VisualStudio.Component.VC.ASAN ^ + --add Microsoft.VisualStudio.Component.VC.CMake.Project ^ + --add Microsoft.VisualStudio.Component.VC.CoreBuildTools ^ + --add Microsoft.VisualStudio.Component.VC.CoreIde ^ + --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest ^ + --add Microsoft.VisualStudio.Component.VC.Tools.ARM64EC ^ + --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 ^ + --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 +) else if "%MSVC_VERSION%" == "14.40" ( + "%INSTALLER_FILE%" --norestart --nocache --quiet --wait --fix --installPath "%DEPENDENCIES_DIR%\VSBuildTools" ^ + --add Microsoft.VisualStudio.Workload.VCTools ^ + --add Microsoft.VisualStudio.Component.Windows10SDK ^ + --add Microsoft.VisualStudio.Component.Windows11SDK.22621 ^ + --add Microsoft.VisualStudio.Component.VC.ASAN ^ + --add Microsoft.VisualStudio.Component.VC.CMake.Project ^ + --add Microsoft.VisualStudio.Component.VC.CoreBuildTools ^ + --add Microsoft.VisualStudio.Component.VC.CoreIde ^ + --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest ^ + --add Microsoft.VisualStudio.Component.VC.14.40.17.10.ARM64 ^ + --add Microsoft.VisualStudio.Component.VC.14.40.17.10.x86.x64 +) else if "%MSVC_VERSION%" == "14.36" ( + "%INSTALLER_FILE%" --norestart --nocache --quiet --wait --fix --installPath "%DEPENDENCIES_DIR%\VSBuildTools" ^ + --add Microsoft.VisualStudio.Workload.VCTools ^ + --add Microsoft.VisualStudio.Component.Windows10SDK ^ + --add Microsoft.VisualStudio.Component.Windows11SDK.22621 ^ + --add Microsoft.VisualStudio.Component.VC.ASAN ^ + --add Microsoft.VisualStudio.Component.VC.CMake.Project ^ + --add Microsoft.VisualStudio.Component.VC.CoreBuildTools ^ + --add Microsoft.VisualStudio.Component.VC.CoreIde ^ + --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest ^ + --add Microsoft.VisualStudio.Component.VC.14.36.17.6.ARM64 ^ + --add Microsoft.VisualStudio.Component.VC.14.36.17.6.x86.x64 +) + +echo exitcode = %errorlevel% + +:: Check if installation was successful +if %errorlevel% neq 0 ( + echo Failed to install Visual Studio Build Tools with C++ components. + exit /b 1 +) + +echo Dependency Visual Studio Build Tools with C++ installation finished. \ No newline at end of file diff --git a/.github/scripts/winx64test/bootstrap_buildtools.sh b/.github/scripts/winx64test/bootstrap_buildtools.sh new file mode 100644 index 0000000000..a2a1a7166e --- /dev/null +++ b/.github/scripts/winx64test/bootstrap_buildtools.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +echo "Dependency MSVC Build Tools with C++ with ARM64/ARM64EC components installation started." + +# Pre-check for downloads and dependencies folders +mkdir -p "$DOWNLOADS_DIR" +mkdir -p "$DEPENDENCIES_DIR" + +# Set download URL for the Visual Studio Installer +DOWNLOAD_URL="https://aka.ms/vs/17/release/vs_BuildTools.exe" +INSTALLER_FILE="$DOWNLOADS_DIR/vs_BuildTools.exe" + +# Download installer +echo "Downloading Visual Studio Build Tools with C++ installer..." +curl -L -o "$INSTALLER_FILE" "$DOWNLOAD_URL" + +# Install the Visual Studio Build Tools with C++ components +echo "Installing Visual Studio Build Tools with C++ components..." +echo "Installing MSVC $MSVC_VERSION" +"$INSTALLER_FILE" --norestart --quiet --wait --installPath "$DEPENDENCIES_DIR/VSBuildTools" \ + --add Microsoft.VisualStudio.Workload.VCTools \ + --add Microsoft.VisualStudio.Component.Windows10SDK \ + --add Microsoft.VisualStudio.Component.Windows11SDK.22621 \ + --add Microsoft.VisualStudio.Component.VC.ASAN \ + --add Microsoft.VisualStudio.Component.VC.CMake.Project \ + --add Microsoft.VisualStudio.Component.VC.CoreBuildTools \ + --add Microsoft.VisualStudio.Component.VC.CoreIde \ + --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest \ + --add Microsoft.VisualStudio.Component.VC.Tools.ARM64EC \ + --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 \ + --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 + +# Check if installation was successful +if [[ $? -ne 0 ]]; then + echo "Failed to install Visual Studio Build Tools with C++ components." + exit 1 +fi + +echo "Dependency Visual Studio Build Tools with C++ installation finished." \ No newline at end of file diff --git a/.github/scripts/winx64test/bootstrap_git.bat b/.github/scripts/winx64test/bootstrap_git.bat new file mode 100644 index 0000000000..3a0ad5b1b6 --- /dev/null +++ b/.github/scripts/winx64test/bootstrap_git.bat @@ -0,0 +1,44 @@ +@echo off +setlocal enabledelayedexpansion + +echo Dependency Git installation started. + +:: Pre-check for downloads and dependencies folders +set "DOWNLOADS_DIR=c:\temp\downloads" +set "DEPENDENCIES_DIR=c:\temp\dependencies" + +if not exist "%DOWNLOADS_DIR%" mkdir "%DOWNLOADS_DIR%" +if not exist "%DEPENDENCIES_DIR%" mkdir "%DEPENDENCIES_DIR%" + +:: Set download URL for the Git +set "DOWNLOAD_URL=https://github.com/git-for-windows/git/releases/download/v2.48.1.windows.1/Git-2.48.1-64-bit.exe" +set "INSTALLER_FILE=%DOWNLOADS_DIR%\Git-2.48.1-64-bit.exe" + +:: Download installer +echo Downloading Git... +curl -L -o "%INSTALLER_FILE%" "%DOWNLOAD_URL%" + +:: Verify download success +if not exist "%INSTALLER_FILE%" ( + echo Failed to download Git! + exit /b 1 +) + +:: Install Git +echo Installing Git... +"%INSTALLER_FILE%" /VERYSILENT /DIR="%DEPENDENCIES_DIR%\git" + +:: Verify installation success +if %errorlevel% neq 0 ( + echo Failed to install Git. (exitcode = %errorlevel%) + exit /b 1 +) + +:: Enable long paths +call "%DEPENDENCIES_DIR%\git\cmd\git.exe" config --system core.longpaths true + +:: Add Git to PATH (temporary for this session) +set "PATH=%DEPENDENCIES_DIR%\git\cmd\;%DEPENDENCIES_DIR%\git\bin\;%PATH%" + +echo Dependency Git installation finished. +exit /b 0 \ No newline at end of file diff --git a/.github/scripts/winx64test/bootstrap_python.bat b/.github/scripts/winx64test/bootstrap_python.bat new file mode 100644 index 0000000000..2de5e76584 --- /dev/null +++ b/.github/scripts/winx64test/bootstrap_python.bat @@ -0,0 +1,47 @@ +@echo off + +echo Dependency Python installation started. + +:: Pre-check for downloads and dependencies folders +if not exist "%DOWNLOADS_DIR%" mkdir %DOWNLOADS_DIR% +if not exist "%DEPENDENCIES_DIR%" mkdir %DEPENDENCIES_DIR% + +echo PYTHON_VERSION is %PYTHON_VERSION% + +if "%PYTHON_VERSION%" == "3.13" ( + echo Python version is set to 3.13 + set DOWNLOAD_URL=https://www.python.org/ftp/python/3.13.2/python-3.13.2-amd64.exe +) +else if "%PYTHON_VERSION%" == "3.12" ( + echo Python version is set to 3.12 + set DOWNLOAD_URL=https://www.python.org/ftp/python/3.12.7/python-3.12.7-amd64.exe +) else if "%PYTHON_VERSION%" == "3.11" ( + echo Python version is set to 3.11 + set DOWNLOAD_URL=https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe +) else ( + echo PYTHON_VERSION not defined, Python version is set to 3.12 + set DOWNLOAD_URL=https://www.python.org/ftp/python/3.12.7/python-3.12.7-amd64.exe +) + +set INSTALLER_FILE=%DOWNLOADS_DIR%\python-installer.exe + +:: Download installer +echo Downloading Python... +curl -L -o "%INSTALLER_FILE%" "%DOWNLOAD_URL%" + +:: Install Python +echo Installing Python... +"%INSTALLER_FILE%" /quiet Include_debug=1 TargetDir="%DEPENDENCIES_DIR%\Python" + +:: Check if installation was successful +if %errorlevel% neq 0 ( + echo "Failed to install Python. (exitcode = %errorlevel%)" + exit /b 1 +) + +:: Add to PATH +echo %DEPENDENCIES_DIR%\Python\>> %GITHUB_PATH% +echo %DEPENDENCIES_DIR%\Python\scripts\>> %GITHUB_PATH% +echo %DEPENDENCIES_DIR%\Python\libs\>> %GITHUB_PATH% + +echo Dependency Python installation finished. \ No newline at end of file diff --git a/.github/scripts/winx64test/build_audio.bat b/.github/scripts/winx64test/build_audio.bat new file mode 100644 index 0000000000..192a04809c --- /dev/null +++ b/.github/scripts/winx64test/build_audio.bat @@ -0,0 +1,60 @@ +@echo on +set SRC_PATH=%GITHUB_WORKSPACE%\%SRC_DIR% +set CMAKE_BUILD_TYPE=%BUILD_TYPE% +set VCVARSALL_PATH=%DEPENDENCIES_DIR%\VSBuildTools\VC\Auxiliary\Build\vcvarsall.bat +set CONDA_PREFIX=%DEPENDENCIES_DIR% +set PATH=%PATH%;%CONDA_PREFIX%\Library\bin +set DISTUTILS_USE_SDK=1 +set USE_FFMPEG=1 +set FFMPEG_ROOT=%DEPENDENCIES_DIR%\Library + +:: find torch file name by searching +for /f "delims=" %%f in ('dir /b "%DOWNLOADS_DIR%" ^| findstr "torch-"') do set "PYTORCH_PATH=%DOWNLOADS_DIR%\%%f" + +:: Dependencies +if not exist "%DOWNLOADS_DIR%" mkdir %DOWNLOADS_DIR% +if not exist "%DEPENDENCIES_DIR%" mkdir %DEPENDENCIES_DIR% +echo * > %DOWNLOADS_DIR%\.gitignore +echo * > %DEPENDENCIES_DIR%\.gitignore + +:: install vcpkg +cd %DOWNLOADS_DIR% +:: for ffmpeg 6.1.1 - pinning the version of vcpkg +:: https://pytorch.org/audio/stable/installation.html +git clone https://github.com/microsoft/vcpkg.git -b 2024.07.12 +cd vcpkg +call bootstrap-vcpkg.bat + +:: install dependencies +vcpkg install ffmpeg[ffmpeg]:x64-windows --x-install-root=%DEPENDENCIES_DIR% +robocopy /E %DEPENDENCIES_DIR%\x64-windows %DEPENDENCIES_DIR%\Library +robocopy /E %DEPENDENCIES_DIR%\Library\tools\ffmpeg %DEPENDENCIES_DIR%\Library\bin +robocopy /E %DEPENDENCIES_DIR%\Library\bin %SRC_PATH%\src\torio\lib + +:: test ffmpeg +echo %FFMPEG_ROOT% +ffmpeg -version + +:: Source directory +cd %SRC_PATH% + +:: Virtual environment +python -m pip install --upgrade pip +python -m venv .venv +echo * > .venv\.gitignore +call .\.venv\Scripts\activate + +:: Install dependencies +pip install %PYTORCH_PATH% + +:: Activate visual studio +call "%VCVARSALL_PATH%" x64 + +:: Creates wheel under dist folder +python setup.py bdist_wheel + +:: Check if installation was successful +if %errorlevel% neq 0 ( + echo "Failed on build_audio. (exitcode = %errorlevel%)" + exit /b 1 +) \ No newline at end of file diff --git a/.github/scripts/winx64test/build_audio.sh b/.github/scripts/winx64test/build_audio.sh new file mode 100644 index 0000000000..7508dba213 --- /dev/null +++ b/.github/scripts/winx64test/build_audio.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +echo "Building audio dependencies and wheel started." + +# Set environment variables +export SRC_PATH="$GITHUB_WORKSPACE/$SRC_DIR" +export VCVARSALL_PATH="$DEPENDENCIES_DIR/VSBuildTools/VC/Auxiliary/Build/vcvarsall.bat" +export CONDA_PREFIX="$DEPENDENCIES_DIR" +export PATH="$PATH:$CONDA_PREFIX/Library/bin" +export DISTUTILS_USE_SDK=1 +export USE_FFMPEG=1 +export FFMPEG_ROOT="$DEPENDENCIES_DIR/Library" +export TRIPLET_FILE="triplets/x64-windows.cmake" + +echo "CONDA_PREFIX: $CONDA_PREFIX" +echo "SRC_PATH: $SRC_PATH" +echo "VCVARSALL_PATH: $VCVARSALL_PATH" +echo "PATH: $PATH" + +# Dependencies +mkdir -p "$DOWNLOADS_DIR" +mkdir -p "$DEPENDENCIES_DIR" +echo "*" > "$DOWNLOADS_DIR/.gitignore" +echo "*" > "$DEPENDENCIES_DIR/.gitignore" + +# Install vcpkg +cd "$DOWNLOADS_DIR" || exit +git clone https://github.com/microsoft/vcpkg.git -b 2024.07.12 +cd vcpkg || exit +./bootstrap-vcpkg.sh + +# Set vcpkg to only build release packages +echo "set(VCPKG_BUILD_TYPE release)" >> "$TRIPLET_FILE" + +# Install dependencies using vcpkg +./vcpkg install ffmpeg[ffmpeg]:x64-windows --x-install-root="$DEPENDENCIES_DIR" + +# Copy files using cp (replace robocopy) +mkdir -p "$DEPENDENCIES_DIR/Library/" +cp -r "$DEPENDENCIES_DIR/x64-windows/"* "$DEPENDENCIES_DIR/Library" +cp -r "$DEPENDENCIES_DIR/Library/tools/ffmpeg/"* "$DEPENDENCIES_DIR/Library/bin" +cp -r "$DEPENDENCIES_DIR/Library/bin/"* "$SRC_PATH/src/torio/lib" + +# Test ffmpeg installation +echo "$FFMPEG_ROOT" +ffmpeg -version + +# Source directory +cd "$SRC_PATH" || exit + +# Create virtual environment +python -m pip install --upgrade pip +python -m venv .venv +echo "*" > .venv/.gitignore +source .venv/Scripts/activate + +# Install dependencies +pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + +# Create wheel under dist folder +python setup.py bdist_wheel + +# Check if build was successful +if [[ $? -ne 0 ]]; then + echo "Failed on build_audio. (exitcode = $?)" + exit 1 +fi + +echo "Build finished successfully." diff --git a/.github/scripts/winx64test/build_vision.bat b/.github/scripts/winx64test/build_vision.bat new file mode 100644 index 0000000000..9eedf3b994 --- /dev/null +++ b/.github/scripts/winx64test/build_vision.bat @@ -0,0 +1,75 @@ +echo "Starting build_vision.bat" +@echo on +set SRC_PATH=%GITHUB_WORKSPACE%\%SRC_DIR% +set CMAKE_BUILD_TYPE=%BUILD_TYPE% +@REM set VCVARSALL_PATH=%DEPENDENCIES_DIR%\VSBuildTools\VC\Auxiliary\Build\vcvarsall.bat +set CONDA_PREFIX=%DEPENDENCIES_DIR% +set PATH=%PATH%;%CONDA_PREFIX%\Library\bin +set DISTUTILS_USE_SDK=1 +@REM :: find toch file name by searching +@REM for /f "delims=" %%f in ('dir /b "%DOWNLOADS_DIR%" ^| findstr "torch-"') do set "PYTORCH_PATH=%DOWNLOADS_DIR%\%%f" + +:: Dependencies +if not exist "%DOWNLOADS_DIR%" mkdir %DOWNLOADS_DIR% +if not exist "%DEPENDENCIES_DIR%" mkdir %DEPENDENCIES_DIR% +echo * > %DOWNLOADS_DIR%\.gitignore +echo * > %DEPENDENCIES_DIR%\.gitignore + +:: install vcpkg +cd %DOWNLOADS_DIR% +git clone https://github.com/microsoft/vcpkg.git +cd vcpkg +call bootstrap-vcpkg.bat +echo "VCPKG Installed" + +:: install dependencies +vcpkg install libjpeg-turbo:x64-windows --x-install-root=%DEPENDENCIES_DIR% +vcpkg install libwebp:x64-windows --x-install-root=%DEPENDENCIES_DIR% +vcpkg install libpng[tools]:x64-windows --x-install-root=%DEPENDENCIES_DIR% +:: https://pytorch.org/vision/stable/index.html +:: Building with FFMPEG is disabled by default in the latest main +:: vcpkg install ffmpeg[ffmpeg]:x64-windows --x-install-root=%DEPENDENCIES_DIR% +copy %DEPENDENCIES_DIR%\x64-windows\lib\libpng16.lib %DEPENDENCIES_DIR%\x64-windows\lib\libpng.lib +copy %DEPENDENCIES_DIR%\x64-windows\bin\libpng16.dll %DEPENDENCIES_DIR%\x64-windows\bin\libpng.dll +copy %DEPENDENCIES_DIR%\x64-windows\bin\libpng16.pdb %DEPENDENCIES_DIR%\x64-windows\bin\libpng.pdb +robocopy /E %DEPENDENCIES_DIR%\x64-windows %DEPENDENCIES_DIR%\Library +robocopy /E %DEPENDENCIES_DIR%\Library\tools\libpng %DEPENDENCIES_DIR%\Library\bin +robocopy /E %DEPENDENCIES_DIR%\Library\bin %SRC_PATH%\torchvision *.dll + +:: Source directory +cd %SRC_PATH% + +:: Virtual environment +python -m pip install --upgrade pip +python -m venv .venv --upgrade-deps +echo * > .venv\.gitignore +call .\.venv\Scripts\activate + +:: Install dependencies +pip install numpy +pip3 install torch + +:: Activate visual studio +set VC_VERSION_LOWER=17 +set VC_VERSION_UPPER=18 + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( + if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VS15INSTALLDIR=%%i" + set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" + goto vswhere + ) +) + +:vswhere +call "%VS15VCVARSALL%" x64 || exit /b 1 + +:: Creates wheel under dist folder +python setup.py bdist_wheel + +:: Check if installation was successful +if %errorlevel% neq 0 ( + echo "Failed on build_vision. (exitcode = %errorlevel%)" + exit /b 1 +) +echo Finished running build_vision.bat" \ No newline at end of file diff --git a/.github/scripts/winx64test/build_vision.sh b/.github/scripts/winx64test/build_vision.sh new file mode 100644 index 0000000000..9eec973dc6 --- /dev/null +++ b/.github/scripts/winx64test/build_vision.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +echo "Building vision dependencies and wheel started." + +# Set environment variables +export SRC_PATH="$GITHUB_WORKSPACE/$SRC_DIR" +export CMAKE_BUILD_TYPE="$BUILD_TYPE" +export VCVARSALL_PATH="$DEPENDENCIES_DIR/VSBuildTools/VC/Auxiliary/Build/vcvarsall.bat" +export CONDA_PREFIX="$DEPENDENCIES_DIR" +export PATH="$PATH:$CONDA_PREFIX/Library/bin" +export DISTUTILS_USE_SDK=1 +export TRIPLET_FILE="triplets/x64-windows.cmake" +export PYTORCH_VERSION="$PYTORCH_VERSION" +export CHANNEL="$CHANNEL" + +echo "channel: $CHANNEL" + +# Dependencies +mkdir -p "$DOWNLOADS_DIR" +mkdir -p "$DEPENDENCIES_DIR" +echo "*" > "$DOWNLOADS_DIR/.gitignore" +echo "*" > "$DEPENDENCIES_DIR/.gitignore" + +# Install vcpkg +cd "$DOWNLOADS_DIR" || exit +git clone https://github.com/microsoft/vcpkg.git +cd vcpkg || exit +./bootstrap-vcpkg.sh + +# # Set vcpkg to only build release packages +echo "set(VCPKG_BUILD_TYPE release)" >> "$TRIPLET_FILE" + +# Install dependencies using vcpkg +./vcpkg install libjpeg-turbo:x64-windows --x-install-root="$DEPENDENCIES_DIR" +./vcpkg install libwebp:x64-windows --x-install-root="$DEPENDENCIES_DIR" +./vcpkg install libpng[tools]:x64-windows --x-install-root="$DEPENDENCIES_DIR" + +# Copy files using cp (replace robocopy) +cp "$DEPENDENCIES_DIR/x64-windows/lib/libpng16.lib" "$DEPENDENCIES_DIR/x64-windows/lib/libpng.lib" +cp "$DEPENDENCIES_DIR/x64-windows/bin/libpng16.dll" "$DEPENDENCIES_DIR/x64-windows/bin/libpng.dll" +cp "$DEPENDENCIES_DIR/x64-windows/bin/libpng16.pdb" "$DEPENDENCIES_DIR/x64-windows/bin/libpng.pdb" +mkdir -p "$DEPENDENCIES_DIR/Library/" +cp -r "$DEPENDENCIES_DIR/x64-windows/"* "$DEPENDENCIES_DIR/Library/" +cp -r "$DEPENDENCIES_DIR/Library/tools/libpng/"* "$DEPENDENCIES_DIR/Library/bin/" +cp -r "$DEPENDENCIES_DIR/Library/bin/"* "$SRC_PATH/torchvision" + +# Source directory +cd "$SRC_PATH" || exit + +# Create virtual environment +python -m pip install --upgrade pip +python -m venv .venv +echo "*" > .venv/.gitignore +source .venv/Scripts/activate + +# Install dependencies +pip install numpy==2.2.3 + +if [ "$CHANNEL" = "release" ]; then + echo "Installing latest stable version of PyTorch." + pip3 install torch +elif [ "$CHANNEL" = "test" ]; then + echo "Installing PyTorch version $PYTORCH_VERSION." + pip3 install torch=="$PYTORCH_VERSION" +else + echo "CHANNEL is not set, installing PyTorch from nightly." + pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu +fi + +# Create wheel under dist folder +python setup.py bdist_wheel + +# Check if build was successful +if [[ $? -ne 0 ]]; then + echo "Failed on build_vision. (exit code = $?)" + exit 1 +fi + +echo "Build finished successfully." diff --git a/.github/workflows/build_wheels_windows.yml b/.github/workflows/build_wheels_windows.yml index 6cf0405a05..a8b955ccdb 100644 --- a/.github/workflows/build_wheels_windows.yml +++ b/.github/workflows/build_wheels_windows.yml @@ -68,6 +68,10 @@ on: description: 'Timeout for the job (in minutes)' default: 60 type: number + architecture: + description: 'CPU architecture to build for' + default: "x64" + type: string permissions: id-token: write @@ -85,15 +89,21 @@ jobs: REF: ${{ inputs.ref }} CU_VERSION: ${{ matrix.desired_cuda }} UPLOAD_TO_BASE_BUCKET: ${{ matrix.upload_to_base_bucket }} - name: build-${{ matrix.build_name }} + name: build-${{ matrix.build_name }}-${{ matrix.gpu_arch_type }}- runs-on: ${{ matrix.validation_runner }} defaults: + run: shell: bash -l {0} # If a build is taking longer than 60 minutes on these runners we need # to have a conversation timeout-minutes: ${{ inputs.timeout }} steps: + - name: Echo variables + run: | + echo "ENV ARCHITECTURE=${{ env.ARCH }}" + echo "ARCHITECTURE=${{ inputs.architecture }}" + echo "CUDA VERSION=${{ ENV.CU_VERSION }}" - uses: actions/checkout@v4 with: # Support the use case where we need to checkout someone's fork @@ -107,16 +117,26 @@ jobs: activate-with-label: false instructions: "SSH with rdesktop using ssh -L 3389:localhost:3389 %%username%%@%%hostname%%" - name: Add Conda scripts to GitHub path + if: inputs.architecture == 'x64' run: | echo "C:/Jenkins/Miniconda3/Scripts" >> $GITHUB_PATH - uses: ./test-infra/.github/actions/set-channel - - name: Set PYTORCH_VERSION - if: ${{ env.CHANNEL == 'test' }} + - name: Setup Git for Windows' minimal SDK + env: + DEPENDENCIES_DIR: c:\temp\dependencies\ + if: inputs.architecture == 'arm64' + uses: git-for-windows/setup-git-for-windows-sdk@v1 + with: + architecture: x86_64 + path: "${{env.DEPENDENCIES_DIR}}\\git" + - name: Set PYTORCH_VERSION + if: env.CHANNEL == 'test' run: | # When building RC, set the version to be the current candidate version, # otherwise, leave it alone so nightly will pick up the latest echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}" - uses: ./test-infra/.github/actions/setup-binary-builds + if: inputs.architecture == 'x64' with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} @@ -124,20 +144,43 @@ jobs: setup-miniconda: false python-version: ${{ env.PYTHON_VERSION }} cuda-version: ${{ env.CU_VERSION }} - arch: ${{ env.ARCH }} + arch: ${{ inputs.architecture }} - name: Install XPU support package if: ${{ matrix.gpu_arch_type == 'xpu' }} env: XPU_VERSION: '2025.0' run: | cmd //c .\\test-infra\\.github\\scripts\\install_xpu.bat + - name: Checkout Target Repository (${{ env.REPOSITORY }}) + if: inputs.architecture == 'arm64' + uses: actions/checkout@v4 + with: + repository: ${{ env.REPOSITORY }} + ref: ${{ env.REF }} + path: ${{ env.REPOSITORY }} + submodules: recursive + - name: Bootstrap python + if: inputs.architecture == 'arm64' + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + architecture: x64 + - name: Bootstrap Build Tools + if: inputs.architecture == 'arm64' + env: + DOWNLOADS_DIR: c:\temp\downloads\ + DEPENDENCIES_DIR: c:\temp\dependencies\ + SCRIPTS_DIR: test-infra\\.github\\scripts\\winx64test + run: | + "${{ env.SCRIPTS_DIR }}\bootstrap_buildtools.sh" - name: Install torch dependency + if: inputs.architecture == 'x64' run: | source "${BUILD_ENV_FILE}" # shellcheck disable=SC2086 ${CONDA_RUN} ${PIP_INSTALL_TORCH} - name: Run Pre-Script with Caching - if: ${{ inputs.pre-script != '' }} + if: ${{ inputs.pre-script != '' && inputs.architecture == 'x64' }} uses: ./test-infra/.github/actions/run-script-with-cache with: cache-path: ${{ inputs.cache-path }} @@ -145,7 +188,24 @@ jobs: repository: ${{ inputs.repository }} script: ${{ inputs.pre-script }} is_windows: 'enabled' + - name: Run Pre-Script arm64 + if: ${{ inputs.pre-script != '' && inputs.architecture == 'arm64' }} + env: + DOWNLOADS_DIR: c:\temp\downloads\ + DEPENDENCIES_DIR: c:\temp\dependencies\ + SCRIPTS_DIR: test-infra\\.github\\scripts\\winx64test + SRC_DIR: ${{ inputs.repository }} + PRE_SCRIPT: ${{ inputs.pre-script }} + shell: cmd + run: | + set VS_PATH=%DEPENDENCIES_DIR%\VSBuildTools\VC\Auxiliary\Build\vcvarsall.bat + set GIT_BASH=%DEPENDENCIES_DIR%\git\usr\bin\bash.exe + cd %SRC_DIR% + + call "%VS_PATH%" x64 + "%GIT_BASH%" -c "bash --noprofile --norc %PRE_SCRIPT%" - name: Build clean + if: inputs.architecture == 'x64' working-directory: ${{ inputs.repository }} env: ENV_SCRIPT: ${{ inputs.env-script }} @@ -161,7 +221,8 @@ jobs: ${CONDA_RUN} ${ENV_SCRIPT} python setup.py clean fi fi - - name: Build the wheel (bdist_wheel) + - name: Build the wheel (bdist_wheel) X64 + if: inputs.architecture == 'x64' working-directory: ${{ inputs.repository }} env: ENV_SCRIPT: ${{ inputs.env-script }} @@ -182,17 +243,26 @@ jobs: else ${CONDA_RUN} ${ENV_SCRIPT} python setup.py bdist_wheel ${BUILD_PARAMS} fi + - name: Build the wheel (bdist_wheel) Arm64 + if: inputs.architecture == 'arm64' + env: + SRC_DIR: ${{ inputs.repository }} + run: | + cd $SRC_DIR + source .venv/Scripts/activate + python setup.py bdist_wheel - name: Run post-script working-directory: ${{ inputs.repository }} env: POST_SCRIPT: ${{ inputs.post-script }} ENV_SCRIPT: ${{ inputs.env-script }} - if: ${{ inputs.post-script != '' }} + if: ${{ inputs.post-script != '' && inputs.architecture == 'x64'}} run: | set -euxo pipefail source "${BUILD_ENV_FILE}" ${CONDA_RUN} ${ENV_SCRIPT} ${POST_SCRIPT} - - name: Smoke Test + - name: Smoke Test X64 + if: inputs.architecture == 'x64' env: ENV_SCRIPT: ${{ inputs.env-script }} PACKAGE_NAME: ${{ inputs.package-name }} @@ -209,7 +279,36 @@ jobs: echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" ${CONDA_RUN} "${{ inputs.repository }}/${ENV_SCRIPT}" python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi + - name: Smoke Test ARM64 + if: inputs.architecture == 'test' + env: + PACKAGE_NAME: ${{ inputs.package-name }} + SMOKE_TEST_SCRIPT: ${{ inputs.smoke-test-script }} + SRC_DIR: ${{ inputs.repository }} + run: | + cd $SRC_DIR + source .venv/Scripts/activate + whl=$(find dist -name "${{env.PACKAGE_NAME}}-*.whl" | head -n 1) + pip install $whl + + if [[ ! -f ${SMOKE_TEST_SCRIPT} ]]; then + echo "${SMOKE_TEST_SCRIPT} not found" + python -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)" + else + echo "${SMOKE_TEST_SCRIPT} found" + python "$SMOKE_TEST_SCRIPT" + fi # NB: Only upload to GitHub after passing smoke tests + - name: Get Artifact name + if: inputs.architecture == 'arm64' + env: + REPOSITORY: ${{ inputs.repository }} + REF: ${{ inputs.ref }} + PYTHON_VERSION: ${{ matrix.python_version }} + CU_VERSION: ${{ env.CU_VERSION }} + ARCH: ${{ inputs.architecture }} + run: | + echo "ARTIFACT_NAME=${REPOSITORY//\//_}_${REF//\//_}_${PYTHON_VERSION}_${CU_VERSION}_${ARCH}" >> "${GITHUB_ENV}" - name: Upload wheel to GitHub continue-on-error: true uses: actions/upload-artifact@v4 @@ -217,7 +316,7 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ inputs.repository }}/dist/ - uses: ./test-infra/.github/actions/teardown-windows - if: always() + if: inputs.architecture == 'x64' name: Teardown Windows upload: @@ -231,7 +330,8 @@ jobs: test-infra-ref: ${{ inputs.test-infra-ref }} build-matrix: ${{ inputs.build-matrix }} trigger-event: ${{ inputs.trigger-event }} + architecture: ${{ inputs.architecture }} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true + cancel-in-progress: true \ No newline at end of file diff --git a/.github/workflows/build_wheels_windows_x64_test.yml b/.github/workflows/build_wheels_windows_x64_test.yml new file mode 100644 index 0000000000..db83ad081f --- /dev/null +++ b/.github/workflows/build_wheels_windows_x64_test.yml @@ -0,0 +1,109 @@ +name: Build Windows X64 Wheels - test + +on: + workflow_call: + inputs: + # python_version: + # required: false + # default: "3.12" + # type: string + repository_name: # change NAME + required: true + type: string + repository_branch: # rename with ref + default value change + required: true + type: string +# add choice input + +env: + DOWNLOADS_DIR: c:\temp\downloads\ + DEPENDENCIES_DIR: c:\temp\dependencies\ + SCRIPTS_DIR: test-infra\\.github\\scripts\\winx64test + SRC_DIR: src\\${{ inputs.repository_name }} + PYTHON_VERSION: 3.12 ## update here ${{ inputs.python_version }} + EXTRACTED_REPO_NAME: ${{ endsWith(inputs.repository_name, 'audio') && 'audio' || endsWith(inputs.repository_name, 'vision') && 'vision' || '' }} + +permissions: + id-token: write + contents: read + +jobs: + build: + runs-on: windows-latest + name: Build wheel + steps: + - name: Print inputs + run: echo "${{ toJSON(github.action.inputs) }}" + - name: Setup Git for Windows' minimal SDK + uses: git-for-windows/setup-git-for-windows-sdk@v1 + with: + architecture: x86_64 + path: "${{env.DEPENDENCIES_DIR}}\\git" + - name: Checkout test-infra repository + uses: actions/checkout@v4 + with: + repository: alinpahontu2912/test-infra + ref: x64windows_test + path: test-infra + - name: Checkout Target Repository (${{ inputs.repository_name }}) + uses: actions/checkout@v4 + with: + repository: ${{ inputs.repository_name }} + ref: ${{ inputs.repository_branch }} + path: src/${{ inputs.repository_name }} + submodules: recursive + - name: Bootstrap python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + architecture: x64 + - name: Bootstrap Build Tools + shell: bash + run: | + "${{ env.SCRIPTS_DIR }}\bootstrap_buildtools.sh" + - name: Call Build Script + shell: cmd + run: | + set VS_PATH=%DEPENDENCIES_DIR%\VSBuildTools\VC\Auxiliary\Build\vcvarsall.bat + set GIT_BASH=%DEPENDENCIES_DIR%\git\usr\bin\bash.exe + + cd %SCRIPTS_DIR% + + if "${{ env.EXTRACTED_REPO_NAME }}"=="audio" ( + call "%VS_PATH%" x64 + "%GIT_BASH%" -c "bash --noprofile --norc build_audio.sh" + ) else if "${{ env.EXTRACTED_REPO_NAME }}"=="vision" ( + call "%VS_PATH%" x64 + "%GIT_BASH%" -c "bash --noprofile --norc build_vision.sh" + ) else ( + echo Invalid repository name: ${{ inputs.repository_name }} + exit /b 1 + ) + - name: Install wheel + shell: bash + run: | + cd $SRC_DIR + source .venv/Scripts/activate + whl=$(find dist -name "torc*-*.whl" | head -n 1) + pip install $whl + - name: Run smoke test + shell: cmd + run: | + cd %SRC_DIR% + call .venv/Scripts/activate + + if "${{ env.EXTRACTED_REPO_NAME }}"=="audio" ( + python test/smoke_test/smoke_test.py + ) else if "${{ env.EXTRACTED_REPO_NAME }}"=="vision" ( + python test/smoke_test.py + ) else ( + echo Invalid repository name: ${{ inputs.repository_name }} + exit /b 1 + ) + - name: Archive wheel + if: success() + uses: actions/upload-artifact@v4 + with: + path: ${{ env.SRC_DIR }}/dist/* + compression-level: 0 + retention-days: 3 \ No newline at end of file diff --git a/.github/workflows/test_build_wheels_windows_arm64.yml b/.github/workflows/test_build_wheels_windows_arm64.yml new file mode 100644 index 0000000000..0c6db18bcd --- /dev/null +++ b/.github/workflows/test_build_wheels_windows_arm64.yml @@ -0,0 +1,46 @@ +name: Test Build Windows Wheels without CUDA + +on: + pull_request: + paths: + - .github/actions/setup-binary-builds/action.yml + - .github/workflows/test_build_wheels_windows_arm64.yml + - .github/workflows/build_wheels_windows_x64.yml + - .github/workflows/generate_binary_build_matrix.yml + - tools/scripts/generate_binary_build_matrix.py + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + generate-matrix: + uses: ./.github/workflows/generate_binary_build_matrix.yml + with: + package-type: wheel + os: windows + test-infra-repository: ${{ github.repository }} + test-infra-ref: ${{ github.ref }} + with-cuda: disable + architecture: arm64 + test: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + smoke-test-script: test//smoke_test.py + package-name: torchvision + uses: ./.github/workflows/build_wheels_windows.yml + name: ${{ matrix.repository }} + with: + repository: ${{ matrix.repository }} + ref: nightly + test-infra-repository: ${{ github.repository }} + test-infra-ref: ${{ github.ref }} + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + smoke-test-script: ${{ matrix.smoke-test-script }} + package-name: ${{ matrix.package-name }} + trigger-event: "${{ github.event_name }}" diff --git a/tools/scripts/generate_binary_build_matrix.py b/tools/scripts/generate_binary_build_matrix.py index adc0b4e10a..a2fdd21515 100755 --- a/tools/scripts/generate_binary_build_matrix.py +++ b/tools/scripts/generate_binary_build_matrix.py @@ -64,6 +64,7 @@ LINUX_AARCH64 = "linux-aarch64" MACOS_ARM64 = "macos-arm64" WINDOWS = "windows" +WINDOWS_ARM64 = "windows-arm64" # Accelerator architectures CPU = "cpu" @@ -94,9 +95,11 @@ LINUX_AARCH64_GPU_RUNNER = "linux.arm64.m7g.4xlarge" WIN_GPU_RUNNER = "windows.g4dn.xlarge" WIN_CPU_RUNNER = "windows.4xlarge" +WIN_ARM64_RUNNER = "windows-latest" MACOS_M1_RUNNER = "macos-m1-stable" PACKAGES_TO_INSTALL_WHL = "torch torchvision torchaudio" +PACKAGES_TO_INSTALL_WHL_WIN_ARM64 = "torch" WHL_INSTALL_BASE = "pip3 install" DOWNLOAD_URL_BASE = "https://download.pytorch.org" @@ -133,8 +136,10 @@ def validation_runner(arch_type: str, os: str) -> str: elif os == WINDOWS: if arch_type == CUDA: return WIN_GPU_RUNNER - else: + elif arch_type == CPU: return WIN_CPU_RUNNER + elif os == WINDOWS_ARM64: + return WIN_ARM64_RUNNER elif os == MACOS_ARM64: return MACOS_M1_RUNNER else: # default to linux cpu runner @@ -296,11 +301,14 @@ def get_wheel_install_command( ): return f"{WHL_INSTALL_BASE} {PACKAGES_TO_INSTALL_WHL}" else: - whl_install_command = ( - f"{WHL_INSTALL_BASE} --pre {PACKAGES_TO_INSTALL_WHL}" - if channel == "nightly" - else f"{WHL_INSTALL_BASE} {PACKAGES_TO_INSTALL_WHL}" - ) + whl_install_command = "" + if os == WINDOWS_ARM64: + # winarm64 only has only nightly torch package for now + whl_install_command = f"{WHL_INSTALL_BASE} --pre {PACKAGES_TO_INSTALL_WHL_WIN_ARM64}" + elif channel == "nightly": + whl_install_command = f"{WHL_INSTALL_BASE} --pre {PACKAGES_TO_INSTALL_WHL}" + else: + whl_install_command = f"{WHL_INSTALL_BASE} {PACKAGES_TO_INSTALL_WHL}" return f"{whl_install_command} --index-url {get_base_download_url_for_repo('whl', channel, gpu_arch_type, desired_cuda)}" # noqa: E501 @@ -432,6 +440,9 @@ def generate_wheels_matrix( if not python_versions: # Define default python version python_versions = list(PYTHON_ARCHES) + + if os == WINDOWS_ARM64: + python_versions = ["3.12"] # only version supported for now if os == LINUX: # NOTE: We only build manywheel packages for linux