Update release scripts (dmlc#11289)

- Generate the variants instead of using patches.
- Update links to s3 buckets.
- Build the glibc warning into XGBoost instead of using a patch
- Build xgboost-cpu on Windows

---------

Co-authored-by: Hyunsu Cho <[email protected]>

Author: trivialfis

Diff for: .github/workflows/python_wheels_macos.yml

@@ -43,6 +43,9 @@ jobs:
           environment-file: ops/conda_env/minimal.yml
       - name: Build wheels
         run: bash ops/pipeline/build-python-wheels-macos.sh ${{ matrix.platform_id }} ${{ github.sha }}
+      - name: Try installing XGBoost
+        run: |
+          python -m pip install -vvv wheelhouse/*.whl
       - name: Upload Python wheel
         if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')
         run: |

Diff for: .github/workflows/windows.yml

@@ -39,6 +39,18 @@ jobs:
             build/testxgboost.exe xgboost.exe `
             (Get-ChildItem python-package/dist/*.whl | Select-Object -Expand FullName)
 
+  build-win64-cpu:
+    name: Build XGBoost for Windows (minimal)
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - runner=windows-cpu
+      - tag=windows-build-win64-cpu
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "true"
+      - run: powershell ops/pipeline/build-win64-cpu.ps1
+
   test-win64-gpu:
     name: Test XGBoost on Windows
     needs: build-win64-gpu

Diff for: dev/release-artifacts.py

@@ -1,7 +1,7 @@
 """
 Simple script for managing Python, R, and source release packages.
 
-tqdm, sh are required to run this script.
+tqdm, sh, and build are required to run this script.
 """
 
 import argparse
@@ -154,7 +154,7 @@ def download_python_wheels(branch: str, commit_hash: str, outdir: Path) -> None:
         "manylinux2014_aarch64",
         "manylinux_2_28_x86_64",
         "manylinux_2_28_aarch64",
-        "macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64",
+        "macosx_10_15_x86_64",
         "macosx_12_0_arm64",
     ]
     minimal_platforms = [
@@ -163,14 +163,14 @@ def download_python_wheels(branch: str, commit_hash: str, outdir: Path) -> None:
         "manylinux2014_aarch64",
     ]
 
-    dir_url = f"{S3_BUCKET_URL}/{branch}/"
+    # https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/release_3.0.0/4bfd4bf60d32e2d62426cc4070ccb5a5ba1ed078/xgboost-3.0.0rc1-py3-none-manylinux_2_28_x86_64.whl
+    dir_url = f"{S3_BUCKET_URL}/{branch}/{commit_hash}/"
     wheels = []
-
     for pkg_name, platforms in [
         ("xgboost", full_platforms),
         ("xgboost_cpu", minimal_platforms),
     ]:
-        src_filename_prefix = f"{pkg_name}-{args.release}%2B{commit_hash}-py3-none-"
+        src_filename_prefix = f"{pkg_name}-{args.release}-py3-none-"
         target_filename_prefix = f"{pkg_name}-{args.release}-py3-none-"
         wheels.extend(
             _download_python_wheels(
@@ -188,24 +188,20 @@ def download_python_wheels(branch: str, commit_hash: str, outdir: Path) -> None:
 
 
 def download_r_artifacts(
-    release: str, branch: str, rc: str, commit: str, outdir: Path
+    release: str, branch: str, commit: str, outdir: Path
 ) -> Tuple[Dict[str, str], List[str]]:
     """Download R package artifacts for the specified release and branch."""
     platforms = ["linux"]
     rpkg_dir = outdir / "r-packages"
     rpkg_dir.mkdir(exist_ok=True)
 
     artifacts = []
-    branch = branch.split("_")[1]  # release_x.y.z
     urls = {}
 
+    # https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/release_3.0.0/4bfd4bf60d32e2d62426cc4070ccb5a5ba1ed078/xgboost_r_gpu_linux.tar.gz
     for plat in platforms:
-        url = f"{S3_BUCKET_URL}/{branch}/xgboost_r_gpu_{plat}_{commit}.tar.gz"
-        artifact_name = (
-            f"xgboost_r_gpu_{plat}_{release}-{rc}.tar.gz"
-            if rc
-            else f"xgboost_r_gpu_{plat}_{release}.tar.gz"
-        )
+        url = f"{S3_BUCKET_URL}/{branch}/{commit}/xgboost_r_gpu_{plat}.tar.gz"
+        artifact_name = f"xgboost_r_gpu_{plat}.tar.gz"
         artifact_path = rpkg_dir / artifact_name
         retrieve(url=url, filename=artifact_path)
         artifacts.append(artifact_path)
@@ -356,7 +352,6 @@ def main(args: argparse.Namespace) -> None:
     urls, hashes = download_r_artifacts(
         release,
         branch,
-        "" if rc is None else f"rc{rc_ver}",
         commit_hash,
         outdir,
     )

Diff for: doc/install.rst

@@ -89,6 +89,8 @@ You may choose to reduce the size of the installed package and save the disk spa
 The ``xgboost-cpu`` variant will have drastically smaller disk footprint, but does not provide some features, such as the GPU algorithms and
 federated learning.
 
+Currently, ``xgboost-cpu`` package is provided for x86_64 (amd64) Linux and Windows platforms.
+
 Conda
 *****
 

Diff for: ops/patch/cpu_only_pypkg.patch

@@ -27,15 +27,14 @@ PYTHON_BIN="/opt/python/cp310-cp310/bin/python"
 
 echo "--- Build binary wheel for ${WHEEL_TAG}"
 set -x
-# Patch to add warning about manylinux2014 variant
-patch -p0 < ops/patch/remove_nccl_dep.patch
-patch -p0 < ops/patch/manylinux2014_warning.patch
+
+python3 ops/script/pypi_variants.py --variant=manylinux2014
 python3 ops/docker_run.py \
   --image-uri "${IMAGE_URI}" \
   -- bash -c \
   "cd python-package && ${PYTHON_BIN} -m pip wheel --no-deps -v . --wheel-dir dist/"
-git checkout python-package/pyproject.toml python-package/xgboost/core.py
-  # discard the patch
+# discard the patch
+python3 ops/script/pypi_variants.py --variant=default
 
 python3 ops/docker_run.py \
   --image-uri "${IMAGE_URI}" \
@@ -49,13 +48,13 @@ mv -v wheelhouse/*.whl python-package/dist/
 
 echo "--- Build binary wheel for ${WHEEL_TAG} (CPU only)"
 # Patch to rename pkg to xgboost-cpu
-patch -p0 < ops/patch/remove_nccl_dep.patch
-patch -p0 < ops/patch/cpu_only_pypkg.patch
+python3 ops/script/pypi_variants.py --variant=cpu
 python3 ops/docker_run.py \
   --image-uri "${IMAGE_URI}" \
   -- bash -c \
   "cd python-package && ${PYTHON_BIN} -m pip wheel --no-deps -v . --wheel-dir dist/"
-git checkout python-package/pyproject.toml  # discard the patch
+# discard the patch
+python3 ops/script/pypi_variants.py --variant=default
 
 python3 ops/docker_run.py \
   --image-uri "${IMAGE_URI}" \

Diff for: ops/patch/manylinux2014_warning.patch

@@ -13,13 +13,11 @@ commit_id=$2
 if [[ "$platform_id" == macosx_* ]]; then
     if [[ "$platform_id" == macosx_arm64 ]]; then
         # MacOS, Apple Silicon
-        WHEEL_TAG=macosx_12_0_arm64
         cpython_ver=310
         cibw_archs=arm64
         export MACOSX_DEPLOYMENT_TARGET=12.0
     elif [[ "$platform_id" == macosx_x86_64 ]]; then
         # MacOS, Intel
-        WHEEL_TAG=macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64
         cpython_ver=310
         cibw_archs=x86_64
         export MACOSX_DEPLOYMENT_TARGET=10.15
@@ -37,7 +35,7 @@ else
     exit 2
 fi
 
-# Tell delocate-wheel to not vendor libomp.dylib into the wheel"
+# Tell delocate-wheel to not vendor libomp.dylib into the wheel
 export CIBW_REPAIR_WHEEL_COMMAND_MACOS="delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} --exclude libomp.dylib"
 
 python -m pip install cibuildwheel

Diff for: ops/patch/remove_nccl_dep.patch

@@ -0,0 +1,40 @@
+## Build Python package xgboost-cpu (minimal install)
+
+$ErrorActionPreference = "Stop"
+
+. ops/pipeline/enforce-ci.ps1
+
+Write-Host "--- Build libxgboost on Windows (minimal)"
+
+mkdir build
+cd build
+cmake .. -G"Visual Studio 17 2022" -A x64
+if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
+cmake --build . --config Release -- /m /nodeReuse:false `
+  "/consoleloggerparameters:ShowCommandLine;Verbosity=minimal"
+if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
+
+Write-Host "--- Build binary wheel"
+cd ..
+# Patch to rename pkg to xgboost-cpu
+conda activate
+python ops/script/pypi_variants.py --variant=cpu
+if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
+
+cd python-package
+& pip wheel --no-deps -v . --wheel-dir dist/
+if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
+python -m wheel tags --python-tag py3 --abi-tag none `
+  --platform win_amd64 --remove `
+  (Get-ChildItem dist/*.whl | Select-Object -Expand FullName)
+if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
+
+Write-Host "--- Upload Python wheel"
+cd ..
+if ( $is_release_branch -eq 1 ) {
+  python ops/pipeline/manage-artifacts.py upload `
+    --s3-bucket 'xgboost-nightly-builds' `
+    --prefix "$Env:BRANCH_NAME/$Env:GITHUB_SHA" --make-public `
+    (Get-ChildItem python-package/dist/*.whl | Select-Object -Expand FullName)
+  if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
+}

Diff for: ops/pipeline/build-manylinux2014.sh

@@ -1,3 +1,5 @@
+## Build Python package and gtest with GPU enabled
+
 $ErrorActionPreference = "Stop"
 
 . ops/pipeline/enforce-ci.ps1

Diff for: ops/pipeline/build-python-wheels-macos.sh

@@ -16,6 +16,7 @@
 import sys
 import tempfile
 
+from pypi_variants import make_pyproject
 from test_utils import JVM_PACKAGES, PY_PACKAGE, R_PACKAGE, ROOT, cd
 
 
@@ -51,7 +52,7 @@ def pypkg(
     with open(pyver_path, "w") as fd:
         fd.write(pyver + "\n")
 
-    pyprj_path = os.path.join("pyproject.toml")
+    pyprj_path = os.path.join("pyproject.toml.in")
     with open(pyprj_path, "r") as fd:
         pyprj = fd.read()
     matched = re.search('version = "' + r"([0-9]+\.[0-9]+\.[0-9]+.*)" + '"', pyprj)
@@ -60,6 +61,8 @@ def pypkg(
     with open(pyprj_path, "w") as fd:
         fd.write(pyprj)
 
+    make_pyproject("default")
+
 
 @cd(R_PACKAGE)
 def rpkg(major: int, minor: int, patch: int, is_dev: bool) -> None: