[CI] Fix JVM tests on Windows (dmlc#10404)

hcho3 · web-flow · commit 1ace9c66ecac · 2024-06-15T00:21:40.000-07:00
diff --git a/.github/workflows/jvm_tests.yml b/.github/workflows/jvm_tests.yml
@@ -23,6 +23,11 @@ jobs:
       with:
         submodules: 'true'
 
+    - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9  # v4.2.1
+      with:
+        distribution: 'temurin'
+        java-version: '8'
+
     - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca  # v3.0.4
       with:
         miniforge-variant: Mambaforge
@@ -38,18 +43,11 @@ jobs:
         key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}
         restore-keys: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}
 
-    - name: Build xgboost4j.dll
-      run: |
-        mkdir build
-        cd build
-        cmake .. -G"Visual Studio 17 2022" -A x64 -DJVM_BINDINGS=ON
-        cmake --build . --config Release
-      if: matrix.os == 'windows-latest'
-
     - name: Test XGBoost4J (Core)
       run: |
         cd jvm-packages
         mvn test -B -pl :xgboost4j_2.12
+      if: matrix.os == 'windows-latest'
 
     - name: Extract branch name
       shell: bash
@@ -87,22 +85,16 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }}
 
-
     - name: Test XGBoost4J (Core, Spark, Examples)
       run: |
         rm -rfv build/
         cd jvm-packages
         mvn -B test
       if: matrix.os == 'ubuntu-latest'  # Distributed training doesn't work on Windows
-      env:
-        RABIT_MOCK: ON
-
 
     - name: Build and Test XGBoost4J with scala 2.13
       run: |
         rm -rfv build/
         cd jvm-packages
         mvn -B clean install test -Pdefault,scala-2.13
       if: matrix.os == 'ubuntu-latest'  # Distributed training doesn't work on Windows
-      env:
-        RABIT_MOCK: ON
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -265,11 +265,18 @@ if(USE_NCCL)
   find_package(Nccl REQUIRED)
 endif()
 
-# dmlc-core
-msvc_use_static_runtime()
-if(FORCE_SHARED_CRT)
-  set(DMLC_FORCE_SHARED_CRT ON)
+if(MSVC)
+  if(FORCE_SHARED_CRT)
+    message(STATUS "XGBoost: Using dynamically linked MSVC runtime...")
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL")
+  else()
+    message(STATUS "XGBoost: Using statically linked MSVC runtime...")
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+  endif()
 endif()
+
+# dmlc-core
+set(DMLC_FORCE_SHARED_CRT ${FORCE_SHARED_CRT})
 add_subdirectory(${xgboost_SOURCE_DIR}/dmlc-core)
 
 if(MSVC)
@@ -489,11 +496,6 @@ if(GOOGLE_TEST)
   endif()
 endif()
 
-# For MSVC: Call msvc_use_static_runtime() once again to completely
-# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462
-# for issues caused by mixing of /MD and /MT flags
-msvc_use_static_runtime()
-
 # Add xgboost.pc
 if(ADD_PKGCONFIG)
   configure_file(${xgboost_SOURCE_DIR}/cmake/xgboost.pc.in ${xgboost_BINARY_DIR}/xgboost.pc @ONLY)
diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake
@@ -13,45 +13,6 @@ function(auto_source_group SOURCES)
   endforeach()
 endfunction()
 
-# Force static runtime for MSVC
-function(msvc_use_static_runtime)
-  if(MSVC AND (NOT BUILD_SHARED_LIBS) AND (NOT FORCE_SHARED_CRT))
-      set(variables
-          CMAKE_C_FLAGS_DEBUG
-          CMAKE_C_FLAGS_MINSIZEREL
-          CMAKE_C_FLAGS_RELEASE
-          CMAKE_C_FLAGS_RELWITHDEBINFO
-          CMAKE_CXX_FLAGS_DEBUG
-          CMAKE_CXX_FLAGS_MINSIZEREL
-          CMAKE_CXX_FLAGS_RELEASE
-          CMAKE_CXX_FLAGS_RELWITHDEBINFO
-      )
-      foreach(variable ${variables})
-          if(${variable} MATCHES "/MD")
-              string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
-              set(${variable} "${${variable}}"  PARENT_SCOPE)
-          endif()
-      endforeach()
-      set(variables
-          CMAKE_CUDA_FLAGS
-          CMAKE_CUDA_FLAGS_DEBUG
-          CMAKE_CUDA_FLAGS_MINSIZEREL
-          CMAKE_CUDA_FLAGS_RELEASE
-          CMAKE_CUDA_FLAGS_RELWITHDEBINFO
-      )
-      foreach(variable ${variables})
-          if(${variable} MATCHES "-MD")
-              string(REGEX REPLACE "-MD" "-MT" ${variable} "${${variable}}")
-              set(${variable} "${${variable}}"  PARENT_SCOPE)
-          endif()
-          if(${variable} MATCHES "/MD")
-              string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
-              set(${variable} "${${variable}}"  PARENT_SCOPE)
-          endif()
-      endforeach()
-  endif()
-endfunction()
-
 # Set output directory of target, ignoring debug or release
 function(set_output_directory target dir)
   set_target_properties(${target} PROPERTIES
diff --git a/dev/prepare_jvm_release.py b/dev/prepare_jvm_release.py
@@ -216,8 +216,7 @@ def main():
     )
     print(
         "   # Skip native build, since we have all needed native binaries from CI\n"
-        "   export MAVEN_SKIP_NATIVE_BUILD=1\n"
-        "   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"
+        "   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"
     )
     print(
         "4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
@@ -228,9 +227,8 @@ def main():
     )
     print(
         "5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:\n"
-        "   export MAVEN_SKIP_NATIVE_BUILD=1\n"
         "   python dev/change_scala_version.py --scala-version 2.13 --purge-artifacts\n"
-        "   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"
+        "   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"
     )
     print(
         "6. Go to https://oss.sonatype.org/ to release the Scala 2.13 artifacts. "
diff --git a/dmlc-core b/dmlc-core
@@ -1 +1 @@
-Subproject commit ea21135fbb141ae103fb5fc960289b5601b468f2
+Subproject commit 13341857549852a9a86b1894b5ba84c6276ab381
diff --git a/jvm-packages/create_jni.py b/jvm-packages/create_jni.py
@@ -51,7 +51,7 @@ def maybe_makedirs(path):
 
 def run(command, **kwargs):
     print(command)
-    subprocess.check_call(command, shell=True, **kwargs)
+    subprocess.run(command, shell=True, check=True, env=os.environ, **kwargs)
 
 
 def cp(source, target):
@@ -85,6 +85,8 @@ def native_build(args):
 
         if sys.platform == "linux":
             maybe_parallel_build = " -- -j $(nproc)"
+        elif sys.platform == "win32":
+            maybe_parallel_build = ' -- /m /nodeReuse:false "/consoleloggerparameters:ShowCommandLine;Verbosity=minimal"'
         else:
             maybe_parallel_build = ""
 
@@ -176,15 +178,10 @@ def native_build(args):
 
 
 if __name__ == "__main__":
-    if "MAVEN_SKIP_NATIVE_BUILD" in os.environ:
-        print("MAVEN_SKIP_NATIVE_BUILD is set. Skipping native build...")
-    else:
-        parser = argparse.ArgumentParser()
-        parser.add_argument(
-            "--log-capi-invocation", type=str, choices=["ON", "OFF"], default="OFF"
-        )
-        parser.add_argument(
-            "--use-cuda", type=str, choices=["ON", "OFF"], default="OFF"
-        )
-        cli_args = parser.parse_args()
-        native_build(cli_args)
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--log-capi-invocation", type=str, choices=["ON", "OFF"], default="OFF"
+    )
+    parser.add_argument("--use-cuda", type=str, choices=["ON", "OFF"], default="OFF")
+    cli_args = parser.parse_args()
+    native_build(cli_args)
diff --git a/jvm-packages/pom.xml b/jvm-packages/pom.xml
@@ -49,6 +49,7 @@
         <cudf.classifier>cuda12</cudf.classifier>
         <scalatest.version>3.2.18</scalatest.version>
         <scala-collection-compat.version>2.12.0</scala-collection-compat.version>
+        <skip.native.build>false</skip.native.build>
 
         <!-- SPARK-36796 for JDK-17 test-->
         <extraJavaTestArgs>
diff --git a/jvm-packages/xgboost4j/pom.xml b/jvm-packages/xgboost4j/pom.xml
@@ -98,6 +98,7 @@
                             <argument>${log.capi.invocation}</argument>
                           </arguments>
                           <workingDirectory>${user.dir}</workingDirectory>
+                          <skip>${skip.native.build}</skip>
                       </configuration>
                   </execution>
               </executions>
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -30,11 +30,6 @@ if(LOG_CAPI_INVOCATION)
   target_compile_definitions(objxgboost PRIVATE -DLOG_CAPI_INVOCATION=1)
 endif()
 
-# For MSVC: Call msvc_use_static_runtime() once again to completely
-# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462
-# for issues caused by mixing of /MD and /MT flags
-msvc_use_static_runtime()
-
 # This grouping organises source files nicely in visual studio
 auto_source_group("${CUDA_SOURCES}")
 auto_source_group("${CPU_SOURCES}")
diff --git a/src/common/io.h b/src/common/io.h
@@ -48,11 +48,12 @@ struct MemoryFixSizeBuffer : public dmlc::SeekStream {
     curr_ptr_ += nread;
     return nread;
   }
-  void Write(const void *ptr, std::size_t size) override {
-    if (size == 0) return;
+  std::size_t Write(const void *ptr, std::size_t size) override {
+    if (size == 0) return 0;
     CHECK_LE(curr_ptr_ + size, buffer_size_);
     std::memcpy(p_buffer_ + curr_ptr_, ptr, size);
     curr_ptr_ += size;
+    return size;
   }
   void Seek(std::size_t pos) override {
     if (pos == kSeekEnd) {
@@ -91,13 +92,14 @@ struct MemoryBufferStream : public dmlc::SeekStream {
     curr_ptr_ += nread;
     return nread;
   }
-  void Write(const void *ptr, size_t size) override {
-    if (size == 0) return;
+  std::size_t Write(const void *ptr, size_t size) override {
+    if (size == 0) return 0;
     if (curr_ptr_ + size > p_buffer_->length()) {
       p_buffer_->resize(curr_ptr_+size);
     }
     std::memcpy(&(*p_buffer_)[0] + curr_ptr_, ptr, size);
     curr_ptr_ += size;
+    return size;
   }
   void Seek(size_t pos) override {
     curr_ptr_ = static_cast<size_t>(pos);
@@ -127,8 +129,9 @@ class PeekableInStream : public dmlc::Stream {
   size_t Read(void* dptr, size_t size) override;
   virtual size_t PeekRead(void* dptr, size_t size);
 
-  void Write(const void*, size_t) override {
+  std::size_t Write(const void*, size_t) override {
     LOG(FATAL) << "Not implemented";
+    return 0;
   }
 
  private:
@@ -155,8 +158,9 @@ class FixedSizeStream : public PeekableInStream {
   [[nodiscard]] std::size_t Tell() const { return pointer_; }
   void Seek(size_t pos);
 
-  void Write(const void*, size_t) override {
+  std::size_t Write(const void*, size_t) override {
     LOG(FATAL) << "Not implemented";
+    return 0;
   }
 
   /*!
diff --git a/tests/buildkite/build-jvm-packages-gpu.sh b/tests/buildkite/build-jvm-packages-gpu.sh
@@ -4,7 +4,7 @@ set -euo pipefail
 
 source tests/buildkite/conftest.sh
 
-echo "--- Build XGBoost JVM packages with CUDA"
+echo "--- Build and test XGBoost JVM packages with CUDA"
 
 if [[ ($is_pull_request == 1) || ($is_release_branch == 0) ]]
 then
diff --git a/tests/buildkite/build-jvm-packages.sh b/tests/buildkite/build-jvm-packages.sh
@@ -4,7 +4,7 @@ set -euo pipefail
 
 source tests/buildkite/conftest.sh
 
-echo "--- Build XGBoost JVM packages scala 2.12"
+echo "--- Build and test XGBoost JVM packages with Scala 2.12"
 tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \
   ${SPARK_VERSION}
 
@@ -14,7 +14,7 @@ buildkite-agent artifact upload "jvm-packages/xgboost4j-spark/target/*.jar"
 buildkite-agent artifact upload "jvm-packages/xgboost4j-flink/target/*.jar"
 buildkite-agent artifact upload "jvm-packages/xgboost4j-example/target/*.jar"
 
-echo "--- Build XGBoost JVM packages scala 2.13"
+echo "--- Build and test XGBoost JVM packages with Scala 2.13"
 
 tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \
   ${SPARK_VERSION} "" "" "true"
diff --git a/tests/buildkite/build-win64-gpu.ps1 b/tests/buildkite/build-win64-gpu.ps1
@@ -12,13 +12,11 @@ if ( $is_release_branch -eq 0 ) {
 }
 mkdir build
 cd build
-cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON `
+cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON `
   -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DBUILD_DEPRECATED_CLI=ON ${arch_flag}
-$msbuild = -join @(
-  "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\MSBuild\\Current"
-  "\\Bin\\MSBuild.exe"
-)
-& $msbuild xgboost.sln /m /p:Configuration=Release /nodeReuse:false
+if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
+cmake --build . --config Release -- /m /nodeReuse:false `
+  "/consoleloggerparameters:ShowCommandLine;Verbosity=minimal"
 if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
 
 Write-Host "--- Build binary wheel"
diff --git a/tests/buildkite/infrastructure/aws-stack-creator/metadata.py b/tests/buildkite/infrastructure/aws-stack-creator/metadata.py
@@ -1,27 +1,27 @@
 AMI_ID = {
     # Managed by XGBoost team
     "linux-amd64-gpu": {
-        "us-west-2": "ami-070080d04e81c5e39",
+        "us-west-2": "ami-0b4079c15bbbd0faf",
     },
     "linux-amd64-mgpu": {
-        "us-west-2": "ami-070080d04e81c5e39",
+        "us-west-2": "ami-0b4079c15bbbd0faf",
     },
     "windows-gpu": {
-        "us-west-2": "ami-07c14abcf529d816a",
+        "us-west-2": "ami-0123456bcf4cdfb82",
     },
     "windows-cpu": {
-        "us-west-2": "ami-07c14abcf529d816a",
+        "us-west-2": "ami-0123456bcf4cdfb82",
     },
     # Managed by BuildKite
     # from https://s3.amazonaws.com/buildkite-aws-stack/latest/aws-stack.yml
     "linux-amd64-cpu": {
-        "us-west-2": "ami-0180f7fb0f07eb0bc",
+        "us-west-2": "ami-0083e0ae73c175ec6",
     },
     "pipeline-loader": {
-        "us-west-2": "ami-0180f7fb0f07eb0bc",
+        "us-west-2": "ami-0083e0ae73c175ec6",
     },
     "linux-arm64-cpu": {
-        "us-west-2": "ami-00686bdc2043a5505",
+        "us-west-2": "ami-0dbf1f9da54222f21",
     },
 }
 
diff --git a/tests/buildkite/infrastructure/worker-image-pipeline/windows-gpu-bootstrap.yml b/tests/buildkite/infrastructure/worker-image-pipeline/windows-gpu-bootstrap.yml
@@ -36,7 +36,12 @@ phases:
 
               # Install Java 11
               Write-Host '>>> Installing Java 11...'
-              choco install openjdk11jre
+              choco install openjdk11
+              if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
+
+              # Install Maven
+              Write-Host '>>> Installing Maven...'
+              choco install maven
               if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
 
               # Install GraphViz
diff --git a/tests/buildkite/pipeline-mgpu.yml b/tests/buildkite/pipeline-mgpu.yml
@@ -29,7 +29,7 @@ steps:
     key: build-cuda
     agents:
       queue: linux-amd64-cpu
-  - label: ":console: Build JVM packages with CUDA"
+  - label: ":console: Build and test JVM packages with CUDA"
     command: "tests/buildkite/build-jvm-packages-gpu.sh"
     key: build-jvm-packages-gpu
     agents:

Original file line number	Diff line number	Diff line change
`@@ -216,8 +216,7 @@ def main():`
`216`	`216`	`)`
`217`	`217`	`print(`
`218`	`218`	`" # Skip native build, since we have all needed native binaries from CI\n"`
`219`		`- " export MAVEN_SKIP_NATIVE_BUILD=1\n"`
`220`		`- " GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"`
	`219`	`+ " GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"`
`221`	`220`	`)`
`222`	`221`	`print(`
`223`	`222`	`"4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "`
`@@ -228,9 +227,8 @@ def main():`
`228`	`227`	`)`
`229`	`228`	`print(`
`230`	`229`	`"5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:\n"`
`231`		`- " export MAVEN_SKIP_NATIVE_BUILD=1\n"`
`232`	`230`	`" python dev/change_scala_version.py --scala-version 2.13 --purge-artifacts\n"`
`233`		`- " GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"`
	`231`	`+ " GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"`
`234`	`232`	`)`
`235`	`233`	`print(`
`236`	`234`	`"6. Go to https://oss.sonatype.org/ to release the Scala 2.13 artifacts. "`