Skip to content

Commit 1ace9c6

Browse files
authored
[CI] Fix JVM tests on Windows (dmlc#10404)
1 parent dc14f98 commit 1ace9c6

File tree

16 files changed

+63
-109
lines changed

16 files changed

+63
-109
lines changed

.github/workflows/jvm_tests.yml

+6-14
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@ jobs:
2323
with:
2424
submodules: 'true'
2525

26+
- uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
27+
with:
28+
distribution: 'temurin'
29+
java-version: '8'
30+
2631
- uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4
2732
with:
2833
miniforge-variant: Mambaforge
@@ -38,18 +43,11 @@ jobs:
3843
key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}
3944
restore-keys: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}
4045

41-
- name: Build xgboost4j.dll
42-
run: |
43-
mkdir build
44-
cd build
45-
cmake .. -G"Visual Studio 17 2022" -A x64 -DJVM_BINDINGS=ON
46-
cmake --build . --config Release
47-
if: matrix.os == 'windows-latest'
48-
4946
- name: Test XGBoost4J (Core)
5047
run: |
5148
cd jvm-packages
5249
mvn test -B -pl :xgboost4j_2.12
50+
if: matrix.os == 'windows-latest'
5351

5452
- name: Extract branch name
5553
shell: bash
@@ -87,22 +85,16 @@ jobs:
8785
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }}
8886
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }}
8987

90-
9188
- name: Test XGBoost4J (Core, Spark, Examples)
9289
run: |
9390
rm -rfv build/
9491
cd jvm-packages
9592
mvn -B test
9693
if: matrix.os == 'ubuntu-latest' # Distributed training doesn't work on Windows
97-
env:
98-
RABIT_MOCK: ON
99-
10094

10195
- name: Build and Test XGBoost4J with scala 2.13
10296
run: |
10397
rm -rfv build/
10498
cd jvm-packages
10599
mvn -B clean install test -Pdefault,scala-2.13
106100
if: matrix.os == 'ubuntu-latest' # Distributed training doesn't work on Windows
107-
env:
108-
RABIT_MOCK: ON

CMakeLists.txt

+11-9
Original file line numberDiff line numberDiff line change
@@ -265,11 +265,18 @@ if(USE_NCCL)
265265
find_package(Nccl REQUIRED)
266266
endif()
267267

268-
# dmlc-core
269-
msvc_use_static_runtime()
270-
if(FORCE_SHARED_CRT)
271-
set(DMLC_FORCE_SHARED_CRT ON)
268+
if(MSVC)
269+
if(FORCE_SHARED_CRT)
270+
message(STATUS "XGBoost: Using dynamically linked MSVC runtime...")
271+
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL")
272+
else()
273+
message(STATUS "XGBoost: Using statically linked MSVC runtime...")
274+
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
275+
endif()
272276
endif()
277+
278+
# dmlc-core
279+
set(DMLC_FORCE_SHARED_CRT ${FORCE_SHARED_CRT})
273280
add_subdirectory(${xgboost_SOURCE_DIR}/dmlc-core)
274281

275282
if(MSVC)
@@ -489,11 +496,6 @@ if(GOOGLE_TEST)
489496
endif()
490497
endif()
491498

492-
# For MSVC: Call msvc_use_static_runtime() once again to completely
493-
# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462
494-
# for issues caused by mixing of /MD and /MT flags
495-
msvc_use_static_runtime()
496-
497499
# Add xgboost.pc
498500
if(ADD_PKGCONFIG)
499501
configure_file(${xgboost_SOURCE_DIR}/cmake/xgboost.pc.in ${xgboost_BINARY_DIR}/xgboost.pc @ONLY)

cmake/Utils.cmake

-39
Original file line numberDiff line numberDiff line change
@@ -13,45 +13,6 @@ function(auto_source_group SOURCES)
1313
endforeach()
1414
endfunction()
1515

16-
# Force static runtime for MSVC
17-
function(msvc_use_static_runtime)
18-
if(MSVC AND (NOT BUILD_SHARED_LIBS) AND (NOT FORCE_SHARED_CRT))
19-
set(variables
20-
CMAKE_C_FLAGS_DEBUG
21-
CMAKE_C_FLAGS_MINSIZEREL
22-
CMAKE_C_FLAGS_RELEASE
23-
CMAKE_C_FLAGS_RELWITHDEBINFO
24-
CMAKE_CXX_FLAGS_DEBUG
25-
CMAKE_CXX_FLAGS_MINSIZEREL
26-
CMAKE_CXX_FLAGS_RELEASE
27-
CMAKE_CXX_FLAGS_RELWITHDEBINFO
28-
)
29-
foreach(variable ${variables})
30-
if(${variable} MATCHES "/MD")
31-
string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
32-
set(${variable} "${${variable}}" PARENT_SCOPE)
33-
endif()
34-
endforeach()
35-
set(variables
36-
CMAKE_CUDA_FLAGS
37-
CMAKE_CUDA_FLAGS_DEBUG
38-
CMAKE_CUDA_FLAGS_MINSIZEREL
39-
CMAKE_CUDA_FLAGS_RELEASE
40-
CMAKE_CUDA_FLAGS_RELWITHDEBINFO
41-
)
42-
foreach(variable ${variables})
43-
if(${variable} MATCHES "-MD")
44-
string(REGEX REPLACE "-MD" "-MT" ${variable} "${${variable}}")
45-
set(${variable} "${${variable}}" PARENT_SCOPE)
46-
endif()
47-
if(${variable} MATCHES "/MD")
48-
string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}")
49-
set(${variable} "${${variable}}" PARENT_SCOPE)
50-
endif()
51-
endforeach()
52-
endif()
53-
endfunction()
54-
5516
# Set output directory of target, ignoring debug or release
5617
function(set_output_directory target dir)
5718
set_target_properties(${target} PROPERTIES

dev/prepare_jvm_release.py

+2-4
Original file line numberDiff line numberDiff line change
@@ -216,8 +216,7 @@ def main():
216216
)
217217
print(
218218
" # Skip native build, since we have all needed native binaries from CI\n"
219-
" export MAVEN_SKIP_NATIVE_BUILD=1\n"
220-
" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"
219+
" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"
221220
)
222221
print(
223222
"4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
@@ -228,9 +227,8 @@ def main():
228227
)
229228
print(
230229
"5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:\n"
231-
" export MAVEN_SKIP_NATIVE_BUILD=1\n"
232230
" python dev/change_scala_version.py --scala-version 2.13 --purge-artifacts\n"
233-
" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests"
231+
" GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"
234232
)
235233
print(
236234
"6. Go to https://oss.sonatype.org/ to release the Scala 2.13 artifacts. "

jvm-packages/create_jni.py

+10-13
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ def maybe_makedirs(path):
5151

5252
def run(command, **kwargs):
5353
print(command)
54-
subprocess.check_call(command, shell=True, **kwargs)
54+
subprocess.run(command, shell=True, check=True, env=os.environ, **kwargs)
5555

5656

5757
def cp(source, target):
@@ -85,6 +85,8 @@ def native_build(args):
8585

8686
if sys.platform == "linux":
8787
maybe_parallel_build = " -- -j $(nproc)"
88+
elif sys.platform == "win32":
89+
maybe_parallel_build = ' -- /m /nodeReuse:false "/consoleloggerparameters:ShowCommandLine;Verbosity=minimal"'
8890
else:
8991
maybe_parallel_build = ""
9092

@@ -176,15 +178,10 @@ def native_build(args):
176178

177179

178180
if __name__ == "__main__":
179-
if "MAVEN_SKIP_NATIVE_BUILD" in os.environ:
180-
print("MAVEN_SKIP_NATIVE_BUILD is set. Skipping native build...")
181-
else:
182-
parser = argparse.ArgumentParser()
183-
parser.add_argument(
184-
"--log-capi-invocation", type=str, choices=["ON", "OFF"], default="OFF"
185-
)
186-
parser.add_argument(
187-
"--use-cuda", type=str, choices=["ON", "OFF"], default="OFF"
188-
)
189-
cli_args = parser.parse_args()
190-
native_build(cli_args)
181+
parser = argparse.ArgumentParser()
182+
parser.add_argument(
183+
"--log-capi-invocation", type=str, choices=["ON", "OFF"], default="OFF"
184+
)
185+
parser.add_argument("--use-cuda", type=str, choices=["ON", "OFF"], default="OFF")
186+
cli_args = parser.parse_args()
187+
native_build(cli_args)

jvm-packages/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
<cudf.classifier>cuda12</cudf.classifier>
5050
<scalatest.version>3.2.18</scalatest.version>
5151
<scala-collection-compat.version>2.12.0</scala-collection-compat.version>
52+
<skip.native.build>false</skip.native.build>
5253

5354
<!-- SPARK-36796 for JDK-17 test-->
5455
<extraJavaTestArgs>

jvm-packages/xgboost4j/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
<argument>${log.capi.invocation}</argument>
9999
</arguments>
100100
<workingDirectory>${user.dir}</workingDirectory>
101+
<skip>${skip.native.build}</skip>
101102
</configuration>
102103
</execution>
103104
</executions>

src/CMakeLists.txt

-5
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,6 @@ if(LOG_CAPI_INVOCATION)
3030
target_compile_definitions(objxgboost PRIVATE -DLOG_CAPI_INVOCATION=1)
3131
endif()
3232

33-
# For MSVC: Call msvc_use_static_runtime() once again to completely
34-
# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462
35-
# for issues caused by mixing of /MD and /MT flags
36-
msvc_use_static_runtime()
37-
3833
# This grouping organises source files nicely in visual studio
3934
auto_source_group("${CUDA_SOURCES}")
4035
auto_source_group("${CPU_SOURCES}")

src/common/io.h

+10-6
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,12 @@ struct MemoryFixSizeBuffer : public dmlc::SeekStream {
4848
curr_ptr_ += nread;
4949
return nread;
5050
}
51-
void Write(const void *ptr, std::size_t size) override {
52-
if (size == 0) return;
51+
std::size_t Write(const void *ptr, std::size_t size) override {
52+
if (size == 0) return 0;
5353
CHECK_LE(curr_ptr_ + size, buffer_size_);
5454
std::memcpy(p_buffer_ + curr_ptr_, ptr, size);
5555
curr_ptr_ += size;
56+
return size;
5657
}
5758
void Seek(std::size_t pos) override {
5859
if (pos == kSeekEnd) {
@@ -91,13 +92,14 @@ struct MemoryBufferStream : public dmlc::SeekStream {
9192
curr_ptr_ += nread;
9293
return nread;
9394
}
94-
void Write(const void *ptr, size_t size) override {
95-
if (size == 0) return;
95+
std::size_t Write(const void *ptr, size_t size) override {
96+
if (size == 0) return 0;
9697
if (curr_ptr_ + size > p_buffer_->length()) {
9798
p_buffer_->resize(curr_ptr_+size);
9899
}
99100
std::memcpy(&(*p_buffer_)[0] + curr_ptr_, ptr, size);
100101
curr_ptr_ += size;
102+
return size;
101103
}
102104
void Seek(size_t pos) override {
103105
curr_ptr_ = static_cast<size_t>(pos);
@@ -127,8 +129,9 @@ class PeekableInStream : public dmlc::Stream {
127129
size_t Read(void* dptr, size_t size) override;
128130
virtual size_t PeekRead(void* dptr, size_t size);
129131

130-
void Write(const void*, size_t) override {
132+
std::size_t Write(const void*, size_t) override {
131133
LOG(FATAL) << "Not implemented";
134+
return 0;
132135
}
133136

134137
private:
@@ -155,8 +158,9 @@ class FixedSizeStream : public PeekableInStream {
155158
[[nodiscard]] std::size_t Tell() const { return pointer_; }
156159
void Seek(size_t pos);
157160

158-
void Write(const void*, size_t) override {
161+
std::size_t Write(const void*, size_t) override {
159162
LOG(FATAL) << "Not implemented";
163+
return 0;
160164
}
161165

162166
/*!

tests/buildkite/build-jvm-packages-gpu.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ set -euo pipefail
44

55
source tests/buildkite/conftest.sh
66

7-
echo "--- Build XGBoost JVM packages with CUDA"
7+
echo "--- Build and test XGBoost JVM packages with CUDA"
88

99
if [[ ($is_pull_request == 1) || ($is_release_branch == 0) ]]
1010
then

tests/buildkite/build-jvm-packages.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ set -euo pipefail
44

55
source tests/buildkite/conftest.sh
66

7-
echo "--- Build XGBoost JVM packages scala 2.12"
7+
echo "--- Build and test XGBoost JVM packages with Scala 2.12"
88
tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \
99
${SPARK_VERSION}
1010

@@ -14,7 +14,7 @@ buildkite-agent artifact upload "jvm-packages/xgboost4j-spark/target/*.jar"
1414
buildkite-agent artifact upload "jvm-packages/xgboost4j-flink/target/*.jar"
1515
buildkite-agent artifact upload "jvm-packages/xgboost4j-example/target/*.jar"
1616

17-
echo "--- Build XGBoost JVM packages scala 2.13"
17+
echo "--- Build and test XGBoost JVM packages with Scala 2.13"
1818

1919
tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \
2020
${SPARK_VERSION} "" "" "true"

tests/buildkite/build-win64-gpu.ps1

+4-6
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,11 @@ if ( $is_release_branch -eq 0 ) {
1212
}
1313
mkdir build
1414
cd build
15-
cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON `
15+
cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON `
1616
-DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DBUILD_DEPRECATED_CLI=ON ${arch_flag}
17-
$msbuild = -join @(
18-
"C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\MSBuild\\Current"
19-
"\\Bin\\MSBuild.exe"
20-
)
21-
& $msbuild xgboost.sln /m /p:Configuration=Release /nodeReuse:false
17+
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
18+
cmake --build . --config Release -- /m /nodeReuse:false `
19+
"/consoleloggerparameters:ShowCommandLine;Verbosity=minimal"
2220
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
2321

2422
Write-Host "--- Build binary wheel"

tests/buildkite/infrastructure/aws-stack-creator/metadata.py

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,27 @@
11
AMI_ID = {
22
# Managed by XGBoost team
33
"linux-amd64-gpu": {
4-
"us-west-2": "ami-070080d04e81c5e39",
4+
"us-west-2": "ami-0b4079c15bbbd0faf",
55
},
66
"linux-amd64-mgpu": {
7-
"us-west-2": "ami-070080d04e81c5e39",
7+
"us-west-2": "ami-0b4079c15bbbd0faf",
88
},
99
"windows-gpu": {
10-
"us-west-2": "ami-07c14abcf529d816a",
10+
"us-west-2": "ami-0123456bcf4cdfb82",
1111
},
1212
"windows-cpu": {
13-
"us-west-2": "ami-07c14abcf529d816a",
13+
"us-west-2": "ami-0123456bcf4cdfb82",
1414
},
1515
# Managed by BuildKite
1616
# from https://s3.amazonaws.com/buildkite-aws-stack/latest/aws-stack.yml
1717
"linux-amd64-cpu": {
18-
"us-west-2": "ami-0180f7fb0f07eb0bc",
18+
"us-west-2": "ami-0083e0ae73c175ec6",
1919
},
2020
"pipeline-loader": {
21-
"us-west-2": "ami-0180f7fb0f07eb0bc",
21+
"us-west-2": "ami-0083e0ae73c175ec6",
2222
},
2323
"linux-arm64-cpu": {
24-
"us-west-2": "ami-00686bdc2043a5505",
24+
"us-west-2": "ami-0dbf1f9da54222f21",
2525
},
2626
}
2727

tests/buildkite/infrastructure/worker-image-pipeline/windows-gpu-bootstrap.yml

+6-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,12 @@ phases:
3636
3737
# Install Java 11
3838
Write-Host '>>> Installing Java 11...'
39-
choco install openjdk11jre
39+
choco install openjdk11
40+
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
41+
42+
# Install Maven
43+
Write-Host '>>> Installing Maven...'
44+
choco install maven
4045
if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
4146
4247
# Install GraphViz

tests/buildkite/pipeline-mgpu.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ steps:
2929
key: build-cuda
3030
agents:
3131
queue: linux-amd64-cpu
32-
- label: ":console: Build JVM packages with CUDA"
32+
- label: ":console: Build and test JVM packages with CUDA"
3333
command: "tests/buildkite/build-jvm-packages-gpu.sh"
3434
key: build-jvm-packages-gpu
3535
agents:

0 commit comments

Comments
 (0)