Skip to content

Commit 8cffe88

Browse files
edsavageedsavage
authored
[ML] Add a script to run each unit test separately (#2859)
Add a script to provide a wrapper around the call to "cmake" that runs the test cases and provides some flexibility as to how the tests should be run in terms of how they are spread across processes. See `CONTRIBUTING.md` for more details.
1 parent 8944db3 commit 8cffe88

File tree

15 files changed

+316
-61
lines changed

15 files changed

+316
-61
lines changed

‎.buildkite/pipelines/build_linux.json.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ def main(args):
7272
"RUN_TESTS": "true",
7373
"BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
7474
},
75-
"artifact_paths": "*/**/unittest/boost_test_results.junit",
7675
"plugins": {
7776
"test-collector#v1.2.0": {
7877
"files": "*/*/unittest/boost_test_results.junit",

‎.buildkite/scripts/steps/build_and_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ fi
106106
if [[ -z "$CPP_CROSS_COMPILE" ]] ; then
107107
OS=$(uname -s | tr "A-Z" "a-z")
108108
TEST_RESULTS_ARCHIVE=${OS}-${HARDWARE_ARCH}-unit_test_results.tgz
109-
find . -path "*/**/ml_test_*.out" -o -path "*/**/*.junit" | xargs tar cvzf ${TEST_RESULTS_ARCHIVE}
109+
find . \( -path "*/**/ml_test_*.out" -o -path "*/**/*.junit" \) -print0 | tar czf ${TEST_RESULTS_ARCHIVE} --null -T -
110110
buildkite-agent artifact upload "${TEST_RESULTS_ARCHIVE}"
111111
fi
112112

‎3rd_party/3rd_party.cmake‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@ if(NOT INSTALL_DIR)
2525
message(FATAL_ERROR "INSTALL_DIR not specified")
2626
endif()
2727

28+
STRING(REPLACE "//" "/" INSTALL_DIR ${INSTALL_DIR})
29+
30+
message(STATUS "3rd_party: CMAKE_CXX_COMPILER_VERSION_MAJOR=${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
31+
2832
string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} HOST_SYSTEM_NAME)
2933
message(STATUS "3rd_party: HOST_SYSTEM_NAME=${HOST_SYSTEM_NAME}")
3034

@@ -43,7 +47,9 @@ set(ARCH ${HOST_SYSTEM_PROCESSOR})
4347
if ("${HOST_SYSTEM_NAME}" STREQUAL "darwin")
4448
message(STATUS "3rd_party: Copying macOS 3rd party libraries")
4549
set(BOOST_LOCATION "/usr/local/lib")
46-
set(BOOST_COMPILER "clang")
50+
set(BOOST_COMPILER "clang-darwin${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
51+
message(STATUS "3rd_party: BOOST_COMPILER=${BOOST_COMPILER}")
52+
4753
if( "${ARCH}" STREQUAL "x86_64" )
4854
set(BOOST_ARCH "x64")
4955
else()
@@ -63,7 +69,7 @@ elseif ("${HOST_SYSTEM_NAME}" STREQUAL "linux")
6369
if(NOT DEFINED ENV{CPP_CROSS_COMPILE} OR "$ENV{CPP_CROSS_COMPILE}" STREQUAL "")
6470
message(STATUS "3rd_party: NOT cross compiling. Copying Linux 3rd party libraries")
6571
set(BOOST_LOCATION "/usr/local/gcc133/lib")
66-
set(BOOST_COMPILER "gcc")
72+
set(BOOST_COMPILER "gcc${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
6773
if( "${ARCH}" STREQUAL "aarch64" )
6874
set(BOOST_ARCH "a64")
6975
else()
@@ -93,7 +99,7 @@ elseif ("${HOST_SYSTEM_NAME}" STREQUAL "linux")
9399
message(STATUS "3rd_party: Cross compile for macosx: Copying macOS 3rd party libraries")
94100
set(SYSROOT "/usr/local/sysroot-x86_64-apple-macosx10.14")
95101
set(BOOST_LOCATION "${SYSROOT}/usr/local/lib")
96-
set(BOOST_COMPILER "clang")
102+
set(BOOST_COMPILER "clang-darwin${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
97103
set(BOOST_EXTENSION "mt-x64-1_86.dylib")
98104
set(BOOST_LIBRARIES "atomic" "chrono" "date_time" "filesystem" "iostreams" "log" "log_setup" "program_options" "regex" "system" "thread" "unit_test_framework")
99105
set(XML_LOCATION)
@@ -108,7 +114,7 @@ elseif ("${HOST_SYSTEM_NAME}" STREQUAL "linux")
108114
message(STATUS "3rd_party: Cross compile for linux-aarch64: Copying Linux 3rd party libraries")
109115
set(SYSROOT "/usr/local/sysroot-$ENV{CPP_CROSS_COMPILE}-linux-gnu")
110116
set(BOOST_LOCATION "${SYSROOT}/usr/local/gcc133/lib")
111-
set(BOOST_COMPILER "gcc")
117+
set(BOOST_COMPILER "gcc${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
112118
if("$ENV{CPP_CROSS_COMPILE}" STREQUAL "aarch64")
113119
set(BOOST_ARCH "a64")
114120
else()
@@ -188,6 +194,9 @@ function(install_libs _target _source_dir _prefix _postfix)
188194

189195
set(LIBRARIES ${ARGN})
190196

197+
message(STATUS "_target=${_target} _source_dir=${_source_dir} _prefix=${_prefix} _postfix=${_postfix} LIBRARIES=${LIBRARIES}")
198+
199+
191200
file(GLOB _LIBS ${_source_dir}/*${_prefix}*${_postfix})
192201

193202
if(_LIBS)
@@ -219,7 +228,7 @@ function(install_libs _target _source_dir _prefix _postfix)
219228
endif()
220229
file(CHMOD ${INSTALL_DIR}/${_LIB} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
221230
else()
222-
file(COPY ${_RESOLVED_PATH} DESTINATION ${INSTALL_DIR})
231+
file(COPY ${_RESOLVED_PATH} DESTINATION "${INSTALL_DIR}")
223232
file(CHMOD ${INSTALL_DIR}/${_RESOLVED_LIB} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
224233
endif()
225234
endforeach()

‎3rd_party/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ add_custom_target(licenses ALL
2222
# as part of the CMake configuration step - avoiding
2323
# the need for it to be done on every build
2424
execute_process(
25-
COMMAND ${CMAKE_COMMAND} -DINSTALL_DIR=${INSTALL_DIR} -P ./3rd_party.cmake
25+
COMMAND ${CMAKE_COMMAND} -DINSTALL_DIR=${INSTALL_DIR} -DCMAKE_CXX_COMPILER_VERSION_MAJOR=${CMAKE_CXX_COMPILER_VERSION_MAJOR} -P ./3rd_party.cmake
2626
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
2727
)
2828

‎CONTRIBUTING.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,17 @@ Note that we configure the build to be of type `RelWithDebInfo` in order to obta
9292
1. It is also possible to control the behaviour of the test framework by passing any other arbitrary flags via the
9393
`TEST_FLAGS` environment variable , e.g. `TEST_FLAGS="--random" cmake --build cmake-build-relwithdebinfo -t test`
9494
(use TEST_FLAGS="--help" to see the full list).
95+
1. On Linux and maOS it is possible to run individual tests within a Boost Test suite in separate processes.
96+
1. This is convenient for several reasons:
97+
1. Isolation: Prevent one test's failures (e.g., memory corruption, unhandled exceptions) from affecting subsequent tests.
98+
1. Resource Management: Clean up of resources (memory, file handles, network connections) between tests more effectively.
99+
1. Stability: Improve the robustness of test suites, especially for long-running or complex tests.
100+
1. Parallelization: A means to run individual test cases in parallel has been provided:
101+
1. For all tests associated with a library or executable, e.g.
102+
`cmake --build cmake-build-relwithdebinfo -j 8 -t test_api_individually`
103+
1. For all tests in the `ml-cpp` repo:
104+
`cmake --build cmake-build-relwithdebinfo -j 8 -t test_individually`
105+
1. **Care should be taken that tests don't modify common resources.**
95106
1. As a convenience, there exists a `precommit` target that both formats the code and runs the entire test suite, e.g.
96107
1. `./gradlew precommit`
97108
1. `cmake --build cmake-build-relwithdebinfo -j 8 -t precommit`

‎cmake/compiler/clang.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ set(CMAKE_RANLIB "ranlib")
1616
set(CMAKE_STRIP "strip")
1717

1818

19-
list(APPEND ML_C_FLAGS
19+
list(APPEND ML_C_FLAGS
2020
${CROSS_FLAGS}
2121
${ARCHCFLAGS}
2222
"-fstack-protector"

‎cmake/functions.cmake‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,13 @@ function(ml_add_test_executable _target)
392392
COMMENT "Running test: ml_test_${_target}"
393393
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
394394
)
395+
396+
add_custom_target(test_${_target}_individually
397+
DEPENDS ml_test_${_target}
398+
COMMAND ${CMAKE_SOURCE_DIR}/run_tests_as_seperate_processes.sh ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR} test_${_target}
399+
COMMENT "Running test: ml_test_${_target}_individually"
400+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
401+
)
395402
endif()
396403
endfunction()
397404

@@ -420,8 +427,10 @@ function(ml_add_test _directory _target)
420427
add_subdirectory(../${_directory} ${_directory})
421428
list(APPEND ML_BUILD_TEST_DEPENDS ml_test_${_target})
422429
list(APPEND ML_TEST_DEPENDS test_${_target})
430+
list(APPEND ML_TEST_INDIVIDUALLY_DEPENDS test_${_target}_individually)
423431
set(ML_BUILD_TEST_DEPENDS ${ML_BUILD_TEST_DEPENDS} PARENT_SCOPE)
424432
set(ML_TEST_DEPENDS ${ML_TEST_DEPENDS} PARENT_SCOPE)
433+
set(ML_TEST_INDIVIDUALLY_DEPENDS ${ML_TEST_INDIVIDUALLY_DEPENDS} PARENT_SCOPE)
425434
endfunction()
426435

427436

‎cmake/test-runner.cmake‎

Lines changed: 33 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,30 +9,44 @@
99
# limitation.
1010
#
1111

12-
if(TEST_NAME STREQUAL "ml_test_seccomp")
13-
execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS} --logger=HRF,all --report_format=HRF --show_progress=no --no_color_output OUTPUT_FILE ${TEST_DIR}/${TEST_NAME}.out ERROR_FILE ${TEST_DIR}/${TEST_NAME}.out RESULT_VARIABLE TEST_SUCCESS)
14-
else()
15-
# Turn the TEST_FLAGS environment variable into a CMake list variable
16-
if (DEFINED ENV{TEST_FLAGS} AND NOT "$ENV{TEST_FLAGS}" STREQUAL "")
17-
string(REPLACE " " ";" TEST_FLAGS $ENV{TEST_FLAGS})
18-
endif()
12+
execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f ${TEST_DIR}/*.out)
13+
execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f ${TEST_DIR}/*.failed)
14+
execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f boost_test_results*.xml)
15+
execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f boost_test_results*.junit)
1916

20-
# Special case for specifying a subset of tests to run (can be regex)
21-
if (DEFINED ENV{TESTS} AND NOT "$ENV{TESTS}" STREQUAL "")
22-
set(TESTS "--run_test=$ENV{TESTS}")
23-
endif()
17+
# Turn the TEST_FLAGS environment variable into a CMake list variable
18+
if (DEFINED ENV{TEST_FLAGS} AND NOT "$ENV{TEST_FLAGS}" STREQUAL "")
19+
string(REPLACE " " ";" TEST_FLAGS $ENV{TEST_FLAGS})
20+
endif()
21+
22+
set(SAFE_TEST_NAME "")
23+
set(TESTS "")
24+
# Special case for specifying a subset of tests to run (can be regex)
25+
if (DEFINED ENV{TESTS} AND NOT "$ENV{TESTS}" STREQUAL "")
26+
set(TESTS "--run_test=$ENV{TESTS}")
27+
string(REGEX REPLACE "[^a-zA-Z0-9_]" "_" SAFE_TEST_NAME "$ENV{TESTS}")
28+
set(SAFE_TEST_NAME "_${SAFE_TEST_NAME}")
29+
endif()
30+
31+
string(REPLACE "boost_test_results" "boost_test_results${SAFE_TEST_NAME}" BOOST_TEST_OUTPUT_FORMAT_FLAGS "$ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS}")
32+
set(OUTPUT_FILE "${TEST_DIR}/${TEST_NAME}${SAFE_TEST_NAME}.out")
33+
set(FAILED_FILE "${TEST_DIR}/${TEST_NAME}${SAFE_TEST_NAME}.failed")
2434

25-
# If any special command line args are present run the tests in the foreground
26-
if (DEFINED TEST_FLAGS OR DEFINED TESTS)
27-
message(STATUS "executing process ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS}")
28-
execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS} RESULT_VARIABLE TEST_SUCCESS)
35+
# If env var RUN_BOOST_TESTS_IN_FOREGROUND is defined run the tests in the foreground
36+
if(TEST_NAME STREQUAL "ml_test_seccomp")
37+
execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} ${BOOST_TEST_OUTPUT_FORMAT_FLAGS} --logger=HRF,all --report_format=HRF --show_progress=no --no_color_output OUTPUT_FILE ${OUTPUT_FILE} ERROR_FILE ${OUTPUT_FILE} RESULT_VARIABLE TEST_SUCCESS)
38+
else()
39+
if(NOT DEFINED ENV{RUN_BOOST_TESTS_IN_FOREGROUND})
40+
execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} ${BOOST_TEST_OUTPUT_FORMAT_FLAGS} --no_color_output OUTPUT_FILE ${OUTPUT_FILE} ERROR_FILE ${OUTPUT_FILE} RESULT_VARIABLE TEST_SUCCESS)
2941
else()
30-
execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} $ENV{TEST_FLAGS} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS}
31-
--no_color_output OUTPUT_FILE ${TEST_DIR}/${TEST_NAME}.out ERROR_FILE ${TEST_DIR}/${TEST_NAME}.out RESULT_VARIABLE TEST_SUCCESS)
42+
execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} ${BOOST_TEST_OUTPUT_FORMAT_FLAGS} RESULT_VARIABLE TEST_SUCCESS)
3243
endif()
3344
endif()
3445

3546
if (NOT TEST_SUCCESS EQUAL 0)
36-
execute_process(COMMAND ${CMAKE_COMMAND} -E cat ${TEST_DIR}/${TEST_NAME}.out)
37-
file(WRITE "${TEST_DIR}/${TEST_NAME}.failed" "")
47+
if (EXISTS ${TEST_DIR}/${TEST_NAME})
48+
execute_process(COMMAND ${CMAKE_COMMAND} -E cat ${OUTPUT_FILE})
49+
file(WRITE "${FAILED_FILE}" "")
50+
endif()
3851
endif()
52+

‎dev-tools/docker/docker_entrypoint.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,6 @@ if [ "x$1" = "x--test" ] ; then
6666
# failure is the unit tests, and then the detailed test results can be
6767
# copied from the image
6868
echo passed > build/test_status.txt
69-
cmake --build cmake-build-docker ${CMAKE_VERBOSE} -j`nproc` -t test || echo failed > build/test_status.txt
69+
cmake --build cmake-build-docker ${CMAKE_VERBOSE} -j $(nproc) -t test_individually || echo failed > build/test_status.txt
7070
fi
7171

‎dev-tools/docker_test.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,10 @@ do
9292
# Using tar to copy the build and test artifacts out of the container seems
9393
# more reliable than docker cp, and also means the files end up with the
9494
# correct uid/gid
95-
docker run --rm --workdir=/ml-cpp $TEMP_TAG bash -c "find . $EXTRACT_FIND | xargs tar cf - $EXTRACT_EXPLICIT && sleep 30" | tar xvf -
95+
docker run --rm --workdir=/ml-cpp $TEMP_TAG bash -c "find . \( $EXTRACT_FIND \) -print0 | tar cf - $EXTRACT_EXPLICIT --null -T -" | tar xvf -
96+
if [ $? != 0 ]; then
97+
echo "Copying build and test artifacts from docker container failed"
98+
fi
9699
docker rmi --force $TEMP_TAG
97100
# The image build is set to return zero (i.e. succeed as far as Docker is
98101
# concerned) when the only problem is that the unit tests fail, as this

0 commit comments

Comments
 (0)