diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 910bd884e..f6c1e5332 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -37,21 +37,114 @@ ci_registry_image=${CI_REGISTRY_IMAGE:-"czregistry.llnl.gov:5050/radiuss/umpire" export ci_registry_user=${CI_REGISTRY_USER:-"${USER}"} export ci_registry_token=${CI_JOB_TOKEN:-"${registry_token}"} -timed_message () +# Track script start time for elapsed time calculations +script_start_time=$(date +%s) + +# Storage for section start times (supports nesting) +declare -A section_start_times + +# Section stack for tracking nested sections +section_id_stack=() +section_counter=0 +section_indent="" + +# Helper function to print errors in red +print_error () +{ + local error_msg="${1}" + echo -e "\e[31m[Error]: ${error_msg}\e[0m" +} + +# Helper function to print warnings in gray +print_warning () { - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~ $(date --rfc-3339=seconds) ~ ${1}" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + local warning_msg="${1}" + echo -e "\e[1;30m[Warning]: ${warning_msg}\e[0m" +} + +# Helper function to print information +print_info () +{ + local info_msg="${1}" + echo -e "[Information]: ${info_msg}" +} + +# GitLab CI collapsible section helpers with nesting support +section_start () +{ + local section_name="${1}" + local section_title="${2}" + local section_state="${3:-""}" + + local collapsed="false" + if [[ "${section_state}" == "collapsed" ]] + then + collapsed="true" + fi + + # Generate unique section ID + section_counter=$((section_counter + 1)) + local section_id="${section_name}_${section_counter}" + + local timestamp=$(date +%s) + local current_time=$(date -d @${timestamp} --rfc-3339=seconds) + local total_elapsed=$((timestamp - script_start_time)) + local total_elapsed_formatted=$(date -d @${total_elapsed} -u +%H:%M:%S) + + # Store section start time for later calculation + section_start_times[${section_id}]=${timestamp} + + # Push section ID onto stack + section_id_stack+=("${section_id}") + + echo "${section_indent}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "${section_indent}~ TIME | TOTAL | SECTION " + echo "${section_indent}~ ${current_time} | ${total_elapsed_formatted} | ${section_title}" + echo -e "\e[0Ksection_start:${timestamp}:${section_id}[collapsed=${collapsed}]\r\e[0K${section_indent}~ ${section_title}" + + # Increase indentation for nested sections + section_indent="${section_indent} " +} + +section_end () +{ + # Pop section ID from stack + if [[ ${#section_id_stack[@]} -eq 0 ]]; then + echo "[Warning]: section_end called with empty stack" + return 1 + fi + + # Decrease indentation before displaying + section_indent="${section_indent% }" + + local stack_index=$((${#section_id_stack[@]} - 1)) + local section_id="${section_id_stack[$stack_index]}" + unset section_id_stack[$stack_index] + + local timestamp=$(date +%s) + local current_time=$(date -d @${timestamp} --rfc-3339=seconds) + local total_elapsed=$((timestamp - script_start_time)) + local total_elapsed_formatted=$(date -d @${total_elapsed} -u +%H:%M:%S) + + # Calculate section elapsed time + local section_start=${section_start_times[${section_id}]:-${timestamp}} + local section_elapsed=$((timestamp - section_start)) + local section_elapsed_formatted=$(date -d @${section_elapsed} -u +%H:%M:%S) + + echo -e "\e[0Ksection_end:${timestamp}:${section_id}\r\e[0K" + echo "${section_indent}~ ${current_time} | ${total_elapsed_formatted} | ${section_elapsed_formatted}" + echo "${section_indent}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + + # Clean up stored time + unset section_start_times[${section_id}] } if [[ ${debug_mode} == true ]] then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Debug mode:" - echo "~~~~~ - Spack debug mode." - echo "~~~~~ - Deactivated shared memory." - echo "~~~~~ - Do not push to buildcache." - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + print_info "Debug mode:" + print_info "- Spack debug mode." + print_info "- Deactivated shared memory." + print_info "- Do not push to buildcache." use_dev_shm=false spack_debug=true push_to_registry=false @@ -59,7 +152,7 @@ fi if [[ -n ${module_list} ]] then - timed_message "Modules to load: ${module_list}" + print_info "Loading modules: ${module_list}" module load ${module_list} fi @@ -82,14 +175,14 @@ else prefix="${project_dir}/../spack-and-build-root" fi -echo "Creating directory ${prefix}" -echo "project_dir: ${project_dir}" +print_info "Creating directory ${prefix}" +print_info "project_dir: ${project_dir}" mkdir -p ${prefix} spack_cmd="${prefix}/spack/bin/spack" spack_env_path="${prefix}/spack_env" -uberenv_cmd="./scripts/uberenv/uberenv.py" +uberenv_cmd="${project_dir}/scripts/uberenv/uberenv.py" if [[ ${spack_debug} == true ]] then spack_cmd="${spack_cmd} --debug --stacktrace" @@ -99,11 +192,12 @@ fi # Dependencies if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]] then - timed_message "Building dependencies" + section_start "dependencies" "Building Dependencies" if [[ -z ${spec} ]] then - echo "[Error]: SPEC is undefined, aborting..." + section_end + print_error "SPEC is undefined, aborting..." exit 1 fi @@ -118,25 +212,29 @@ then mkdir -p ${spack_user_cache} # generate cmake cache file with uberenv and radiuss spack package - timed_message "Spack setup and environment" + section_start "spack_setup" "Spack setup and environment" "collapsed" ${uberenv_cmd} --setup-and-env-only --spec="${spec}" ${prefix_opt} + section_end if [[ -n ${ci_registry_token} ]] then - timed_message "GitLab registry as Spack Buildcache" + section_start "registry_setup" "GitLab registry as Spack Buildcache" "collapsed" ${spack_cmd} -D ${spack_env_path} mirror add --unsigned --oci-username-variable ci_registry_user --oci-password-variable ci_registry_token gitlab_ci oci://${ci_registry_image} + section_end fi - timed_message "Spack build of dependencies" + section_start "spack_build" "Spack build of dependencies" "collapsed" ${uberenv_cmd} --skip-setup-and-env --spec="${spec}" ${prefix_opt} + section_end if [[ -n ${ci_registry_token} && ${push_to_registry} == true ]] then - timed_message "Push dependencies to buildcache" + section_start "buildcache_push" "Push dependencies to buildcache" "collapsed" ${spack_cmd} -D ${spack_env_path} buildcache push --only dependencies gitlab_ci + section_end fi - timed_message "Dependencies built" + section_end fi # Find cmake cache file (hostconfig) @@ -150,13 +248,13 @@ then hostconfig_path=${hostconfigs[0]} elif [[ ${#hostconfigs[@]} == 0 ]] then - echo "[Error]: No result for: ${project_dir}/*.cmake" - echo "[Error]: Spack generated host-config not found." + print_error "No result for: ${project_dir}/*.cmake" + print_error "Spack generated host-config not found." exit 1 else - echo "[Error]: More than one result for: ${project_dir}/*.cmake" - echo "[Error]: ${hostconfigs[@]}" - echo "[Error]: Please specify one with HOST_CONFIG variable" + print_error "More than one result for: ${project_dir}/*.cmake" + print_error "${hostconfigs[@]}" + print_error "Please specify one with HOST_CONFIG variable" exit 1 fi else @@ -165,7 +263,7 @@ else fi hostconfig=$(basename ${hostconfig_path}) -echo "[Information]: Found hostconfig ${hostconfig_path}" +print_info "Found hostconfig ${hostconfig_path}" # Build Directory # When using /dev/shm, we use prefix for both spack builds and source build, unless BUILD_ROOT was defined @@ -179,16 +277,13 @@ cmake_exe=`grep 'CMake executable' ${hostconfig_path} | cut -d ':' -f 2 | xargs` # Build if [[ "${option}" != "--deps-only" && "${option}" != "--test-only" ]] then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Prefix: ${prefix}" - echo "~~~~~ Host-config: ${hostconfig_path}" - echo "~~~~~ Build Dir: ${build_dir}" - echo "~~~~~ Project Dir: ${project_dir}" - echo "~~~~~ Install Dir: ${install_dir}" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "" - timed_message "Cleaning working directory" + print_info "Prefix ${prefix}" + print_info "Host-config ${hostconfig_path}" + print_info "Build Dir ${build_dir}" + print_info "Project Dir ${project_dir}" + print_info "Install Dir ${install_dir}" + section_start "clean" "Cleaning working directory" "collapsed" # Map CPU core allocations declare -A core_counts=(["lassen"]=40 ["poodle"]=28 ["dane"]=28 ["matrix"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32 ["tuolumne"]=48) @@ -198,8 +293,8 @@ then # use max cores. rm -rf ${build_dir} 2>/dev/null mkdir -p ${build_dir} && cd ${build_dir} + section_end - timed_message "Building Umpire" # We set the MPI tests command to allow overlapping. # Shared allocation: Allows build_and_test.sh to run within a sub-allocation (see CI config). # Use /dev/shm: Prevent MPI tests from running on a node where the build dir doesn't exist. @@ -209,22 +304,52 @@ then cmake_options="-DBLT_MPI_COMMAND_APPEND:STRING=--overlap" fi - $cmake_exe \ + section_start "cmake_config" "CMake Configuration" "collapsed" + if ! $cmake_exe \ -C ${hostconfig_path} \ ${cmake_options} \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ ${project_dir} + then + status=$? + section_end + print_error "CMake configuration failed, dumping output..." + + $cmake_exe \ + -C ${hostconfig_path} \ + ${cmake_options} \ + -DCMAKE_INSTALL_PREFIX=${install_dir} \ + ${project_dir} --debug-output --trace-expand + + exit ${status} + fi + section_end + + section_start "build" "Building Umpire" "collapsed" if ! $cmake_exe --build . -j ${core_counts[$truehostname]} then - echo "[Error]: Compilation failed, building with verbose output..." - timed_message "Re-building with --verbose" + status=$? + section_end + print_error "Compilation failed, building with verbose output..." + + section_start "build_verbose" "Verbose Rebuild" $cmake_exe --build . --verbose -j 1 - else - timed_message "Installing" - $cmake_exe --install . + section_end + + exit ${status} fi + section_end - timed_message "Umpire built and installed" + section_start "install" "Installing Umpire" "collapsed" + if ! $cmake_exe --install . + then + status=$? + section_end + print_error "Installation failed." + + exit ${status} + fi + section_end fi # Test @@ -233,13 +358,15 @@ then if [[ ! -d ${build_dir} ]] then - echo "[Error]: Build directory not found : ${build_dir}" && exit 1 + print_error "Build directory not found : ${build_dir}" + exit 1 fi cd ${build_dir} - timed_message "Testing Umpire" + section_start "tests" "Running Tests" "collapsed" ctest --output-on-failure --no-compress-output -T test -VV 2>&1 | tee tests_output.txt + ctest_status=${PIPESTATUS[0]} # If Developer benchmarks enabled, run the no-op benchmark and show output if [[ "${option}" != "--build-only" ]] && grep -q -i "UMPIRE_ENABLE_DEVELOPER_BENCHMARKS.*ON" ${hostconfig_path} @@ -252,40 +379,51 @@ then no_test_str="No tests were found!!!" if [[ "$(tail -n 1 tests_output.txt)" == "${no_test_str}" ]] then - echo "[Error]: No tests were found" && exit 1 + section_end + print_error "No tests were found" + exit ${ctest_status} fi - timed_message "Preparing tests xml reports for export" tree Testing xsltproc -o junit.xml ${project_dir}/scripts/radiuss-spack-configs/utilities/ctest-to-junit.xsl Testing/*/Test.xml mv junit.xml ${project_dir}/junit.xml if grep -q "Errors while running CTest" ./tests_output.txt then - echo "[Error]: Failure(s) while running CTest" && exit 1 + section_end + print_error "Failure(s) while running CTest" + exit ${ctest_status} fi + section_end + section_start "install_test" "Testing Installed Examples" "collapsed" if grep -q -i "ENABLE_HIP.*ON" ${hostconfig_path} then + section_end echo "[Warning]: Not testing install with HIP" else if [[ ! -d ${install_dir} ]] then - echo "[Error]: Install directory not found : ${install_dir}" && exit 1 + section_end + print_error "Install directory not found : ${install_dir}" + exit 1 fi cd ${install_dir}/examples/umpire/using-with-cmake mkdir build && cd build if ! $cmake_exe -C ../host-config.cmake ..; then - echo "[Error]: Running $cmake_exe for using-with-cmake test" && exit 1 + section_end + print_error "Running $cmake_exe for using-with-cmake test" + exit 1 fi if ! make; then - echo "[Error]: Running make for using-with-cmake test" && exit 1 + section_end + print_error "Running make for using-with-cmake test" + exit 1 fi + section_end fi - - timed_message "Umpire tests completed" fi #timed_message "Cleaning up" @@ -293,4 +431,6 @@ fi cd ${project_dir} -timed_message "Build and test completed" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo "~ Build and test completed" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"