Skip to content

Commit 9444c31

Browse files
wrtobinwrtobin
authored
Initial HIP/ROCM Support (#248)
1 parent 4294084 commit 9444c31

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1755
-248
lines changed

‎CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ if( NOT is_submodule )
3535

3636
option( ENABLE_ADDR2LINE "Enable addr2line usage in stacktraces" ON )
3737
option( ENABLE_CUDA "Build with CUDA" OFF )
38+
option( ENABLE_HIP "Build with HIP" OFF )
3839
option( ENABLE_UMPIRE "Build with UMPIRE" OFF )
3940
option( ENABLE_CHAI "Build with CHAI" OFF )
4041
option( ENABLE_CALIPER "Build with Caliper" OFF )
@@ -80,6 +81,8 @@ blt_list_append( TO lvarray_dependencies ELEMENTS chai IF ENABLE_CHAI )
8081

8182
blt_list_append( TO lvarray_dependencies ELEMENTS cuda IF ENABLE_CUDA )
8283

84+
blt_list_append( TO lvarray_dependencies ELEMENTS blt::hip IF ENABLE_HIP )
85+
8386
blt_list_append( TO lvarray_dependencies ELEMENTS caliper IF ENABLE_CALIPER )
8487

8588

‎cmake/CMakeBasics.cmake‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ option( ENABLE_TOTALVIEW_OUTPUT "" OFF )
1212
set( LVARRAY_BUILD_OBJ_LIBS OFF CACHE BOOL "" )
1313

1414

15-
if( NOT BLT_CXX_STD STREQUAL c++14 )
16-
MESSAGE(FATAL_ERROR "c++14 is NOT enabled. LvArray requires c++14")
17-
endif()
15+
# if( NOT BLT_CXX_STD STREQUAL c++14 )
16+
# MESSAGE(FATAL_ERROR "c++14 is NOT enabled. LvArray requires c++14")
17+
# endif()
1818

1919

2020
blt_append_custom_compiler_flag( FLAGS_VAR CMAKE_CXX_FLAGS DEFAULT "${OpenMP_CXX_FLAGS}")

‎cmake/Config.cmake‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
set( PREPROCESSOR_DEFINES UMPIRE
33
CHAI
44
CUDA
5+
HIP
56
TOTALVIEW_OUTPUT
67
CALIPER )
78

‎cmake/blt‎

Submodule blt updated 163 files

‎host-configs/LLNL/lassen-base.cmake‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ set(ENABLE_CHAI ON CACHE BOOL "")
1212
set(CHAI_DIR ${GEOSX_TPL_DIR}/chai CACHE PATH "")
1313

1414
set(ENABLE_CALIPER ON CACHE BOOL "")
15+
set(ENABLE_ADIAK ON CACHE BOOL "" )
1516
set(CALIPER_DIR ${GEOSX_TPL_DIR}/caliper CACHE PATH "")
1617

1718
set(ENABLE_ADDR2LINE ON CACHE BOOL "")
@@ -21,14 +22,14 @@ set(ENABLE_CUDA ON CACHE BOOL "")
2122
set(CUDA_TOOLKIT_ROOT_DIR /usr/tce/packages/cuda/cuda-10.1.243 CACHE STRING "")
2223
set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "")
2324
set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
24-
set(CUDA_ARCH sm_70 CACHE STRING "")
25+
set(CUDA_ARCHITECTURES sm_70 CACHE STRING "")
2526
set(CMAKE_CUDA_STANDARD 14 CACHE STRING "")
26-
set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "")
27+
set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCHITECTURES} --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "")
2728
set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le" CACHE STRING "")
2829
set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
2930
set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")
3031

31-
set(CHAI_CUDA_FLAGS "-arch ${CUDA_ARCH}" CACHE STRING "" FORCE)
32+
set(CHAI_CUDA_FLAGS "-arch ${CUDA_ARCHITECTURES}" CACHE STRING "" FORCE)
3233

3334
# Uncomment this line to make nvcc output register usage for each kernel.
3435
# set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --resource-usage" CACHE STRING "" FORCE)

‎host-configs/LLNL/lassen-clang13-cuda11.cmake‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
set(CONFIG_NAME "lassen-clang13-cuda11" CACHE PATH "")
1+
set(CONFIG_NAME "lassen-clang13-cuda11" CACHE PATH "")
22

33

44
set(COMPILER_DIR /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1 )
@@ -14,7 +14,7 @@ set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "")
1414

1515
# Set up the tpls
1616
set(GEOSX_TPL_ROOT_DIR /usr/gapps/GEOSX/thirdPartyLibs CACHE PATH "")
17-
set(GEOSX_TPL_DIR ${GEOSX_TPL_ROOT_DIR}/2023-01-23/install-${CONFIG_NAME}-release CACHE PATH "")
17+
set(GEOSX_TPL_DIR ${GEOSX_TPL_ROOT_DIR}/2023-03-15/install-${CONFIG_NAME}-release CACHE PATH "")
1818

1919
set(CAMP_DIR ${GEOSX_TPL_DIR}/raja CACHE PATH "")
2020
set(RAJA_DIR ${GEOSX_TPL_DIR}/raja CACHE PATH "")

‎host-configs/LLNL/quartz-base.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@ set(ENABLE_GTEST_DEATH_TESTS ON CACHE BOOL "")
2626

2727
set(ENABLE_ADDR2LINE ON CACHE BOOL "")
2828

29-
set(CUDA_ENABLED OFF CACHE BOOL "")
29+
set(ENABLE_CUDA OFF CACHE BOOL "")
3030

3131
set(ENABLE_TOTALVIEW_OUTPUT OFF CACHE BOOL "Enables Totalview custom view" FORCE)

‎host-configs/LLNL/tioga-base.cmake‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
set(CMAKE_CXX_STANDARD 14 CACHE STRING "")
2+
3+
set( ENABLE_MPI ON CACHE BOOL "" FORCE )
4+
set( ENABLE_FIND_MPI ON CACHE BOOL "" FORCE )
5+
6+
# HIP Options
7+
set( ENABLE_HIP ON CACHE BOOL "" FORCE )
8+
9+
# suppress -Werror for now
10+
set( ENABLE_WARNINGS_AS_ERRORS FALSE CACHE BOOL "" FORCE )
11+
12+
# GTEST
13+
set(ENABLE_GTEST_DEATH_TESTS OFF CACHE BOOL "")
14+
set(gtest_disable_pthreads ON CACHE BOOL "")
15+
16+
# disable most binaries and doc generation
17+
set(ENABLE_TESTS OFF CACHE BOOL "" FORCE)
18+
set(DISABLE_UNIT_TESTS ON CACHE BOOL "" FORCE)
19+
set(ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE)
20+
set(ENABLE_BENCHMARKS OFF CACHE BOOL "" FORCE)
21+
set(ENABLE_DOCS OFF CACHE BOOL "" FORCE)
22+
23+
# BLT trying to find MPI fails on cray with cce
24+
set(ENABLE_FIND_MPI FALSE CACHE BOOL "")

‎host-configs/LLNL/[email protected]‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
2+
set(CONFIG_NAME "[email protected]" CACHE PATH "")
3+
include( ${CMAKE_CURRENT_LIST_DIR}/tioga-base.cmake )
4+
5+
# Set up the tpls
6+
set(TPL_INSTALL_DATE 2023-01-19)
7+
set(GEOSX_TPL_DIR "/usr/WS1/GEOS/GEOSX/TPLs_${TPL_INSTALL_DATE}/install-${CONFIG_NAME}" CACHE PATH "")
8+
9+
set(CAMP_DIR "${GEOSX_TPL_DIR}/camp" CACHE PATH "" )
10+
set(RAJA_DIR "${GEOSX_TPL_DIR}/raja" CACHE PATH "" )
11+
12+
set(ENABLE_UMPIRE TRUE CACHE BOOL "" )
13+
set(UMPIRE_DIR "${GEOSX_TPL_DIR}/umpire" CACHE PATH "" )
14+
15+
set(ENABLE_CHAI TRUE CACHE BOOL "" )
16+
set(CHAI_DIR "${GEOSX_TPL_DIR}/chai" CACHE PATH "" )
17+
18+
set(METIS_DIR "${GEOSX_TPL_DIR}/metis" CACHE PATH "" )
19+
set(PARMETIS_DIR "${GEOSX_TPL_DIR}/parmetis" CACHE PATH "" )
20+
21+
# C++ options
22+
set(CRAYPE_VERSION "2.7.19")
23+
set(CMAKE_C_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/cc" CACHE PATH "")
24+
set(CMAKE_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH "")
25+
set(CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "")
26+
27+
if( ENABLE_HIP )
28+
set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE ) # don't invoke hipcc, rely on cce link-time compilation
29+
30+
set( HIP_VERSION_STRING "5.4.0" CACHE STRING "" )
31+
set( HIP_ROOT "/opt/rocm-${HIP_VERSION_STRING}" CACHE PATH "" )
32+
set( ROCM_PATH ${HIP_ROOT} CACHE PATH "" )
33+
34+
set( CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "" FORCE )
35+
set( CMAKE_CXX_FLAGS "-mno-unsafe-fp-atomics -fgpu-rdc" CACHE STRING "" FORCE )
36+
set( CMAKE_CXX_LINK_FLAGS "-fgpu-rdc --hip-link" CACHE STRING "" FORCE )
37+
endif()

‎host-configs/[email protected]‎ renamed to ‎host-configs/ORNL/[email protected]‎

File renamed without changes.

0 commit comments

Comments
 (0)