From d03450e9eae19d8ace548bfb38c9398373585db3 Mon Sep 17 00:00:00 2001 From: Liwei Ji Date: Thu, 28 Mar 2024 15:04:06 -0400 Subject: [PATCH] Docs: Rename config files for cuda on frontera --- Docs/compile-notes/frontera/README.md | 18 ++-- .../frontera/configs/config-frontera-cuda.cfg | 97 ------------------- ....cfg => config-frontera-gcc-cuda-impi.cfg} | 36 +++---- .../frontera/cuda-11.8.0/spack.yaml | 70 ------------- .../spack.yaml | 8 +- 5 files changed, 31 insertions(+), 198 deletions(-) delete mode 100644 Docs/compile-notes/frontera/configs/config-frontera-cuda.cfg rename Docs/compile-notes/frontera/configs/{config-frontera-cuda-intel.cfg => config-frontera-gcc-cuda-impi.cfg} (66%) delete mode 100644 Docs/compile-notes/frontera/cuda-11.8.0/spack.yaml rename Docs/compile-notes/frontera/{cuda-11.8.0-intel => gcc11.2.0-cuda11.8.0-impi19.0.9}/spack.yaml (87%) diff --git a/Docs/compile-notes/frontera/README.md b/Docs/compile-notes/frontera/README.md index 8a47814f..03872235 100644 --- a/Docs/compile-notes/frontera/README.md +++ b/Docs/compile-notes/frontera/README.md @@ -46,7 +46,7 @@ spack load cuda@11.8.0 cd Cactus - gmake SpacetimeX-cuda options=repos/SpacetimeX/Docs/compile-notes/frontera/configs/config-frontera-cuda-intel.cfg + gmake SpacetimeX-cuda options=repos/SpacetimeX/Docs/compile-notes/frontera/configs/config-frontera-gcc-cuda-impi.cfg cp repos/SpacetimeX/Docs/thornlist/spacetimex.th configs/SpacetimeX-cuda/ThornList gmake -j16 SpacetimeX-cuda ``` @@ -117,19 +117,19 @@ spack install gcc@11.2.0 %gcc@4.8.5 spack compiler add ... # ... is the last line of previous command ``` -* Create a dir where you want put `view` in (say `/work2/.../username/frontera/SpackView/cuda-intel`) +* Create a dir where you want put `view` in (say `/work2/.../username/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9`) -* Replace the last line of `cuda-11.8.0-intel/spack_yaml` with your dir (say `/work2/.../username/frontera/SpackView/cuda-intel/view`) +* Replace the last line of `gcc11.2.0-cuda11.8.0-impi19.0.9/spack_yaml` with your dir (say `/work2/.../username/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view`) -* Replace the dir `/work2/08708/liwei/frontera/SpackView/cuda-intel/view` (with say `/work2/.../username/frontera/SpackView/cuda-intel/view`) -in `config-frontera-cuda-intel.cfg` +* Replace the dir `/work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view` (with say `/work2/.../username/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view`) +in `config-frontera-gcc11.2.0-cuda11.8.0-impi19.0.9.cfg` * Install other required packages ```bash -env TMPDIR=$WORK/tmp spack --env-dir ./cuda-11.8.0-intel compiler find view-cuda-compilers -env TMPDIR=$WORK/tmp spack --env-dir ./cuda-11.8.0-intel concretize --force -env TMPDIR=$WORK/tmp spack --env-dir ./cuda-11.8.0-intel install --fail-fast +env TMPDIR=$WORK/tmp spack --env-dir ./gcc11.2.0-cuda11.8.0-impi19.0.9 compiler find view-cuda-compilers +env TMPDIR=$WORK/tmp spack --env-dir ./gcc11.2.0-cuda11.8.0-impi19.0.9 concretize --force +env TMPDIR=$WORK/tmp spack --env-dir ./gcc11.2.0-cuda11.8.0-impi19.0.9 install --fail-fast ``` * Install SpacetimeX @@ -138,7 +138,7 @@ env TMPDIR=$WORK/tmp spack --env-dir ./cuda-11.8.0-intel install --fail-fast spack load gcc@11.2.0 spack load cuda@11.8.0 cd Cactus -gmake SpacetimeX-cuda options=config-frontera-cuda-intel.cfg +gmake SpacetimeX-cuda options=config-frontera-gcc-cuda-impi.cfg cp repos/SpacetimeX/Docs/thornlist/spacetimex.th configs/SpacetimeX-cuda/ThornList gmake -j16 SpacetimeX-cuda ``` diff --git a/Docs/compile-notes/frontera/configs/config-frontera-cuda.cfg b/Docs/compile-notes/frontera/configs/config-frontera-cuda.cfg deleted file mode 100644 index ea1657c3..00000000 --- a/Docs/compile-notes/frontera/configs/config-frontera-cuda.cfg +++ /dev/null @@ -1,97 +0,0 @@ -# Configuration for an frontera installation: cuda - -# Whenever this version string changes, the application is configured -# and rebuilt from scratch -VERSION = frontera-2023-09-30 - -CPP = cpp -CC = gcc -CXX = nvcc -x cu -FPP = cpp -FC = gfortran -F90 = gfortran -LD = nvcc - -CPPFLAGS = -DSIMD_CPU -CFLAGS = -pipe -g -std=gnu11 -CXXFLAGS = -pipe -g0 -std=c++17 --compiler-options -std=gnu++17 --expt-relaxed-constexpr --extended-lambda --gpu-architecture sm_75 --forward-unknown-to-host-compiler --Werror ext-lambda-captures-this --relocatable-device-code=true --objdir-as-tempdir -FPPFLAGS = -traditional -F90FLAGS = -pipe -g -fcray-pointer -ffixed-line-length-none -LIBS = nvToolsExt gfortran -#CUCCFLAGS = -std=c++17 - -#CUCC = nvcc -#CUCCFLAGS = -std=c++14 -I/opt/apps/cuda/11.3/include #-L/opt/apps/cuda/11.3/lib64 -#LDFLAGS = -rdynamic -#LDFLAGS = -Wl,-rpath,/opt/apps/cuda/11.3/lib64 -Wl,-rpath,/work2/08708/liwei/frontera/SpackView/cuda/view/lib -Wl,-rpath,/work2/08708/liwei/frontera/SpackView/cuda/view/lib64 -#LIBDIRS = /opt/apps/cuda/11.3/lib64 - -C_LINE_DIRECTIVES = yes -F_LINE_DIRECTIVES = yes - -INTEGER_PRECISION = 4 -REAL_PRECISION = 8 - -# CUDA does not support these -DISABLE_INT16 = yes -DISABLE_REAL16 = yes - -DEBUG = no -CPP_DEBUG_FLAGS = -DCARPET_DEBUG -C_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv -CXX_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv -lineinfo -FPP_DEBUG_FLAGS = -DCARPET_DEBUG -F90_DEBUG_FLAGS = -fcheck=bounds,do,mem,pointer,recursion -finit-character=65 -finit-integer=42424242 -finit-real=nan -fsanitize=undefined -fstack-protector-all -ftrapv - -OPTIMISE = yes -C_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations -CXX_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations -F90_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations - -OPENMP = yes -CPP_OPENMP_FLAGS = -fopenmp -FPP_OPENMP_FLAGS = -D_OPENMP - -WARN = yes -CPP_WARN_FLAGS = -Wall -C_WARN_FLAGS = -CXX_WARN_FLAGS = -FPP_WARN_FLAGS = -Wall -F90_WARN_FLAGS = -Wall -Wshadow -Wsurprising - -VECTORISE = no - -MPI_DIR = /opt/intel/oneapi/mpi/2021.9.0 -MPI_LIB_DIRS = /opt/intel/oneapi/mpi/2021.9.0/lib -MPI_INC_DIRS = /opt/intel/oneapi/mpi/2021.9.0/include -MPI_LIBS = mpi - -HWLOC_DIR = /opt/apps/hwloc/1.11.12 - -BLAS_DIR = NO_BUILD -BLAS_LIBS = # /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64 -LAPACK_DIR = NO_BUILD -LAPACK_LIBS = # /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64 -OPENBLAS_DIR = NO_BUILD -OPENBLAS_LIBS = # /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64 - -PTHREADS_DIR = NO_BUILD - -GSL_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -FFTW3_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -HDF5_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -ADIOS2_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -ADIOS2_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/cuda/view/lib64 -ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11 -AMREX_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -NSIMD_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -NSIMD_INC_DIRS = /work2/08708/liwei/frontera/SpackView/cuda/view/include -NSIMD_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/cuda/view/lib -NSIMD_ARCH = AVX2 -NSIMD_OPTIONS = #AVX2 FMA -OPENPMD_API_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -OPENPMD_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -SSHT_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -SILO_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -YAML_CPP_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view -ZLIB_DIR = /work2/08708/liwei/frontera/SpackView/cuda/view diff --git a/Docs/compile-notes/frontera/configs/config-frontera-cuda-intel.cfg b/Docs/compile-notes/frontera/configs/config-frontera-gcc-cuda-impi.cfg similarity index 66% rename from Docs/compile-notes/frontera/configs/config-frontera-cuda-intel.cfg rename to Docs/compile-notes/frontera/configs/config-frontera-gcc-cuda-impi.cfg index 935b60f0..f422a40e 100644 --- a/Docs/compile-notes/frontera/configs/config-frontera-cuda-intel.cfg +++ b/Docs/compile-notes/frontera/configs/config-frontera-gcc-cuda-impi.cfg @@ -77,26 +77,26 @@ OPENBLAS_LIBS = # /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/in PTHREADS_DIR = NO_BUILD -FFTW3_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view -HDF5_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view +FFTW3_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view +HDF5_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view HDF5_ENABLE_CXX = yes HDF5_ENABLE_FORTRAN = yes -HDF5_INC_DIRS = /work2/08708/liwei/frontera/SpackView/cuda-intel/view/include -HDF5_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/cuda-intel/view/lib +HDF5_INC_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/include +HDF5_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/lib HDF5_LIBS = hdf5_hl_cpp hdf5_cpp hdf5hl_fortran hdf5_fortran hdf5_hl hdf5 -ADIOS2_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view -ADIOS2_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/cuda-intel/view/lib64 +ADIOS2_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view +ADIOS2_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/lib64 ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11 -AMREX_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view -NSIMD_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view -NSIMD_INC_DIRS = /work2/08708/liwei/frontera/SpackView/cuda-intel/view/include -NSIMD_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/cuda-intel/view/lib +AMREX_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view +NSIMD_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view +NSIMD_INC_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/include +NSIMD_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/lib NSIMD_ARCH = AVX2 -OPENPMD_API_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view -OPENPMD_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view -SSHT_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view -SILO_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view -#SILO_INC_DIRS = /work2/08708/liwei/frontera/SpackView/cuda-intel/view/include -#SILO_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/cuda-intel/view/lib -YAML_CPP_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view -ZLIB_DIR = /work2/08708/liwei/frontera/SpackView/cuda-intel/view +OPENPMD_API_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view +OPENPMD_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view +SSHT_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view +SILO_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view +#SILO_INC_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/include +#SILO_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/lib +YAML_CPP_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view +ZLIB_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view diff --git a/Docs/compile-notes/frontera/cuda-11.8.0/spack.yaml b/Docs/compile-notes/frontera/cuda-11.8.0/spack.yaml deleted file mode 100644 index ba76558f..00000000 --- a/Docs/compile-notes/frontera/cuda-11.8.0/spack.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# source $SPACK_DIR/share/spack/setup-env.sh -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/cuda-11.8.0 compiler find view-cuda-compilers && -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/cuda-11.8.0 concretize --force && -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/cuda-11.8.0 install --fail-fast -spack: - definitions: - - compilers: - - gcc @11.2.0 #arch=linux-centos7-broadwell - - packages: - - adios2 - - amrex +cuda cuda_arch=70 ~fortran +hdf5 +openmp +particles - - cuda @11.8.0 +allow-unsupported-compilers - - fftw +mpi +openmp - - hwloc @1.11.12 - - nsimd @3.0.1 - - openpmd-api @0.15.1 - - ssht - - hdf5 @1.10.9 +cxx +fortran +hl +mpi +threadsafe - - silo@4.10.2 +hdf5 - - yaml-cpp - - zlib - - gsl - # - petsc @3.17 +cuda +fftw +hwloc +openmp - # - openblas - packages: - hwloc: - buildable: false - externals: - - spec: hwloc@1.11.12 - prefix: /opt/apps/hwloc/1.11.12 - intel-mkl: - buildable: false - externals: - - spec: intel-mkl@23.1.0 - prefix: /opt/intel/oneapi/mkl/2023.1.0 - libfabric: - buildable: false - externals: - - spec: libfabric@2021.9.0 - prefix: /opt/intel/oneapi/mpi/2021.9.0/libfabric/lib - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.9.0 - prefix: /opt/intel/oneapi - mpi: - buildable: false - #petsc: - # buildable: false - # externals: - # - spec: petsc@3.17 - # prefix: /home1/apps/intel19/impi19_0/petsc/3.17 - #cmake: - # buildable: false - # externals: - # - spec: cmake@3.24.2 - # prefix: /opt/apps/cmake/3.24.2 - #fftw: - # buildable: false - # externals: - # - spec: fftw@3.3.10 - # prefix: /opt/apps/intel19/impi19_0/fftw3/3.3.10 - specs: - - matrix: - - [$%compilers] - - [$packages] - concretizer: - reuse: false - unify: true - view: /work2/08708/liwei/frontera/SpackView/cuda/view diff --git a/Docs/compile-notes/frontera/cuda-11.8.0-intel/spack.yaml b/Docs/compile-notes/frontera/gcc11.2.0-cuda11.8.0-impi19.0.9/spack.yaml similarity index 87% rename from Docs/compile-notes/frontera/cuda-11.8.0-intel/spack.yaml rename to Docs/compile-notes/frontera/gcc11.2.0-cuda11.8.0-impi19.0.9/spack.yaml index 75438959..cc0c0a91 100644 --- a/Docs/compile-notes/frontera/cuda-11.8.0-intel/spack.yaml +++ b/Docs/compile-notes/frontera/gcc11.2.0-cuda11.8.0-impi19.0.9/spack.yaml @@ -1,7 +1,7 @@ # source $WORK/SpackSource/spack/share/spack/setup-env.sh -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/cuda-11.8.0-intel compiler find view-cuda-compilers && -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/cuda-11.8.0-intel concretize --force && -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/cuda-11.8.0-intel install --fail-fast +# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/gcc11.2.0-cuda11.8.0-impi19.0.9 compiler find view-cuda-compilers && +# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/gcc11.2.0-cuda11.8.0-impi19.0.9 concretize --force && +# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/gcc11.2.0-cuda11.8.0-impi19.0.9 install --fail-fast spack: definitions: - compilers: @@ -72,4 +72,4 @@ spack: concretizer: reuse: false unify: true - view: /work2/08708/liwei/frontera/SpackView/cuda-intel/view + view: /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view