diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbcfc66c3..c28984170 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: uses: actions/cache@v4 with: path: dependencies - key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} + key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('spack.yaml', 'scripts/spack', 'scripts/install-dependencies.sh') }} - name: Install dependencies if: ${{ steps.cache.outputs.cache-hit != 'true' }} env: @@ -130,12 +130,13 @@ jobs: uses: actions/cache@v4 with: path: /dependencies - key: ${{ matrix.os.image }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} + key: ${{ matrix.os.image }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('spack.yaml', 'scripts/spack', 'scripts/install-dependencies.sh') }} - name: Install dependencies if: ${{ steps.cache.outputs.cache-hit != 'true' }} env: JULEA_SPACK_COMPILER: ${{ matrix.os.compiler_spack }}@${{ matrix.os.compiler_version }} JULEA_SPACK_DIR: /dependencies + JULEA_SPACK_ENV: . run: | ./scripts/install-dependencies.sh build: @@ -187,7 +188,7 @@ jobs: uses: actions/cache/restore@v4 with: path: dependencies - key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} + key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('spack.yaml', 'scripts/spack', 'scripts/install-dependencies.sh') }} - name: Check dependencies if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }} run: | @@ -313,7 +314,7 @@ jobs: uses: actions/cache/restore@v4 with: path: dependencies - key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} + key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('spack.yaml', 'scripts/spack', 'scripts/install-dependencies.sh') }} - name: Check dependencies if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }} run: | @@ -405,7 +406,7 @@ jobs: uses: actions/cache/restore@v4 with: path: dependencies - key: ubuntu-24.04-gcc-13.3.0-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} + key: ubuntu-24.04-gcc-13.3.0-${{ hashFiles('spack.yaml', 'scripts/spack', 'scripts/install-dependencies.sh') }} - name: Check dependencies if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | diff --git a/.gitignore b/.gitignore index 3a3f02351..8c04d908a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,10 @@ bld*/ # Visual Studio Code .vscode/ + +# CLion +.idea/ + +# Spack-Env +.spack-env/ +spack.lock diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index c35de58ae..d79bcd69e 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -24,8 +24,6 @@ SELF_BASE="${SELF_PATH##*/}" # shellcheck source=scripts/common . "${SELF_DIR}/common" -# shellcheck source=scripts/spack -. "${SELF_DIR}/spack" usage () { @@ -54,6 +52,7 @@ then JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" fi -spack_load_dependencies +# shellcheck source=scripts/environment.sh +. "${SELF_DIR}/environment.sh" run_benchmark "$@" diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 0639f2b1a..903fce7ff 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -21,19 +21,24 @@ set -e MODE="$1" DEPS="$2" -GDBM_PREFIX='' - +echo "$PATH" . scripts/environment.sh +echo "$PATH" + +ls + +spack env status + +printf "\n\n" +ls .spack-env/ -if test "${DEPS}" = 'spack' -then - GDBM_PREFIX="-Dgdbm_prefix=$(spack location --install-dir gdbm)" -fi +printf "\n\n" +ls .spack-env/view/bin/ case "${MODE}" in release) # shellcheck disable=SC2086 - meson setup --prefix="${GITHUB_WORKSPACE}/julea-install" --buildtype=release --werror ${GDBM_PREFIX} bld + meson setup --prefix="${GITHUB_WORKSPACE}/julea-install" --buildtype=release --werror bld ninja -C bld ninja -C bld install ;; @@ -44,12 +49,12 @@ case "${MODE}" in CLANG_LUNDEF='-Db_lundef=false' fi # shellcheck disable=SC2086 - meson setup -Db_sanitize=address,undefined ${CLANG_LUNDEF} ${GDBM_PREFIX} bld + meson setup -Db_sanitize=address,undefined ${CLANG_LUNDEF} bld ninja -C bld ;; coverage) # shellcheck disable=SC2086 - meson setup -Db_coverage=true -Db_sanitize=address,undefined ${GDBM_PREFIX} bld + meson setup -Db_coverage=true -Db_sanitize=address,undefined bld ninja -C bld ;; *) diff --git a/scripts/environment.sh b/scripts/environment.sh index 10e3b75ce..e004ba567 100755 --- a/scripts/environment.sh +++ b/scripts/environment.sh @@ -25,8 +25,6 @@ SELF_BASE="${SELF_PATH##*/}" # shellcheck source=scripts/common . "${SELF_DIR}/common" -# shellcheck source=scripts/spack -. "${SELF_DIR}/spack" JULEA_ENVIRONMENT_SOURCED=1 @@ -63,7 +61,19 @@ then JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" fi -spack_load_dependencies +# shellcheck source=scripts/spack +. "${SELF_DIR}/spack" + +# activate spack if its installed +if spack_init +then + # Activate spack shell support. + # shellcheck source=/dev/null + . "${JULEA_SPACK_DIR}/share/spack/setup-env.sh" + # Activate spack environment. + spack env activate . + spack env status +fi # Do not filter out paths contained in CPATH and LIBRARY_PATH. PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh index 46a4ef24a..dc0c67a4d 100755 --- a/scripts/install-dependencies.sh +++ b/scripts/install-dependencies.sh @@ -38,4 +38,9 @@ then JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" fi +if test -z "${JULEA_SPACK_ENV}" +then + JULEA_SPACK_ENV="$(get_directory "${SELF_DIR}/..")" +fi + spack_install_dependencies diff --git a/scripts/setup b/scripts/setup index 6d1189e27..910899969 100644 --- a/scripts/setup +++ b/scripts/setup @@ -160,6 +160,7 @@ setup_start () host="$(get_host "${server}")" port="$(get_port "${server}")" + # TODO: These sometimes dont match which causes the script to silently fail. Fix this. if test "${host}" = "${HOSTNAME}" then port_option='' diff --git a/scripts/setup.sh b/scripts/setup.sh index a041a458a..f79de74a8 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -27,8 +27,6 @@ SELF_BASE="${SELF_PATH##*/}" . "${SELF_DIR}/common" # shellcheck source=scripts/setup . "${SELF_DIR}/setup" -# shellcheck source=scripts/spack -. "${SELF_DIR}/spack" usage () { @@ -76,7 +74,9 @@ then JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" fi -spack_load_dependencies + +# shellcheck source=scripts/environment.sh +. "${SELF_DIR}/environment.sh" setup_init diff --git a/scripts/spack b/scripts/spack index 56e375b79..1f531abfb 100644 --- a/scripts/spack +++ b/scripts/spack @@ -33,28 +33,20 @@ spack_clone () if test ! -d "${JULEA_SPACK_DIR}" then - git clone --config feature.manyFiles=true https://github.com/spack/spack.git "${JULEA_SPACK_DIR}" + git clone --config feature.manyFiles=true --depth=1 --branch=releases/v1.0 https://github.com/spack/spack.git "${JULEA_SPACK_DIR}" fi test -d "${JULEA_SPACK_DIR}" || return 1 if test ! -d "${JULEA_SPACK_DIR}/packages" then - git clone --config feature.manyFiles=true https://github.com/spack/spack-packages.git "${JULEA_SPACK_DIR}/packages" + git clone --config feature.manyFiles=true --branch=develop https://github.com/spack/spack-packages.git "${JULEA_SPACK_DIR}/packages" fi test -d "${JULEA_SPACK_DIR}/packages" || return 1 spack_set_env - ( - cd "${JULEA_SPACK_DIR}" || return 1 - - git fetch - git checkout releases/v1.0 - git reset --hard origin/releases/v1.0 - ) - ( cd "${JULEA_SPACK_DIR}/packages" || return 1 @@ -63,7 +55,7 @@ spack_clone () git reset --hard "${spack_packages_commit}" ) - spack_cmd repo set --destination "${JULEA_SPACK_DIR}/packages" builtin + spack_cmd repo set --destination "${JULEA_SPACK_DIR}/packages" builtin return 0 } @@ -99,101 +91,7 @@ spack_cmd () ) } -spack_load () -{ - local pkg_loaded - local spack_pkg - - pkg_loaded='YES' - spack_pkg="$1" - - test -n "${spack_pkg}" || return 1 - - spack_pkg="$(printf '%s' "${spack_pkg}" | sed 's/#/ /g')" - - spack load "${spack_pkg}" || pkg_loaded='NO' - - printf 'Dependency "%s" loaded: %s\n' "${spack_pkg}" "${pkg_loaded}" -} - -spack_install () -{ - local spack_pkg - - spack_pkg="$1" - - test -n "${spack_pkg}" || return 1 - - spack_pkg="$(printf '%s' "${spack_pkg}" | sed 's/#/ /g')" - - printf 'Installing "%s"\n' "${spack_pkg}" - # FIXME ignore errors? - spack_cmd install "${spack_pkg}" -} - -spack_get_dependencies () -{ - local dependencies - - dependencies='' - - # Required for building JULEA - dependencies="${dependencies} meson" - dependencies="${dependencies} pkgconf" - - # Mandatory dependencies - dependencies="${dependencies} glib" - dependencies="${dependencies} libbson" - dependencies="${dependencies} libfabric#fabrics=sockets,tcp,udp,verbs,rxd,rxm" - - # Recommended dependencies - dependencies="${dependencies} hdf5~mpi@1.14" - dependencies="${dependencies} libfuse~utils" - dependencies="${dependencies} lmdb" - dependencies="${dependencies} sqlite" - - # Optional dependencies - # FIXME https://github.com/spack/spack-packages/pull/1109 - dependencies="${dependencies} gdbm@1.23" - dependencies="${dependencies} leveldb" - dependencies="${dependencies} mariadb-c-client" - dependencies="${dependencies} mongo-c-driver" - dependencies="${dependencies} otf" - dependencies="${dependencies} rocksdb~static" - - if test -n "${CI}" - then - dependencies="${dependencies} py-gcovr" - fi - - #dependencies="${dependencies} enzo@main" - #dependencies="${dependencies} mpi" - - printf '%s' "${dependencies}" -} - -spack_load_dependencies () -{ - if test -z "${JULEA_SPACK_DEPENDENCIES_LOADED}" - then - if spack_init - then - for dependency in $(spack_get_dependencies) - do - spack_load "${dependency}" - done - - # FIXME this could be more clever by storing the actual dependencies loaded - JULEA_SPACK_DEPENDENCIES_LOADED=1 - - export JULEA_SPACK_DEPENDENCIES_LOADED - fi - else - printf 'Dependencies have already been loaded, skipping.\n' >&2 - fi -} - -spack_install_dependencies () +spack_install_dependencies() { # FIXME does not fail if Git is not installed if spack_clone @@ -222,14 +120,17 @@ spack_install_dependencies () spack_cmd config --scope site add "packages:all:require:\"%${JULEA_SPACK_COMPILER}\"" fi - spack_cmd mark --implicit --all + spack_cmd mark --implicit --all - for dependency in $(spack_get_dependencies) - do - spack_install "${dependency}" - done + if spack_init + then + echo "Installing Spack packages" + spack env activate "${JULEA_SPACK_ENV}" + spack env status + spack install --fail-fast --deprecated + fi - spack_cmd gc --yes-to-all + spack_cmd gc --yes-to-all if test -n "${CI}" then diff --git a/scripts/test.sh b/scripts/test.sh index d300dac68..ae6d53d8a 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -24,8 +24,6 @@ SELF_BASE="${SELF_PATH##*/}" # shellcheck source=scripts/common . "${SELF_DIR}/common" -# shellcheck source=scripts/spack -. "${SELF_DIR}/spack" usage () { @@ -63,6 +61,7 @@ then JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" fi -spack_load_dependencies +# shellcheck source=scripts/environment.sh +. "${SELF_DIR}/environment.sh" run_test "$@" diff --git a/spack.yaml b/spack.yaml new file mode 100644 index 000000000..253d13461 --- /dev/null +++ b/spack.yaml @@ -0,0 +1,32 @@ +# This is a Spack Environment file. +# +# It describes a set of packages to be installed, along with +# configuration settings. +spack: + # add package specs to the `specs` list + # TODO: py-gcovr is only used for coverage and thus is only needed for CI + specs: + - meson + - pkgconf + + - glib + - libbson + - libfabric fabrics=sockets,tcp,udp,verbs,rxd,rxm + + - hdf5@1.14~mpi + - libfuse~utils + - lmdb + - sqlite + + - gdbm@1.23 + - leveldb + - mariadb-c-client + - mongo-c-driver + - otf + - rocksdb~static + + - py-gcovr + + view: true + concretizer: + unify: true