diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 22366b897..c882ccca2 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -111,7 +111,7 @@ jobs: set +e spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set +e + set -e echo "Packages in spack binary cache:" spack buildcache list fi diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index a8f295b7c..14a80693e 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -51,12 +51,14 @@ jobs: # Set up spack-stack source ./setup.sh - declare -a TEMPLATES=("unified-dev" "skylab-dev") + declare -a TEMPLATES=("unified-dev" "skylab-dev" "cylc-dev") for TEMPLATE in "${TEMPLATES[@]}"; do if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then export ENVNAME=ue-gcc-11.4.0-buildcache elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then export ENVNAME=se-gcc-11.4.0-buildcache + elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then + export ENVNAME=ce-gcc-11.4.0-buildcache fi echo "Creating environment ${ENVNAME} from template ${TEMPLATE}" @@ -68,7 +70,8 @@ jobs: # Find external packages spack external find --scope system \ --exclude bison --exclude openssl \ - --exclude curl --exclude python + --exclude curl --exclude python \ + --exclude meson spack external find --scope system grep spack external find --scope system sed spack external find --scope system perl @@ -104,7 +107,7 @@ jobs: set +e spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set +e + set -e echo "Packages in spack binary cache:" spack buildcache list fi @@ -113,20 +116,22 @@ jobs: # This allows us to "spin up" builds that altogether take longer than # six hours, and/or fail later in the build process. - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env - - # jedi-base-env - echo "jedi-base-env ..." - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env - - # jedi-ufs-env - echo "jedi-ufs-env ..." - spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env - spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + if [[ "${TEMPLATE}" == *"unified-dev"* || "${TEMPLATE}" == *"skylab-dev"* ]]; then + # base-env + echo "base-env ..." + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env + + # jedi-base-env + echo "jedi-base-env ..." + spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + fi # the rest echo "${TEMPLATE} ..." diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index 708efb581..00b39a81d 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -59,7 +59,8 @@ jobs: # Find external packages spack external find --scope system \ --exclude bison --exclude openssl \ - --exclude curl --exclude python + --exclude curl --exclude python \ + --exclude meson spack external find --scope system grep spack external find --scope system sed spack external find --scope system perl @@ -144,7 +145,7 @@ jobs: set +e spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set +e + set -e echo "Packages in spack binary cache:" spack buildcache list fi diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml index 417dd8029..17779f482 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml @@ -60,7 +60,7 @@ jobs: spack external find --scope system \ --exclude bison --exclude openssl \ --exclude curl --exclude python \ - --exclude gmake + --exclude meson --exclude gmake spack external find --scope system grep spack external find --scope system sed spack external find --scope system perl @@ -150,7 +150,7 @@ jobs: set +e spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set +e + set -e echo "Packages in spack binary cache:" spack buildcache list fi diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 7c11fbb2e..e2c79175d 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -60,7 +60,7 @@ jobs: spack external find --scope system \ --exclude bison --exclude openssl \ --exclude curl --exclude python \ - --exclude gmake + --exclude meson --exclude gmake spack external find --scope system grep spack external find --scope system sed spack external find --scope system perl @@ -150,7 +150,7 @@ jobs: set +e spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set +e + set -e echo "Packages in spack binary cache:" spack buildcache list fi diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index a5a0b29ea..366bee3d5 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -215,10 +215,9 @@ packages: # Avoid full rust dependency: version 43+ requires py-maturin py-cryptography: require: '@:42 +rust_bootstrap' - # Introduced in https://github.com/JCSDA/spack-stack/pull/894, pin py-cython - # to avoid duplicate packages being built (cylc dependencies soft-want @3:) + # Pin py-cython to avoid duplicate packages py-cython: - require: '@0.29.36' + require: '@3' # As long as we need the "delayed" variant for py-dask, # we are restricted to version 2021.03.0 (last one that has it) # and newer than 2021.01.0 because of these issues: @@ -233,13 +232,15 @@ packages: require: '~mpi' # https://github.com/JCSDA/spack-stack/issues/1276 py-matplotlib: - require: '@3.7.4' + require: '@3.7.4' # Variant ~mpi not working for latest py-netcdf4 # https://github.com/spack/spack/issues/47652 py-netcdf4: require: '@1.7.1 +mpi' - py-pandas: - require: '+excel' + # To avoid duplicate packages + py-numpy: + require: + - '@1.26' # To avoid duplicate packages py-ruamel-yaml: require: '@0.17.16' diff --git a/configs/common/packages_intel.yaml b/configs/common/packages_intel.yaml index 56c0dbbde..e853f7a12 100644 --- a/configs/common/packages_intel.yaml +++ b/configs/common/packages_intel.yaml @@ -30,6 +30,9 @@ packages: boost: require: - '%gcc' + qt: + require: + - '%gcc' ecflow: require: - '%gcc' diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index 6dec75f43..f148f1ecf 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -21,3 +21,6 @@ packages: require: '%gcc' libbsd: require: '%gcc' + qt: + require: + - '%gcc' diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index ec9b170c0..2ef73aab1 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -48,10 +48,16 @@ compilers:: - compiler: spec: gcc@10.3.0 paths: + # Still need Cray wrappers for most environments cc: cc cxx: CC f77: ftn fc: ftn + # For cylc-dev, can't use Cray wrappers + #cc: /opt/cray/pe/gcc/10.3.0/snos/bin/gcc + #cxx: /opt/cray/pe/gcc/10.3.0/snos/bin/g++ + #f77: /opt/cray/pe/gcc/10.3.0/snos/bin/gfortran + #fc: /opt/cray/pe/gcc/10.3.0/snos/bin/gfortran flags: {} operating_system: sles15 modules: diff --git a/configs/templates/cylc-dev/spack.yaml b/configs/templates/cylc-dev/spack.yaml new file mode 100644 index 000000000..c77313021 --- /dev/null +++ b/configs/templates/cylc-dev/spack.yaml @@ -0,0 +1,33 @@ +# Template for CYLC (flow, rose, GUI). Supported with GCC only. +# Creates a view that allows users to define (manually for now) +# simple wrappers for cylc that can be aliased as "cylc": +# +##!/bin/bash +##CYLC_INSTALL_DIR=/path/to/env/view/bin +##PATH=${CYLC_INSTALL_DIR}:$PATH +##unset PYTHONPATH +##cylc "$@" +# +spack: + concretizer: + unify: true + + view: + cylc: + root: view + select: [^python] + link: run + link_type: symlink + include: [] + + definitions: + - compilers: ['%gcc'] + - packages: + - py-cylc-flow@8.3.6 + - py-cylc-rose@1.4.2 + - py-cylc-uiserver@1.5.1 + + specs: + - matrix: + - [$packages] + - [$compilers] diff --git a/spack b/spack index 57960d2c0..dd99fc1e1 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 57960d2c0ed456571ced05c9b12f111df445bc32 +Subproject commit dd99fc1e177cbae412e1753f3a81499df90cee27 diff --git a/spack-ext/repos/spack-stack/packages/jedi-tools-env/package.py b/spack-ext/repos/spack-stack/packages/jedi-tools-env/package.py index 5e0ac1578..a373bbe3c 100644 --- a/spack-ext/repos/spack-stack/packages/jedi-tools-env/package.py +++ b/spack-ext/repos/spack-stack/packages/jedi-tools-env/package.py @@ -25,7 +25,7 @@ class JediToolsEnv(BundlePackage): # depends_on("aws-parallelcluster", type="run") depends_on("py-click", type="run") depends_on("py-openpyxl", type="run") - depends_on("py-pandas +excel", type="run") + depends_on("py-pandas", type="run") depends_on("py-pygithub", type="run") depends_on("py-scipy", type="run") depends_on("py-sphinx", type="run")