diff --git a/.gitignore b/.gitignore index 9c064ff8b..38038e6e3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __pycache__ cache .vscode doc/build +deploy_logs/ diff --git a/configs/sites/tier1/acorn/compilers.yaml b/configs/sites/tier1/acorn/compilers.yaml deleted file mode 100644 index da0e33a1b..000000000 --- a/configs/sites/tier1/acorn/compilers.yaml +++ /dev/null @@ -1,133 +0,0 @@ -compilers: -- compiler: - spec: intel@19.1.3.304 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - modules: - - PrgEnv-intel/8.5.0 - - craype/2.7.17 - - intel/19.1.3.304 - - libfabric - environment: - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] -- compiler: - spec: intel@2022.2.0.262 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - modules: - - PrgEnv-intel/8.5.0 - - craype/2.7.17 - - intel-classic/2022.2.0.262 - - libfabric - environment: - prepend_path: - PATH: /opt/cray/pe/gcc/10.2.0/bin - LD_LIBRARY_PATH: /opt/cray/pe/gcc/10.2.0/snos/lib64 - CPATH: /opt/cray/pe/gcc/10.2.0/snos/include - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] -- compiler: - spec: gcc@11.2.0 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - modules: - - PrgEnv-gnu/8.3.3 - - craype/2.7.13 - - gcc/11.2.0 - - libfabric - environment: - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] -# Needed for Intel 19/2022 backend -- compiler: - spec: gcc@10.2.0 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - modules: - - PrgEnv-gnu/8.3.3 - - craype/2.7.13 - - gcc/10.2.0 - - libfabric - environment: - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] -- compiler: - spec: oneapi@2024.2.1 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - modules: - - PrgEnv-intel/8.5.0 - - craype/2.7.17 - - intel-oneapi/2024.2 - - libfabric - environment: - prepend_path: - PATH: /opt/cray/pe/gcc/12.1.0/bin - LD_LIBRARY_PATH: /opt/cray/pe/gcc/12.1.0/snos/lib64 - LIBRARY_PATH: /opt/cray/pe/gcc/12.1.0/snos/lib64 - CPATH: /opt/cray/pe/gcc/12.1.0/snos/include - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] -- compiler: - spec: gcc@12.1.0 - paths: - cc: /opt/cray/pe/gcc/12.1.0/bin/gcc - cxx: /opt/cray/pe/gcc/12.1.0/bin/g++ - f77: /opt/cray/pe/gcc/12.1.0/bin/gfortran - fc: /opt/cray/pe/gcc/12.1.0/bin/gfortran - flags: {} - operating_system: sles15 - modules: [] - environment: - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] diff --git a/configs/sites/tier1/acorn/packages.yaml b/configs/sites/tier1/acorn/packages.yaml index 42e0165dd..0161b2569 100644 --- a/configs/sites/tier1/acorn/packages.yaml +++ b/configs/sites/tier1/acorn/packages.yaml @@ -43,3 +43,5 @@ externals: - spec: grep@3.1 prefix: /usr + libxslt: + require: '~crypto' diff --git a/configs/sites/tier1/acorn/packages_gcc.yaml b/configs/sites/tier1/acorn/packages_gcc.yaml deleted file mode 100644 index b801a9758..000000000 --- a/configs/sites/tier1/acorn/packages_gcc.yaml +++ /dev/null @@ -1,14 +0,0 @@ - packages: - all: - compiler:: [gcc@11.2.0] - providers: - mpi:: [cray-mpich@8.1.9] - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.9~wrappers - modules: - - libfabric - - craype-network-ofi - - cray-mpich/8.1.9 diff --git a/configs/sites/tier1/acorn/packages_intel.yaml b/configs/sites/tier1/acorn/packages_intel.yaml deleted file mode 100644 index 28b327d1a..000000000 --- a/configs/sites/tier1/acorn/packages_intel.yaml +++ /dev/null @@ -1,60 +0,0 @@ - packages: - all: - compiler:: [intel@19.1.3.304,intel@2022.0.2.262,gcc@10.2.0] - providers: - mpi:: [cray-mpich@8.1.9] - # Remove the next three lines to switch to intel-oneapi-mkl - blas:: [openblas] - fftw-api:: [fftw] - lapack:: [openblas] - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.9~wrappers - modules: - - libfabric - - craype-network-ofi - - cray-mpich/8.1.9 - - cray-pals - intel-oneapi-mkl: - # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl - buildable: False - #externals: - #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0 - # prefix: /apps/oneapi - # DH* Remove this section to switch to intel-oneapi-mkl - ectrans: - require:: - - '@1.2.0 ~mkl +fftw' - gsibec: - require:: - - '@1.2.1 ~mkl' - py-numpy: - require:: - - '^[virtuals=lapack,blas] openblas' - - '@1.26 %gcc' - # *DH - eckit: - require: - - any_of: ["@1.23.1"] - when: "%intel@19.1.3.304" - eccodes: - require: - - any_of: ["@2.25.0"] - when: "%intel@19.1.3.304" - message: "2.25.0 is the last version to use C++11 (as opposed to C++17)" - gettext: - require: ['%gcc', '@0.21.1'] - py-scipy: - require: - - any_of: ["@1.10.1"] - when: "%intel@19.1.3.304" - cdo: - require: - - any_of: ["@2.3.0"] - when: "%intel@2022.0.2.262" - message: "2.3.0 is the last version to use C++17" - - any_of: ["@2.0.5"] - when: "%intel@19.1.3.304" - message: "2.0.5 is the last version to use C++14" diff --git a/configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml new file mode 100644 index 000000000..2553756bc --- /dev/null +++ b/configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml @@ -0,0 +1,75 @@ +packages: + ### Compilers / compiler runtimes + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2024.2.1 ~envmods + modules: + - PrgEnv-intel/8.5.0 + - craype/2.7.17 + - intel-oneapi/2024.2 + - libfabric + extra_attributes: + compilers: + c: cc + cxx: CC + fortran: ftn + flags: + cflags: '--gcc-toolchain=/opt/cray/pe/gcc/12.1.0/snos' + cxxflags: '--gcc-toolchain=/opt/cray/pe/gcc/12.1.0/snos' + fflags: '-gcc-name=/opt/cray/pe/gcc/12.1.0/snos/bin/gcc -gxx-name=/opt/cray/pe/gcc/12.1.0/snos/bin/g++' + environment: + prepend_path: + PATH: /opt/cray/pe/gcc/12.1.0/snos/bin + LD_LIBRARY_PATH: /opt/cray/pe/gcc/12.1.0/snos/lib64 + LIBRARY_PATH: /opt/cray/pe/gcc/12.1.0/snos/lib64 + CPATH: /opt/cray/pe/gcc/12.1.0/snos/include + set: + CONFIG_SITE: '' + INTEL_COMPILER_TYPE: RECOMMENDED + gcc: + buildable: false + externals: + - spec: gcc@12.1.0 languages:='c,c++,fortran' + prefix: /opt/cray/pe/gcc/12.1.0/snos/ + extra_attributes: + compilers: + c: /opt/cray/pe/gcc/12.1.0/snos/bin/gcc + cxx: /opt/cray/pe/gcc/12.1.0/snos/bin/g++ + fortran: /opt/cray/pe/gcc/12.1.0/snos/bin/gfortran + environment: + set: + # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so + # Automake-based builds are installed in lib64 + # which confuses some packages. + CONFIG_SITE: '' + gcc-runtime: + require: '%gcc@12.1.0' + ### General package/provider settings & MPI external + mpi: + buildable: false + require: ['cray-mpich@8.1.29'] + cray-mpich: + externals: + - spec: cray-mpich@8.1.29~wrappers + prefix: /opt/cray/pe/mpich/8.1.29/ofi/intel/2022.1 + modules: + - libfabric + - craype-network-ofi + - cray-mpich/8.1.29 + - cray-pals + ### Disable MKL + intel-oneapi-mkl: + buildable: false + blas: + require: ['openblas'] + lapack: + require: ['openblas'] + fftw-api: + require: ['fftw'] + ectrans: + require:: ['~mkl', '+fftw'] + gsibec: + require: ['~mkl'] + py-numpy: + require:: ['^[virtuals=lapack,blas] openblas'] diff --git a/configs/sites/tier1/acorn/packages_oneapi.yaml b/configs/sites/tier1/acorn/packages_oneapi.yaml deleted file mode 100644 index 01e4538e1..000000000 --- a/configs/sites/tier1/acorn/packages_oneapi.yaml +++ /dev/null @@ -1,37 +0,0 @@ - packages: - all: - compiler:: [oneapi@2024.2] - providers: - mpi:: [cray-mpich@8.1.29] - # Remove the next three lines to switch to intel-oneapi-mkl - blas:: [openblas] - fftw-api:: [fftw] - lapack:: [openblas] - gcc-runtime: - require: '%gcc@12.1.0' - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.29~wrappers - modules: - - libfabric - - craype-network-ofi - - cray-mpich/8.1.29 - - cray-pals - intel-oneapi-mkl: - # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl - buildable: False - #externals: - #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0 - # prefix: /apps/oneapi - # DH* Remove this section to switch to intel-oneapi-mkl - ectrans: - require:: - - '@1.2.0 ~mkl +fftw' - gsibec: - require:: - - '@1.2.1 ~mkl' - py-numpy: - require:: - - '^[virtuals=lapack,blas] openblas' diff --git a/configs/sites/tier1/acorn/setup.sh b/configs/sites/tier1/acorn/setup.sh index 86ba92b44..966263370 100644 --- a/configs/sites/tier1/acorn/setup.sh +++ b/configs/sites/tier1/acorn/setup.sh @@ -1,5 +1,4 @@ -## Load modules to avoid various issues with old Python versions -module load gcc/10.3.0 python/3.11.7 +module load cray-python/3.11.7 ## Go, Rust repo setup for fetcher scripts export GOMODCACHE=${SPACK_STACK_DIR}/cache/go diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/cmd/stack_cmds/create.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/cmd/stack_cmds/create.py index 31eae3615..e91ae2679 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/cmd/stack_cmds/create.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/cmd/stack_cmds/create.py @@ -3,7 +3,7 @@ import shutil from sys import platform -import llnl.util.tty as tty +import spack.llnl.util.tty as tty from spack.extensions.stack.container_env import StackContainer from spack.extensions.stack.stack_env import StackEnv, stack_path diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index e3dc258d6..45917d361 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -349,13 +349,14 @@ def write(self): "version of spack-stack, or really, really know what you are doing.") ) new_envrepo = os.path.join(self.env_dir(), "envrepo") + config_envrepo = False for upstream_path in all_upstreams[::-1]: envrepo_path = os.path.realpath(os.path.join(upstream_path, "../envrepo")) if os.path.isdir(envrepo_path): shutil.copytree(envrepo_path, new_envrepo, dirs_exist_ok=True) - if os.path.isdir(new_envrepo): - repo_cfg = "repos:[$env/envrepo]" - spack.config.add(repo_cfg, scope=env_scope) + config_envrepo = True + if config_envrepo: + spack.config.add("repos:[$env/envrepo]", scope=env_scope) if self.modifypkg: logging.info("Creating custom repo with packages %s" % ", ".join(self.modifypkg)) @@ -388,8 +389,16 @@ def write(self): if not pkg_found: logging.warning(f"WARNING: package '{pkg_name}' could not be found") logging.info("Adding custom repo 'envrepo' to env config") - repo_cfg = "repos:[$env/envrepo]" - spack.config.add(repo_cfg, scope=env_scope) + spack.config.add("repos:[$env/envrepo]", scope=env_scope) + + # Copy any envrepo directory associated with the template (i.e., add-on env) + if self.template: + template_envrepo_path = os.path.join(self.template_path, "envrepo") + if os.path.isdir(template_envrepo_path): + logging.info(f"Copying envrepo/ from {self.template_path}") + target_envrepo_path = os.path.join(env_dir, "envrepo") + shutil.copytree(template_envrepo_path, target_envrepo_path, dirs_exist_ok=True) + spack.config.add("repos:[$env/envrepo]", scope=env_scope) # Merge the original spack.yaml template back in # so it has the highest precedence diff --git a/util/acorn/deploy/deploy.py b/util/acorn/deploy/deploy.py new file mode 100755 index 000000000..96c72d750 --- /dev/null +++ b/util/acorn/deploy/deploy.py @@ -0,0 +1,230 @@ +#!/usr/bin/env spack-python + +import argparse + +parser = argparse.ArgumentParser(description="Your script description") + +parser.add_argument('-n', '--no-scheduler', action='store_true', help="Run installation on local node (no job scheduler)") +parser.add_argument('-s', '--skip-go-rust-handling', action='store_true', help="Skip handling of Go/Rust dep fetching when using parallel job scheduler") +parser.add_argument('-c', '--concretize-args', type=str, help="Concretize arguments (provide a single string)") +parser.add_argument('-i', '--install-args', type=str, help="Install arguments (provide a single string)") +parser.add_argument('-r', '--redeploy-existing', action='store_true', help="Redeploy existing deployments") + +parser.add_argument('deployments', nargs='*', help="List of deployments to apply (default is all)") + +args = parser.parse_args() + +import collections +from datetime import datetime +import logging +import os +import subprocess +import yaml +from contextlib import redirect_stdout, redirect_stderr +from types import SimpleNamespace + +logging.basicConfig(level=logging.CRITICAL) + +import spack.extensions +spack.extensions.load_extension("stack") +from spack.extensions.stack.cmd.stack_cmds.create import * +from spack.extensions.stack.meta_modules import setup_meta_modules +import spack.environment +from spack.cmd.concretize import concretize +from spack.cmd.fetch import fetch +from spack.cmd.module import module +from spack import modules +from spack.util.executable import which + +spack_stack_dir = os.getenv("SPACK_STACK_DIR") + +nowdate = datetime.now().strftime("%Y%m%d-%H%M") +logdir = os.path.join(spack_stack_dir, "deploy_logs") +os.makedirs(logdir, exist_ok=True) + +def get_env_dir_basename(deployment): + base = deployment["template"] + base = base.replace("unified-dev", "ue") + base = base.replace("-dev", "") + return "-".join([base, deployment["compiler"]]) + +def deployment_already_exists(env_dir_basename, args): + path_to_check = os.path.join(spack_stack_dir, "envs", env_dir_basename) + return os.path.isdir(path_to_check) + +def is_deployment_requested(env_dir_basename, deployment, args): + if deployment["template"] in args.deployments: + return True + if deployment["template"] + "/" + deployment["compiler"] in args.deployments: + return True + if args.redeploy_existing: + return True + return not deployment_already_exists(env_dir_basename, args) + +def get_create_env_settings(env_dir_basename, deployment, deployments): + config_dict = {} + config_dict["site"] = "acorn" + config_dict["template"] = deployment["template"] + config_dict["dir"] = os.path.join(spack_stack_dir, "envs") + config_dict["name"] = env_dir_basename + if "upstreams" in deployment: + upstream_full_paths = [] + for upstream_template in deployment["upstreams"]: + for _candidate_upstream in deployments.values(): + if (_candidate_upstream["template"] == upstream_template) and (_candidate_upstream["compiler"] == deployment["compiler"]): + upstream_basename = get_env_dir_basename(_candidate_upstream) + upstream_full_path = os.path.join(spack_stack_dir, "envs", upstream_basename, "install") + upstream_full_paths.append([upstream_full_path]) + break + config_dict["upstreams"] = upstream_full_paths + config_dict["compiler"] = deployment["compiler"] + + return config_dict + +def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, logfile, logfilepath, suffix=".batch_install"): + if "walltime" in deployment: + walltime = deployment["walltime"] + elif "default_walltime" in batch_config: + walltime = batch_config["default_walltime"] + else: + assert False, "Set deployment-specific walltime or batch_config:default_walltime" + cmd = [ + "qsub", + "-o", logfilepath + suffix, + "-e", logfilepath + suffix, + "-A", batch_config["account"], + "-q", batch_config["queue"], + "-l", "walltime=" + walltime + ",select=1:ncpus=12", + "-V", "-Wblock=true", "--", + which("spack").path, "--env", env_dir_full_path, + "install", "--concurrent-packages", "3", "--jobs", "4", + ] + cmd.extend(specs_str) + subprocess.run(cmd, stdout=logfile, stderr=logfile, check=True) + +# Load deployments.yaml configuration +print("Loading deployments.yaml") +script_directory = os.path.dirname(os.path.abspath(__file__)) +deployments_yaml_path = os.path.join(script_directory, "deployments.yaml") +with open(deployments_yaml_path, "r") as f: + deployments_yaml = yaml.safe_load(f) + +# Generate deployments object, including iterating over compilers +deployments = collections.OrderedDict() + +for _deployment in deployments_yaml["deployments"]: + for _compiler in _deployment["compilers"]: + deployment = _deployment.copy() + del(deployment["compilers"]) + deployment["compiler"] = _compiler + env_dir_basename = get_env_dir_basename(deployment) + deployments[env_dir_basename] = deployment + print(f"Registered deployment: {deployment['template']}/{deployment['compiler']} ({env_dir_basename})") + +print("="*30) + +# Create and install each deployment +for env_dir_basename, deployment in deployments.items(): + if not is_deployment_requested(env_dir_basename, deployment, args): + print(f"Skipping existing deployment: {deployment['template']}/{deployment['compiler']} ({env_dir_basename})") + continue + print("="*30) + # Create env based on config + stack_settings = get_create_env_settings(env_dir_basename, deployment, deployments) + env_dir_full_path = os.path.join(spack_stack_dir, "envs", env_dir_basename) + if os.path.isdir(env_dir_full_path) and args.redeploy_existing: + backup_dir_full_path = os.path.join(spack_stack_dir, "envs", env_dir_basename + "_bkp") + assert not os.path.isdir(backup_dir_full_path), f"Backup dir {backup_dir_full_path} already exists" + print(f"Moving {env_dir_full_path}\n to {backup_dir_full_path}") + os.rename(env_dir_full_path, backup_dir_full_path) + logfilepath = os.path.join(logdir, nowdate + f".{deployment['template']}.{deployment['compiler']}.log") + print(f"Log file: {logfilepath}") + logfile = open(logfilepath, "a") + logfile.write(str(deployment) + "\n") + logfile.write(str(stack_settings) + "\n") + print(f"Creating environment for {deployment['template']}/{deployment['compiler']}") + print(f" at {env_dir_full_path} ...") + # Use equivalent of 'spack stack create env' + stack_env = StackEnv(**stack_settings) + stack_env.write() + stack_env.check_umask() + env = spack.environment.Environment(env_dir_full_path) + spack.environment.activate(env) + + # Concretize environment + concretize_args = SimpleNamespace( + test = False, ### + quiet = False, + fresh = True, ### + force = True, + ) + print(f"... concretizing ...") + with redirect_stdout(logfile), redirect_stderr(logfile): + concretize(None, concretize_args) + + # Fetch packages + if "packages_to_install" in deployment: + specs_str = deployment["packages_to_install"] + else: + specs_str = [] + fetch_args = SimpleNamespace( + missing = True, + no_checksum = False, + dependencies = bool(specs_str), + specs = specs_str, + ) + print(f"... fetching packages ...") + with redirect_stdout(logfile), redirect_stderr(logfile): + fetch(None, fetch_args) + + # Install packages + print("... installing" + (" specs: " +" ".join(specs_str) if specs_str else "") + " ...") + logfile.flush() + if args.no_scheduler: + specs = env.all_matching_specs(*specs_str) + env.install_specs(specs) + else: + if not args.skip_go_rust_handling: + run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, ["rust", "go"], logfile, logfilepath, suffix=".rustgo") + logfile.flush() + shell_env = os.environ.copy() + shell_env["SPACK_ENV"] = env_dir_full_path + subprocess.run( + os.path.join(spack_stack_dir, "util", "fetch_cargo_deps.py"), + env=shell_env, + stdout=logfile, + stderr=logfile, + check=True, + text=True, + ) + logfile.flush() + subprocess.run( + os.path.join(spack_stack_dir, "util", "fetch_go_deps.py"), + env=shell_env, + stdout=logfile, + stderr=logfile, + check=True, + text=True, + ) + logfile.flush() + run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, specs_str, logfile, logfilepath) + logfile.flush() + + # Generate modules + print(f"... writing package modules ...") + subprocess.run( + ["spack", "-e", env_dir_full_path, "module", "lmod", "refresh", "--yes-to-all", "--upstream-modules"], + stdout=logfile, + stderr=logfile, + check=True, + text=True, + ) + logfile.flush() + + # Meta modules + print(f"... writing metamodules ...") + setup_meta_modules() + + logfile.close() + + print(f"... done.") diff --git a/util/acorn/deploy/deployments.yaml b/util/acorn/deploy/deployments.yaml new file mode 100644 index 000000000..e875d3ef1 --- /dev/null +++ b/util/acorn/deploy/deployments.yaml @@ -0,0 +1,14 @@ +deployments: + - template: "unified-dev" + compilers: ["oneapi-2024.2.1"] + packages_to_install: + - ufs-weather-model-env + - global-workflow-env + - gsi-env + - madis + walltime: 03:00:00 + +batch_config: + account: NCEPLIBS-DEV + queue: dev + default_walltime: 02:00:00 diff --git a/util/fetch_cargo_deps.py b/util/fetch_cargo_deps.py index 250a94804..9e1091a94 100755 --- a/util/fetch_cargo_deps.py +++ b/util/fetch_cargo_deps.py @@ -21,12 +21,13 @@ this_dir = os.path.dirname(os.path.abspath(sys.argv[0])) from spack.environment import active_environment -from spack.build_systems.cargo import CargoPackage -from spack.build_systems.python import PythonPackage +from spack_repo.builtin.build_systems.cargo import CargoPackage +from spack_repo.builtin.build_systems.python import PythonPackage from spack.package_base import PackageBase from spack.util.executable import Executable, which from llnl.util.filesystem import working_dir from spack.store import find +from spack.util.environment import EnvironmentModifications from spack.error import SpackError # Load the current environment @@ -75,11 +76,17 @@ pkg = spec.package pkg.do_stage() - if which("cargo"): + cargo_path = os.path.join(spec["rust"].prefix.bin, "cargo") + if os.path.exists(cargo_path): + cargo_bin = spec["rust"].prefix.bin + cargo_exe = Executable(cargo_path) + elif which("cargo"): + cargo_bin = "" cargo_exe = Executable("cargo") else: # cargo/rustup - cargo_path = os.path.join(cargo_home, "bin", "cargo") + cargo_bin = os.path.join(cargo_home, "bin") + cargo_path = os.path.join(cargo_bin, "cargo") print(f"Checking for {cargo_path} ...") if which(cargo_path): print(f" ... found {cargo_path}") @@ -109,7 +116,10 @@ os.chmod(cargo_wrapper, 0o744) cargo_exe = Executable(cargo_wrapper) + env = EnvironmentModifications() + if cargo_bin: + env.prepend_path("PATH", cargo_bin) for root, dirs, files in os.walk(pkg.stage.source_path): if "Cargo.toml" in files: with working_dir(root): - cargo_exe("fetch") \ No newline at end of file + cargo_exe("fetch", env=env)