diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index fc2412f38..bee0b4709 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -1,16 +1,12 @@ -''' -FRE Checkout Script Generator - -Retrieves information from the resolved YAML configuration to generate a -checkout.sh script that git clones the model source code. - -The checkout script will clone component repositories defined in the -compile YAML to build the model. - -Note, a bare-metal build defaults to a parallel checkout. -A container build defaults to a non-parallel checkout. - -''' +""" +Create_checkout_script provides methods to generate a checkout.sh script from a YAML configuration +file, where checkout.sh git clone's all component source repositories listed under the +src key of the compile YAML. + +The method checkout_create is the entry point called by fre make checkout-script and +fre make all. Checkout_create then calls baremetal_checkout_write for a bare-metal platform or +container_checkout_write for a container platform to write checkout.sh. +""" import shutil from pathlib import Path from datetime import datetime @@ -26,23 +22,31 @@ def baremetal_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, jobs: str, parallel_cmd: str, execute: bool): """ - This function baremetal_checkout_write is called by checkout_create in order to - - - Extract compilation specifications from the parsed YAML configuration - - Generate a checkout script to the source directory. The source directory is - defined within the 'modelRoot' variable in the "platforms" section of the combined YAML - - - :param model_yaml: "freyaml" class object containing a parsed and validated yaml dictionary - containing the "compile" specification + Baremetal_checkout_write generates the checkout.sh script and optionally executes the script to + git clone the component repositories in preparation for model compilation. + + Called by checkout_create for each bare-metal platform, this method + - reads the compile section of the resolved YAML to determine source repositories to clone, + - writes checkout.sh into src_dir, + - optionally executes checkout.sh afterwards + + :param model_yaml: is the parsed and validated YAML object containing the compile + specifications (source repositories, experiment name, etc.). :type model_yaml: yamlfre.freyaml - :param src_dir: Absolute directory path to git clone the source code + :param src_dir: is the absolute path of the directory where checkout.sh will be + written and where the source repositories will be cloned. Typically, + src_dir = [modelRoot]/[experiment]/src where modelRoot is defined in + platforms.yaml. :type src_dir: str - :param jobs: Number of git submodules to clone simultaneously (TO CLARIFY) + :param jobs: is the number of git submodules to fetch simultaneously, passed to git clone + --jobs (relevant only if the component repository contain submodules.) :type jobs: str - :param parallel_cmd: Set to " &" for parallel checkouts and "" for non-parallel checkouts + :param parallel_cmd: is the shell suffix appended to each git clone command to control + concurrency. Pass " &" to background each clone (parallel + checkout) or "" to clone sequentially. :type parallel_cmd: str - :param execute: If True, run the generated checkout.sh + :param execute: is a flag where if True, checkout.sh is executed immediately after creation. + Defaults to False. :type execute: bool """ fre_checkout = checkout.checkout("checkout.sh", src_dir) @@ -60,23 +64,28 @@ def baremetal_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, jobs: st def container_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, tmp_dir: str, jobs: str, parallel_cmd: str): """ - This function container_checkout_write is called by checkout_create in order to - - - Extract compilation specifications from the parsed YAML configuration - - Generate a checkout script in a local ./tmp directory, where it will later be - copied to the directory of the container image filesystem for execution - - :param model_yaml: "freyaml" class object containing a parsed and validated yaml dictionary - containing the "compile" specification + Container_checkout_write generates checkout.sh for a container build. + + Called by checkout_create for each container platform, this method + writes checkout.sh into a temporary directory on the host (tmp/[platform-name]/) + where the script eventually will be COPY-ed to the container image filesystem. + The script will be executed during the container build to git clone the component + repositories serially. + + :param model_yaml: is the parsed and validated YAML object containing the compile + specifications (source repositories, experiment name, etc.). :type model_yaml: yamlfre.freyaml - :param src_dir: Internal path for source code in the running container. The source directory is - defined within the 'modelRoot' variable in the "platforms" section of the combined YAML + :param src_dir: is the source-code path inside the running container where repositories will + be cloned. Defined by modelRoot in platforms.yaml. :type src_dir: str - :param tmp_dir: Temporary directory (outside of container) that hosts the created checkout script + :param tmp_dir: is the local temporary directory on the host (outside the container) where + checkout.sh is staged before being COPYed into the image. + Typically tmp/[platform-name]. :type tmp_dir: str - :param jobs: Number of git submodules to clone simultaneously (TO CLARIFY) + :param jobs: is the number of git submodules to fetch simultaneously, passed to git clone + --jobs. :type jobs: str - :param parallel_cmd: Since container builds are not parallelized, set to "" + :param parallel_cmd: is a flag not used in this method and should be removed. :type parallel_cmd: str """ fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) @@ -88,31 +97,43 @@ def checkout_create(yamlfile: str, platform: tuple, target: tuple, no_parallel_checkout: Optional[bool] = None, njobs: int = 4, execute: Optional[bool] = False, force_checkout: Optional[bool] = False): """ - Calls baremetal_checkout_write or container_checkout_write to create checkout.sh - for baremetal or container builds, respectively. + Checkout_create is the entry point for fre make checkout-script. The method resolves + the YAML configuration and calls baremetal_checkout_write or container_checkout_write + for each specified platform. - :param yamlfile: Model YAML file path + :param yamlfile: is the path to the model YAML configuration file (e.g. am5.yaml). + The experiment name is derived by stripping the .yaml extension. :type yamlfile: str - :param platform: FRE platform(s) that are defined in the platforms.yaml - :type platform: tuple - :param target: Predefined FRE target(s) - :type target: tuple - :param no_parallel_checkout: Option to disable parallel checkouts - :type no_parallel_checkout: bool - :param njobs: Used in the recursive clone; number of submodules to fetch simultaneously (default 4) (TO CLARIFY) + :param platform: is one or more FRE platform strings as defined in platforms.yaml. + :type platform: tuple[str] + :param target: is one or more mkmf target strings (e.g. debug, repro, prod). + :type target: tuple[str] + :param no_parallel_checkout: is a flag where if True, git clone component repositories sequentially. + Defaults to False to enable parallel checkout for bare-metal builds; + Is not used for container builds + :type no_parallel_checkout: bool, optional + :param njobs: is the number of git submodules to fetch simultaneously, passed to + git clone --jobs. Defaults to 4. :type njobs: int - :param execute: If True, run checkout.sh - :type execute: bool - :param force_checkout: If True, for bare-metal build: add timestamp to source directory and create a new checkout script - If True, for container build: overwrite locally existing checkout script before COPY-ing to the - container image filesystem - :type force_checkout: bool + :param execute: If True, execute checkout.sh immediately after writing it + (bare-metal only). Defaults to False. + :type execute: bool, optional + :param force_checkout: is a flag to controls behavior when checkout.sh already exists. + For bare-metal build, renames the existing src directory with a + YYYYmmdd.HHMMSS timestamp suffix, then writes a fresh + checkout.sh in a new src directory. + For container build, deletes the existing tmp/[platform]/checkout.sh + and writes a new one in its place. + Defaults to False. + :type force_checkout: bool, optional :raises ValueError: - - If 'njobs' is not an integer - - If 'platform' does not exist in the platforms.yaml configuration - :raises OSError: If executing checkout.sh returns an error - + - If njobs is passed as a boolean while --execute is also set (ambiguous + intent — njobs must be an explicit integer). + - If a specified platform name does not exist in platforms.yaml. + :raises OSError: If --execute is set and running an existing checkout.sh + returns a non-zero exit code (e.g. the source directory already + contains conflicting content). """ # Standardize inputs jobs_str = str(njobs) diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index b40cf228f..37a6edb2e 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -1,20 +1,33 @@ -''' -Retrieves information from the resolved YAML configuration to generate the compile.sh -in the ``[modelRoot]/[experiment name]/[platform-target]/exec`` directory, where +""" +Create_compile_script retrieves information from the resolved YAML configuration to generate compile.sh +for bare-metal builds. The method compile_create is the entry point called by fre make all and +fre make compile-script. -- ``modelRoot`` is defined in the `platforms.yaml` -- ``experiment name`` is defined in `compile.yaml` -- ``platform`` and ``target`` are passed via Click options +The generated script is written to:: -The compile.sh script + [modelRoot]/[experiment]/[platform]-[target]/exec/compile.sh -1. Sets the ``src_dir`` -2. Sets the ``bld_dir`` -3. Sets the ``mkmf_template`` -4. Loads/unloads modules to set-up the compile environment -5. Calls ``mkmf`` to generate Makefiles for each model component defined in the `compile.yaml` -6. Calls ``make`` to generate the model executable -''' +where + +- modelRoot is defined in platforms.yaml +- experiment is the basename of the model YAML file (e.g. am5 from am5.yaml) +- platform and target are passed via the -p / -t CLI options + to fre make compile-script and fre make all + +When executed, compile.sh + + 1. Sets src_dir (where source code was checked out by checkout.sh) + 2. Sets bld_dir (the exec/ directory where the executable is placed) + 3. Sets the path to the mkmf template (mkTemplate from platforms.yaml) + 4. Loads the correct environment modules to set the compile environment + (see envSetup from platforms.yaml) + 5. Calls mkmf for each model component listed under src in compile.yaml + to generate per-component Makefiles + 6. Calls make (with -j [makejobs]) to build the model executable + +Container platforms are silently skipped — compilation inside a container image +is handled by the Dockerfile generated by fre make dockerfile. +""" import logging from multiprocessing.dummy import Pool @@ -38,26 +51,41 @@ def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejob nparallel: int = 1, execute: Optional[bool] = False, verbose: Optional[bool] = None): """ - This function compile_create generates the compile script for bare-metal build. + Generates the compile.sh script for each bare-metal platform and target combination and + optionally executes compile.sh to compile a model executable. + + For each bare-metal platform in the platform yaml, a compile.sh is written to + [modelRoot]/[experiment]/[platform]-[target]/exec/. Container platforms are + silently skipped here; their compilation is handled by the Dockerfile produced by + ``dockerfile_create``. - :param yamlfile: Model compile YAML file + :param yamlfile: is the path to the model YAML file (e.g. am5.yaml). The experiment + name is derived by stripping the `.yaml` extension. :type yamlfile: str - :param platform: FRE platform; defined in the platforms yaml - :type platform: tuple of strings - :param target: Predefined FRE targets - :type target: tuple of strings - :param makejobs: Number of recipes from the Makefile to run in parallel (default 4); - corresponds to -j option in make + :param platform: is one or more FRE platform strings as defined in the platform yaml + (e.g. ncrc5.intel23). Container platforms in this tuple are + silently ignored. + :type platform: tuple[str] + :param target: is one or more mkmf target strings (e.g. prod, debug, + repro, prod-openmp). One compile.sh is generated per + platform/target pair. + :type target: tuple[str] + :param makejobs: is the number of Makefile recipes to run simultaneously, passed to + make -j. Defaults to 4. :type makejobs: int - :param nparallel: Number of compile.sh scripts to run in parallel (default 1) + :param nparallel: is the number of compile.sh scripts to run concurrently when + execute=True. Defaults to 1 (sequential execution). :type nparallel: int - :param execute: If True, execute the created compile.sh script to build a model executable + :param execute: is a flag where if True, run every generated compile.sh after creation. + Defaults to False. :type execute: bool - :param verbose: If True, increase verbosity output + :param verbose: is a flag where if True, set logger level to "DEBUG" for detailed output. + Defaults to False with logger level set to "INFO". :type verbose: bool + :raises ValueError: - - Error if platform does not exist in platforms yaml configuration - - Error if the mkmf template defined in platforms yaml does not exist + - If a specified platform does not exist in the platforms yaml + - If the mkTemplate path defined in the platforms yaml does not exist """ # Define variables diff --git a/fre/make/create_docker_script.py b/fre/make/create_docker_script.py index 95be4d9ee..eb716f509 100644 --- a/fre/make/create_docker_script.py +++ b/fre/make/create_docker_script.py @@ -1,13 +1,29 @@ -''' -Generates a Dockerfile and an accompanying createContainer.sh script that -builds a Docker image containing the compiled model executable and the library -dependencies from the generated Dockerfile. Unless specified, -createContainer.sh will convert the Docker OCI image to a Singularity image -file format (.sif) that can be launched with Singularity/Apptainer. -Note, once the container image is built, the source code and the compiled -executable cannot be modified. -''' +""" +Create_docker_script provides methods to generate a Dockerfile and +an accompanying createContainer.sh script to build container images. +The method dockerfile_create is the entry point called by +fre make dockerfile and by fre make all. + +The Dockerfile uses a two-stage build: + +1. Build stage — starts from the base container image (image field in + platforms yaml), copies in the checkout.sh and Makefile that were + staged under tmp/[platform]/ by fre make checkout-script and + fre make makefile, runs mkmf and make to compile the model + executable. +2. Runtime stage — copies the compiled executable and its runtime dependencies + into a leaner second base image (containerBase2 in platforms.yaml). + +createContainer.sh builds the container image and, unless --no-format-transfer +is specified, converts it to a Singularity Image File (.sif) that can be +launched with Singularity/Apptainer on HPC systems. + +.. note:: + Once a container image is built, the source code and compiled executable + inside it cannot be modified. To incorporate source changes, re-run + fre make all (or the individual sub-commands) and rebuild the image. +""" import logging import os @@ -29,24 +45,35 @@ def dockerfile_create(yamlfile: str, platform: tuple[str], target: tuple[str], execute: bool = False, no_format_transfer: bool = False): """ - This function dockerfile_create creates a Dockerfile and - an accompanying createContainer.sh script that builds a container image containing - the compiled model executable and the library dependencies - - :param yamlfile: model compile YAML file + Dockerfile_create generates a Dockerfile and createContainer.sh for each container platform/target + combination and optionally executes the build script to produce a container image. + + fre make checkout-script and fre make makefile should be invoked + beforehand to stage the checkout.sh script and Makefile in tmp/[platform-name]/. + + :param yamlfile: is the path to the model YAML file (e.g. am5.yaml). The experiment + name is derived by stripping the .yaml extension. :type yamlfile: str - :param platform: FRE container-specific platform(s) that are defined in platforms.yaml - :type platform: tuple(str) - :param target: Predefined FRE targets - :type target: tuple(str) - :param execute: If true, execute createContainer.sh to build the container image + :param platform: is one or more FRE platform strings as defined in platforms.yaml. + Only container platforms (container: true) are processed; bare-metal + platforms are skipped. + :type platform: tuple[str] + :param target: is one or more mkmf target strings (e.g. prod, repro, debug). + One Dockerfile is generated per platform/target pair. + :type target: tuple[str] + :param execute: is a flag where if True, run createContainer.sh immediately after generation + to build the container image. Defaults to False. :type execute: bool - :param no_format_transfer: if True, skip container image format conversion to a .sif file + :param no_format_transfer: is a flag where if True, skip the OCI-to-Singularity (.sif) format + conversion step in createContainer.sh. Defaults to + False. :type no_format_transfer: bool - :raises ValueError: Error if platform does not exist in platforms.yaml - .. note:: If building the container image on GFDL's RDHPCS GAEA with the Podman container engine, - please submit a GFDL helpdesk ticket to request Podman access + :raises ValueError: If a specified platform does not exist in platforms.yaml. + + .. note:: If building the container image on GFDL's RDHPCS GAEA with the Podman + container engine, submit a GFDL helpdesk ticket to request Podman access + before running this command. """ ## Split and store the platforms and targets in a list diff --git a/fre/make/create_makefile_script.py b/fre/make/create_makefile_script.py index 896edb079..db95f1fdb 100644 --- a/fre/make/create_makefile_script.py +++ b/fre/make/create_makefile_script.py @@ -1,36 +1,41 @@ -''' -For a bare-metal build: -Create the Makefile used for model compilation in the -``[modelRoot]/[experiment name]/[platform]-[target]/exec`` -folder. +""" +create_makefile_script contains methods to generate the top-level Makefile used for model compilation. +makefile_create is the entry point called by fre make makefile and by fre make all. -For a container build: -Create the Makefile used for model compilation in the -``./tmp/[platform]`` directory. +The top-level Makefile is outputted to: +- for bare-metal platforms: [modelRoot]/[experiment]/[platform]-[target]/exec/Makefile +- for container platforms: ./tmp/[platform]/Makefile (staged for COPY during container image build) -- ``modelRoot`` is defined in the `platforms.yaml` -- ``experiment name`` is defined in `compile.yaml` -- ``platform`` and ``target`` are passed via Click options +where -The Makefile +- modelRoot is defined in platforms.yaml +- experiment is the basename of the model YAML file (e.g. am5 from am5.yaml) +- platform and target are passed via the -p / -t CLI options to fre make makefile +and fre make all. -1. Sets the ``SRCROOT`` -2. Sets the ``BUILDROOT`` -3. Sets the ``MK_TEMPLATE_PATH`` +The generated Makefile - - This path is defined in the `platforms.yaml` and refers to a template in the - `mkmf repository `_. - -4. Sets the build and linking recipes that adhere to the following structure: - - .. code-block:: makefile +1. Sets SRCROOT — path to the checked-out source code +2. Sets BUILDROOT — path to the directory where the executable is placed +3. Sets MK_TEMPLATE_PATH — path to the mkmf template file (mkTemplate + from platforms.yaml); templates can be found in + mkmf repository +4. Defines build and link recipes (as shown below) for each component listed under src in + compile.yaml: [target]: [prerequisites] [recipe] +Additional library flags are handled as below: + +- Bare-metal: baremetal_linkerflags from compile.yaml are added + directly as -L / -l flags to the Makefile. +- Container: container_addlibs from the compile yaml are resolved at + build time. + For more information about the Makefile, see the fre-cli glossary: https://github.com/NOAA-GFDL/fre-cli/blob/main/docs/glossary.rst -''' +""" import logging from pathlib import Path @@ -42,31 +47,38 @@ def makefile_create(yamlfile: str, platform: tuple[str], target: tuple[str]): """ - This function makefile_create generates the top level Makefile for the source code - that is specified in the model compile YAML file. + Makefile_create generates the top-level Makefile for each platform/target combination. + + For bare-metal platforms, the Makefile is written to + [modelRoot]/[experiment]/[platform]-[target]/exec/. - :param yamlfile: Model compile YAML file + For container platforms, the Makefile is staged in + ./tmp/[platform]/ and is COPYed into the container image at image build time. + + :param yamlfile: is the path to the model YAML file (e.g. am5.yaml). The experiment + name is derived by stripping the .yaml extension. :type yamlfile: str - :param platform: FRE platforms that are defined in the platforms.yaml - :type platform: tuple of strings - :param target: Predefined FRE targets - :type target: tuple of strings - :raises ValueError: Error if platform does not exist in platforms yaml configuration - - .. note:: If additional library dependencies are defined in the compile.yaml file: - - - For a container build, where library dependencies are defined via the "container_addlibs" - key in the `compile.yaml`, a linkline.sh script will be generated to determine paths for the - additional `-L/[path to libraries]` and `-l[library name]` located inside the container to - the Makefile. - - - Example: `container_addlibs: ['darcy']` - - - For a bare-metal build, library flags, `-L/[path to libraries]` and `-l[library name]`, are - defined via the "baremetal_linkerflags" key in the `compile.yaml` and added to the link line - in the Makefile. - - - Example: `baremetal_linkerflags: ["-L/derbyshire/pemberly -ldarcy"]` + :param platform: is one or more FRE platform strings as defined in platforms.yaml + (e.g. ncrc5.intel23). Both bare-metal and container platforms + are supported. + :type platform: tuple[str] + :param target: is one or more mkmf target strings (e.g. prod, debug, + repro, prod-openmp). One Makefile is generated per + platform/target pair. + :type target: tuple[str] + + :raises ValueError: If a specified platform does not exist in platforms.yaml. + + .. note:: Additional library dependencies are handled differently per build type: + + - Container build — list library names under container_addlibs in + compile.yaml (e.g. container_addlibs: ['darcy']). A linkline.sh + script, generated alongside the Makefile, locates the libraries inside + the container and resolves the -L / -l flags at container image build time. + + - Bare-metal build — provide full linker flags under baremetal_linkerflags in compile.yaml + (e.g. baremetal_linkerflags: ["-L/path/to/libs -ldarcy"]). These flags + are added directly to the Makefile. """ name = yamlfile.split(".")[0] diff --git a/fre/make/fremake.py b/fre/make/fremake.py index b71e4a33b..550b1f5ef 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -1,5 +1,5 @@ """ -Module defining the `click `_ interfaces for the following **fre make** subcommands: +fremake is the module defining the `click `_ interfaces for the following **fre make** subcommands: * fre make checkout-script [ARGS]: writes a script that will clone (checkout) the model code from respective git repositories @@ -116,7 +116,7 @@ def make_cli(): def all(yamlfile, platform, target, nparallel, makejobs, gitjobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout): """ - - Perform all fre make functions; for baremetal platforms: create checkout script, makefile, and compile scripts; + fre make all perform all fre make functions; for baremetal platforms: create checkout script, makefile, and compile scripts; for container platforms: create checkout script, makefile, Dockerfile, and createContainer script """ run_fremake_script.fremake_run( @@ -160,7 +160,10 @@ def all(yamlfile, platform, target, nparallel, makejobs, gitjobs, no_parallel_ch default = False, help = "Force a git checkout if the source directory already exists.") def checkout_script(yamlfile, platform, target, no_parallel_checkout, gitjobs, execute, force_checkout): - """ - Write the checkout script """ + """ - Write checkout.sh, which git-clones all component source repositories defined + in the compile YAML. For bare-metal platforms, the script is written to + ``[modelRoot]/[experiment]/src/``; for container platforms it is staged under + ``tmp/[platform]/`` for later inclusion in the Dockerfile. """ create_checkout_script.checkout_create( yamlfile, platform, target, no_parallel_checkout, gitjobs, execute, force_checkout) @@ -181,7 +184,9 @@ def checkout_script(yamlfile, platform, target, no_parallel_checkout, gitjobs, e help = _TARGET_OPT_HELP, required = True) def makefile(yamlfile, platform, target): - """ - Write the makefile """ + """ fre make makefile write the top-level Makefile for model compilation. For bare-metal platforms, + the Makefile is written to ``[modelRoot]/[experiment]/[platform]-[target]/exec/``; + for container platforms it is staged under ``tmp/[platform]/``. """ create_makefile_script.makefile_create(yamlfile, platform, target) @make_cli.command('compile-script') @@ -221,7 +226,10 @@ def makefile(yamlfile, platform, target): is_flag = True, help = _VERBOSE_OPT_HELP) def compile_script(yamlfile, platform, target, makejobs, nparallel, execute, verbose): - """ - Write the compile script """ + """ fre make compile-script write compile.sh for bare-metal platforms. The script configures the compile + environment (loads modules), calls mkmf to generate per-component Makefiles, and + runs make to build the model executable. Written to + ``[modelRoot]/[experiment]/[platform]-[target]/exec/``. """ create_compile_script.compile_create( yamlfile, platform, target, makejobs, nparallel, execute, verbose) @@ -252,5 +260,8 @@ def compile_script(yamlfile, platform, target, makejobs, nparallel, execute, ver help = """Execute the createContainer script immediately following its generation. The default behavior is to generate the script, but not execute.""") def dockerfile(yamlfile, platform, target, no_format_transfer, execute): - """ - Write the Dockerfile and createContainer script""" + """ fre make dockerfile write the Dockerfile and createContainer.sh for container platforms. The + Dockerfile defines a two-stage build (compile + runtime); createContainer.sh builds + the Docker image and converts it to a Singularity Image File (.sif) unless + --no-format-transfer is specified. """ create_docker_script.dockerfile_create(yamlfile, platform, target, no_format_transfer, execute) diff --git a/fre/make/make_helpers.py b/fre/make/make_helpers.py index 243d66a39..d321cb143 100644 --- a/fre/make/make_helpers.py +++ b/fre/make/make_helpers.py @@ -1,6 +1,7 @@ -''' -module of helper/utility functions used in the fre make subtool -''' +""" +make_helpers contain helper/utility functions +create_compile_script, create_makefile_script, and create_docker_script. +""" import logging from pathlib import Path @@ -8,24 +9,27 @@ def get_mktemplate_path(mk_template: str, container_flag: bool, model_root: str = None) -> str: """ - This function get_mktemplate_path generates the full path to the - mkmf mk_template on the bare-metal system or the container image filesystem - - :param mk_template: Full path to or the mkmf mk_template filename with .mk extension - :type mk_template: string - :param model_root: Path to the root for all model install files (TO CLARIFY) - :type model_root: str - :param container_flag: if True and the full path to the mk_template is not specified, - return model_root+"/mkmf/templates/"+mk_template; else if True - and mk_template is the filename, return mk_template - :type container_flag: boolean - - :raises ValueError: Error if the mk_template file does not exist in the generated full path - - :return: Full path to the mkmf mk_template - :rtype: string - - .. note:: model_root must be specified if container_flag is True + get_mktemplate_path resolves the full path to an mkmf template file (.mk) for either a + bare-metal system or a container image filesystem. + + mk_template may be a bare filename (e.g. intel.mk) or an absolute path (e.g. /path/to/intel.mk). + + :param mk_template: is the bare filename (e.g. intel.mk) or absolute path to the + mkmf template. Defined as mkTemplate in platforms yaml. + :type mk_template: str + :param container_flag: is a flag where True for container builds and False for bare-metal builds. + :type container_flag: bool + :param model_root: Root directory for model install files inside the container + (defined as modelRoot in platforms yaml). Required + when container_flag=True and mk_template is a bare + filename; unused otherwise. + :type model_root: str, optional + + :raises ValueError: If container_flag=False and the resolved template path + does not exist on the host filesystem. + + :return: a resolved full path to the mkmf template file. + :rtype: str """ template_path = mk_template diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 9bbff6a91..fc9920d25 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -1,10 +1,24 @@ -''' -For a bare-metal build: Creates and runs the checkout script to check out source code, - creates the makefile, and creates the compile script to generate - a model executable. -For a container build: Creates the checkout script and makefile, and creates and runs a - dockerfile to generate a singularity image file. -''' +""" +run_fremake_script contains the main orchestration function, fremake_run, which executes all +fre make sub-commands in the correct order to compile the model src code into an executable. +fremake_run is the entry point called by fre make all. + +For bare-metal platforms, fremake_run calls: + +1. checkout_create — creates and executes checkout.sh to clone all source repositories + into [modelRoot]/[experiment]/src/ +2. makefile_create — creates the top-level Makefile into + [modelRoot]/[experiment]/[platform]-[target]/exec/ +3. compile_create — creates compile.sh to the + [modelRoot]/[experiment]/[platform]-[target]/exec/ and executes the script + +For container platforms, fremake_run calls: +1. checkout_create — stages checkout.sh under tmp/[platform]/ +2. makefile_create — stages the Makefile under tmp/[platform]/ +3. dockerfile_create — creates the Dockerfile and createContainer.sh, + and, if --execute is set, runs the build script to produce a + Singularity image file (.sif) for each platform/target combination. +""" import logging from typing import Optional from pathlib import Path @@ -25,33 +39,48 @@ def fremake_run(yamlfile:str, platform:str, target:str, verbose: Optional[bool] = None, force_checkout: Optional[bool] = False): """ - Runs all of fre make code + fremake_run runs all fre make sub-commands in sequence to produce a model executable + (bare-metal) or a container image. - :param yamlfile: Model compile YAML file + :param yamlfile: is the path to the model YAML file (e.g. am5.yaml). The experiment + name is derived by stripping the .yaml extension. :type yamlfile: str - :param platform: FRE platform; defined in the platforms yaml - If on gaea c5, a FRE platform may look like ncrc5.intel23-classic - :type platform: str - :param target: Predefined FRE targets; options include [prod/debug/repro]-openmp - :type target: str - :param nparallel: Number of concurrent model builds (default 1) + :param platform: is one or more FRE platform strings as defined in platforms.yaml + (e.g. ncrc5.intel23 for a bare-metal GAEA C5 platform). + :type platform: tuple[str] + :param target: is one or more mkmf target strings (e.g. prod, repro debug) + :type target: tuple[str] + :param nparallel: is the number of compile.sh scripts to execute concurrently when + execute=True (bare-metal only). Defaults to 1 + (sequential). :type nparallel: int - :param makejobs: Number of jobs to run simultaneously; used for parallelism with make (default 4) + :param makejobs: is the number of Makefile recipes to run simultaneously, passed to + make -j. Defaults to 4. :type makejobs: int - :param gitjobs: Number of jobs to run simultaneously; used for parallelism with - recursive cloning with checking out source code (default 4) + :param gitjobs: is the number of git submodules to clone simultaneously, passed to + git clone --jobs inside checkout.sh. Defaults to 4. :type gitjobs: int - :param no_parallel_checkout: Use this option if you do not want a parallel checkout - :type no_parallel_checkout: bool - :param no_format_transfer: Skip the container format conversion to a .sif file - :type no_format_transfer: bool - :param execute: Run the created compile script or dockerfile to create a model executable - or container - :type execute: bool - :param force_checkout: Re-create the checkout script if changes were made to configurations - :type force_checkout: bool - :param verbose: Increase verbosity output - :type verbose: bool + :param no_parallel_checkout: is a flag where if True, clone source repositories sequentially + Defaults to False to enable parallel checkout for bare-metal builds. + Only sequential checkouts are available for container builds. + :type no_parallel_checkout: bool, optional + :param no_format_transfer: is a flag where if True, skip the Docker-to-Singularity (.sif) + format conversion in createContainer.sh. Defaults to False. + :type no_format_transfer: bool, optional + :param execute: is a flag where if True, execute checkout.sh and compile.sh (bare-metal) + or createContainer.sh (container) immediately after generation. + Defaults to False. + :type execute: bool, optional + :param verbose: is a flag where if True, set logger level to DEBUG for detailed output from + compile_create. Defaults to False and sets logger level to INFO. + :type verbose: bool, optional + :param force_checkout: is a flag where if True, for bare-metal: remove existing + compile.sh before regenerating. For container: remove + the existing Dockerfile before regenerating. Defaults + to False. + :type force_checkout: bool, optional + + :raises ValueError: If a specified platform does not exist in platforms.yaml. """ # if verbose: # fre_logger.setLevel(level = logging.DEBUG)