From 939a93c814b4d3509fa90662cee1c335315ca4aa Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 14 May 2026 09:31:07 -0400 Subject: [PATCH 1/8] in progress --- fre/make/create_checkout_script.py | 75 +++++++++++++++--------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index fc2412f38..1a82966d1 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -1,16 +1,12 @@ -''' -FRE Checkout Script Generator - -Retrieves information from the resolved YAML configuration to generate a -checkout.sh script that git clones the model source code. - -The checkout script will clone component repositories defined in the -compile YAML to build the model. - -Note, a bare-metal build defaults to a parallel checkout. -A container build defaults to a non-parallel checkout. - -''' +""" +Create_checkout_script provides methods to generate a checkout.sh script from a resolved YAML +configuration. The script git clones all component repositories defined in the +compile YAML. + +Build type defaults: + - Bare-metal build: parallel checkout (multiple repositories cloned concurrently) + - Container build: non-parallel checkout (repositories cloned sequentially) +""" import shutil from pathlib import Path from datetime import datetime @@ -26,26 +22,27 @@ def baremetal_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, jobs: str, parallel_cmd: str, execute: bool): """ - This function baremetal_checkout_write is called by checkout_create in order to - - - Extract compilation specifications from the parsed YAML configuration - - Generate a checkout script to the source directory. The source directory is - defined within the 'modelRoot' variable in the "platforms" section of the combined YAML - + Baremetal_checkout_write generates and optionally executes a checkout.sh script for + bare-metal builds. This method is called by ``checkout_create`` to extract compilation + specifications from the parsed YAML configuration and write checkout.sh to the source + directory. The source directory is defined by the ``modelRoot`` variable in the ``platforms`` + specifications in the yaml. - :param model_yaml: "freyaml" class object containing a parsed and validated yaml dictionary - containing the "compile" specification + :param model_yaml: is the parsed and validated YAML object containing ``compile`` specifications. :type model_yaml: yamlfre.freyaml - :param src_dir: Absolute directory path to git clone the source code + :param src_dir: is the absolute path to the directory where the checkout script is written + and source code is cloned. :type src_dir: str - :param jobs: Number of git submodules to clone simultaneously (TO CLARIFY) + :param jobs: is the number of git repositories to clone simultaneously. :type jobs: str - :param parallel_cmd: Set to " &" for parallel checkouts and "" for non-parallel checkouts + :param parallel_cmd: is the shell suffix controlling parallelism. Use ``" &"`` for parallel + checkout or ``""`` for sequential checkout. :type parallel_cmd: str - :param execute: If True, run the generated checkout.sh + :param execute: is a flag where if ``True``, checkout.sh is executed after being written. + This is set to ``False`` by default. :type execute: bool """ - fre_checkout = checkout.checkout("checkout.sh", src_dir) + fre_checkout = checkout.checkout("checkout.sh", src_dir) fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(), jobs, parallel_cmd) fre_checkout.finish(model_yaml.compile.getCompileYaml(), parallel_cmd) @@ -60,23 +57,25 @@ def baremetal_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, jobs: st def container_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, tmp_dir: str, jobs: str, parallel_cmd: str): """ - This function container_checkout_write is called by checkout_create in order to - - - Extract compilation specifications from the parsed YAML configuration - - Generate a checkout script in a local ./tmp directory, where it will later be - copied to the directory of the container image filesystem for execution + Container_checkout_write generates a checkout.sh script for container builds. + This method is called by ``checkout_create`` to extract compilation specifications + from the parsed YAML configuration and write checkout.sh to a local temporary directory + on the physical host system. The script is copied elsewhere into the container image filesystem + and is executed when the container is launched. - :param model_yaml: "freyaml" class object containing a parsed and validated yaml dictionary - containing the "compile" specification + :param model_yaml: is the parsed and validated YAML object containing ``compile`` specification. :type model_yaml: yamlfre.freyaml - :param src_dir: Internal path for source code in the running container. The source directory is - defined within the 'modelRoot' variable in the "platforms" section of the combined YAML + :param src_dir: is the internal source-code path used inside the running container. + This value is defined by ``modelRoot`` in the ``platforms`` + specifications in the yaml. :type src_dir: str - :param tmp_dir: Temporary directory (outside of container) that hosts the created checkout script + :param tmp_dir: is the local temporary directory (outside the container) where + checkout.sh is created. :type tmp_dir: str - :param jobs: Number of git submodules to clone simultaneously (TO CLARIFY) + :param jobs: is the number of git repositories to clone simultaneously. :type jobs: str - :param parallel_cmd: Since container builds are not parallelized, set to "" + :param parallel_cmd: is the shell suffix controlling parallelism. For container + builds, this is typically ``""`` (sequential checkout). :type parallel_cmd: str """ fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) From ab03418679be2d031f81dd4ab655a9bee9184c6b Mon Sep 17 00:00:00 2001 From: Miso Potstickers Date: Thu, 14 May 2026 14:06:28 -0400 Subject: [PATCH 2/8] ai --- fre/make/create_checkout_script.py | 165 +++++++++++++++++++---------- fre/make/create_compile_script.py | 82 +++++++++----- fre/make/create_docker_script.py | 80 ++++++++++---- fre/make/create_makefile_script.py | 102 ++++++++++-------- fre/make/fremake.py | 19 +++- fre/make/make_helpers.py | 70 ++++++++---- fre/make/run_fremake_script.py | 103 +++++++++++++----- 7 files changed, 427 insertions(+), 194 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 1a82966d1..7ac8d98b0 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -1,11 +1,20 @@ """ -Create_checkout_script provides methods to generate a checkout.sh script from a resolved YAML -configuration. The script git clones all component repositories defined in the -compile YAML. +Create_checkout_script provides methods to generate a ``checkout.sh`` script from a resolved +YAML configuration. The script ``git clone``'s all component source repositories listed under the +``src`` key of the compile YAML. + +``checkout_create`` is the entry point called by ``fre make checkout-script`` and +``fre make all``. Depending on the user-specified platform type (bare-metal or container), +checkout_create then calls either + +- ``baremetal_checkout_write`` for bare-metal platforms: writes ``checkout.sh`` directly on the local platform + into ``[modelRoot]/[experiment]/src/`` and optionally executes the script. +- ``container_checkout_write`` for container platforms: writes ``checkout.sh`` into a local + temporary directory (``tmp/[platform-name]/``). Build type defaults: - - Bare-metal build: parallel checkout (multiple repositories cloned concurrently) - - Container build: non-parallel checkout (repositories cloned sequentially) + - Bare-metal build: parallel checkout + - Container build: sequential checkout """ import shutil from pathlib import Path @@ -22,24 +31,38 @@ def baremetal_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, jobs: str, parallel_cmd: str, execute: bool): """ - Baremetal_checkout_write generates and optionally executes a checkout.sh script for - bare-metal builds. This method is called by ``checkout_create`` to extract compilation - specifications from the parsed YAML configuration and write checkout.sh to the source - directory. The source directory is defined by the ``modelRoot`` variable in the ``platforms`` - specifications in the yaml. - - :param model_yaml: is the parsed and validated YAML object containing ``compile`` specifications. + Baremetal_checkout_write generates and optionally executes ``checkout.sh`` for + model compilation on the bare-metal. + + Called by ``checkout_create`` for each bare-metal platform. Reads the ``compile`` + section of the parsed YAML to determine which source repositories to clone, writes + ``checkout.sh`` into ``src_dir``, sets its permissions to ``rwxr--r--`` (0o744), and + optionally runs it immediately. + + Because all bare-metal platforms in a single ``fre make`` invocation share the same + physical source tree, ``checkout_create`` calls this function only once per experiment + and skips subsequent bare-metal platforms if the script already exists (unless + ``--force-checkout`` is set). + + :param model_yaml: Parsed and validated YAML object produced by ``yamlfre.freyaml``, + containing the ``compile`` specifications (source repositories, experiment + name, etc.). :type model_yaml: yamlfre.freyaml - :param src_dir: is the absolute path to the directory where the checkout script is written - and source code is cloned. + :param src_dir: Absolute path of the source directory where ``checkout.sh`` is written and + where the source repositories will be cloned. Typically + ``[modelRoot]/[experiment]/src``, with ``modelRoot`` defined in + ``platforms.yaml``. :type src_dir: str - :param jobs: is the number of git repositories to clone simultaneously. + :param jobs: Number of git submodules to fetch simultaneously, passed to ``git clone + --jobs``. Supplied as a string because it is written directly into the + shell script. :type jobs: str - :param parallel_cmd: is the shell suffix controlling parallelism. Use ``" &"`` for parallel - checkout or ``""`` for sequential checkout. + :param parallel_cmd: Shell suffix appended to each ``git clone`` command to control + concurrency. Pass ``" &"`` to background each clone (parallel + checkout) or ``""`` to clone sequentially. :type parallel_cmd: str - :param execute: is a flag where if ``True``, checkout.sh is executed after being written. - This is set to ``False`` by default. + :param execute: If ``True``, run ``checkout.sh`` immediately after writing it. + Defaults to ``False``. :type execute: bool """ fre_checkout = checkout.checkout("checkout.sh", src_dir) @@ -57,25 +80,36 @@ def baremetal_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, jobs: st def container_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, tmp_dir: str, jobs: str, parallel_cmd: str): """ - Container_checkout_write generates a checkout.sh script for container builds. - This method is called by ``checkout_create`` to extract compilation specifications - from the parsed YAML configuration and write checkout.sh to a local temporary directory - on the physical host system. The script is copied elsewhere into the container image filesystem - and is executed when the container is launched. - - :param model_yaml: is the parsed and validated YAML object containing ``compile`` specification. + Generates ``checkout.sh`` for a container build. + + Called by ``checkout_create`` for each container platform. Writes ``checkout.sh`` + into a local temporary directory on the host (``tmp/[platform-name]/``). The + Dockerfile produced by ``fre make dockerfile`` subsequently COPYs this script into + the container image filesystem; it is executed inside the container when the image + is launched to clone the model source code into ``src_dir``. + + Unlike the bare-metal variant, this function never executes the script directly — the + container build step (``createContainer.sh``) handles that. Parallel checkout with + ``&`` is also not used for container builds because backgrounded shell processes are + not reliably supported during the container image build stage. + + :param model_yaml: Parsed and validated YAML object produced by ``yamlfre.freyaml``, + containing the ``compile`` specifications (source repositories, experiment + name, etc.). :type model_yaml: yamlfre.freyaml - :param src_dir: is the internal source-code path used inside the running container. - This value is defined by ``modelRoot`` in the ``platforms`` - specifications in the yaml. + :param src_dir: Source-code path *inside* the running container where repositories will + be cloned. Defined by ``modelRoot`` in ``platforms.yaml``. :type src_dir: str - :param tmp_dir: is the local temporary directory (outside the container) where - checkout.sh is created. + :param tmp_dir: Local temporary directory on the host (outside the container) where + ``checkout.sh`` is staged before being COPYed into the image. + Typically ``tmp/[platform-name]``. :type tmp_dir: str - :param jobs: is the number of git repositories to clone simultaneously. + :param jobs: Number of git submodules to fetch simultaneously, passed to ``git clone + --jobs``. Supplied as a string because it is written directly into the + shell script. :type jobs: str - :param parallel_cmd: is the shell suffix controlling parallelism. For container - builds, this is typically ``""`` (sequential checkout). + :param parallel_cmd: Shell suffix for concurrency control. For container builds this + should always be ``""`` (sequential checkout). :type parallel_cmd: str """ fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) @@ -87,31 +121,54 @@ def checkout_create(yamlfile: str, platform: tuple, target: tuple, no_parallel_checkout: Optional[bool] = None, njobs: int = 4, execute: Optional[bool] = False, force_checkout: Optional[bool] = False): """ - Calls baremetal_checkout_write or container_checkout_write to create checkout.sh - for baremetal or container builds, respectively. + Entry point for ``fre make checkout-script``. Resolves the YAML configuration and + dispatches to ``baremetal_checkout_write`` or ``container_checkout_write`` for each + platform supplied on the command line. - :param yamlfile: Model YAML file path + Bare-metal platforms share a single source directory + (``[modelRoot]/[experiment]/src``), so the checkout script is written once and + reused across targets. Container platforms each receive their own staging directory + (``tmp/[platform-name]/``). + + :param yamlfile: Path to the model YAML file (e.g. ``am5.yaml``). The experiment + name is derived by stripping the ``.yaml`` extension. :type yamlfile: str - :param platform: FRE platform(s) that are defined in the platforms.yaml - :type platform: tuple - :param target: Predefined FRE target(s) - :type target: tuple - :param no_parallel_checkout: Option to disable parallel checkouts - :type no_parallel_checkout: bool - :param njobs: Used in the recursive clone; number of submodules to fetch simultaneously (default 4) (TO CLARIFY) + :param platform: One or more FRE platform strings as defined in ``platforms.yaml``. + Repeat the ``-p`` flag on the command line to specify multiple + platforms. + :type platform: tuple[str] + :param target: One or more ``mkmf`` target strings (e.g. ``debug``, ``prod``). + Repeat the ``-t`` flag to specify multiple targets. + :type target: tuple[str] + :param no_parallel_checkout: If ``True``, clone repositories sequentially (no ``&`` + backgrounding). Defaults to ``False`` (parallel checkout) + for bare-metal builds; container builds are always + sequential regardless of this flag. + :type no_parallel_checkout: bool, optional + :param njobs: Number of git submodules to fetch simultaneously, passed to + ``git clone --jobs``. Defaults to ``4``. :type njobs: int - :param execute: If True, run checkout.sh - :type execute: bool - :param force_checkout: If True, for bare-metal build: add timestamp to source directory and create a new checkout script - If True, for container build: overwrite locally existing checkout script before COPY-ing to the - container image filesystem - :type force_checkout: bool + :param execute: If ``True``, execute ``checkout.sh`` immediately after writing it + (bare-metal only). Defaults to ``False``. + :type execute: bool, optional + :param force_checkout: Controls behaviour when ``checkout.sh`` already exists. + + - **Bare-metal**: renames the existing ``src`` directory with a + ``YYYYmmdd.HHMMSS`` timestamp suffix, then writes a fresh + ``checkout.sh`` in a new ``src`` directory. + - **Container**: deletes the existing ``tmp/[platform]/checkout.sh`` + and writes a new one in its place. + + Defaults to ``False``. + :type force_checkout: bool, optional :raises ValueError: - - If 'njobs' is not an integer - - If 'platform' does not exist in the platforms.yaml configuration - :raises OSError: If executing checkout.sh returns an error - + - If ``njobs`` is passed as a boolean while ``--execute`` is also set (ambiguous + intent — ``njobs`` must be an explicit integer). + - If a specified platform name does not exist in ``platforms.yaml``. + :raises OSError: If ``--execute`` is set and running an existing ``checkout.sh`` + returns a non-zero exit code (e.g. the source directory already + contains conflicting content). """ # Standardize inputs jobs_str = str(njobs) diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index b40cf228f..ad641c068 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -1,19 +1,31 @@ ''' -Retrieves information from the resolved YAML configuration to generate the compile.sh -in the ``[modelRoot]/[experiment name]/[platform-target]/exec`` directory, where +Retrieves information from the resolved YAML configuration to generate ``compile.sh`` +for bare-metal builds. ``compile_create`` is the entry point called by +``fre make compile-script`` and (for bare-metal platforms only) by ``fre make all``. -- ``modelRoot`` is defined in the `platforms.yaml` -- ``experiment name`` is defined in `compile.yaml` -- ``platform`` and ``target`` are passed via Click options +The generated script is written to:: -The compile.sh script + [modelRoot]/[experiment]/[platform]-[target]/exec/compile.sh -1. Sets the ``src_dir`` -2. Sets the ``bld_dir`` -3. Sets the ``mkmf_template`` -4. Loads/unloads modules to set-up the compile environment -5. Calls ``mkmf`` to generate Makefiles for each model component defined in the `compile.yaml` -6. Calls ``make`` to generate the model executable +where + +- ``modelRoot`` is defined in ``platforms.yaml`` +- ``experiment`` is the basename of the model YAML file (e.g. ``am5`` from ``am5.yaml``) +- ``platform`` and ``target`` are passed via the ``-p`` / ``-t`` CLI options + +When executed, ``compile.sh`` + +1. Sets ``src_dir`` (where source code was checked out by ``checkout.sh``) +2. Sets ``bld_dir`` (the ``exec/`` directory where the executable is placed) +3. Sets the path to the ``mkmf`` template (``mkTemplate`` from ``platforms.yaml``) +4. Loads/unloads environment modules to configure the compile environment + (``envSetup`` from ``platforms.yaml``) +5. Calls ``mkmf`` for each model component listed under ``src`` in ``compile.yaml`` + to generate per-component Makefiles +6. Calls ``make`` (with ``-j [makejobs]``) to build the model executable + +Container platforms are silently skipped — compilation inside a container image +is handled by the Dockerfile generated by ``fre make dockerfile``. ''' import logging @@ -38,26 +50,46 @@ def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejob nparallel: int = 1, execute: Optional[bool] = False, verbose: Optional[bool] = None): """ - This function compile_create generates the compile script for bare-metal build. + Generates ``compile.sh`` for each bare-metal platform/target combination and + optionally executes those scripts to produce a model executable. - :param yamlfile: Model compile YAML file + For each bare-metal platform in ``platform``, a ``compile.sh`` is written to + ``[modelRoot]/[experiment]/[platform]-[target]/exec/``. Container platforms are + silently skipped here; their compilation is handled by the Dockerfile produced by + ``dockerfile_create``. + + When ``execute=True``, all generated ``compile.sh`` scripts are run concurrently + using a ``multiprocessing.dummy.Pool`` of size ``nparallel``. + + :param yamlfile: Path to the model YAML file (e.g. ``am5.yaml``). The experiment + name is derived by stripping the ``.yaml`` extension. :type yamlfile: str - :param platform: FRE platform; defined in the platforms yaml - :type platform: tuple of strings - :param target: Predefined FRE targets - :type target: tuple of strings - :param makejobs: Number of recipes from the Makefile to run in parallel (default 4); - corresponds to -j option in make + :param platform: One or more FRE platform strings as defined in ``platforms.yaml`` + (e.g. ``ncrc5.intel23``). Container platforms in this tuple are + silently ignored. + :type platform: tuple[str] + :param target: One or more ``mkmf`` target strings (e.g. ``prod``, ``debug``, + ``repro``, ``prod-openmp``). One ``compile.sh`` is generated per + platform/target pair. + :type target: tuple[str] + :param makejobs: Number of Makefile recipes to run simultaneously, passed to + ``make -j``. Defaults to ``4``. :type makejobs: int - :param nparallel: Number of compile.sh scripts to run in parallel (default 1) + :param nparallel: Number of ``compile.sh`` scripts to run concurrently when + ``execute=True``. Uses a ``multiprocessing.dummy.Pool`` internally. + Defaults to ``1`` (sequential execution). :type nparallel: int - :param execute: If True, execute the created compile.sh script to build a model executable + :param execute: If ``True``, run every generated ``compile.sh`` after writing it. + Defaults to ``False``. :type execute: bool - :param verbose: If True, increase verbosity output + :param verbose: If ``True``, set log level to ``DEBUG`` for detailed output. + Defaults to ``False`` (``INFO`` level). :type verbose: bool + :raises ValueError: - - Error if platform does not exist in platforms yaml configuration - - Error if the mkmf template defined in platforms yaml does not exist + - If a specified platform does not exist in ``platforms.yaml``. + - If the ``mkTemplate`` path defined in ``platforms.yaml`` cannot be resolved to + an existing file (raised by ``get_mktemplate_path``). """ # Define variables diff --git a/fre/make/create_docker_script.py b/fre/make/create_docker_script.py index 95be4d9ee..186576329 100644 --- a/fre/make/create_docker_script.py +++ b/fre/make/create_docker_script.py @@ -1,12 +1,29 @@ ''' -Generates a Dockerfile and an accompanying createContainer.sh script that -builds a Docker image containing the compiled model executable and the library -dependencies from the generated Dockerfile. Unless specified, -createContainer.sh will convert the Docker OCI image to a Singularity image -file format (.sif) that can be launched with Singularity/Apptainer. - -Note, once the container image is built, the source code and the compiled -executable cannot be modified. +Generates a Dockerfile and an accompanying ``createContainer.sh`` script for +container builds. ``dockerfile_create`` is the entry point called by +``fre make dockerfile`` and (for container platforms only) by ``fre make all``. + +The Dockerfile uses a two-stage build: + +1. **Build stage** — starts from the base container image (``image`` field in + ``platforms.yaml``), copies in the ``checkout.sh`` and ``Makefile`` that were + staged under ``tmp/[platform]/`` by ``fre make checkout-script`` and + ``fre make makefile``, runs ``mkmf`` and ``make`` to compile the model + executable. +2. **Runtime stage** — copies the compiled executable and its runtime dependencies + into a leaner second base image (``containerBase2`` in ``platforms.yaml``). + +``createContainer.sh`` builds the Docker image and, unless ``--no-format-transfer`` +is specified, converts it to a Singularity Image File (``.sif``) that can be +launched with Singularity/Apptainer on HPC systems. + +Bare-metal platforms are silently skipped — their compilation is handled by +``compile_create``. + +.. note:: + Once a container image is built, the source code and compiled executable + inside it cannot be modified. To incorporate source changes, re-run + ``fre make all`` (or the individual sub-commands) and rebuild the image. ''' import logging @@ -29,24 +46,45 @@ def dockerfile_create(yamlfile: str, platform: tuple[str], target: tuple[str], execute: bool = False, no_format_transfer: bool = False): """ - This function dockerfile_create creates a Dockerfile and - an accompanying createContainer.sh script that builds a container image containing - the compiled model executable and the library dependencies + Generates a Dockerfile and ``createContainer.sh`` for each container platform/target + combination and optionally executes the build script to produce a container image. - :param yamlfile: model compile YAML file + For each container platform, the following artifacts are written to the current + working directory and ``tmp/[platform-name]/``: + + - ``Dockerfile`` — two-stage build definition (compile stage + runtime stage) + - ``tmp/[platform]/execrunscript.sh`` — script executed when the container is launched + - ``createContainer.sh`` — builds the Docker image and (unless ``--no-format-transfer``) + converts it to a ``.sif`` Singularity image via ``docker2singularity`` or equivalent + + This function relies on ``checkout.sh`` and ``Makefile`` having already been staged + in ``tmp/[platform-name]/`` by ``checkout_create`` and ``makefile_create``. + + Bare-metal platforms in ``platform`` are silently skipped. + + :param yamlfile: Path to the model YAML file (e.g. ``am5.yaml``). The experiment + name is derived by stripping the ``.yaml`` extension. :type yamlfile: str - :param platform: FRE container-specific platform(s) that are defined in platforms.yaml - :type platform: tuple(str) - :param target: Predefined FRE targets - :type target: tuple(str) - :param execute: If true, execute createContainer.sh to build the container image + :param platform: One or more FRE platform strings as defined in ``platforms.yaml``. + Only container platforms (``container: true``) are processed; bare-metal + platforms are skipped. + :type platform: tuple[str] + :param target: One or more ``mkmf`` target strings (e.g. ``prod``, ``debug``). + One Dockerfile is generated per platform/target pair. + :type target: tuple[str] + :param execute: If ``True``, run ``createContainer.sh`` immediately after generating it + to build the container image. Defaults to ``False``. :type execute: bool - :param no_format_transfer: if True, skip container image format conversion to a .sif file + :param no_format_transfer: If ``True``, skip the OCI-to-Singularity (``.sif``) format + conversion step in ``createContainer.sh``. Defaults to + ``False``. :type no_format_transfer: bool - :raises ValueError: Error if platform does not exist in platforms.yaml - .. note:: If building the container image on GFDL's RDHPCS GAEA with the Podman container engine, - please submit a GFDL helpdesk ticket to request Podman access + :raises ValueError: If a specified platform does not exist in ``platforms.yaml``. + + .. note:: If building the container image on GFDL's RDHPCS GAEA with the Podman + container engine, submit a GFDL helpdesk ticket to request Podman access + before running this command. """ ## Split and store the platforms and targets in a list diff --git a/fre/make/create_makefile_script.py b/fre/make/create_makefile_script.py index 896edb079..b45640cdf 100644 --- a/fre/make/create_makefile_script.py +++ b/fre/make/create_makefile_script.py @@ -1,33 +1,40 @@ ''' -For a bare-metal build: -Create the Makefile used for model compilation in the -``[modelRoot]/[experiment name]/[platform]-[target]/exec`` -folder. +Generates the top-level ``Makefile`` used for model compilation. +``makefile_create`` is the entry point called by ``fre make makefile`` and by +``fre make all`` (for both bare-metal and container platforms). -For a container build: -Create the Makefile used for model compilation in the -``./tmp/[platform]`` directory. +Output locations differ by platform type: -- ``modelRoot`` is defined in the `platforms.yaml` -- ``experiment name`` is defined in `compile.yaml` -- ``platform`` and ``target`` are passed via Click options +- **Bare-metal**: ``[modelRoot]/[experiment]/[platform]-[target]/exec/Makefile`` +- **Container**: ``./tmp/[platform]/Makefile`` (staged for inclusion in the Dockerfile) -The Makefile +where -1. Sets the ``SRCROOT`` -2. Sets the ``BUILDROOT`` -3. Sets the ``MK_TEMPLATE_PATH`` +- ``modelRoot`` is defined in ``platforms.yaml`` +- ``experiment`` is the basename of the model YAML file (e.g. ``am5`` from ``am5.yaml``) +- ``platform`` and ``target`` are passed via the ``-p`` / ``-t`` CLI options - - This path is defined in the `platforms.yaml` and refers to a template in the - `mkmf repository `_. +The generated Makefile -4. Sets the build and linking recipes that adhere to the following structure: - - .. code-block:: makefile +1. Sets ``SRCROOT`` — path to the checked-out source code +2. Sets ``BUILDROOT`` — path to the directory where the executable is placed +3. Sets ``MK_TEMPLATE_PATH`` — path to the ``mkmf`` template file (``mkTemplate`` + from ``platforms.yaml``); templates live in the bundled + `mkmf repository `_ +4. Defines build and link recipes for each component listed under ``src`` in + ``compile.yaml``, following standard Makefile structure:: [target]: [prerequisites] [recipe] +Additional library flags are handled differently per build type: + +- **Bare-metal**: ``baremetal_linkerflags`` from ``compile.yaml`` are embedded + directly as ``-L`` / ``-l`` flags in the Makefile link line. +- **Container**: ``container_addlibs`` from ``compile.yaml`` are resolved at + build time via a generated ``linkline.sh`` script that locates the library + paths inside the container image. + For more information about the Makefile, see the fre-cli glossary: https://github.com/NOAA-GFDL/fre-cli/blob/main/docs/glossary.rst ''' @@ -42,31 +49,40 @@ def makefile_create(yamlfile: str, platform: tuple[str], target: tuple[str]): """ - This function makefile_create generates the top level Makefile for the source code - that is specified in the model compile YAML file. - - :param yamlfile: Model compile YAML file + Generates the top-level ``Makefile`` for each platform/target combination and + writes it to the appropriate output directory. + + For bare-metal platforms the Makefile is written to + ``[modelRoot]/[experiment]/[platform]-[target]/exec/``. For container platforms + it is staged in ``./tmp/[platform]/`` so the Dockerfile can COPY it into the + image at build time. One Makefile is produced per platform/target pair. + + :param yamlfile: Path to the model YAML file (e.g. ``am5.yaml``). The experiment + name is derived by stripping the ``.yaml`` extension. :type yamlfile: str - :param platform: FRE platforms that are defined in the platforms.yaml - :type platform: tuple of strings - :param target: Predefined FRE targets - :type target: tuple of strings - :raises ValueError: Error if platform does not exist in platforms yaml configuration - - .. note:: If additional library dependencies are defined in the compile.yaml file: - - - For a container build, where library dependencies are defined via the "container_addlibs" - key in the `compile.yaml`, a linkline.sh script will be generated to determine paths for the - additional `-L/[path to libraries]` and `-l[library name]` located inside the container to - the Makefile. - - - Example: `container_addlibs: ['darcy']` - - - For a bare-metal build, library flags, `-L/[path to libraries]` and `-l[library name]`, are - defined via the "baremetal_linkerflags" key in the `compile.yaml` and added to the link line - in the Makefile. - - - Example: `baremetal_linkerflags: ["-L/derbyshire/pemberly -ldarcy"]` + :param platform: One or more FRE platform strings as defined in ``platforms.yaml`` + (e.g. ``ncrc5.intel23``). Both bare-metal and container platforms + are supported. + :type platform: tuple[str] + :param target: One or more ``mkmf`` target strings (e.g. ``prod``, ``debug``, + ``repro``, ``prod-openmp``). One Makefile is generated per + platform/target pair. + :type target: tuple[str] + + :raises ValueError: If a specified platform does not exist in ``platforms.yaml``. + + .. note:: Additional library dependencies are handled differently per build type: + + - **Container build** — list library names under ``container_addlibs`` in + ``compile.yaml`` (e.g. ``container_addlibs: ['darcy']``). A ``linkline.sh`` + script is generated alongside the Makefile; it resolves the ``-L`` / ``-l`` + flags at container-image build time by locating the libraries inside the + container. + + - **Bare-metal build** — provide full linker flags under + ``baremetal_linkerflags`` in ``compile.yaml`` + (e.g. ``baremetal_linkerflags: ["-L/path/to/libs -ldarcy"]``). These flags + are embedded directly in the Makefile link line. """ name = yamlfile.split(".")[0] diff --git a/fre/make/fremake.py b/fre/make/fremake.py index b71e4a33b..fc229aab8 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -160,7 +160,10 @@ def all(yamlfile, platform, target, nparallel, makejobs, gitjobs, no_parallel_ch default = False, help = "Force a git checkout if the source directory already exists.") def checkout_script(yamlfile, platform, target, no_parallel_checkout, gitjobs, execute, force_checkout): - """ - Write the checkout script """ + """ - Write checkout.sh, which git-clones all component source repositories defined + in the compile YAML. For bare-metal platforms, the script is written to + ``[modelRoot]/[experiment]/src/``; for container platforms it is staged under + ``tmp/[platform]/`` for later inclusion in the Dockerfile. """ create_checkout_script.checkout_create( yamlfile, platform, target, no_parallel_checkout, gitjobs, execute, force_checkout) @@ -181,7 +184,9 @@ def checkout_script(yamlfile, platform, target, no_parallel_checkout, gitjobs, e help = _TARGET_OPT_HELP, required = True) def makefile(yamlfile, platform, target): - """ - Write the makefile """ + """ - Write the top-level Makefile for model compilation. For bare-metal platforms, + the Makefile is written to ``[modelRoot]/[experiment]/[platform]-[target]/exec/``; + for container platforms it is staged under ``tmp/[platform]/``. """ create_makefile_script.makefile_create(yamlfile, platform, target) @make_cli.command('compile-script') @@ -221,7 +226,10 @@ def makefile(yamlfile, platform, target): is_flag = True, help = _VERBOSE_OPT_HELP) def compile_script(yamlfile, platform, target, makejobs, nparallel, execute, verbose): - """ - Write the compile script """ + """ - Write compile.sh for bare-metal platforms. The script configures the compile + environment (loads modules), calls mkmf to generate per-component Makefiles, and + runs make to build the model executable. Written to + ``[modelRoot]/[experiment]/[platform]-[target]/exec/``. """ create_compile_script.compile_create( yamlfile, platform, target, makejobs, nparallel, execute, verbose) @@ -252,5 +260,8 @@ def compile_script(yamlfile, platform, target, makejobs, nparallel, execute, ver help = """Execute the createContainer script immediately following its generation. The default behavior is to generate the script, but not execute.""") def dockerfile(yamlfile, platform, target, no_format_transfer, execute): - """ - Write the Dockerfile and createContainer script""" + """ - Write the Dockerfile and createContainer.sh for container platforms. The + Dockerfile defines a two-stage build (compile + runtime); createContainer.sh builds + the Docker image and converts it to a Singularity Image File (.sif) unless + --no-format-transfer is specified. """ create_docker_script.dockerfile_create(yamlfile, platform, target, no_format_transfer, execute) diff --git a/fre/make/make_helpers.py b/fre/make/make_helpers.py index 243d66a39..0440ac02c 100644 --- a/fre/make/make_helpers.py +++ b/fre/make/make_helpers.py @@ -1,5 +1,8 @@ -''' -module of helper/utility functions used in the fre make subtool +''' +Helper/utility functions shared across the ``fre make`` sub-commands. + +Functions here are imported by ``create_compile_script``, ``create_makefile_script``, +and ``create_docker_script`` to avoid duplicating path-resolution logic. ''' import logging @@ -8,24 +11,51 @@ def get_mktemplate_path(mk_template: str, container_flag: bool, model_root: str = None) -> str: """ - This function get_mktemplate_path generates the full path to the - mkmf mk_template on the bare-metal system or the container image filesystem - - :param mk_template: Full path to or the mkmf mk_template filename with .mk extension - :type mk_template: string - :param model_root: Path to the root for all model install files (TO CLARIFY) - :type model_root: str - :param container_flag: if True and the full path to the mk_template is not specified, - return model_root+"/mkmf/templates/"+mk_template; else if True - and mk_template is the filename, return mk_template - :type container_flag: boolean - - :raises ValueError: Error if the mk_template file does not exist in the generated full path - - :return: Full path to the mkmf mk_template - :rtype: string - - .. note:: model_root must be specified if container_flag is True + Resolves the full path to an ``mkmf`` template file (``.mk``) for either a + bare-metal system or a container image filesystem. + + ``mk_template`` may be either a bare filename (e.g. ``intel.mk``) or an + absolute path (e.g. ``/path/to/intel.mk``). The presence of ``/`` in the + value is used to distinguish the two cases. + + Resolution rules: + + - **Bare-metal** (``container_flag=False``): + + - If ``mk_template`` is a bare filename, the path is constructed as + ``[fre package root]/mkmf/templates/[mk_template]`` using the bundled + ``mkmf`` git submodule. + - If ``mk_template`` is already an absolute path, it is used as-is. + - The resolved path is validated; a ``ValueError`` is raised if the file + does not exist. + + - **Container** (``container_flag=True``): + + - If ``mk_template`` is a bare filename, the path is constructed as + ``[model_root]/mkmf/templates/[mk_template]`` inside the container + image filesystem. ``model_root`` must be provided. + - If ``mk_template`` is already an absolute path, it is used as-is. + - No filesystem validation is performed (the path is inside the container). + + :param mk_template: Bare filename (e.g. ``intel.mk``) or absolute path to the + ``mkmf`` template. Defined as ``mkTemplate`` in + ``platforms.yaml``. + :type mk_template: str + :param container_flag: ``True`` for container builds; ``False`` for bare-metal + builds. Controls both path construction and whether the + resolved path is validated on the host filesystem. + :type container_flag: bool + :param model_root: Root directory for model install files inside the container + (defined as ``modelRoot`` in ``platforms.yaml``). Required + when ``container_flag=True`` and ``mk_template`` is a bare + filename; unused otherwise. + :type model_root: str, optional + + :raises ValueError: If ``container_flag=False`` and the resolved template path + does not exist on the host filesystem. + + :return: Resolved full path to the ``mkmf`` template file. + :rtype: str """ template_path = mk_template diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 9bbff6a91..86056df05 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -1,9 +1,27 @@ ''' -For a bare-metal build: Creates and runs the checkout script to check out source code, - creates the makefile, and creates the compile script to generate - a model executable. -For a container build: Creates the checkout script and makefile, and creates and runs a - dockerfile to generate a singularity image file. +Orchestrates all ``fre make`` sub-commands in the correct order for a complete +build. ``fremake_run`` is the entry point called by ``fre make all``. + +For **bare-metal** platforms, ``fremake_run`` calls in sequence: + +1. ``checkout_create`` — clones all source repositories into + ``[modelRoot]/[experiment]/src/`` +2. ``makefile_create`` — writes the top-level Makefile into + ``[modelRoot]/[experiment]/[platform]-[target]/exec/`` +3. ``compile_create`` — writes ``compile.sh`` to the same ``exec/`` directory + and, if ``--execute`` is set, runs it to build the model executable + +For **container** platforms, ``fremake_run`` calls in sequence: + +1. ``checkout_create`` — stages ``checkout.sh`` under ``tmp/[platform]/`` +2. ``makefile_create`` — stages the Makefile under ``tmp/[platform]/`` +3. ``dockerfile_create`` — writes the Dockerfile and ``createContainer.sh``, + and, if ``--execute`` is set, runs the build script to produce a + Singularity image file (``.sif``) + +When ``--force-checkout`` is specified, previously generated ``compile.sh`` +scripts (bare-metal) or Dockerfiles (container) are removed so they are +regenerated with the updated source configuration. ''' import logging from typing import Optional @@ -25,33 +43,64 @@ def fremake_run(yamlfile:str, platform:str, target:str, verbose: Optional[bool] = None, force_checkout: Optional[bool] = False): """ - Runs all of fre make code + Runs all ``fre make`` sub-commands in sequence to produce a model executable + (bare-metal) or a Singularity container image (container). + + Platforms in ``platform`` are split into bare-metal and container groups. + Both groups share the ``checkout_create`` and ``makefile_create`` steps; they + then diverge — bare-metal proceeds to ``compile_create`` and container proceeds + to ``dockerfile_create``. - :param yamlfile: Model compile YAML file + When ``force_checkout=True``, any previously generated ``compile.sh`` scripts + (bare-metal) or ``Dockerfile`` (container) are deleted before the corresponding + create function is called, ensuring the build reflects the latest YAML + configuration. + + :param yamlfile: Path to the model YAML file (e.g. ``am5.yaml``). The experiment + name is derived by stripping the ``.yaml`` extension. :type yamlfile: str - :param platform: FRE platform; defined in the platforms yaml - If on gaea c5, a FRE platform may look like ncrc5.intel23-classic - :type platform: str - :param target: Predefined FRE targets; options include [prod/debug/repro]-openmp - :type target: str - :param nparallel: Number of concurrent model builds (default 1) + :param platform: One or more FRE platform strings as defined in ``platforms.yaml`` + (e.g. ``ncrc5.intel23`` for a bare-metal GAEA C5 platform). + Repeat the ``-p`` flag to specify multiple platforms. + :type platform: tuple[str] + :param target: One or more ``mkmf`` target strings (e.g. ``prod``, ``debug``, + ``repro``, ``prod-openmp``). Repeat the ``-t`` flag to specify + multiple targets. + :type target: tuple[str] + :param nparallel: Number of ``compile.sh`` scripts to execute concurrently when + ``execute=True`` (bare-metal only). Defaults to ``1`` + (sequential). :type nparallel: int - :param makejobs: Number of jobs to run simultaneously; used for parallelism with make (default 4) + :param makejobs: Number of Makefile recipes to run simultaneously, passed to + ``make -j``. Defaults to ``4``. :type makejobs: int - :param gitjobs: Number of jobs to run simultaneously; used for parallelism with - recursive cloning with checking out source code (default 4) + :param gitjobs: Number of git submodules to clone simultaneously, passed to + ``git clone --jobs`` inside ``checkout.sh``. Defaults to ``4``. :type gitjobs: int - :param no_parallel_checkout: Use this option if you do not want a parallel checkout - :type no_parallel_checkout: bool - :param no_format_transfer: Skip the container format conversion to a .sif file - :type no_format_transfer: bool - :param execute: Run the created compile script or dockerfile to create a model executable - or container - :type execute: bool - :param force_checkout: Re-create the checkout script if changes were made to configurations - :type force_checkout: bool - :param verbose: Increase verbosity output - :type verbose: bool + :param no_parallel_checkout: If ``True``, clone source repositories sequentially + (disables ``&`` backgrounding in ``checkout.sh``). + Defaults to ``False`` (parallel checkout for + bare-metal builds). + :type no_parallel_checkout: bool, optional + :param no_format_transfer: If ``True``, skip the Docker-to-Singularity (``.sif``) + format conversion in ``createContainer.sh`` (container + builds only). Defaults to ``False``. + :type no_format_transfer: bool, optional + :param execute: If ``True``, execute ``checkout.sh`` and ``compile.sh`` (bare-metal) + or ``createContainer.sh`` (container) immediately after generating + them. Defaults to ``False``. + :type execute: bool, optional + :param verbose: If ``True``, set log level to ``DEBUG`` for detailed output from + ``compile_create``. Defaults to ``False`` (``INFO`` level). + :type verbose: bool, optional + :param force_checkout: If ``True``, re-create all build artifacts even if they + already exist. For bare-metal: removes existing + ``compile.sh`` before regenerating. For container: removes + the existing ``Dockerfile`` before regenerating. Defaults + to ``False``. + :type force_checkout: bool, optional + + :raises ValueError: If a specified platform does not exist in ``platforms.yaml``. """ # if verbose: # fre_logger.setLevel(level = logging.DEBUG) From 82e4d9773cb0c64275f5c599cbbb156ee3a1874c Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 14 May 2026 15:56:36 -0400 Subject: [PATCH 3/8] refinement --- fre/make/create_checkout_script.py | 147 ++++++++++++----------------- fre/make/create_compile_script.py | 6 +- 2 files changed, 61 insertions(+), 92 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 7ac8d98b0..d820e090d 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -1,20 +1,13 @@ """ -Create_checkout_script provides methods to generate a ``checkout.sh`` script from a resolved -YAML configuration. The script ``git clone``'s all component source repositories listed under the +Create_checkout_script provides methods to generate a ``checkout.sh`` script from a YAML configuration +file, where ``checkout.sh`` git clone's all component source repositories listed under the ``src`` key of the compile YAML. ``checkout_create`` is the entry point called by ``fre make checkout-script`` and ``fre make all``. Depending on the user-specified platform type (bare-metal or container), -checkout_create then calls either - -- ``baremetal_checkout_write`` for bare-metal platforms: writes ``checkout.sh`` directly on the local platform - into ``[modelRoot]/[experiment]/src/`` and optionally executes the script. -- ``container_checkout_write`` for container platforms: writes ``checkout.sh`` into a local - temporary directory (``tmp/[platform-name]/``). - -Build type defaults: - - Bare-metal build: parallel checkout - - Container build: sequential checkout +checkout_create then calls for baremetal_checkout_write for a bare-metal platform or +``container_checkout_write`` to write ``checkout.sh`` into a local +temporary directory (``tmp/[platform-name]/``) in preparation to build the model in a container. """ import shutil from pathlib import Path @@ -31,41 +24,36 @@ def baremetal_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, jobs: str, parallel_cmd: str, execute: bool): """ - Baremetal_checkout_write generates and optionally executes ``checkout.sh`` for - model compilation on the bare-metal. - - Called by ``checkout_create`` for each bare-metal platform. Reads the ``compile`` - section of the parsed YAML to determine which source repositories to clone, writes - ``checkout.sh`` into ``src_dir``, sets its permissions to ``rwxr--r--`` (0o744), and - optionally runs it immediately. - - Because all bare-metal platforms in a single ``fre make`` invocation share the same - physical source tree, ``checkout_create`` calls this function only once per experiment - and skips subsequent bare-metal platforms if the script already exists (unless - ``--force-checkout`` is set). - - :param model_yaml: Parsed and validated YAML object produced by ``yamlfre.freyaml``, - containing the ``compile`` specifications (source repositories, experiment - name, etc.). + Baremetal_checkout_write generates ``checkout.sh`` and optionally executes the script to + check out the component repositories in preparation for model compilation. + + + Called by ``checkout_create`` for each bare-metal platform, this method + - reads the ``compile`` section of the resolved YAML to determine source repositories to clone, + - writes ``checkout.sh`` into ``src_dir``, + - sets its permissions to ``rwxr--r--`` (0o744), + - optionally executes ``checkout.sh`` afterwards + + :param model_yaml: is the parsed and validated YAML object containing the ``compile`` + specifications (source repositories, experiment name, etc.). :type model_yaml: yamlfre.freyaml - :param src_dir: Absolute path of the source directory where ``checkout.sh`` is written and - where the source repositories will be cloned. Typically - ``[modelRoot]/[experiment]/src``, with ``modelRoot`` defined in - ``platforms.yaml``. + :param src_dir: is the absolute path of the directory where ``checkout.sh`` is + written and where the source repositories will be cloned. Typically, + src_dir = ``[modelRoot]/[experiment]/src`` where ``modelRoot`` is defined in + platforms.yaml. :type src_dir: str - :param jobs: Number of git submodules to fetch simultaneously, passed to ``git clone - --jobs``. Supplied as a string because it is written directly into the - shell script. + :param jobs: is the number of git submodules to fetch simultaneously, passed to ``git clone + --jobs``, if the component repository contain submodules. :type jobs: str - :param parallel_cmd: Shell suffix appended to each ``git clone`` command to control + :param parallel_cmd: is the shell suffix appended to each ``git clone`` command to control concurrency. Pass ``" &"`` to background each clone (parallel checkout) or ``""`` to clone sequentially. :type parallel_cmd: str - :param execute: If ``True``, run ``checkout.sh`` immediately after writing it. + :param execute: is a flag where if ``True``, ``checkout.sh`` is executed immediately after creation. Defaults to ``False``. :type execute: bool """ - fre_checkout = checkout.checkout("checkout.sh", src_dir) + fre_checkout = checkout.checkout("checkout.sh", src_dir) fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(), jobs, parallel_cmd) fre_checkout.finish(model_yaml.compile.getCompileYaml(), parallel_cmd) @@ -80,36 +68,28 @@ def baremetal_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, jobs: st def container_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, tmp_dir: str, jobs: str, parallel_cmd: str): """ - Generates ``checkout.sh`` for a container build. - - Called by ``checkout_create`` for each container platform. Writes ``checkout.sh`` - into a local temporary directory on the host (``tmp/[platform-name]/``). The - Dockerfile produced by ``fre make dockerfile`` subsequently COPYs this script into - the container image filesystem; it is executed inside the container when the image - is launched to clone the model source code into ``src_dir``. - - Unlike the bare-metal variant, this function never executes the script directly — the - container build step (``createContainer.sh``) handles that. Parallel checkout with - ``&`` is also not used for container builds because backgrounded shell processes are - not reliably supported during the container image build stage. - - :param model_yaml: Parsed and validated YAML object produced by ``yamlfre.freyaml``, - containing the ``compile`` specifications (source repositories, experiment - name, etc.). + Container_checkout_write generates ``checkout.sh`` for a container build. + + Called by ``checkout_create`` for each container platform, this method + writes ``checkout.sh`` into a temporary directory on the host (``tmp/[platform-name]/``) + where the script eventually will be COPY-ed to the container image filesystem elsewhere in the + fre make workflow. The script will be executed during the container build to git clone the component + repositories serially. + + :param model_yaml: is the parsed and validated YAML object containing the ``compile`` + specifications (source repositories, experiment name, etc.). :type model_yaml: yamlfre.freyaml - :param src_dir: Source-code path *inside* the running container where repositories will + :param src_dir: is the source-code path inside the running container where repositories will be cloned. Defined by ``modelRoot`` in ``platforms.yaml``. :type src_dir: str - :param tmp_dir: Local temporary directory on the host (outside the container) where + :param tmp_dir: is the local temporary directory on the host (outside the container) where ``checkout.sh`` is staged before being COPYed into the image. Typically ``tmp/[platform-name]``. :type tmp_dir: str - :param jobs: Number of git submodules to fetch simultaneously, passed to ``git clone - --jobs``. Supplied as a string because it is written directly into the - shell script. + :param jobs: is the number of git submodules to fetch simultaneously, passed to ``git clone + --jobs``. :type jobs: str - :param parallel_cmd: Shell suffix for concurrency control. For container builds this - should always be ``""`` (sequential checkout). + :param parallel_cmd: is a flag not used in this method and should be removed. :type parallel_cmd: str """ fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) @@ -121,44 +101,33 @@ def checkout_create(yamlfile: str, platform: tuple, target: tuple, no_parallel_checkout: Optional[bool] = None, njobs: int = 4, execute: Optional[bool] = False, force_checkout: Optional[bool] = False): """ - Entry point for ``fre make checkout-script``. Resolves the YAML configuration and - dispatches to ``baremetal_checkout_write`` or ``container_checkout_write`` for each - platform supplied on the command line. - - Bare-metal platforms share a single source directory - (``[modelRoot]/[experiment]/src``), so the checkout script is written once and - reused across targets. Container platforms each receive their own staging directory - (``tmp/[platform-name]/``). + Checkout_create is the entry point for ``fre make checkout-script``. The method resolves + the YAML configuration and calls ``baremetal_checkout_write`` or ``container_checkout_write`` + for each specified platform. - :param yamlfile: Path to the model YAML file (e.g. ``am5.yaml``). The experiment - name is derived by stripping the ``.yaml`` extension. + :param yamlfile: is the path to the model YAML configuration file (e.g. ``am5.yaml``). + The experiment name is derived by stripping the ``.yaml`` extension. :type yamlfile: str - :param platform: One or more FRE platform strings as defined in ``platforms.yaml``. - Repeat the ``-p`` flag on the command line to specify multiple - platforms. + :param platform: is one or more FRE platform strings as defined in ``platforms.yaml``. :type platform: tuple[str] - :param target: One or more ``mkmf`` target strings (e.g. ``debug``, ``prod``). - Repeat the ``-t`` flag to specify multiple targets. + :param target: is one or more ``mkmf`` target strings (e.g. ``debug``, ``prod``). :type target: tuple[str] - :param no_parallel_checkout: If ``True``, clone repositories sequentially (no ``&`` - backgrounding). Defaults to ``False`` (parallel checkout) - for bare-metal builds; container builds are always - sequential regardless of this flag. + :param no_parallel_checkout: is a flag where if True, git clone component repositories sequentially. + Defaults to ``False`` to enable parallel checkout for bare-metal builds; + Is not used for container builds :type no_parallel_checkout: bool, optional - :param njobs: Number of git submodules to fetch simultaneously, passed to + :param njobs: is the number of git submodules to fetch simultaneously, passed to ``git clone --jobs``. Defaults to ``4``. :type njobs: int :param execute: If ``True``, execute ``checkout.sh`` immediately after writing it (bare-metal only). Defaults to ``False``. :type execute: bool, optional - :param force_checkout: Controls behaviour when ``checkout.sh`` already exists. - - - **Bare-metal**: renames the existing ``src`` directory with a - ``YYYYmmdd.HHMMSS`` timestamp suffix, then writes a fresh - ``checkout.sh`` in a new ``src`` directory. - - **Container**: deletes the existing ``tmp/[platform]/checkout.sh`` - and writes a new one in its place. - + :param force_checkout: is a flag to controls behavior when ``checkout.sh`` already exists. + For bare-metal build, renames the existing ``src`` directory with a + ``YYYYmmdd.HHMMSS`` timestamp suffix, then writes a fresh + ``checkout.sh`` in a new ``src`` directory. + For container build, deletes the existing ``tmp/[platform]/checkout.sh`` + and writes a new one in its place. Defaults to ``False``. :type force_checkout: bool, optional diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index ad641c068..5e8ce8eb8 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -1,5 +1,5 @@ -''' -Retrieves information from the resolved YAML configuration to generate ``compile.sh`` +""" +Create_compile_script retrieves information from the resolved YAML configuration to generate ``compile.sh`` for bare-metal builds. ``compile_create`` is the entry point called by ``fre make compile-script`` and (for bare-metal platforms only) by ``fre make all``. @@ -26,7 +26,7 @@ Container platforms are silently skipped — compilation inside a container image is handled by the Dockerfile generated by ``fre make dockerfile``. -''' +""" import logging from multiprocessing.dummy import Pool From 469f681bde5e1379094da2e0318fdd6294a0e99d Mon Sep 17 00:00:00 2001 From: mlee03 Date: Mon, 18 May 2026 11:48:49 -0400 Subject: [PATCH 4/8] update --- fre/make/create_checkout_script.py | 112 ++++++++++++++--------------- fre/make/create_compile_script.py | 78 ++++++++++---------- 2 files changed, 91 insertions(+), 99 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index d820e090d..bee0b4709 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -1,13 +1,11 @@ """ -Create_checkout_script provides methods to generate a ``checkout.sh`` script from a YAML configuration -file, where ``checkout.sh`` git clone's all component source repositories listed under the -``src`` key of the compile YAML. - -``checkout_create`` is the entry point called by ``fre make checkout-script`` and -``fre make all``. Depending on the user-specified platform type (bare-metal or container), -checkout_create then calls for baremetal_checkout_write for a bare-metal platform or -``container_checkout_write`` to write ``checkout.sh`` into a local -temporary directory (``tmp/[platform-name]/``) in preparation to build the model in a container. +Create_checkout_script provides methods to generate a checkout.sh script from a YAML configuration +file, where checkout.sh git clone's all component source repositories listed under the +src key of the compile YAML. + +The method checkout_create is the entry point called by fre make checkout-script and +fre make all. Checkout_create then calls baremetal_checkout_write for a bare-metal platform or +container_checkout_write for a container platform to write checkout.sh. """ import shutil from pathlib import Path @@ -24,33 +22,31 @@ def baremetal_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, jobs: str, parallel_cmd: str, execute: bool): """ - Baremetal_checkout_write generates ``checkout.sh`` and optionally executes the script to - check out the component repositories in preparation for model compilation. - + Baremetal_checkout_write generates the checkout.sh script and optionally executes the script to + git clone the component repositories in preparation for model compilation. - Called by ``checkout_create`` for each bare-metal platform, this method - - reads the ``compile`` section of the resolved YAML to determine source repositories to clone, - - writes ``checkout.sh`` into ``src_dir``, - - sets its permissions to ``rwxr--r--`` (0o744), - - optionally executes ``checkout.sh`` afterwards + Called by checkout_create for each bare-metal platform, this method + - reads the compile section of the resolved YAML to determine source repositories to clone, + - writes checkout.sh into src_dir, + - optionally executes checkout.sh afterwards - :param model_yaml: is the parsed and validated YAML object containing the ``compile`` + :param model_yaml: is the parsed and validated YAML object containing the compile specifications (source repositories, experiment name, etc.). :type model_yaml: yamlfre.freyaml - :param src_dir: is the absolute path of the directory where ``checkout.sh`` is + :param src_dir: is the absolute path of the directory where checkout.sh will be written and where the source repositories will be cloned. Typically, - src_dir = ``[modelRoot]/[experiment]/src`` where ``modelRoot`` is defined in + src_dir = [modelRoot]/[experiment]/src where modelRoot is defined in platforms.yaml. :type src_dir: str - :param jobs: is the number of git submodules to fetch simultaneously, passed to ``git clone - --jobs``, if the component repository contain submodules. + :param jobs: is the number of git submodules to fetch simultaneously, passed to git clone + --jobs (relevant only if the component repository contain submodules.) :type jobs: str - :param parallel_cmd: is the shell suffix appended to each ``git clone`` command to control - concurrency. Pass ``" &"`` to background each clone (parallel - checkout) or ``""`` to clone sequentially. + :param parallel_cmd: is the shell suffix appended to each git clone command to control + concurrency. Pass " &" to background each clone (parallel + checkout) or "" to clone sequentially. :type parallel_cmd: str - :param execute: is a flag where if ``True``, ``checkout.sh`` is executed immediately after creation. - Defaults to ``False``. + :param execute: is a flag where if True, checkout.sh is executed immediately after creation. + Defaults to False. :type execute: bool """ fre_checkout = checkout.checkout("checkout.sh", src_dir) @@ -68,26 +64,26 @@ def baremetal_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, jobs: st def container_checkout_write(model_yaml: yamlfre.freyaml, src_dir: str, tmp_dir: str, jobs: str, parallel_cmd: str): """ - Container_checkout_write generates ``checkout.sh`` for a container build. + Container_checkout_write generates checkout.sh for a container build. - Called by ``checkout_create`` for each container platform, this method - writes ``checkout.sh`` into a temporary directory on the host (``tmp/[platform-name]/``) - where the script eventually will be COPY-ed to the container image filesystem elsewhere in the - fre make workflow. The script will be executed during the container build to git clone the component + Called by checkout_create for each container platform, this method + writes checkout.sh into a temporary directory on the host (tmp/[platform-name]/) + where the script eventually will be COPY-ed to the container image filesystem. + The script will be executed during the container build to git clone the component repositories serially. - :param model_yaml: is the parsed and validated YAML object containing the ``compile`` + :param model_yaml: is the parsed and validated YAML object containing the compile specifications (source repositories, experiment name, etc.). :type model_yaml: yamlfre.freyaml :param src_dir: is the source-code path inside the running container where repositories will - be cloned. Defined by ``modelRoot`` in ``platforms.yaml``. + be cloned. Defined by modelRoot in platforms.yaml. :type src_dir: str :param tmp_dir: is the local temporary directory on the host (outside the container) where - ``checkout.sh`` is staged before being COPYed into the image. - Typically ``tmp/[platform-name]``. + checkout.sh is staged before being COPYed into the image. + Typically tmp/[platform-name]. :type tmp_dir: str - :param jobs: is the number of git submodules to fetch simultaneously, passed to ``git clone - --jobs``. + :param jobs: is the number of git submodules to fetch simultaneously, passed to git clone + --jobs. :type jobs: str :param parallel_cmd: is a flag not used in this method and should be removed. :type parallel_cmd: str @@ -101,41 +97,41 @@ def checkout_create(yamlfile: str, platform: tuple, target: tuple, no_parallel_checkout: Optional[bool] = None, njobs: int = 4, execute: Optional[bool] = False, force_checkout: Optional[bool] = False): """ - Checkout_create is the entry point for ``fre make checkout-script``. The method resolves - the YAML configuration and calls ``baremetal_checkout_write`` or ``container_checkout_write`` + Checkout_create is the entry point for fre make checkout-script. The method resolves + the YAML configuration and calls baremetal_checkout_write or container_checkout_write for each specified platform. - :param yamlfile: is the path to the model YAML configuration file (e.g. ``am5.yaml``). - The experiment name is derived by stripping the ``.yaml`` extension. + :param yamlfile: is the path to the model YAML configuration file (e.g. am5.yaml). + The experiment name is derived by stripping the .yaml extension. :type yamlfile: str - :param platform: is one or more FRE platform strings as defined in ``platforms.yaml``. + :param platform: is one or more FRE platform strings as defined in platforms.yaml. :type platform: tuple[str] - :param target: is one or more ``mkmf`` target strings (e.g. ``debug``, ``prod``). + :param target: is one or more mkmf target strings (e.g. debug, repro, prod). :type target: tuple[str] :param no_parallel_checkout: is a flag where if True, git clone component repositories sequentially. - Defaults to ``False`` to enable parallel checkout for bare-metal builds; + Defaults to False to enable parallel checkout for bare-metal builds; Is not used for container builds :type no_parallel_checkout: bool, optional :param njobs: is the number of git submodules to fetch simultaneously, passed to - ``git clone --jobs``. Defaults to ``4``. + git clone --jobs. Defaults to 4. :type njobs: int - :param execute: If ``True``, execute ``checkout.sh`` immediately after writing it - (bare-metal only). Defaults to ``False``. + :param execute: If True, execute checkout.sh immediately after writing it + (bare-metal only). Defaults to False. :type execute: bool, optional - :param force_checkout: is a flag to controls behavior when ``checkout.sh`` already exists. - For bare-metal build, renames the existing ``src`` directory with a - ``YYYYmmdd.HHMMSS`` timestamp suffix, then writes a fresh - ``checkout.sh`` in a new ``src`` directory. - For container build, deletes the existing ``tmp/[platform]/checkout.sh`` + :param force_checkout: is a flag to controls behavior when checkout.sh already exists. + For bare-metal build, renames the existing src directory with a + YYYYmmdd.HHMMSS timestamp suffix, then writes a fresh + checkout.sh in a new src directory. + For container build, deletes the existing tmp/[platform]/checkout.sh and writes a new one in its place. - Defaults to ``False``. + Defaults to False. :type force_checkout: bool, optional :raises ValueError: - - If ``njobs`` is passed as a boolean while ``--execute`` is also set (ambiguous - intent — ``njobs`` must be an explicit integer). - - If a specified platform name does not exist in ``platforms.yaml``. - :raises OSError: If ``--execute`` is set and running an existing ``checkout.sh`` + - If njobs is passed as a boolean while --execute is also set (ambiguous + intent — njobs must be an explicit integer). + - If a specified platform name does not exist in platforms.yaml. + :raises OSError: If --execute is set and running an existing checkout.sh returns a non-zero exit code (e.g. the source directory already contains conflicting content). """ diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index 5e8ce8eb8..37a6edb2e 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -1,7 +1,7 @@ """ -Create_compile_script retrieves information from the resolved YAML configuration to generate ``compile.sh`` -for bare-metal builds. ``compile_create`` is the entry point called by -``fre make compile-script`` and (for bare-metal platforms only) by ``fre make all``. +Create_compile_script retrieves information from the resolved YAML configuration to generate compile.sh +for bare-metal builds. The method compile_create is the entry point called by fre make all and +fre make compile-script. The generated script is written to:: @@ -9,23 +9,24 @@ where -- ``modelRoot`` is defined in ``platforms.yaml`` -- ``experiment`` is the basename of the model YAML file (e.g. ``am5`` from ``am5.yaml``) -- ``platform`` and ``target`` are passed via the ``-p`` / ``-t`` CLI options +- modelRoot is defined in platforms.yaml +- experiment is the basename of the model YAML file (e.g. am5 from am5.yaml) +- platform and target are passed via the -p / -t CLI options + to fre make compile-script and fre make all -When executed, ``compile.sh`` +When executed, compile.sh -1. Sets ``src_dir`` (where source code was checked out by ``checkout.sh``) -2. Sets ``bld_dir`` (the ``exec/`` directory where the executable is placed) -3. Sets the path to the ``mkmf`` template (``mkTemplate`` from ``platforms.yaml``) -4. Loads/unloads environment modules to configure the compile environment - (``envSetup`` from ``platforms.yaml``) -5. Calls ``mkmf`` for each model component listed under ``src`` in ``compile.yaml`` - to generate per-component Makefiles -6. Calls ``make`` (with ``-j [makejobs]``) to build the model executable + 1. Sets src_dir (where source code was checked out by checkout.sh) + 2. Sets bld_dir (the exec/ directory where the executable is placed) + 3. Sets the path to the mkmf template (mkTemplate from platforms.yaml) + 4. Loads the correct environment modules to set the compile environment + (see envSetup from platforms.yaml) + 5. Calls mkmf for each model component listed under src in compile.yaml + to generate per-component Makefiles + 6. Calls make (with -j [makejobs]) to build the model executable Container platforms are silently skipped — compilation inside a container image -is handled by the Dockerfile generated by ``fre make dockerfile``. +is handled by the Dockerfile generated by fre make dockerfile. """ import logging @@ -50,46 +51,41 @@ def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejob nparallel: int = 1, execute: Optional[bool] = False, verbose: Optional[bool] = None): """ - Generates ``compile.sh`` for each bare-metal platform/target combination and - optionally executes those scripts to produce a model executable. + Generates the compile.sh script for each bare-metal platform and target combination and + optionally executes compile.sh to compile a model executable. - For each bare-metal platform in ``platform``, a ``compile.sh`` is written to - ``[modelRoot]/[experiment]/[platform]-[target]/exec/``. Container platforms are + For each bare-metal platform in the platform yaml, a compile.sh is written to + [modelRoot]/[experiment]/[platform]-[target]/exec/. Container platforms are silently skipped here; their compilation is handled by the Dockerfile produced by ``dockerfile_create``. - When ``execute=True``, all generated ``compile.sh`` scripts are run concurrently - using a ``multiprocessing.dummy.Pool`` of size ``nparallel``. - - :param yamlfile: Path to the model YAML file (e.g. ``am5.yaml``). The experiment - name is derived by stripping the ``.yaml`` extension. + :param yamlfile: is the path to the model YAML file (e.g. am5.yaml). The experiment + name is derived by stripping the `.yaml` extension. :type yamlfile: str - :param platform: One or more FRE platform strings as defined in ``platforms.yaml`` - (e.g. ``ncrc5.intel23``). Container platforms in this tuple are + :param platform: is one or more FRE platform strings as defined in the platform yaml + (e.g. ncrc5.intel23). Container platforms in this tuple are silently ignored. :type platform: tuple[str] - :param target: One or more ``mkmf`` target strings (e.g. ``prod``, ``debug``, - ``repro``, ``prod-openmp``). One ``compile.sh`` is generated per + :param target: is one or more mkmf target strings (e.g. prod, debug, + repro, prod-openmp). One compile.sh is generated per platform/target pair. :type target: tuple[str] - :param makejobs: Number of Makefile recipes to run simultaneously, passed to - ``make -j``. Defaults to ``4``. + :param makejobs: is the number of Makefile recipes to run simultaneously, passed to + make -j. Defaults to 4. :type makejobs: int - :param nparallel: Number of ``compile.sh`` scripts to run concurrently when - ``execute=True``. Uses a ``multiprocessing.dummy.Pool`` internally. - Defaults to ``1`` (sequential execution). + :param nparallel: is the number of compile.sh scripts to run concurrently when + execute=True. Defaults to 1 (sequential execution). :type nparallel: int - :param execute: If ``True``, run every generated ``compile.sh`` after writing it. - Defaults to ``False``. + :param execute: is a flag where if True, run every generated compile.sh after creation. + Defaults to False. :type execute: bool - :param verbose: If ``True``, set log level to ``DEBUG`` for detailed output. - Defaults to ``False`` (``INFO`` level). + :param verbose: is a flag where if True, set logger level to "DEBUG" for detailed output. + Defaults to False with logger level set to "INFO". :type verbose: bool :raises ValueError: - - If a specified platform does not exist in ``platforms.yaml``. - - If the ``mkTemplate`` path defined in ``platforms.yaml`` cannot be resolved to - an existing file (raised by ``get_mktemplate_path``). + - If a specified platform does not exist in the platforms yaml + - If the mkTemplate path defined in the platforms yaml does not exist """ # Define variables From 7a44906341d5c3d7f5630f0ec47c0893058749db Mon Sep 17 00:00:00 2001 From: mlee03 Date: Mon, 18 May 2026 12:41:33 -0400 Subject: [PATCH 5/8] cleanup create-docker-script --- fre/make/create_docker_script.py | 73 ++++++++++++++------------------ 1 file changed, 31 insertions(+), 42 deletions(-) diff --git a/fre/make/create_docker_script.py b/fre/make/create_docker_script.py index 186576329..eb716f509 100644 --- a/fre/make/create_docker_script.py +++ b/fre/make/create_docker_script.py @@ -1,30 +1,29 @@ -''' -Generates a Dockerfile and an accompanying ``createContainer.sh`` script for -container builds. ``dockerfile_create`` is the entry point called by -``fre make dockerfile`` and (for container platforms only) by ``fre make all``. + +""" +Create_docker_script provides methods to generate a Dockerfile and +an accompanying createContainer.sh script to build container images. +The method dockerfile_create is the entry point called by +fre make dockerfile and by fre make all. The Dockerfile uses a two-stage build: -1. **Build stage** — starts from the base container image (``image`` field in - ``platforms.yaml``), copies in the ``checkout.sh`` and ``Makefile`` that were - staged under ``tmp/[platform]/`` by ``fre make checkout-script`` and - ``fre make makefile``, runs ``mkmf`` and ``make`` to compile the model +1. Build stage — starts from the base container image (image field in + platforms yaml), copies in the checkout.sh and Makefile that were + staged under tmp/[platform]/ by fre make checkout-script and + fre make makefile, runs mkmf and make to compile the model executable. -2. **Runtime stage** — copies the compiled executable and its runtime dependencies - into a leaner second base image (``containerBase2`` in ``platforms.yaml``). +2. Runtime stage — copies the compiled executable and its runtime dependencies + into a leaner second base image (containerBase2 in platforms.yaml). -``createContainer.sh`` builds the Docker image and, unless ``--no-format-transfer`` -is specified, converts it to a Singularity Image File (``.sif``) that can be +createContainer.sh builds the container image and, unless --no-format-transfer +is specified, converts it to a Singularity Image File (.sif) that can be launched with Singularity/Apptainer on HPC systems. -Bare-metal platforms are silently skipped — their compilation is handled by -``compile_create``. - .. note:: Once a container image is built, the source code and compiled executable inside it cannot be modified. To incorporate source changes, re-run - ``fre make all`` (or the individual sub-commands) and rebuild the image. -''' + fre make all (or the individual sub-commands) and rebuild the image. +""" import logging import os @@ -46,41 +45,31 @@ def dockerfile_create(yamlfile: str, platform: tuple[str], target: tuple[str], execute: bool = False, no_format_transfer: bool = False): """ - Generates a Dockerfile and ``createContainer.sh`` for each container platform/target + Dockerfile_create generates a Dockerfile and createContainer.sh for each container platform/target combination and optionally executes the build script to produce a container image. - For each container platform, the following artifacts are written to the current - working directory and ``tmp/[platform-name]/``: - - - ``Dockerfile`` — two-stage build definition (compile stage + runtime stage) - - ``tmp/[platform]/execrunscript.sh`` — script executed when the container is launched - - ``createContainer.sh`` — builds the Docker image and (unless ``--no-format-transfer``) - converts it to a ``.sif`` Singularity image via ``docker2singularity`` or equivalent - - This function relies on ``checkout.sh`` and ``Makefile`` having already been staged - in ``tmp/[platform-name]/`` by ``checkout_create`` and ``makefile_create``. - - Bare-metal platforms in ``platform`` are silently skipped. - - :param yamlfile: Path to the model YAML file (e.g. ``am5.yaml``). The experiment - name is derived by stripping the ``.yaml`` extension. + fre make checkout-script and fre make makefile should be invoked + beforehand to stage the checkout.sh script and Makefile in tmp/[platform-name]/. + + :param yamlfile: is the path to the model YAML file (e.g. am5.yaml). The experiment + name is derived by stripping the .yaml extension. :type yamlfile: str - :param platform: One or more FRE platform strings as defined in ``platforms.yaml``. - Only container platforms (``container: true``) are processed; bare-metal + :param platform: is one or more FRE platform strings as defined in platforms.yaml. + Only container platforms (container: true) are processed; bare-metal platforms are skipped. :type platform: tuple[str] - :param target: One or more ``mkmf`` target strings (e.g. ``prod``, ``debug``). + :param target: is one or more mkmf target strings (e.g. prod, repro, debug). One Dockerfile is generated per platform/target pair. :type target: tuple[str] - :param execute: If ``True``, run ``createContainer.sh`` immediately after generating it - to build the container image. Defaults to ``False``. + :param execute: is a flag where if True, run createContainer.sh immediately after generation + to build the container image. Defaults to False. :type execute: bool - :param no_format_transfer: If ``True``, skip the OCI-to-Singularity (``.sif``) format - conversion step in ``createContainer.sh``. Defaults to - ``False``. + :param no_format_transfer: is a flag where if True, skip the OCI-to-Singularity (.sif) format + conversion step in createContainer.sh. Defaults to + False. :type no_format_transfer: bool - :raises ValueError: If a specified platform does not exist in ``platforms.yaml``. + :raises ValueError: If a specified platform does not exist in platforms.yaml. .. note:: If building the container image on GFDL's RDHPCS GAEA with the Podman container engine, submit a GFDL helpdesk ticket to request Podman access From 2407f94cf64c33d11c4a0bd4b3d27af180260078 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Mon, 18 May 2026 12:55:59 -0400 Subject: [PATCH 6/8] create-makefile-script --- fre/make/create_makefile_script.py | 90 ++++++++++++++---------------- 1 file changed, 43 insertions(+), 47 deletions(-) diff --git a/fre/make/create_makefile_script.py b/fre/make/create_makefile_script.py index b45640cdf..db95f1fdb 100644 --- a/fre/make/create_makefile_script.py +++ b/fre/make/create_makefile_script.py @@ -1,43 +1,41 @@ -''' -Generates the top-level ``Makefile`` used for model compilation. -``makefile_create`` is the entry point called by ``fre make makefile`` and by -``fre make all`` (for both bare-metal and container platforms). +""" +create_makefile_script contains methods to generate the top-level Makefile used for model compilation. +makefile_create is the entry point called by fre make makefile and by fre make all. -Output locations differ by platform type: - -- **Bare-metal**: ``[modelRoot]/[experiment]/[platform]-[target]/exec/Makefile`` -- **Container**: ``./tmp/[platform]/Makefile`` (staged for inclusion in the Dockerfile) +The top-level Makefile is outputted to: +- for bare-metal platforms: [modelRoot]/[experiment]/[platform]-[target]/exec/Makefile +- for container platforms: ./tmp/[platform]/Makefile (staged for COPY during container image build) where -- ``modelRoot`` is defined in ``platforms.yaml`` -- ``experiment`` is the basename of the model YAML file (e.g. ``am5`` from ``am5.yaml``) -- ``platform`` and ``target`` are passed via the ``-p`` / ``-t`` CLI options +- modelRoot is defined in platforms.yaml +- experiment is the basename of the model YAML file (e.g. am5 from am5.yaml) +- platform and target are passed via the -p / -t CLI options to fre make makefile +and fre make all. The generated Makefile -1. Sets ``SRCROOT`` — path to the checked-out source code -2. Sets ``BUILDROOT`` — path to the directory where the executable is placed -3. Sets ``MK_TEMPLATE_PATH`` — path to the ``mkmf`` template file (``mkTemplate`` - from ``platforms.yaml``); templates live in the bundled - `mkmf repository `_ -4. Defines build and link recipes for each component listed under ``src`` in - ``compile.yaml``, following standard Makefile structure:: +1. Sets SRCROOT — path to the checked-out source code +2. Sets BUILDROOT — path to the directory where the executable is placed +3. Sets MK_TEMPLATE_PATH — path to the mkmf template file (mkTemplate + from platforms.yaml); templates can be found in + mkmf repository +4. Defines build and link recipes (as shown below) for each component listed under src in + compile.yaml: [target]: [prerequisites] [recipe] -Additional library flags are handled differently per build type: +Additional library flags are handled as below: -- **Bare-metal**: ``baremetal_linkerflags`` from ``compile.yaml`` are embedded - directly as ``-L`` / ``-l`` flags in the Makefile link line. -- **Container**: ``container_addlibs`` from ``compile.yaml`` are resolved at - build time via a generated ``linkline.sh`` script that locates the library - paths inside the container image. +- Bare-metal: baremetal_linkerflags from compile.yaml are added + directly as -L / -l flags to the Makefile. +- Container: container_addlibs from the compile yaml are resolved at + build time. For more information about the Makefile, see the fre-cli glossary: https://github.com/NOAA-GFDL/fre-cli/blob/main/docs/glossary.rst -''' +""" import logging from pathlib import Path @@ -49,40 +47,38 @@ def makefile_create(yamlfile: str, platform: tuple[str], target: tuple[str]): """ - Generates the top-level ``Makefile`` for each platform/target combination and - writes it to the appropriate output directory. + Makefile_create generates the top-level Makefile for each platform/target combination. - For bare-metal platforms the Makefile is written to - ``[modelRoot]/[experiment]/[platform]-[target]/exec/``. For container platforms - it is staged in ``./tmp/[platform]/`` so the Dockerfile can COPY it into the - image at build time. One Makefile is produced per platform/target pair. + For bare-metal platforms, the Makefile is written to + [modelRoot]/[experiment]/[platform]-[target]/exec/. + + For container platforms, the Makefile is staged in + ./tmp/[platform]/ and is COPYed into the container image at image build time. - :param yamlfile: Path to the model YAML file (e.g. ``am5.yaml``). The experiment - name is derived by stripping the ``.yaml`` extension. + :param yamlfile: is the path to the model YAML file (e.g. am5.yaml). The experiment + name is derived by stripping the .yaml extension. :type yamlfile: str - :param platform: One or more FRE platform strings as defined in ``platforms.yaml`` - (e.g. ``ncrc5.intel23``). Both bare-metal and container platforms + :param platform: is one or more FRE platform strings as defined in platforms.yaml + (e.g. ncrc5.intel23). Both bare-metal and container platforms are supported. :type platform: tuple[str] - :param target: One or more ``mkmf`` target strings (e.g. ``prod``, ``debug``, - ``repro``, ``prod-openmp``). One Makefile is generated per + :param target: is one or more mkmf target strings (e.g. prod, debug, + repro, prod-openmp). One Makefile is generated per platform/target pair. :type target: tuple[str] - :raises ValueError: If a specified platform does not exist in ``platforms.yaml``. + :raises ValueError: If a specified platform does not exist in platforms.yaml. .. note:: Additional library dependencies are handled differently per build type: - - **Container build** — list library names under ``container_addlibs`` in - ``compile.yaml`` (e.g. ``container_addlibs: ['darcy']``). A ``linkline.sh`` - script is generated alongside the Makefile; it resolves the ``-L`` / ``-l`` - flags at container-image build time by locating the libraries inside the - container. + - Container build — list library names under container_addlibs in + compile.yaml (e.g. container_addlibs: ['darcy']). A linkline.sh + script, generated alongside the Makefile, locates the libraries inside + the container and resolves the -L / -l flags at container image build time. - - **Bare-metal build** — provide full linker flags under - ``baremetal_linkerflags`` in ``compile.yaml`` - (e.g. ``baremetal_linkerflags: ["-L/path/to/libs -ldarcy"]``). These flags - are embedded directly in the Makefile link line. + - Bare-metal build — provide full linker flags under baremetal_linkerflags in compile.yaml + (e.g. baremetal_linkerflags: ["-L/path/to/libs -ldarcy"]). These flags + are added directly to the Makefile. """ name = yamlfile.split(".")[0] From 631680a399778834c1a464af1b366b5092114606 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Mon, 18 May 2026 13:02:53 -0400 Subject: [PATCH 7/8] make_helpers.py --- fre/make/make_helpers.py | 52 ++++++++++------------------------------ 1 file changed, 13 insertions(+), 39 deletions(-) diff --git a/fre/make/make_helpers.py b/fre/make/make_helpers.py index 0440ac02c..d321cb143 100644 --- a/fre/make/make_helpers.py +++ b/fre/make/make_helpers.py @@ -1,9 +1,7 @@ -''' -Helper/utility functions shared across the ``fre make`` sub-commands. - -Functions here are imported by ``create_compile_script``, ``create_makefile_script``, -and ``create_docker_script`` to avoid duplicating path-resolution logic. -''' +""" +make_helpers contain helper/utility functions +create_compile_script, create_makefile_script, and create_docker_script. +""" import logging from pathlib import Path @@ -11,50 +9,26 @@ def get_mktemplate_path(mk_template: str, container_flag: bool, model_root: str = None) -> str: """ - Resolves the full path to an ``mkmf`` template file (``.mk``) for either a + get_mktemplate_path resolves the full path to an mkmf template file (.mk) for either a bare-metal system or a container image filesystem. - ``mk_template`` may be either a bare filename (e.g. ``intel.mk``) or an - absolute path (e.g. ``/path/to/intel.mk``). The presence of ``/`` in the - value is used to distinguish the two cases. - - Resolution rules: - - - **Bare-metal** (``container_flag=False``): - - - If ``mk_template`` is a bare filename, the path is constructed as - ``[fre package root]/mkmf/templates/[mk_template]`` using the bundled - ``mkmf`` git submodule. - - If ``mk_template`` is already an absolute path, it is used as-is. - - The resolved path is validated; a ``ValueError`` is raised if the file - does not exist. - - - **Container** (``container_flag=True``): - - - If ``mk_template`` is a bare filename, the path is constructed as - ``[model_root]/mkmf/templates/[mk_template]`` inside the container - image filesystem. ``model_root`` must be provided. - - If ``mk_template`` is already an absolute path, it is used as-is. - - No filesystem validation is performed (the path is inside the container). + mk_template may be a bare filename (e.g. intel.mk) or an absolute path (e.g. /path/to/intel.mk). - :param mk_template: Bare filename (e.g. ``intel.mk``) or absolute path to the - ``mkmf`` template. Defined as ``mkTemplate`` in - ``platforms.yaml``. + :param mk_template: is the bare filename (e.g. intel.mk) or absolute path to the + mkmf template. Defined as mkTemplate in platforms yaml. :type mk_template: str - :param container_flag: ``True`` for container builds; ``False`` for bare-metal - builds. Controls both path construction and whether the - resolved path is validated on the host filesystem. + :param container_flag: is a flag where True for container builds and False for bare-metal builds. :type container_flag: bool :param model_root: Root directory for model install files inside the container - (defined as ``modelRoot`` in ``platforms.yaml``). Required - when ``container_flag=True`` and ``mk_template`` is a bare + (defined as modelRoot in platforms yaml). Required + when container_flag=True and mk_template is a bare filename; unused otherwise. :type model_root: str, optional - :raises ValueError: If ``container_flag=False`` and the resolved template path + :raises ValueError: If container_flag=False and the resolved template path does not exist on the host filesystem. - :return: Resolved full path to the ``mkmf`` template file. + :return: a resolved full path to the mkmf template file. :rtype: str """ From 0f0d4ebe85bb99c9852f287176d332d6627badc5 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Mon, 18 May 2026 13:16:16 -0400 Subject: [PATCH 8/8] additional cleanup --- fre/make/fremake.py | 10 +-- fre/make/run_fremake_script.py | 118 ++++++++++++++------------------- 2 files changed, 54 insertions(+), 74 deletions(-) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index fc229aab8..550b1f5ef 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -1,5 +1,5 @@ """ -Module defining the `click `_ interfaces for the following **fre make** subcommands: +fremake is the module defining the `click `_ interfaces for the following **fre make** subcommands: * fre make checkout-script [ARGS]: writes a script that will clone (checkout) the model code from respective git repositories @@ -116,7 +116,7 @@ def make_cli(): def all(yamlfile, platform, target, nparallel, makejobs, gitjobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout): """ - - Perform all fre make functions; for baremetal platforms: create checkout script, makefile, and compile scripts; + fre make all perform all fre make functions; for baremetal platforms: create checkout script, makefile, and compile scripts; for container platforms: create checkout script, makefile, Dockerfile, and createContainer script """ run_fremake_script.fremake_run( @@ -184,7 +184,7 @@ def checkout_script(yamlfile, platform, target, no_parallel_checkout, gitjobs, e help = _TARGET_OPT_HELP, required = True) def makefile(yamlfile, platform, target): - """ - Write the top-level Makefile for model compilation. For bare-metal platforms, + """ fre make makefile write the top-level Makefile for model compilation. For bare-metal platforms, the Makefile is written to ``[modelRoot]/[experiment]/[platform]-[target]/exec/``; for container platforms it is staged under ``tmp/[platform]/``. """ create_makefile_script.makefile_create(yamlfile, platform, target) @@ -226,7 +226,7 @@ def makefile(yamlfile, platform, target): is_flag = True, help = _VERBOSE_OPT_HELP) def compile_script(yamlfile, platform, target, makejobs, nparallel, execute, verbose): - """ - Write compile.sh for bare-metal platforms. The script configures the compile + """ fre make compile-script write compile.sh for bare-metal platforms. The script configures the compile environment (loads modules), calls mkmf to generate per-component Makefiles, and runs make to build the model executable. Written to ``[modelRoot]/[experiment]/[platform]-[target]/exec/``. """ @@ -260,7 +260,7 @@ def compile_script(yamlfile, platform, target, makejobs, nparallel, execute, ver help = """Execute the createContainer script immediately following its generation. The default behavior is to generate the script, but not execute.""") def dockerfile(yamlfile, platform, target, no_format_transfer, execute): - """ - Write the Dockerfile and createContainer.sh for container platforms. The + """ fre make dockerfile write the Dockerfile and createContainer.sh for container platforms. The Dockerfile defines a two-stage build (compile + runtime); createContainer.sh builds the Docker image and converts it to a Singularity Image File (.sif) unless --no-format-transfer is specified. """ diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 86056df05..fc9920d25 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -1,28 +1,24 @@ -''' -Orchestrates all ``fre make`` sub-commands in the correct order for a complete -build. ``fremake_run`` is the entry point called by ``fre make all``. - -For **bare-metal** platforms, ``fremake_run`` calls in sequence: - -1. ``checkout_create`` — clones all source repositories into - ``[modelRoot]/[experiment]/src/`` -2. ``makefile_create`` — writes the top-level Makefile into - ``[modelRoot]/[experiment]/[platform]-[target]/exec/`` -3. ``compile_create`` — writes ``compile.sh`` to the same ``exec/`` directory - and, if ``--execute`` is set, runs it to build the model executable - -For **container** platforms, ``fremake_run`` calls in sequence: - -1. ``checkout_create`` — stages ``checkout.sh`` under ``tmp/[platform]/`` -2. ``makefile_create`` — stages the Makefile under ``tmp/[platform]/`` -3. ``dockerfile_create`` — writes the Dockerfile and ``createContainer.sh``, - and, if ``--execute`` is set, runs the build script to produce a - Singularity image file (``.sif``) - -When ``--force-checkout`` is specified, previously generated ``compile.sh`` -scripts (bare-metal) or Dockerfiles (container) are removed so they are -regenerated with the updated source configuration. -''' +""" +run_fremake_script contains the main orchestration function, fremake_run, which executes all +fre make sub-commands in the correct order to compile the model src code into an executable. +fremake_run is the entry point called by fre make all. + +For bare-metal platforms, fremake_run calls: + +1. checkout_create — creates and executes checkout.sh to clone all source repositories + into [modelRoot]/[experiment]/src/ +2. makefile_create — creates the top-level Makefile into + [modelRoot]/[experiment]/[platform]-[target]/exec/ +3. compile_create — creates compile.sh to the + [modelRoot]/[experiment]/[platform]-[target]/exec/ and executes the script + +For container platforms, fremake_run calls: +1. checkout_create — stages checkout.sh under tmp/[platform]/ +2. makefile_create — stages the Makefile under tmp/[platform]/ +3. dockerfile_create — creates the Dockerfile and createContainer.sh, + and, if --execute is set, runs the build script to produce a + Singularity image file (.sif) for each platform/target combination. +""" import logging from typing import Optional from pathlib import Path @@ -43,64 +39,48 @@ def fremake_run(yamlfile:str, platform:str, target:str, verbose: Optional[bool] = None, force_checkout: Optional[bool] = False): """ - Runs all ``fre make`` sub-commands in sequence to produce a model executable - (bare-metal) or a Singularity container image (container). + fremake_run runs all fre make sub-commands in sequence to produce a model executable + (bare-metal) or a container image. - Platforms in ``platform`` are split into bare-metal and container groups. - Both groups share the ``checkout_create`` and ``makefile_create`` steps; they - then diverge — bare-metal proceeds to ``compile_create`` and container proceeds - to ``dockerfile_create``. - - When ``force_checkout=True``, any previously generated ``compile.sh`` scripts - (bare-metal) or ``Dockerfile`` (container) are deleted before the corresponding - create function is called, ensuring the build reflects the latest YAML - configuration. - - :param yamlfile: Path to the model YAML file (e.g. ``am5.yaml``). The experiment - name is derived by stripping the ``.yaml`` extension. + :param yamlfile: is the path to the model YAML file (e.g. am5.yaml). The experiment + name is derived by stripping the .yaml extension. :type yamlfile: str - :param platform: One or more FRE platform strings as defined in ``platforms.yaml`` - (e.g. ``ncrc5.intel23`` for a bare-metal GAEA C5 platform). - Repeat the ``-p`` flag to specify multiple platforms. + :param platform: is one or more FRE platform strings as defined in platforms.yaml + (e.g. ncrc5.intel23 for a bare-metal GAEA C5 platform). :type platform: tuple[str] - :param target: One or more ``mkmf`` target strings (e.g. ``prod``, ``debug``, - ``repro``, ``prod-openmp``). Repeat the ``-t`` flag to specify - multiple targets. + :param target: is one or more mkmf target strings (e.g. prod, repro debug) :type target: tuple[str] - :param nparallel: Number of ``compile.sh`` scripts to execute concurrently when - ``execute=True`` (bare-metal only). Defaults to ``1`` + :param nparallel: is the number of compile.sh scripts to execute concurrently when + execute=True (bare-metal only). Defaults to 1 (sequential). :type nparallel: int - :param makejobs: Number of Makefile recipes to run simultaneously, passed to - ``make -j``. Defaults to ``4``. + :param makejobs: is the number of Makefile recipes to run simultaneously, passed to + make -j. Defaults to 4. :type makejobs: int - :param gitjobs: Number of git submodules to clone simultaneously, passed to - ``git clone --jobs`` inside ``checkout.sh``. Defaults to ``4``. + :param gitjobs: is the number of git submodules to clone simultaneously, passed to + git clone --jobs inside checkout.sh. Defaults to 4. :type gitjobs: int - :param no_parallel_checkout: If ``True``, clone source repositories sequentially - (disables ``&`` backgrounding in ``checkout.sh``). - Defaults to ``False`` (parallel checkout for - bare-metal builds). + :param no_parallel_checkout: is a flag where if True, clone source repositories sequentially + Defaults to False to enable parallel checkout for bare-metal builds. + Only sequential checkouts are available for container builds. :type no_parallel_checkout: bool, optional - :param no_format_transfer: If ``True``, skip the Docker-to-Singularity (``.sif``) - format conversion in ``createContainer.sh`` (container - builds only). Defaults to ``False``. + :param no_format_transfer: is a flag where if True, skip the Docker-to-Singularity (.sif) + format conversion in createContainer.sh. Defaults to False. :type no_format_transfer: bool, optional - :param execute: If ``True``, execute ``checkout.sh`` and ``compile.sh`` (bare-metal) - or ``createContainer.sh`` (container) immediately after generating - them. Defaults to ``False``. + :param execute: is a flag where if True, execute checkout.sh and compile.sh (bare-metal) + or createContainer.sh (container) immediately after generation. + Defaults to False. :type execute: bool, optional - :param verbose: If ``True``, set log level to ``DEBUG`` for detailed output from - ``compile_create``. Defaults to ``False`` (``INFO`` level). + :param verbose: is a flag where if True, set logger level to DEBUG for detailed output from + compile_create. Defaults to False and sets logger level to INFO. :type verbose: bool, optional - :param force_checkout: If ``True``, re-create all build artifacts even if they - already exist. For bare-metal: removes existing - ``compile.sh`` before regenerating. For container: removes - the existing ``Dockerfile`` before regenerating. Defaults - to ``False``. + :param force_checkout: is a flag where if True, for bare-metal: remove existing + compile.sh before regenerating. For container: remove + the existing Dockerfile before regenerating. Defaults + to False. :type force_checkout: bool, optional - :raises ValueError: If a specified platform does not exist in ``platforms.yaml``. + :raises ValueError: If a specified platform does not exist in platforms.yaml. """ # if verbose: # fre_logger.setLevel(level = logging.DEBUG)