diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000..45c42e3e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,90 @@ +name: Bug report +description: Report something that is incorrect or broken +type: "bug" +labels: ["bug"] + +body: + - type: markdown + attributes: + value: | + Thanks for helping improve EAGLE. + + Before opening a bug report, please search the existing issues to check whether the problem has already been reported. + Enhancement requests should be opened as an Enhancement Discussion instead of an issue. + + - type: textarea + id: current_behavior + attributes: + label: What is wrong? + description: Describe the incorrect or broken behavior. + placeholder: Include short log snippets here. For long logs, create a GitHub gist and link it. + validations: + required: true + + - type: textarea + id: expected_behavior + attributes: + label: What should have happened? + description: Describe the behavior you expected. + validations: + required: true + + - type: dropdown + id: area + attributes: + label: Area affected + multiple: true + options: + - Data prep / UFS2ARCO + - Training + - Inference + - Verification / WXVX + - Visualization / plotting + - Configuration + - Documentation + - CI/CD or tooling + - Other + validations: + required: true + + - type: dropdown + id: system + attributes: + label: System or environment + multiple: true + options: + - Ursa + - Cloud + - Local workstation + - Other HPC + - Unknown / not applicable + validations: + required: true + + - type: input + id: version + attributes: + label: EAGLE version, branch, or commit + placeholder: main, v1.1.0, or a commit SHA + validations: + required: true + + - type: textarea + id: reproduction_steps + attributes: + label: Steps to reproduce + description: Provide enough detail for another developer to reproduce the problem. + placeholder: | + 1. + 2. + 3. + validations: + required: true + + - type: textarea + id: additional_context + attributes: + label: Additional context + description: Add links, logs, screenshots, or related issues. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..2bcb69ef --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Enhancement requests + url: https://github.com/NOAA-EPIC/EAGLE/discussions/new?category=enhancement-discussion + about: Please propose new capabilities in an Enhancement Discussion instead of opening an issue. + - name: Questions and answers + url: https://github.com/NOAA-EPIC/EAGLE/discussions/categories/q-a + about: Please use GitHub Discussions for usage questions and general help. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 42ad9110..1ab7a715 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -21,18 +21,22 @@ Refs NOAA-EPIC/repo#5678 ## Type of change: - [ ] Bug fix -- [ ] New feature +- [ ] Enhancement / new feature - [ ] Refactor / cleanup - [ ] Documentation - [ ] CI/CD or tooling +- [ ] Subcomponent update - [ ] Other: ## Area(s) affected - + - [ ] nested_eagle workflow +- [ ] Training +- [ ] Inference - [ ] Verification / evaluation (via WXVX) - [ ] Data prep / UFS2ARCO - [ ] Config (YAML) @@ -51,6 +55,7 @@ Delete options that are not applicable. - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings - [ ] I have made corresponding changes to the system documentation if necessary +- [ ] I checked whether this PR requires subcomponent PRs and completed the subcomponent checklist below ## Testing / Verification: + +* EAGLE - * UFS2ARCO - * WXVX (verification) - @@ -92,8 +96,8 @@ Example: * WXVX: NOAA-EPIC/WXVX#33 Delete sections that are not needed. --> +- [ ] No subcomponent PRs are required for this change. +- [ ] Required subcomponent PRs are linked below. * EAGLE: NOAA-EPIC/EAGLE# * UFS2ARCO: NOAA-PSL/UFS2ARCO# * WXVX (verification): NOAA-EPIC/WXVX# -* None - diff --git a/.github/scripts/build b/.github/scripts/build index 68d2b3d1..8143f9ad 100755 --- a/.github/scripts/build +++ b/.github/scripts/build @@ -3,5 +3,7 @@ # shellcheck disable=1091 set -euo pipefail +export EAGLE_CLEAN_CONDA_PKGS=1 +export PIP_NO_CACHE_DIR=1 time make devenv cudascript=dummy . conda/etc/profile.d/conda.sh diff --git a/.github/scripts/fsclean b/.github/scripts/fsclean index 082eb414..fb16bf64 100755 --- a/.github/scripts/fsclean +++ b/.github/scripts/fsclean @@ -2,7 +2,7 @@ # shellcheck disable=2068,2086 -GIGSREQD=57 +GIGSREQD=47 gigsfree() { df . -BG --output=avail | tail -n1 | sed 's/[^0-9]//g'; } msg() { echo $GIGSREQD GB NEEDED, $1 GB AVAILABLE: $2; } @@ -38,12 +38,12 @@ else ) dirs=( /home/runner/{.cargo,.dotnet,.rustup} - /opt/hostedtoolcache/{Ruby,PyPy,node,go,CodeQL} - /opt/{az,google,microsoft,pipx} + /opt/hostedtoolcache/{Python,Ruby,PyPy,node,go,CodeQL} + /opt/{az,ghc,google,microsoft,pipx} /usr/lib/{firefox,google-cloud-sdk,jvm,snapd} /usr/local/lib/android - /usr/local/share/{chromium,powershell} - /usr/local/{aws-*,julia*} + /usr/local/share/{boost,chromium,powershell,vcpkg} + /usr/local/{.ghcup,aws-*,julia*} /usr/share/{az_*,dotnet,gradle-*,kotlinc,miniconda,swift} ) ( diff --git a/README.md b/README.md index 91c05c83..d0c97979 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,14 @@ To learn about EAGLE and how to use the provided workflows, please see our [docu ## Collaboration - If you encounter a problem using EAGLE that appears to be a bug, please open an [issue](https://github.com/NOAA-EPIC/EAGLE/issues) with us. +- For enhancement requests, please start an [Enhancement Discussion](https://github.com/NOAA-EPIC/EAGLE/discussions/new?category=enhancement-discussion). - For free-form sharing of ideas, questions, tips and tricks, etc., please start or join a [discussion](https://github.com/NOAA-EPIC/EAGLE/discussions). - To contribute to the codebase, please see our [docs](https://epic-eagle.readthedocs.io/en/latest/contributing.html). +## Citation + +The Earth Prediction Innovation Center (EPIC) and the NOAA AI for Numerical Weather Prediction (AI4NWP) Working Group. (2026). The nested-EAGLE (Experimental AI Global and Limited-area Ensemble forecast system) (v1.0.0). Zenodo. https://doi.org/10.5281/zenodo.19672026 + ## Acknowledgments ufs2arco: Tim Smith (NOAA Physical Sciences Laboratory) diff --git a/docs/community/contributing.rst b/docs/community/contributing.rst index 211f7325..a37e38f6 100644 --- a/docs/community/contributing.rst +++ b/docs/community/contributing.rst @@ -9,18 +9,20 @@ Contributing Development ------------------------------------------------------------------------------ -First, clone the main :term:`EAGLE` repository and create a branch on the machine where you will -do the development work. Contributions should be submitted as pull requests from a -branch separate from the main branch. +First, fork the main :term:`EAGLE` repository into your GitHub account, then +clone your fork on the machine where you will do the development work. External +contributions should be submitted as pull requests from a branch in your fork. .. code-block:: text - git clone https://github.com/NOAA-EPIC/EAGLE.git + git clone https://github.com//EAGLE.git cd EAGLE + git remote add upstream https://github.com/NOAA-EPIC/EAGLE.git + git fetch upstream .. code-block:: text - git checkout -b + git checkout -b upstream/main To build the runtime virtual environments **and** install all required development packages in each environment: @@ -100,7 +102,8 @@ Contributions to the ``EAGLE`` project are made through a fork and pull request #. Create or identify a GitHub issue to document the proposed change. #. Fork the `EAGLE repository `_ into your personal GitHub account. #. Clone your fork onto your development system. -#. Create a branch in your clone for the change. All development should take place on a branch, not on ``main``. +#. Add the upstream remote, if your clone does not already have one: ``git remote add upstream https://github.com/NOAA-EPIC/EAGLE.git``. +#. Create a branch in your clone for the change. All development should take place on a branch in your fork, not on ``main``. #. Make, commit, and push your changes to that branch in your fork. #. Open a pull request to merge your changes into the upstream repository. #. When merging your PR, select "Squash and merge" unless there's a reason to preserve all individual commits from the feature branch. @@ -114,13 +117,13 @@ For future contributions, keep your fork current by syncing it with the upstream Development and Testing Process ============================================================================== -#. **Branch and develop:** Work on a branch dedicated to a single change or closely related set of changes. +#. **Branch and develop:** Work on a fork branch dedicated to a single change or closely related set of changes. #. **Build the development environment:** Use the commands in the `Development` section above to create the required environments and install development tools. #. **Format code/data and run code-quality checks:** Before opening a pull request, format code and data and perform code-quality checks by running ``make format && make test``. #. **Update documentation:** If your change affects workflow behavior, capabilities, or developer setup, update the appropriate RST files in ``docs/``. #. **Open the pull request:** Push your branch to GitHub and open a pull request against the upstream repository. -When your changes are ready, commit them on your feature branch and push the branch to GitHub: +When your changes are ready, commit them on your feature branch and push the branch to your fork: .. code-block:: bash @@ -128,115 +131,39 @@ When your changes are ready, commit them on your feature branch and push the bra git commit -m "" git push origin -Then open a pull request through this repository's `PR page `_. For general guidance on creating pull requests, see this `GitHub documentation `_. +Then open a pull request from your fork branch to the upstream ``NOAA-EPIC/EAGLE`` repository through this repository's `PR page `_. For general guidance on creating pull requests, see this `GitHub documentation `_. .. _PRTemplate: PR Template ============================================================================== -Use the following pull request template when opening a PR: - -.. code-block:: md - - - - ## Description: - - - - - ## Type of change: - - - [ ] Bug fix - - [ ] New feature - - [ ] Refactor / cleanup - - [ ] Documentation - - [ ] CI/CD or tooling - - [ ] Other: - - ## Area(s) affected - - - [ ] nested_eagle workflow - - [ ] Verification / evaluation (via WXVX) - - [ ] Data prep / UFS2ARCO - - [ ] Config (YAML) - - [ ] Plotting / post-processing - - [ ] Infrastructure / Slurm scripts - - [ ] Other: - - ## Commit Requirements: - - - [ ] This PR addresses a relevant NOAA-EPIC/EAGLE issue (if not, create an issue); a person responsible for submitting the update has been assigned to the issue (link issue) - - [ ] Fill out all sections of this template. - - [ ] I have performed a self-review of my own code - - [ ] My changes generate no new warnings - - [ ] I have made corresponding changes to the system documentation if necessary - - ## Testing / Verification: - - - [ ] I ran and/or verified the changes (or provided a test plan) - - Commands/config used: - - - - Evidence (logs, key output paths, screenshots if relevant): - - - - ## Runtime Environment: - - - System/HPC: - - Account/role: - - Conda env: - - Key versions (optional): - - `python --version`: - - `wxvx --version` (if applicable): - - MET version (if applicable): - - ## Commit Message: - - * UFS2ARCO - - * WXVX (verification) - - - ## Subcomponent Pull Requests: - - * EAGLE: NOAA-EPIC/EAGLE# - * UFS2ARCO: NOAA-PSL/UFS2ARCO# - * WXVX (verification): NOAA-EPIC/WXVX# - * None +GitHub will automatically populate the PR description with the repository's +`pull request template `_. +Complete the checklist, including the subcomponent PR check, before requesting +review. + +.. _ForkPRCI: + +CI for Fork-Based Pull Requests +============================================================================== + +Pull requests from forks use the repository's normal GitHub Actions checks, +subject to GitHub's maintainer approval flow for fork-based contributions. +Maintainers should inspect the proposed changes before approving workflow runs, +especially when a PR changes files under ``.github/workflows/``. + +The Ursa end-to-end workflow is intentionally opt-in. After a maintainer has +reviewed the PR and is comfortable running it on the self-hosted Ursa runner, +they can add the ``eagle-ursa`` label to trigger the label-gated workflow. If +GitHub marks the workflow as awaiting approval because the PR came from a fork, +a maintainer with write access must approve the workflow run from the PR checks +or Actions page. + +If repository or organization settings prevent the Ursa workflow from running +directly from a fork, a maintainer can create a branch in the upstream +``NOAA-EPIC/EAGLE`` repository from the contributor's changes and run the +label-gated workflow from that branch. .. _Docs: diff --git a/docs/community/support.rst b/docs/community/support.rst index 15f0806a..5bf31ee1 100644 --- a/docs/community/support.rst +++ b/docs/community/support.rst @@ -9,10 +9,15 @@ Support Issues ------------------------------------------------------------------------------ -If you encounter a problem, please +If you encounter a problem that appears to be a bug, please `open an issue `_ and include enough detail for the team to understand and reproduce the problem. +An enhancement request should be opened as an +`Enhancement Discussion `_ +instead of an issue. This gives maintainers and users space to discuss scope, +design options, and acceptance criteria before implementation work begins. + .. _QA: Questions and Answers diff --git a/docs/eagle_models/nested_eagle.rst b/docs/eagle_models/nested_eagle.rst index 4e5f2984..97f23789 100644 --- a/docs/eagle_models/nested_eagle.rst +++ b/docs/eagle_models/nested_eagle.rst @@ -77,16 +77,22 @@ starting point for NRT runs of nested-EAGLE. To run NRT: #. Check out the ``nested-eagle-v1`` branch. -#. Follow the NRT workflow in :ref:`Quickstart` (see "Run inference in near-real-time (NRT)"). + + .. code-block:: bash + + git checkout nested-eagle-v1 + +#. Follow the :ref:`NRT workflow `. + #. EPIC hosts the checkpoint on Azure. To download the checkpoint to your machine, simply run: -.. code-block:: bash + .. code-block:: bash - wget -O inference-last.ckpt https://eaglecheckpoints.blob.core.windows.net/eagle-checkpoints/nested-eagle/inference-last.ckpt + wget -O inference-last.ckpt https://eaglecheckpoints.blob.core.windows.net/eagle-checkpoints/nested-eagle/inference-last.ckpt Before running ``make realize``, update: - * ``app.base`` to the absolute path of your local ``src/`` directory + * ``app.base`` to the absolute path of your local repository root * ``inference.anemoi.checkpoint_dir`` to the checkpoint you downloaded from Azure (inference-last.ckpt) After those updates, realize the config and continue with the remaining quickstart diff --git a/docs/ml_pipeline_overview/data_creation.rst b/docs/ml_pipeline_overview/data_creation.rst index b0822333..cd95befe 100644 --- a/docs/ml_pipeline_overview/data_creation.rst +++ b/docs/ml_pipeline_overview/data_creation.rst @@ -140,6 +140,25 @@ in the ``source`` block of ``recipe.yaml``. See the `ufs2arco documentation `_ for the supported variables and configuration details. +Using a Different Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +EAGLE expects training and inference data to be available in a format compatible +with the Anemoi data stack. There are two common ways to use a different +dataset: + +* Generate a new Zarr dataset with ``ufs2arco`` by updating the appropriate + ``zarrs..zarr.ufs2arco`` config block. In most cases this means + changing the source archive, date range, variables, levels, transforms, and + target Zarr path. +* Point EAGLE at an existing compatible Zarr dataset by updating the training + dataloader dataset path and the inference input dataset settings in the + composed EAGLE config. + +When changing datasets, make sure the variable names, vertical levels, grid, +time frequency, forcing fields, and normalization/statistics settings are +consistent with the model configuration and any checkpoint used for inference. + .. _MPIUsage: MPI Usage diff --git a/docs/ml_pipeline_overview/inference.rst b/docs/ml_pipeline_overview/inference.rst index a2134603..88e0739c 100644 --- a/docs/ml_pipeline_overview/inference.rst +++ b/docs/ml_pipeline_overview/inference.rst @@ -33,3 +33,26 @@ simplified YAML configuration looks like: output: 2026-01-01T00.240hr.nc This simple setup will successfully execute inference. + +Using the Official Nested-EAGLE Checkpoint +-------------------------------------------------- + +The official pretrained nested-EAGLE checkpoint can be downloaded with: + +.. code-block:: bash + + wget -O inference-last.ckpt https://eaglecheckpoints.blob.core.windows.net/eagle-checkpoints/nested-eagle/inference-last.ckpt + +To use this checkpoint for inference, set ``inference.anemoi.checkpoint_path`` +to the downloaded file: + +.. code-block:: yaml + + inference: + anemoi: + checkpoint_path: /path/to/inference-last.ckpt + +If ``inference.checkpoint_dir`` is also present, the ``Inference`` driver will +use the latest checkpoint from that directory instead of the explicit +``checkpoint_path``. Remove or override ``checkpoint_dir`` when you want to +force use of a specific pretrained checkpoint. diff --git a/docs/reference/faq.rst b/docs/reference/faq.rst index d9cb8635..6d65f6ed 100644 --- a/docs/reference/faq.rst +++ b/docs/reference/faq.rst @@ -18,3 +18,40 @@ What platforms are currently supported? =============================================================================== At present, :term:`Ursa` is the only supported platform for EAGLE, but support for additional platforms is coming soon. + +Can I generate Anemoi training plots only for the nested HRRR domain? +=============================================================================== + +Yes. Users can define multiple bounding boxes to specify subset regions, such +as :term:`CONUS` or Europe, by adding focus-area configuration to the +``PlotSample`` callback in the EAGLE pipeline ``base.yaml``. This configuration +generates plots that exclude the surrounding global domain. Define named boxes +under ``focus_areas`` and reference the selected box from ``PlotSample`` with +``focus_area``. + +Although users can modify ``base.yaml`` directly for reference, it is +recommended that they instead add the configuration to the ``eagle.yaml`` file +created during the Quickstart workflow. Changes made directly to ``base.yaml`` +may be overwritten during configuration composition, whereas updates made in +``eagle.yaml`` are preserved. + +For example, for the :term:`HRRR`/:term:`CONUS` domain, use +``latlon_bbox: [22.0, -135.0, 50.0, -60.0]`` or adjust as needed. Once added, +the diagnostics will generate plots restricted to the configured region. For +more information, see the `plot adaptor compatibility section +`_ +of the `Anemoi Training diagnostic documentation +`_. + +Can I add an HRRR-only variable, such as reflectivity, to nested-EAGLE training? +================================================================================ + +Not directly with the current nested :term:`HRRR`-over-:term:`GFS` training +setup. The nested pipeline combines HRRR and GFS data into a single Anemoi +training dataset, so the variables used for training must be compatible across +the nested domains. + +To train with an HRRR-only variable, users would need a :term:`CONUS`-only +configuration or a design that provides a compatible global counterpart, +placeholder, or derived field, along with corresponding data, normalization, and +model configuration changes. diff --git a/docs/reference/glossary.rst b/docs/reference/glossary.rst index be7e7d6e..21212381 100644 --- a/docs/reference/glossary.rst +++ b/docs/reference/glossary.rst @@ -7,15 +7,30 @@ Glossary .. glossary:: :sorted: + AORC + Analysis of Record for Calibration. A NOAA gridded meteorological forcing dataset used for land surface and hydrologic applications. + anemoi The ECMWF-led machine learning weather prediction software stack used by EAGLE for training and inference. See the `anemoi-core repository `__. + checkpoint + A saved snapshot of a trained machine learning model that can be used to resume training or perform inference. + + CONUS + Continental United States. + ECMWF European Centre for Medium-Range Weather Forecasts. EAGLE Experimental AI Global and Limited-area Ensemble forecast system. + ERA5 + The fifth-generation atmospheric reanalysis produced by the European Centre for Medium-Range Weather Forecasts (ECMWF). + + GEFS + Global Ensemble Forecast System. NOAA's global ensemble weather prediction system. + GFS Global Forecast System. See the `NOAA GFS page `__. @@ -34,6 +49,18 @@ Glossary MPI Message Passing Interface. A standard used for parallel computing across multiple processes. + MET + Model Evaluation Tools. A community-developed verification package used to evaluate numerical weather prediction and machine learning weather forecasts. + + netCDF + Network Common Data Form. A file format commonly used for storing and sharing scientific array-based data. + + NRT + Near real-time. Refers to workflows that generate forecasts using the latest available input data shortly after it becomes available. + + PrepBUFR + A BUFR-formatted observational dataset used for numerical weather prediction and forecast verification. + prewxvx The component used by EAGLE to prepare forecast output for :term:`wxvx` verification steps. diff --git a/docs/user_guide/configuration.rst b/docs/user_guide/configuration.rst index 0ddf112f..1941e006 100644 --- a/docs/user_guide/configuration.rst +++ b/docs/user_guide/configuration.rst @@ -73,6 +73,12 @@ Configuration for the ``Inference`` driver. ``checkpoint_dir`` The ``checkpoint_dir:`` parameter specifies the location of the checkpoints created by the training step. +``anemoi.checkpoint_path`` + The ``anemoi.checkpoint_path:`` parameter can be set to an explicit checkpoint file, such as the official nested-EAGLE pretrained checkpoint. If ``checkpoint_dir`` is configured, it takes precedence by selecting the latest checkpoint in that directory. + +``anemoi.input_dataset_kwargs`` + The ``anemoi.input_dataset_kwargs:`` block controls the dataset used as input for inference. Update this block when using a different initial-condition dataset. + platform ------------------------------------------------------------------------------ @@ -151,6 +157,11 @@ Configuration for the ``Zarr`` driver. ``gfs`` and ``hrrr`` The ``gfs:`` and ``hrrr:`` sub-blocks provide refinements for ingesting GFS and HRRR data, respectively, for EAGLE. +To ingest a different data source, update or add a ``zarrs.`` block with +the source, transforms, and target settings needed by ``ufs2arco``. Downstream +training and inference dataset paths must be updated to point to the resulting +Zarr output. + .. _ConfigRealization: Config Realization diff --git a/docs/user_guide/quickstart.rst b/docs/user_guide/quickstart.rst index 39d1427f..278c6f1c 100644 --- a/docs/user_guide/quickstart.rst +++ b/docs/user_guide/quickstart.rst @@ -11,10 +11,18 @@ platform. Future development will include additional platforms. GNU ``make`` version 3.82 or higher is required. -**Complete the following steps from the repository root.** - .. note:: The EAGLE runtime software environment currently requires over 50 GB of disk space. Consider available space, quota, etc. when choosing where to clone the EAGLE repository and run the following steps. +If you do not already have the repository, clone it and change into the +repository root: + +.. code-block:: bash + + git clone https://github.com/NOAA-EPIC/EAGLE.git + cd EAGLE + +**Complete the following steps from the repository root.** + .. _QuickstartWorkflow: Building and Running :term:`EAGLE` @@ -34,15 +42,15 @@ Building and Running :term:`EAGLE` create the same environments but also install additional code-quality tools for formatting, linting, shellchecking, typechecking, unit testing, and :term:`YAML` linting. -.. note:: + .. note:: - EAGLE virtual environments are built using both conda and pip packages. If you examine the output from the ``make env`` command above, you may see messages like the following, from pip: + EAGLE virtual environments are built using both conda and pip packages. If you examine the output from the ``make env`` command above, you may see messages like the following, from pip: - .. code-block:: text + .. code-block:: text - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. + ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - This will be followed by a report of packages pip believes are not installed or are installed with incompatible versions. These messages are due to fundamental differences in how conda and pip operate, and are generally safe to ignore. But please open an :ref:`Issue ` if you later encounter problems you believe are related to package versions. + This will be followed by a report of packages pip believes are not installed or are installed with incompatible versions. These messages are due to fundamental differences in how conda and pip operate, and are generally safe to ignore. But please open an :ref:`Issue ` if you later encounter problems you believe are related to package versions. #. Create the EAGLE YAML config @@ -123,7 +131,9 @@ Building and Running :term:`EAGLE` For the grid-based ``vis-grid-global`` and ``vis-grid-lam`` targets, additional error plots (forecast vs truth differences) will be created under ``run//visualization/grid2grid/{global,lam}/plots-spatial-stats/``. These plots depend on 1. The config value at key-path ``vx.grid2grid.{global,lam}.wxvx.wxvx.ncdiffs`` being set to ``true``, which instructs MET to produce netCDF difference files during verification; and 2. The config block at key-path ``visualization.grid2grid.{global,lam}.visualization.spatial_stat_plots``, which enables and configures plot generation, being present. -#. Run inference in near-real-time (NRT) +#. .. _NRT: + + Run inference in near-real-time (NRT) a. Create the EAGLE NRT config diff --git a/envs/anemoi.yaml b/envs/anemoi.yaml index e9054bb4..b256acbf 100644 --- a/envs/anemoi.yaml +++ b/envs/anemoi.yaml @@ -11,10 +11,13 @@ dependencies: - numpy 2.2.6.* - pip - python 3.12 + - pytorch 2.10.* - setuptools 81.0.* + - triton 3.6.* - uwtools 2.16.* - pip: - anemoi-inference==0.10.* - anemoi-models==0.13.* - anemoi-training==0.10.* + - anemoi-transform==0.1.29 - eagle-tools==0.8.1 diff --git a/envs/data.yaml b/envs/data.yaml index 0d585520..a03ea3bd 100644 --- a/envs/data.yaml +++ b/envs/data.yaml @@ -3,18 +3,18 @@ channels: - conda-forge - ufs-community dependencies: - - cf-xarray 0.10.11 + - cf-xarray 0.10.11.* - impi_rt 2021.13.* - mpi4py 4.1.* - - numpy 1.26.4 - - pandas 2.3.3 + - numpy 2.2.6.* + - pandas 2.3.3.* - pip - setuptools 81.0.* - types-pyyaml - ufs2arco 0.19.* - uwtools 2.16.* - xarray 2026.2.* - - xesmf 0.9.2 + - xesmf 0.9.2.* - pip: - anemoi-datasets==0.5.* - anemoi-graphs==0.9.* diff --git a/envs/visualization.yaml b/envs/visualization.yaml index dafcd87a..3d3df455 100644 --- a/envs/visualization.yaml +++ b/envs/visualization.yaml @@ -5,7 +5,8 @@ channels: dependencies: - cartopy 0.25.* - matplotlib 3.10.* - - numpy 2.2.6 + - numpy 2.2.6.* + - pandas 2.3.* - pip - python 3.13 - uwtools 2.16.* diff --git a/envs/wxvx.yaml b/envs/wxvx.yaml index 5fb41778..af431c6d 100644 --- a/envs/wxvx.yaml +++ b/envs/wxvx.yaml @@ -6,8 +6,9 @@ channels: dependencies: - pip - python 3.13 + - requests 2.34.* - uwtools 2.16.* - - wxvx >=0.7.2 + - wxvx >=0.7.3 - xesmf 0.8.* - zarr 2.18.* - pip: diff --git a/setup b/setup index 9d7ea28e..577f981a 100755 --- a/setup +++ b/setup @@ -16,6 +16,16 @@ cachedir() { ) } +clean_conda_pkgs_maybe() { + if [[ -n ${EAGLE_CLEAN_CONDA_PKGS+x} ]]; then + ( + env_activate base + msg Cleaning conda package cache + mamba clean --all --yes + ) + fi +} + cuda_command_file() { local path val val=${ARGS[cudascript]} @@ -113,6 +123,7 @@ prep_env() { mamba env update ${args[@]} | { grep --line-buffered -v "Requirement already satisfied" || true; } else msg Creating environment: $name + export PIP_VERBOSE=1 mamba env create ${args[@]} fi ) @@ -147,6 +158,7 @@ if [[ -v EAGLE_OLD_XDG_CACHE_HOME ]]; then fi EOF ) + clean_conda_pkgs_maybe } prep_env_anemoi() { @@ -182,6 +194,7 @@ prep_env_base() { msg Updating environment: $name mamba env update ${MAMBAFLAGS[@]} --file envs/$name.yaml install_devpkgs_maybe $name + clean_conda_pkgs_maybe ) }