diff --git a/.github/ISSUE_TEMPLATE/support_request.md b/.github/ISSUE_TEMPLATE/support_request.md index aee27a860..1de09b98e 100644 --- a/.github/ISSUE_TEMPLATE/support_request.md +++ b/.github/ISSUE_TEMPLATE/support_request.md @@ -21,7 +21,7 @@ If applicable, add screenshots to help explain your problem. **System (please complete the following information):** - Supercomputer: [which machine you are on] - - Version [Please run `esm_versions check`] + - Version [Please run `esm_tools --version`] **Additional context** diff --git a/README.rst b/README.rst index edcf98fba..86acae11c 100644 --- a/README.rst +++ b/README.rst @@ -29,8 +29,9 @@ glogin.hlrn.de / blogin.hlrn.de:: juwels.fz-juelich.de:: + $ module load Stages/2022 $ module load git - $ module load Python-3.6.8 + $ module load Python/3.9.6 aleph:: diff --git a/configs/components/fesom/fesom-2.0.yaml b/configs/components/fesom/fesom-2.0.yaml index 662174c4a..e542a987e 100644 --- a/configs/components/fesom/fesom-2.0.yaml +++ b/configs/components/fesom/fesom-2.0.yaml @@ -149,7 +149,7 @@ choose_resolution: nx: 1306775 jane: nx: 33348172 - SO3: + SO3: nx: 11087062 mesh_dir: "${pool_dir}/${resolution}/" @@ -167,6 +167,8 @@ restart_in_in_work: par_ice_restart: fesom.${parent_date!syear}.ice.restart/*.nc fesom_raw_restart_info: fesom_raw_restart/*.info fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* restart_in_sources: oce_restart: fesom.${parent_date!syear}.oce.restart.nc ice_restart: fesom.${parent_date!syear}.ice.restart.nc @@ -174,6 +176,8 @@ restart_in_sources: par_ice_restart: fesom.${parent_date!syear}.ice.restart/*.nc fesom_raw_restart_info: fesom_raw_restart/*.info fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* restart_out_files: oce_restart: oce_restart @@ -182,6 +186,8 @@ restart_out_files: par_ice_restart: par_ice_restart fesom_raw_restart_info: fesom_raw_restart_info fesom_raw_restart: fesom_raw_restart + fesom_bin_restart_info: fesom_bin_restart_info + fesom_bin_restart: fesom_bin_restart restart_out_in_work: oce_restart: fesom.${end_date!syear}.oce.restart.nc @@ -190,6 +196,8 @@ restart_out_in_work: par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc fesom_raw_restart_info: fesom_raw_restart/*.info fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* restart_out_sources: oce_restart: fesom.${end_date!syear}.oce.restart.nc @@ -198,6 +206,8 @@ restart_out_sources: par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc fesom_raw_restart_info: fesom_raw_restart/*.info fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* outdata_sources: @@ -226,6 +236,14 @@ file_movements: all_directions: move fesom_raw_restart_info_out: all_directions: move + fesom_bin_restart_in: + all_directions: move + fesom_bin_restart_out: + all_directions: move + fesom_bin_restart_info_in: + all_directions: move + fesom_bin_restart_info_out: + all_directions: move # Is it a branchoff experiment? branchoff: "$(( ${lresume} and ${general.run_number}==1 ))" @@ -241,6 +259,8 @@ choose_branchoff: add_restart_in_files: fesom_raw_restart_info: fesom_raw_restart_info fesom_raw_restart: fesom_raw_restart + fesom_bin_restart_info: fesom_bin_restart_info + fesom_bin_restart: fesom_bin_restart diff --git a/configs/components/fesom/fesom-2.1.yaml b/configs/components/fesom/fesom-2.1.yaml index 81e969519..9ecf12945 100644 --- a/configs/components/fesom/fesom-2.1.yaml +++ b/configs/components/fesom/fesom-2.1.yaml @@ -132,15 +132,30 @@ choose_resolution: restart_in_files: oce_restart: oce_restart ice_restart: ice_restart + par_oce_restart: par_oce_restart + par_ice_restart: par_ice_restart + restart_in_in_work: oce_restart: fesom.${parent_date!syear}.oce.restart.nc ice_restart: fesom.${parent_date!syear}.ice.restart.nc + par_oce_restart: fesom.${parent_date!syear}.oce.restart/*.nc + par_ice_restart: fesom.${parent_date!syear}.ice.restart/*.nc + fesom_raw_restart_info: fesom_raw_restart/*.info + fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* wiso_restart: fesom.${parent_date!syear}.wiso.restart.nc icb_restart: iceberg.restart #.${parent_date!syear!month} icb_restart_ISM: iceberg.restart.ISM restart_in_sources: oce_restart: fesom.${parent_date!syear}.oce.restart.nc ice_restart: fesom.${parent_date!syear}.ice.restart.nc + par_oce_restart: fesom.${parent_date!syear}.oce.restart/*.nc + par_ice_restart: fesom.${parent_date!syear}.ice.restart/*.nc + fesom_raw_restart_info: fesom_raw_restart/*.info + fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* wiso_restart: fesom.${parent_date!syear}.wiso.restart.nc icb_restart: iceberg.restart #.${parent_date!syear!month} icb_restart_ISM: iceberg.restart.ISM @@ -148,15 +163,34 @@ restart_in_sources: restart_out_files: oce_restart: oce_restart ice_restart: ice_restart + par_oce_restart: par_oce_restart + par_ice_restart: par_ice_restart + fesom_raw_restart_info: fesom_raw_restart_info + fesom_raw_restart: fesom_raw_restart + fesom_bin_restart_info: fesom_bin_restart_info + fesom_bin_restart: fesom_bin_restart + restart_out_in_work: oce_restart: fesom.${end_date!syear}.oce.restart.nc ice_restart: fesom.${end_date!syear}.ice.restart.nc + par_oce_restart: fesom.${end_date!syear}.oce.restart/*.nc + par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc + fesom_raw_restart_info: fesom_raw_restart/*.info + fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* wiso_restart: fesom.${end_date!syear}.wiso.restart.nc icb_restart: iceberg.restart #.${parent_date!syear} icb_restart_ISM: iceberg.restart.ISM restart_out_sources: oce_restart: fesom.${end_date!syear}.oce.restart.nc ice_restart: fesom.${end_date!syear}.ice.restart.nc + par_oce_restart: fesom.${end_date!syear}.oce.restart/*.nc + par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc + fesom_raw_restart_info: fesom_raw_restart/*.info + fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* wiso_restart: fesom.${end_date!syear}.wiso.restart.nc icb_restart: iceberg.restart #.${parent_date!syear} icb_restart_ISM: iceberg.restart.ISM @@ -247,6 +281,23 @@ log_sources: clock: fesom.clock mesh_diag: fesom.mesh.diag.nc +file_movements: + fesom_raw_restart_in: + all_directions: move + fesom_raw_restart_out: + all_directions: move + fesom_raw_restart_info_in: + all_directions: move + fesom_raw_restart_info_out: + all_directions: move + fesom_bin_restart_in: + all_directions: move + fesom_bin_restart_out: + all_directions: move + fesom_bin_restart_info_in: + all_directions: move + fesom_bin_restart_info_out: + all_directions: move # Is it a branchoff experiment? branchoff: "$(( ${lresume} and ${general.run_number}==1 ))" @@ -259,6 +310,12 @@ choose_branchoff: false: daynew: "${initial_date!sdoy}" yearnew: "${initial_date!syear}" + add_restart_in_files: + fesom_raw_restart_info: fesom_raw_restart_info + fesom_raw_restart: fesom_raw_restart + fesom_bin_restart_info: fesom_bin_restart_info + fesom_bin_restart: fesom_bin_restart + namelist_changes: diff --git a/configs/components/oifs/oifs.env.yaml b/configs/components/oifs/oifs.env.yaml index c70c37e91..9e64a6b0f 100644 --- a/configs/components/oifs/oifs.env.yaml +++ b/configs/components/oifs/oifs.env.yaml @@ -341,18 +341,13 @@ compiletime_environment_changes: OIFS_CCDEFS: '"LINUX LITTLE INTEGER_IS_INT _ABI64 BLAS _OPENMP"' albedo: -# add_module_actions: -# - "load libaec/1.0.5-intel-2021.5.0" add_export_vars: -# SZIPROOT: "/sw/spack-levante/libaec-1.0.5-gij7yv" - HDF5_ROOT: $HDF5ROOT #TODO remove! - ECCODESROOT: "/albedo/home/mandresm/.spack/sw/eccodes/2.25.0-hwsa4h3/" - #ECCODESROOT: "/albedo/soft/sw/spack-sw/eccodes/2.25.0-e3frvr6/" + HDF5_ROOT: $HDF5ROOT - HDF5_C_INCLUDE_DIRECTORIES: $HDF5_ROOT/include #TODO remove! - NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFFROOT/include #TODO remove! - NETCDF_C_INCLUDE_DIRECTORIES: $NETCDFROOT/include #TODO remove! + HDF5_C_INCLUDE_DIRECTORIES: $HDF5_ROOT/include + NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFFROOT/include + NETCDF_C_INCLUDE_DIRECTORIES: $NETCDFROOT/include OASIS3MCT_FC_LIB: '"-L$NETCDFFROOT/lib -lnetcdff"' # TODO: figure out whether those two are still needed ESM_NETCDF_C_DIR: "$NETCDFROOT" diff --git a/configs/components/oifs/oifs.yaml b/configs/components/oifs/oifs.yaml index 667598d58..4b92218d5 100644 --- a/configs/components/oifs/oifs.yaml +++ b/configs/components/oifs/oifs.yaml @@ -339,6 +339,14 @@ choose_resolution: res_number: 639 res_number_tl: "639_4" truncation: "TCO" + TCO1279: + nx: 6599680 + ny: 1 + time_step: 300 + oasis_grid_name: 128 + res_number: 1279 + res_number_tl: "1279_4" + truncation: "TCO" TL511: nx: 348528 ny: 1 diff --git a/configs/machines/albedo.yaml b/configs/machines/albedo.yaml index 2a5e37a7f..e482cd483 100644 --- a/configs/machines/albedo.yaml +++ b/configs/machines/albedo.yaml @@ -95,9 +95,10 @@ choose_compiler_suite: add_export_vars: # I/O libraries - HDF5ROOT: "/albedo/home/mandresm/.spack/sw/hdf5/1.12.2-7bogsh7/" - NETCDFROOT: "/albedo/home/mandresm/.spack/sw/netcdf-c/4.8.1-2u6p2ge/" - NETCDFFROOT: "/albedo/home/mandresm/.spack/sw/netcdf-fortran/4.5.4-2gawmpc/" + HDF5ROOT: "/albedo/soft/sw/spack-sw/hdf5/1.12.2-rgostku/" + NETCDFROOT: "/albedo/soft/sw/spack-sw/netcdf-c/4.8.1-i5n4n63/" + NETCDFFROOT: "/albedo/soft/sw/spack-sw/netcdf-fortran/4.5.4-yb7woqz/" + ECCODESROOT: "/albedo/soft/sw/spack-sw/eccodes/2.25.0-vhmiess/" intel-oneapi: compiler_module: intel-oneapi-compilers/2022.1.0 @@ -142,7 +143,7 @@ module_actions: - "load cdo/2.0.5" - "load nco/5.0.1" - "load git/2.35.2" - #- "load perl/5.35.0-gcc12.1.0" + - "load perl/5.35.0-gcc12.1.0" - "load python/3.10.4" # Show what is there in the log: - "list" @@ -163,9 +164,7 @@ export_vars: FESOM_PLATFORM_STRATEGY: albedo # PERL library - # TODO: change that to lib systems - #PERL5LIB: "/albedo/soft/sw/spack-sw/perl/5.35.0-asf6m5t/lib/5.35.0/" - PERL5LIB: "/albedo/home/mandresm/my_libs/perl-5.32.0/lib" + PERL5LIB: "/albedo/soft/sw/spack-sw/perl-uri/1.72-epj7s32/lib/perl5:/albedo/soft/sw/spack-sw/perl/5.35.0-asf6m5t/lib" # I/O libraries HDF5ROOT: "" @@ -201,7 +200,7 @@ warning: environment variables, pool problems, incorrect or not optimal compiler flags... Feel free to try it a report and issue in https://github.com/esm-tools/esm_tools/issues labelling it with the ``albedo`` tag, if things don't work as expected. By - running ``esm_tools --test-state`` you'll get information of what you can + running ``esm_tools test-state`` you'll get information of what you can expect to work and what not. Bare in mind that the ESM-Tools configurations specific to Albedo will be changing fast on the following weeks, until Albedo is fully supported. We recommend against using it for production diff --git a/configs/templates/component_template.yaml b/configs/templates/component_template.yaml new file mode 100644 index 000000000..c421af18e --- /dev/null +++ b/configs/templates/component_template.yaml @@ -0,0 +1,99 @@ +# NAME OF YOUR MODEL OR COMPONENT HERE +# +# For more information about the extended YAML syntax, please consult: +# https://esm-tools.readthedocs.io/en/latest/yaml.html#esm-tools-extended-yaml-syntax +# +# For more information about the ESM-Tools feature variables available, please consult: +# https://esm-tools.readthedocs.io/en/latest/esm_variables.html#esm-tools-variables + +model: name of your component +version: default version + +metadata: + Institute: where it was developed + Description: + A brief description here + Authors: Authors + Publications: + - "Main publication for this component here " + License: + License details here + +git-repository: if your code is hosted in git place here the address +branch: default branch +comp_command: command used to compile your component +install_bins: subpath within ``model_dir`` where the comp command produces the binaries +clean_command: ${defaults.clean_command} +executable: medusa_recom_paleo + +choose_version: + # Here you can place all the variables you would want to control via the version + # variable above (for example, you could also add ``git-repository``, + # ``comp_command``, ``destination``, ...) + a_version_of_your_choice: + branch: name of the branch/tag associated to that version if you use git + another_version_of_your_choice: + branch: name of the branch/tag associated to that version if you use git + +# ``model_dir`` let's ESM-Tools know where is the source code of your component (must +# be an absolute path). Normally, changed via the runscript. We recommend you leave it +# as below, changing only the ``name_of_your_model`` part to whatever you defined as +# the ``model`` var. This will automatically select the source code for esm_master if +# the command is operating with a coupled setup +model_dir: "${general.esm_master.dir}/name_of_your_model-${version}" + +# This section takes care of dealing with environment changes you might want to do, to +# deviated from the defaults define for the specific machine +# (``configs/machine/.yaml``). For more information consult: +# https://esm-tools.readthedocs.io/en/latest/esm_environment.html#esm-environment +environment_changes: + a_variable_in_the_computer_yaml: here you can over + add_module_actions: + - module command without ``module`` word (e.g. ``load hdf5``) + - module command without ``module`` word (e.g. ``load netcdf``) + add_export_vars: + VARIABLE_TO_EXPORT: '"value of the variable you want to export"' + VARIABLE_TO_EXPORT: '"value of the variable you want to export"' + +# Some recommended defaults +lresume: false + +# Some commonly used variables in other ESM-Tools components (but optional) +input_dir: ${pool_dir}/path/within/the/pool/dir/to/the/input/of/your/component +namelist_dir: /absolute/path/to/the/namelist/folder + +# If your components has namelist the following syntax allows you to control the values +# of their values. For more information consult: +# https://esm-tools.readthedocs.io/en/latest/yaml.html#changing-namelists +namelist_changes: + namelist_name: + section_in_the_namelist: + variable_in_the_namelist: its new value here + +namelists: + - name of the namelist + +# File dictionaries to control the copying/moving/linking of all files associated to +# the simulation. This syntax will change for something better soon. For more details +# about this syntax consult: +# https://esm-tools.readthedocs.io/en/latest/yaml.html#file-dictionaries +input_files: + file_name: file_tag +input_source: + file_tag: /absolute/path/to/the/file +input_in_work: + file_tag: /relative/path/to/the/file/in/the/work/directory + +config_files: + namelist_name: namelist_tag # Or any other configuration file +config_sources: + namelist_tag: ${namelist_dir}/ # Use ${namelist_dir} here if you + # defined it above + +create_config: + name_of_the_config_you_want_to_create: + - "<--append-- line 1" + - "<--append-- line 2" + +bin_sources: + bin_tag: ${model_dir}/bin/${executable} # This would be the normal way to do it diff --git a/configs/templates/machine_template.yaml b/configs/templates/machine_template.yaml new file mode 100644 index 000000000..8f0b91b60 --- /dev/null +++ b/configs/templates/machine_template.yaml @@ -0,0 +1,155 @@ +####################################################################################### +# The dummy yaml machine config file for esm-tools +# +# This is a dummy machine file. It is intend to be used for creating a new config file +# for a new machine setup. It provides a minimal structure to be filled in with +# the oppropriate values of the new HPC system. +# +# Usage: +# - Please replace the placeholder indicated by <...> +# with your PHC specific values +# - Please uncomment sections of this yaml file if appropriate. +# Possible sections are indicating this (see below). +# - See also files for other machines that are already implemented in esm-tools. +# - After editing this file, save it as .yaml in configs/machines. Add also +# an entry in the file configs/machines/all_machines.yaml (in this folder) for the +# new machine. +####################################################################################### + +# GENERIC YAML CONFIGURATION FILES +# +# Set hostname of the machine +name: + +# General information about operating system +# Operating system +#operating_system: "" +# Shell interpreter (e.g. "/usr/bin/bash") +sh_interpreter: "" + +# Information about the job scheduler +# Set batch system (e.g. slurm/pbs) +batch_system: "" + +# Set whether an account needs to be set for batch system (e.g. true or false). +# The actual account that will be used for the experiment will be set in the runscript. +accounting: true +# +#hetjob_flag: "?" # either hetjob or packjob, this depends on your SLURM version. More info can be found: https://slurm.schedmd.com/heterogeneous_jobs.html +# +#jobtype: compute +# +# Information about the partitions of the machine +# +# Specify if hyperthreading should be used. +use_hyperthreading: False +# Set further options depending on hyperthreading variable 'use_hyperthreading' by +# uncommenting and editing the following 'choose_use_hyperthreading' section: +#choose_use_hyperthreading: +# True: +# hyperthreading_flag: "" +# mt_launcher_flag: "" +# False: +# : + +# Set default partition (e.g. compute) +partition: +# +# Set detail information for available partitions on the machine. +# These variables will be used in other parts of this yaml file. +choose_partition: + : + partition_name: "" + partition_cpn: # integer number +# +# Define available partitions of the machine for different jobtypes (e.g. for computing [compute], post-processing [PP], etc.) +# +partitions: + compute: + name: ${computer.partition_name} # this uses the variable set in choose_partition section (see above) + #name: "compute" # or set the partition name as string + cores_per_node: ${computer.partition_cpn} +# +#logical_cpus_per_core: 2 +# +#threads_per_core: 1 +# +# Specify different pool directories for this machine. +# +pool_directories: +# focipool: "" +# pool: "" +# + +# Setup environment +# Load necessary modules (e.g. git, cdo, nco, compiler). Edit and extent the following lines. +module_actions: + - "purge" + - "load " +# - "load " +# - "load " +# +# Export environment variables. Extend the following lines to export needed environment variables. +export_vars: + # Locale Settings + LC_ALL: en_US.UTF-8 + # Compiler + FC: + F77: + MPICC: + MPIFC: + CC: + CXX: + # Other environemnt variables + +## Some other yaml files use a file 'computer.fc', etc to identify the compiler, so we need to add them here. +fc: "$FC" +cc: "$CC" +mpifc: "$MPIFC" +mpicc: "$MPICC" +cxx: "$CXX" +# +# +# Launcher flags (i.e. flags for the srun/mpirun/aprun command) +launcher_flags: "-l" + +# Choose another configuration file, that should be evaluated (e.g. slurm.yaml, pbs.yaml) +further_reading: + - batch_system/slurm.yaml + + +#################################################################################### +# Further functionality: +# +# If there are modules or environment variables that are different for +# different e.g. compilers, you can use the funcionality of +# declaring choose_ blocks in this yaml file. +# (For more details about yaml syntax please also see the documentation +# https://esm-tools.readthedocs.io/en/latest/yaml.html#switches-choose) +# +# The following lines show an example of how such a choose_ block can be +# set up depending on a given compiler value/key given by the variable compiler_mpi. +# + +#compiler_mpi: + +#choose_compiler_mpi: +# : +# : +# # load additional modules specific for compiler +# add_module_actions: +# - "load " +# - "load " +# - "load " +# # add additional variables to exported: change and extend following lines: +# add_export_vars: +# # Compiler +# FC: +# F77: +# MPICC: +# MPIFC: +# CC: +# CXX: +#################################################################################### diff --git a/docs/cookbook.rst b/docs/cookbook.rst index 9f4c6770c..aabc8e4b0 100644 --- a/docs/cookbook.rst +++ b/docs/cookbook.rst @@ -41,6 +41,7 @@ documentation issue on `our GitHub repository + Creating a new component configuration for my_new_thing + + ...EDITOR OPENS.... + + Thank you! The new configuration has been saved. Please commit it (and get in touch with the + esm-tools team if you need help)! + +You can also specify if you are creating a new ``setup`` or a new ``component`` with:: + + $ esm_tools create-new-config --type setup + +Note however that there is (as of this writing) no template available for setups! diff --git a/docs/index.rst b/docs/index.rst index 428e1e1c8..2729163f7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,6 +13,7 @@ Welcome to ESM Tools's documentation! yaml_hierarchy esm_variables Supported_Models + esm_tools esm_master esm_versions esm_runscripts diff --git a/docs/recipes/add_machine.rst b/docs/recipes/add_machine.rst new file mode 100644 index 000000000..fb68d8a1c --- /dev/null +++ b/docs/recipes/add_machine.rst @@ -0,0 +1,50 @@ +Implement a New HPC Machine +=========================== + +To implement a new HPC machine to `ESM-Tools`, two files need to be updated and created, respectively: + +- ``/esm_tools/configs/machines/all_machines.yaml`` +- ``/esm_tools/configs/machines/.yaml`` + +1. Add an additional entry for the new machine. + + Use your favourite text editor and open the file ``/esm_tools/configs/machines/all_machines.yaml``:: + + $ /esm_tools/configs/machines/all_machines.yaml + + and add a new entry for the new machine (replace placeholders indicated by <...>) + + .. code-block:: yaml + + : + login_nodes: '*' # A regex pattern that matches the hostname of login nodes + compute_nodes: '' # A regex pattern that matches the hostname of compute nodes + + +2. Create a new machine file. + + Use your favourite text editor to create and edit a new machine file ``.yaml`` in the + ``esm_tools/configs/machines/`` folder:: + + $ /esm_tools/configs/machines/.yaml + + A template file (``machine_template.yaml``) is available in ``configs/templates``, so you can alternatively copy + this file into the ``configs/machines`` folder edit the relevant entries:: + + $ cp /esm_tools/configs/templates/machine_template.yaml /esm_tools/configs/machines/.yaml + $ /esm_tools/configs/machines/.yaml + + You can also reproduce the two steps above simply by running the following ``esm_tools`` command:: + + $ esm_tools create-new-config /esm_tools/configs/machines/.yaml -t machine + + This will copy the ``machine_template.yaml`` in the target location and open the file in your default editor. + +See also +~~~~~~~~ + +.. links to relevant parts of the documentation + +- :ref:`esm_variables:ESM-Tools Variables` +- :ref:`yaml:Switches (\`\`choose_\`\`)` +- :ref:`yaml:What Is YAML?` diff --git a/docs/recipes/add_model_setup.rst b/docs/recipes/add_model_setup.rst index bbcb137b3..d3bb4e066 100644 --- a/docs/recipes/add_model_setup.rst +++ b/docs/recipes/add_model_setup.rst @@ -3,6 +3,8 @@ Implement a New Model **Feature available since version:** 4.2 +.. note:: since version 6.20.2 a template is available in + ``esm_tools/configs/templates/component_template.yaml`` 1. Upload your model into a repository such us `gitlab.awi.de`, `gitlab.dkrz.de` or `GitHub`. Make sure to set up the right access permissions, so that you comply with the licensing of diff --git a/install.sh b/install.sh index 9b99e8dcf..ea0cf72dd 100755 --- a/install.sh +++ b/install.sh @@ -46,7 +46,7 @@ if hash git 2>/dev/null; then echo $git_error_message echo "git version found: ${git_version}" else - if test ${minor_git_version} -lt "13"; then + if test ${minor_git_version} -lt "10"; then echo $git_error_message echo "git version found: ${git_version}" fi diff --git a/namelists/fesom2/2.0/awicm3/DART/namelist.config b/namelists/fesom2/2.0/awicm3/DART/namelist.config index b283fdd8d..ca42ecb72 100755 --- a/namelists/fesom2/2.0/awicm3/DART/namelist.config +++ b/namelists/fesom2/2.0/awicm3/DART/namelist.config @@ -30,7 +30,7 @@ logfile_outfreq=960 !in logfile info. output frequency, # steps &ale_def which_ALE='zstar' ! 'linfs','zlevel', 'zstar' -use_partial_cell=.true. +use_partial_cell=.false. / &geometry diff --git a/namelists/fesom2/2.0/awicm3/DART/namelist.io b/namelists/fesom2/2.0/awicm3/DART/namelist.io index 341614c52..6c834f1b7 100644 --- a/namelists/fesom2/2.0/awicm3/DART/namelist.io +++ b/namelists/fesom2/2.0/awicm3/DART/namelist.io @@ -2,7 +2,7 @@ ldiag_solver =.false. lcurt_stress_surf=.false. ldiag_curl_vel3 =.false. -ldiag_energy =.false. +ldiag_energy =.true. ldiag_salt3D =.false. ldiag_dMOC =.false. ldiag_DVD =.false. @@ -19,26 +19,30 @@ io_listsize=100 !number of streams to allocate. shallbe large or equal to the nu ! 'otracers' - all other tracers if applicable ! for 'dMOC' to work ldiag_dMOC must be .true. otherwise no output &nml_list -io_list = 'sst ',1, 'm', 4, - 'sss ',1, 'm', 4, - 'ssh ',1, 'm', 4, - 'uice ',1, 'm', 4, - 'vice ',1, 'm', 4, - 'a_ice ',1, 'm', 4, - 'm_ice ',1, 'm', 4, - 'm_snow ',1, 'm', 4, +io_list = 'sst ',90,'s', 4, + 'sss ',1, 'd', 4, + 'ssh ',1, 'd', 4, + 'uice ',1, 'd', 4, + 'vice ',1, 'd', 4, + 'a_ice ',1, 'd', 4, + 'm_ice ',1, 'd', 4, + 'm_snow ',1, 'd', 4, 'MLD1 ',1, 'm', 4, 'MLD2 ',1, 'm', 4, 'tx_sur ',1, 'm', 4, 'ty_sur ',1, 'm', 4, - 'temp ',1, 'y', 4, - 'salt ',1, 'y', 4, - 'N2 ',1, 'y', 4, - 'Kv ',1, 'y', 4, - 'u ',1, 'y', 4, - 'v ',1, 'y', 4, - 'w ',1, 'y', 4, - 'Av ',1, 'y', 4, + 'temp ',1, 'm', 4, + 'salt ',1, 'm', 4, + 'N2 ',1, 'm', 4, + 'Kv ',1, 'm', 4, + 'u ',1, 'm', 4, + 'v ',1, 'm', 4, + 'w ',1, 'm', 4, + 'temp1-31 ',1, 'd', 4, + 'salt1-31 ',1, 'd', 4, + 'u1-31 ',1, 'd', 4, + 'v1-31 ',1, 'd', 4, + 'w1-31 ',1, 'd', 4, 'bolus_u ',1, 'y', 4, 'bolus_v ',1, 'y', 4, 'bolus_w ',1, 'y', 4, diff --git a/namelists/fesom2/2.0/awicm3/DART/namelist.oce b/namelists/fesom2/2.0/awicm3/DART/namelist.oce index 1874bf363..be283831e 100644 --- a/namelists/fesom2/2.0/awicm3/DART/namelist.oce +++ b/namelists/fesom2/2.0/awicm3/DART/namelist.oce @@ -1,85 +1,75 @@ &oce_dyn -C_d=0.0025 ! Bottom drag, nondimensional -gamma0=0.001 ! [m/s], backgroung viscosity= gamma0*len, it should be as small as possible (keep it < 0.01 m/s). -gamma1=0.05 !0.0285 ! [nodim], for computation of the flow aware viscosity -gamma2=0.5 !0.285 ! [s/m], is only used in easy backscatter option -Div_c=.5 ! the strength of the modified Leith viscosity, nondimensional, 0.3 -- 1.0 -Leith_c=.05 ! the strength of the Leith viscosity -visc_option=5 ! 1=Harmonic Leith parameterization; - ! 2=Laplacian+Leith+biharmonic background - ! 3=Biharmonic Leith parameterization - ! 4=Biharmonic flow aware - ! 5=Kinematic (easy) Backscatter - ! 6=Biharmonic flow aware (viscosity depends on velocity Laplacian) - ! 7=Biharmonic flow aware (viscosity depends on velocity differences) - ! 8=Dynamic Backscatter -easy_bs_return= 0.9 !1.5 ! coefficient for returned sub-gridscale energy, to be used with visc_option=5 (easy backscatter) -A_ver= 1.e-4 ! Vertical viscosity, m^2/s -scale_area=5.8e9 ! Visc. and diffus. are for an element with scale_area -mom_adv=2 ! 1=vector CV, p1 vel, 2=sca. CV, 3=vector inv. -free_slip=.false. ! Switch on free slip -i_vert_visc=.true. -w_split=.true. -w_max_cfl=0.8 ! maximum allowed CFL criteria in vertical (0.5 < w_max_cfl < 1.) ! in older FESOM it used to be w_exp_max=1.e-3 -SPP=.false. ! Salt Plume Parameterization -Fer_GM=.false. !.true. ! to swith on/off GM after Ferrari et al. 2010 -K_GM_max = 3000.0 ! max. GM thickness diffusivity (m2/s) -K_GM_min = 2.0 ! max. GM thickness diffusivity (m2/s) -K_GM_bvref = 2 ! def of bvref in ferreira scaling 0=srf,1=bot mld,2=mean over mld,3=weighted mean over mld -K_GM_rampmax = 40.0 ! Resol >K_GM_rampmax[km] GM on -K_GM_rampmin = 30.0 ! Resol TB04 mixing -momix_lat = -50.0 ! latitidinal treshhold for TB04, =90 --> global -momix_kv = 0.01 ! PP/KPP, mixing coefficient within MO length -use_instabmix = .true. ! enhance convection in case of instable stratification -instabmix_kv = 0.1 -use_windmix = .false. ! enhance mixing trough wind only for PP mixing (for stability) -windmix_kv = 1.e-3 -windmix_nl = 2 - -diff_sh_limit=5.0e-3 ! for KPP, max diff due to shear instability -Kv0_const=.false. -double_diffusion=.false. ! for KPP,dd switch -K_ver=1.0e-5 -K_hor=0. !3000. -surf_relax_T=0.0 -surf_relax_S=0. !1.929e-06 ! 50m/300days 6.43e-07! m/s 10./(180.*86400.) -balance_salt_water =.true. ! balance virtual-salt or freshwater flux or not -clim_relax=0.0 ! 1/s, geometrical information has to be supplied -ref_sss_local=.true. -ref_sss=34. -i_vert_diff =.true. ! true -tra_adv_hor ='MFCT' !'MUSCL', 'UPW1' -tra_adv_ver ='QR4C' !'QR4C', 'CDIFF', 'UPW1' -tra_adv_lim ='FCT' !'FCT', 'NONE' (default) -tra_adv_ph = 1. ! a parameter to be used in horizontal advection (for MUSCL it is the fraction of fourth-order contribution in the solution) -tra_adv_pv = 1. ! a parameter to be used in horizontal advection (for QR4C it is the fraction of fourth-order contribution in the solution) -! Implemented trassers (3d restoring): -! 301 - Fram strait. -! 302 - Bering Strait -! 303 - BSO -num_tracers=2 !number of all tracers -tracer_ID =0,1 !their IDs (0 and 1 are reserved for temperature and salinity) +&oce_tra + use_momix = .true. + momix_lat = -50.0 + momix_kv = 0.01 + use_instabmix = .true. + instabmix_kv = 0.1 + use_windmix = .false. + windmix_kv = 0.001 + windmix_nl = 2 + diff_sh_limit = 0.005 + kv0_const = .true. + double_diffusion = .false. + k_ver = 1e-05 + k_hor = 0. + surf_relax_t = 0.0 + surf_relax_s = 0 + balance_salt_water = .true. + clim_relax = 0.0 + ref_sss_local = .true. + ref_sss = 34.0 + i_vert_diff = .true. + tra_adv_hor = 'MFCT' + tra_adv_ver = 'QR4C' + tra_adv_lim = 'FCT' + tra_adv_ph = 0.0 + tra_adv_pv = 1.0 + gamma0_tra = 0.0005 + gamma1_tra = 0.0125 + gamma2_tra = 0.0 + num_tracers = 2 + tracer_id = 0, 1 / -&oce_init3d ! initial conditions for tracers -n_ic3d = 2 ! number of tracers to initialize -idlist = 1, 0 ! their IDs (0 is temperature, 1 is salinity, etc.). The reading order is defined here! -filelist = 'phc3.0_winter.nc', 'phc3.0_winter.nc' ! list of files in ClimateDataPath to read (one file per tracer), same order as idlist -varlist = 'salt', 'temp' ! variables to read from specified files -t_insitu = .true. ! if T is insitu it will be converted to potential after reading it +&oce_init3d + n_ic3d = 2 + idlist = 1, 0 + filelist = 'phc3.0_winter.nc', 'phc3.0_winter.nc' + varlist = 'salt', 'temp' + t_insitu = .true. / diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/axis_def.xml b/namelists/oifs/43r3/xios/TCO1279_DART/axis_def.xml new file mode 100644 index 000000000..334b9bb8b --- /dev/null +++ b/namelists/oifs/43r3/xios/TCO1279_DART/axis_def.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/context_ifs.xml b/namelists/oifs/43r3/xios/TCO1279_DART/context_ifs.xml new file mode 100644 index 000000000..df526436e --- /dev/null +++ b/namelists/oifs/43r3/xios/TCO1279_DART/context_ifs.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + false + false + + + + -3 + -3 + + + + 2 + 2 + 2 + + + + 0 + false + false + 60000.0 + + + diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/domain_def.xml b/namelists/oifs/43r3/xios/TCO1279_DART/domain_def.xml new file mode 100644 index 000000000..971951c2c --- /dev/null +++ b/namelists/oifs/43r3/xios/TCO1279_DART/domain_def.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/field_def.xml b/namelists/oifs/43r3/xios/TCO1279_DART/field_def.xml new file mode 100644 index 000000000..c31e980ca --- /dev/null +++ b/namelists/oifs/43r3/xios/TCO1279_DART/field_def.xml @@ -0,0 +1,277 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml b/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml new file mode 100644 index 000000000..ec82f96fa --- /dev/null +++ b/namelists/oifs/43r3/xios/TCO1279_DART/file_def.xml @@ -0,0 +1,317 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/grid_def.xml b/namelists/oifs/43r3/xios/TCO1279_DART/grid_def.xml new file mode 100644 index 000000000..4e314a0c8 --- /dev/null +++ b/namelists/oifs/43r3/xios/TCO1279_DART/grid_def.xml @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/namelists/oifs/43r3/xios/TCO1279_DART/iodef.xml b/namelists/oifs/43r3/xios/TCO1279_DART/iodef.xml new file mode 100644 index 000000000..d34b854b0 --- /dev/null +++ b/namelists/oifs/43r3/xios/TCO1279_DART/iodef.xml @@ -0,0 +1,29 @@ + + + + + + + + true + 50 + 1 + false + 0.5 + + + + performance + 2.0 + + + + true + true + oifs + 50 + true + + + + diff --git a/runscripts/awicm3/v3.1/awicm3-v3.1-levante-TCO1279L137-DART_inital.yaml b/runscripts/awicm3/v3.1/awicm3-v3.1-levante-TCO1279L137-DART_inital.yaml new file mode 100644 index 000000000..69fc5cf5e --- /dev/null +++ b/runscripts/awicm3/v3.1/awicm3-v3.1-levante-TCO1279L137-DART_inital.yaml @@ -0,0 +1,85 @@ +general: + user: !ENV ${USER} + setup_name: "awicm3" + version: "v3.1" + account: "ab0995" + compute_time: "00:30:00" + initial_date: "2000-01-01" + final_date: "2004-01-01" + base_dir: "/work/ab0246/${user}/runtime/${general.setup_name}-${general.version}/" + nday: 1 + nmonth: 0 + nyear: 0 + +computer: + taskset: true + +awicm3: + postprocessing: false + model_dir: "/home/a/${user}/model_codes/${general.setup_name}-${general.version}//" + +fesom: + resolution: "DART" + namelist_dir: "/home/a/a270092/esm_tools/namelists/fesom2/2.0/awicm3/DART" + pool_dir: "/work/ab0246/a270092/input/fesom2/" + mesh_dir: "${pool_dir}/dart/" + restart_rate: 1 + restart_unit: "d" + restart_first: 1 + lresume: false + + time_step: 60 + nproc: 1024 + add_namelist_changes: + namelist.oce: + oce_dyn: + div_c: 5 + leith_c: 0.5 + +oifs: + resolution: "TCO1279" + levels: "L137" + prepifs_expid: hf05 + input_expid: awi3 + wam: true + lresume: false + time_step: 180 + nproc: 1280 + omp_num_threads: 4 + add_namelist_changes: + fort.4: + NAMCT0: + LXIOS: false + +oasis3mct: + lresume: false # Set to false to generate the rst files for first leg + time_step: 7200 + use_lucia: true + coupling_methods: + gauswgt_i: + time_transformation: instant + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" + gauswgt_gss: + time_transformation: conserv + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 9 + weight: "2" + postprocessing: + conserv: + method: gsspos + algorithm: opt + +xios: + with_model: oifs + nproc: 1 + omp_num_threads: 128 + + diff --git a/setup.cfg b/setup.cfg index 8f1520709..4a0e23bf7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 6.20.0 +current_version = 6.21.11 commit = True tag = True diff --git a/setup.py b/setup.py index 8f8ef61ab..4c1b0d6d5 100644 --- a/setup.py +++ b/setup.py @@ -98,6 +98,6 @@ tests_require=test_requirements, extras_require=extras, url="https://github.com/esm-tools/esm_tools", - version="6.20.0", + version="6.21.11", zip_safe=False, ) diff --git a/src/esm_archiving/__init__.py b/src/esm_archiving/__init__.py index e47770ee9..92fc916c9 100644 --- a/src/esm_archiving/__init__.py +++ b/src/esm_archiving/__init__.py @@ -4,7 +4,7 @@ __author__ = """Paul Gierz""" __email__ = "pgierz@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from .esm_archiving import (archive_mistral, check_tar_lists, delete_original_data, determine_datestamp_location, diff --git a/src/esm_calendar/__init__.py b/src/esm_calendar/__init__.py index 26b95e7fd..a5e46333b 100644 --- a/src/esm_calendar/__init__.py +++ b/src/esm_calendar/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from .esm_calendar import * diff --git a/src/esm_calendar/esm_calendar.py b/src/esm_calendar/esm_calendar.py index 0afaa7b66..8abcaceb1 100644 --- a/src/esm_calendar/esm_calendar.py +++ b/src/esm_calendar/esm_calendar.py @@ -701,59 +701,41 @@ def format( self, form="SELF", givenph=None, givenpm=None, givenps=None ): # basically format_date """ - Beautifully returns a ``Date`` object as a string. - - Parameters: - ----------- - form : str or int - Some cryptic that Dirk over-took from MPI-Met - givenph : bool-ish - Print hours - givenpm : bool-ish - Print minutes - givenps : bool-ish - Print seconds - - Notes: - ------ - How to use the ``form`` argument: - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The following forms are accepted: - + SELF: uses the format which was given when constructing the date - + 0: A Date formatted as YYYY + Needs a docstring! + The following forms are accepted: + + SELF: uses the format which was given when constructing the date + + 0: A Date formated as YYYY - In [5]: test.format(form=1) - Out[5]: '1850-01-01_00:00:00' + In [5]: test.format(form=1) + Out[5]: '1850-01-01_00:00:00' - In [6]: test.format(form=2) - Out[6]: '1850-01-01T00:00:00' + In [6]: test.format(form=2) + Out[6]: '1850-01-01T00:00:00' - In [7]: test.format(form=3) - Out[7]: '1850-01-01 00:00:00' + In [7]: test.format(form=3) + Out[7]: '1850-01-01 00:00:00' - In [8]: test.format(form=4) - Out[8]: '1850 01 01 00 00 00' + In [8]: test.format(form=4) + Out[8]: '1850 01 01 00 00 00' - In [9]: test.format(form=5) - Out[9]: '01 Jan 1850 00:00:00' + In [9]: test.format(form=5) + Out[9]: '01 Jan 1850 00:00:00' - In [10]: test.format(form=6) - Out[10]: '18500101_00:00:00' + In [10]: test.format(form=6) + Out[10]: '18500101_00:00:00' - In [11]: test.format(form=7) - Out[11]: '1850-01-01_000000' + In [11]: test.format(form=7) + Out[11]: '1850-01-01_000000' - In [12]: test.format(form=8) - Out[12]: '18500101000000' + In [12]: test.format(form=8) + Out[12]: '18500101000000' - In [13]: test.format(form=9) - Out[13]: '18500101_000000' + In [13]: test.format(form=9) + Out[13]: '18500101_000000' - In [14]: test.format(form=10) - Out[14]: '01/01/1850 00:00:00' + In [14]: test.format(form=10) + Out[14]: '01/01/1850 00:00:00' """ - # Programmer notes to not be ever included in the doc-string: Paul really, Really, REALLY - # dislikes this function. It rubs me in the wrong way. if form == "SELF": form = self._date_format.form @@ -761,12 +743,11 @@ def format( pm = self._date_format.printminutes ps = self._date_format.printseconds - # These variables are....utterly pointless - if givenph is not None: + if not givenph == None: ph = givenph - if givenpm is not None: + if not givenpm == None: pm = givenpm - if givenps is not None: + if not givenps == None: ps = givenps ndate = list( map( diff --git a/src/esm_cleanup/__init__.py b/src/esm_cleanup/__init__.py index 3712c99fc..985efe003 100644 --- a/src/esm_cleanup/__init__.py +++ b/src/esm_cleanup/__init__.py @@ -2,4 +2,4 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" diff --git a/src/esm_database/__init__.py b/src/esm_database/__init__.py index f1d7f504b..8479a10ed 100644 --- a/src/esm_database/__init__.py +++ b/src/esm_database/__init__.py @@ -2,4 +2,4 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" diff --git a/src/esm_database/location_database.py b/src/esm_database/location_database.py index f169cdc03..d69d1b7eb 100644 --- a/src/esm_database/location_database.py +++ b/src/esm_database/location_database.py @@ -1,9 +1,9 @@ -from sqlalchemy import create_engine, Column, Integer, String, Sequence, DateTime -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker - import os +from sqlalchemy import (Column, DateTime, Integer, Sequence, String, + create_engine) +from sqlalchemy.orm import declarative_base, sessionmaker + main_database_file = os.path.expanduser("~") + "/.esm_tools/esmtools.db" if not os.path.isdir(os.path.expanduser("~") + "/.esm_tools"): os.mkdir(os.path.expanduser("~") + "/.esm_tools") diff --git a/src/esm_environment/__init__.py b/src/esm_environment/__init__.py index 1fbf074f2..a93e70d8b 100644 --- a/src/esm_environment/__init__.py +++ b/src/esm_environment/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from .esm_environment import * diff --git a/src/esm_master/__init__.py b/src/esm_master/__init__.py index 367c5473c..27167741d 100644 --- a/src/esm_master/__init__.py +++ b/src/esm_master/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from . import database diff --git a/src/esm_motd/__init__.py b/src/esm_motd/__init__.py index c110fe013..6768675d0 100644 --- a/src/esm_motd/__init__.py +++ b/src/esm_motd/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from .esm_motd import * diff --git a/src/esm_motd/esm_motd.py b/src/esm_motd/esm_motd.py index 1cb08045b..37fdcf42b 100644 --- a/src/esm_motd/esm_motd.py +++ b/src/esm_motd/esm_motd.py @@ -29,7 +29,7 @@ def __init__(self): url = "https://raw.githubusercontent.com/esm-tools/esm_tools/release/motd/motd.yaml" try: self.motdfile = urllib.request.urlopen(url) - except urllib.error.HTTPError: + except (urllib.error.HTTPError, urllib.error.URLError): timeout = 1 # seconds to wait # print(f"HTTP Error: Connection to file {url} containing update messages could not be established") # print(" Please check the URL by manually...") diff --git a/src/esm_parser/__init__.py b/src/esm_parser/__init__.py index c437098c1..53a3951dd 100644 --- a/src/esm_parser/__init__.py +++ b/src/esm_parser/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from .esm_parser import * diff --git a/src/esm_plugin_manager/__init__.py b/src/esm_plugin_manager/__init__.py index 61dbef8e8..b58c50ed0 100644 --- a/src/esm_plugin_manager/__init__.py +++ b/src/esm_plugin_manager/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi, Paul Gierz, Sebastian Wahl""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from .esm_plugin_manager import * diff --git a/src/esm_profile/__init__.py b/src/esm_profile/__init__.py index 1dd3403e4..94becfafe 100644 --- a/src/esm_profile/__init__.py +++ b/src/esm_profile/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from .esm_profile import * diff --git a/src/esm_runscripts/__init__.py b/src/esm_runscripts/__init__.py index ee182fa4f..d1c256eb8 100644 --- a/src/esm_runscripts/__init__.py +++ b/src/esm_runscripts/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from .batch_system import * from .chunky_parts import * diff --git a/src/esm_runscripts/database.py b/src/esm_runscripts/database.py index eb60fd021..cf5653919 100644 --- a/src/esm_runscripts/database.py +++ b/src/esm_runscripts/database.py @@ -1,9 +1,9 @@ -from sqlalchemy import create_engine, Column, Integer, String, Sequence, DateTime -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker - import os +from sqlalchemy import (Column, DateTime, Integer, Sequence, String, + create_engine) +from sqlalchemy.orm import declarative_base, sessionmaker + # database_file = os.path.dirname(os.path.abspath(__file__)) + "/../database/esm_runscripts.db" database_file = os.path.expanduser("~") + "/.esm_tools/esm_runscripts.db" if not os.path.isdir(os.path.expanduser("~") + "/.esm_tools"): diff --git a/src/esm_runscripts/filedicts.py b/src/esm_runscripts/filedicts.py index 154d251a4..ea9f4254d 100644 --- a/src/esm_runscripts/filedicts.py +++ b/src/esm_runscripts/filedicts.py @@ -1,5 +1,5 @@ """ -This module contains the description of a SimulationFile. +The file-dictionary implementation Developer Notes --------------- @@ -14,28 +14,46 @@ import os import pathlib import shutil -import sys from enum import Enum, auto -from typing import Any, AnyStr +from typing import Any, AnyStr, Dict import dpath.util import yaml from loguru import logger +from esm_calendar import Date +# These should be relative from esm_parser import ConfigSetup, user_error -logger.remove() -LEVEL = "ERROR" # "WARNING" # "INFO" # "DEBUG" -LOGGING_FORMAT = "[{time:HH:mm:ss DD/MM/YYYY}] |{level}| [{file} -> {function}() line:{line: >3}] >> {message}" -logger.add(sys.stderr, level=LEVEL, format=LOGGING_FORMAT) -# Enumeration of file types -class FileTypes(Enum): +class DotDict(dict): """ - Describes which type a particular file might have, e.g. ``FILE``, - ``NOT_EXISTS``, ``BROKEN_LINK``. + A dictionary subclass that allows accessing data via dot-attributes and keeps changes between dictionary + keys and dot-attributes in sync. + + This class inherits from the built-in `dict` class and overrides the `__getattr__` and `__setattr__` methods + to provide dot-attribute access to dictionary items. When an attribute is accessed using dot notation, the + corresponding dictionary key is returned. Similarly, when an attribute is set using dot notation, the corresponding + dictionary key is updated. Changes made using dictionary keys are also reflected in the dot-attributes. + + Note that this implementation assumes that the keys in the dictionary are strings, since dot-attributes can + only be strings in Python. """ + def __getattr__(self, attr): + try: + return self[attr] + except KeyError: + raise AttributeError( + f"'{self.__class__.__name__}' object has no attribute '{attr}'" + ) + + def __setattr__(self, attr, value): + self[attr] = value + + +# Enumeration of file types +class FileTypes(Enum): FILE = auto() # ordinary file DIR = auto() # directory LINK = auto() # symbolic link @@ -52,13 +70,21 @@ def _allowed_to_be_missing(method): If a method is decorated with ``@_allowed_to_be_missing``, it will return ``None`` instead of executing if the file has a attribute of - ``allowed_to_be_missing`` set to ``True. You get a warning via the logger + ``_allowed_to_be_missing`` set to ``True``. You get a warning via the logger giving the full method name that was decorated and a representation of the - file that was trying to be moved, linked, or copied. + file that was attempted to be moved, linked, or copied. Usage Example ------------- - Given you have an instanciated simulation file under ``sim_file`` with the following property:: + Given you have an instansiated simulation file under ``sim_file`` with + the following property in YAML:: + + echam: + files: + sim_file: + allowed_to_be_missing: True + ...other properties... + >>> sim_file.allowed_to_be_missing # doctest: +SKIP True @@ -91,7 +117,8 @@ def inner_method(self, *args, **kwargs): logger.warning( f"Skipping {method.__qualname__} as this file ({self}) is allowed to be missing!" ) - return None # None is the default return, but let us be explicit here, as it is a bit confusing + # None is the default return, but let us be explicit here, as it is a bit confusing + return None else: return method(self, *args, **kwargs) @@ -135,7 +162,7 @@ def _fname_has_date_stamp_info(fname, date, reqs=["%Y", "%m", "%d"]): return fname.count("checked") == len(reqs) -def globbing(method): +def _globbing(method): """ Decorator method for ``SimulationFile``'s methods ``cp``, ``mv``, ``ln``, that enables globbing. If a ``*`` is found on the ``source`` or ``target`` the globbing @@ -168,10 +195,10 @@ def inner_method(self, source, target, *args, **kwargs): target_pattern = target_name.split("*") # Check wild cards syntax - self._wild_card_check(source_pattern, target_pattern) + self.wild_card_check(source_pattern, target_pattern) # Obtain source files - glob_source_paths = self._find_globbing_files(source) + glob_source_paths = self.find_globbing_files(source) # Extract globbing source names glob_source_names = [ @@ -192,13 +219,21 @@ def inner_method(self, source, target, *args, **kwargs): glob_source_names, glob_target_names ): # Create a new simulation file object for this specific glob file - glob_config = copy.deepcopy(self._config) - glob_dict = dpath.util.get( - glob_config, self.attrs_address, separator=".", default={} + glob_file = copy.deepcopy(self) + # set source and target names: + glob_file[f"absolute_path_in_{source}"] = pathlib.Path( + str(glob_file[f"absolute_path_in_{source}"]).replace( + source_name, glob_source_name + ) + ) + glob_file[f"absolute_path_in_{target}"] = pathlib.Path( + str(glob_file[f"absolute_path_in_{target}"]).replace( + target_name, glob_target_name + ) ) - glob_dict[f"name_in_{source}"] = glob_source_name - glob_dict[f"name_in_{target}"] = glob_target_name - glob_file = SimulationFile(glob_config, self.attrs_address) + # Also need to replace names: + glob_file[f"name_in_{source}"] = glob_source_name + glob_file[f"name_in_{target}"] = glob_target_name # Use method this_method = getattr(glob_file, method_name) this_method(source, target, *args, **kwargs) @@ -208,42 +243,10 @@ def inner_method(self, source, target, *args, **kwargs): return inner_method -class SimulationFile(dict): +class SimulationFile(DotDict): """ Describes a file used within a ESM Simulation. - A ``SimulationFile`` object describes one particular file used within an - ``esm-tools`` run. This description is similar to a standard Python - dictionary. Beyond the standard dictionary methods and attributes, there - are a variety of attributes that describe how the file should behave, as - well as a few additional methods you can use to relocate the file around on - the system. Please see the detailed documentation on each of the methods - for more specifics, but in summary, a ``SimulationFile`` has the following - additional functions:: - - >>> sim_file = SimulationFile(...) # doctest: +SKIP - >>> sim_file.mv("computer", "work") # doctest: +SKIP - >>> sim_file.ln("work", "run_tree") # doctest: +SKIP - >>> sim_file.cp("run_tree", "exp_tree") # doctest: +SKIP - - You get extra functions for moving, copying, or linking a file from one - location to another. Location keys are desccribed in detail in the Notes - section. - - Furthermore, there are a few attributes that you should be aware of. These - include: - - * ``name`` : A human readable name for the file. - * ``allowed_to_be_missing`` : A ``bool`` value to set a certain file as - allowed to be missing or not. In case it is, the cp/ln/mv command will not - fail if the original file is not found. - * ``datestamp_method`` : Sets how a datestamp should be added. See - ``_allowed_datestamp_methods`` for more information. - * ``datestamp_format`` : Sets how a datestamp should be formatted. See - ``_allowed_datestamp_methods`` for more information. - - Example - ------- Given a config, you should be able to use this in YAML:: $ cat dummy_config.yaml @@ -261,30 +264,101 @@ class SimulationFile(dict): And, assuming config is as described above:: - >>> sim_file = SimulationFile(config, 'echam.files.jan_surf') # doctest: +SKIP + >>> sim_file = SimulationFile(config, ['echam']['files']['jan_surf']) # doctest: +SKIP You could then copy the file to the experiment folder:: >>> sim_file.cp_to_exp_tree() # doctest: +SKIP - - Notes - ----- - A file can be located in one of these categories (``LOCATION_KEYS``): - - computer: pool/source directory (for input files) - - exp_tree: file in the category directory in experiment directory (eg. input, output, ...) - - run_tree: file in the experiment/run_// directory - - work: file in the current work directory. Eg. experiment/run_/work/ - - LOCATION_KEY is one of the strings defined in LOCATION_KEY list - - name_in : file name (without path) in the LOCATION_KEY - - eg. name_in_computer: T63CORE2_jan_surf.nc - - eg. name_in_work: unit.24 - - absolute_path_in_ : absolute path in the LOCATION_KEY - - eg. absolute_path_in_run_tree: - - /work/ollie/pgierz/some_exp/run_20010101-20010101/input/echam/T63CORE2_jan_surf.nc """ - def __init__(self, full_config: dict, attrs_address: str): + input_file_kinds = [ + "config", + "forcing", + "input", + ] + output_file_kinds = [ + "analysis", + "couple", + "log", + "mon", + "outdata", + "restart", + "viz", + "ignore", + ] + all_model_filekinds = ( + input_file_kinds + output_file_kinds + ["src"] + ) # FIXME: In review, someone should check this + + def __init__( + self, + name="", + component="", + paths={}, + kind=None, + allowed_to_be_missing=False, + description="", + filetype="", + datestamp_method="avoid_overwrite", + **kwargs, + ): + # self.name = name + # self.paths = paths + # self.kind = kind + # self.allowed_to_be_missing = allowed_to_be_missing + # self.description = description + # self.filetype = filetype + # self._datestamp_method = datestamp_method + # self.locations = {k: v.parent for k, v in self.paths.items()} + + super().__init__( + name=name, + component=component, + paths={k: pathlib.Path(v) for k, v in paths.items()}, + kind=kind, + allowed_to_be_missing=allowed_to_be_missing, + description=description, + filetype=filetype, + _datestamp_method=datestamp_method, + locations={k: pathlib.Path(v).parent for k, v in paths.items()}, + **kwargs, + ) + + for location, path in paths.items(): + for attr_name, attr_value in { + f"absolute_path_in_{location}": path.resolve(), + f"name_in_{location}": path.name, + }.items(): + if attr_name not in self: + self[attr_name] = attr_value + + # possible paths for files: + + # location_keys = ["computer", "exp_tree", "run_tree", "work"] + # initialize the locations and complete paths for all possible locations + # self.locations = dict.fromkeys(location_keys, None) + + # Current Attributes: + # {'absolute_path_in_computer': PosixPath('/work/ollie/pool/ECHAM/T63/T63CORE2_jan_surf.nc'), + # 'absolute_path_in_exp_tree': PosixPath('/work/ollie/pgierz/some_exp/input/echam/T63CORE2_jan_surf.nc'), + # 'absolute_path_in_run_tree': PosixPath('/work/ollie/pgierz/some_exp/run_20000101-20000101/input/echam/T63CORE2_jan_surf.nc'), + # 'absolute_path_in_work': PosixPath('/work/ollie/pgierz/some_exp/run_20010101-20010101/work/unit.24'), + # 'allowed_to_be_missing': False, + # 'description': 'Initial values used for the simulation, including properties such as geopotential, temperature, pressure\n', + # 'filetype': 'NetCDF', + # 'name_in_computer': 'T63CORE2_jan_surf.nc', + # 'name_in_exp_tree': 'T63CORE2_jan_surf.nc', + # 'name_in_run_tree': 'T63CORE2_jan_surf.nc', + # 'name_in_work': 'unit.24', + # 'path_in_computer': '/work/ollie/pool/ECHAM/T63', + # 'type': 'input'} + + ############################################################################################## + # Initialize from esm-tools config + ############################################################################################## + + @classmethod + def from_config(cls, full_config: dict, attrs_address: str): """ - Initiates the properties of the object - Triggers basic checks @@ -295,52 +369,194 @@ def __init__(self, full_config: dict, attrs_address: str): The full simulation configuration attrs_address : str The address of this specific file in the full config, separated by dots. + + Note + ---- + A file can be located in one of these categories (``LOCATION_KEYS``): + - computer: pool/source directory (for input files) + - exp_tree: file in the category directory in experiment directory (eg. input, output, ...) + - run_tree: file in the experiment/run_// directory + - work: file in the current work directory. Eg. experiment/run_/work/ + + LOCATION_KEY is one of the strings defined in LOCATION_KEY list + - name_in : file name (without path) in the LOCATION_KEY + - eg. name_in_computer: T63CORE2_jan_surf.nc + - eg. name_in_work: unit.24 + - absolute_path_in_ : absolute path in the LOCATION_KEY + - eg. absolute_path_in_run_tree: + - /work/ollie/pgierz/some_exp/run_20010101-20010101/input/echam/T63CORE2_jan_surf.nc """ attrs_dict = dpath.util.get( full_config, attrs_address, separator=".", default={} ) - super().__init__(attrs_dict) - self._original_filedict = copy.deepcopy(attrs_dict) - self._config = full_config - self.attrs_address = attrs_address - self._sim_date = full_config["general"][ - "current_date" - ] # NOTE: we might have to change this in the future, depending on whether SimulationFile is access through tidy ("end_date") or prepcompute ("start_date") - self.name = attrs_address.split(".")[-1] - self.component = attrs_address.split(".")[0] - self.all_model_filetypes = full_config["general"]["all_model_filetypes"] - self.path_in_computer = self.get("path_in_computer") - self._datestamp_method = self.get( - "datestamp_method", "avoid_overwrite" - ) # This is the old default behaviour - if self.path_in_computer: - self.path_in_computer = pathlib.Path(self.path_in_computer) - - self._check_file_syntax() + _original_filedict = copy.deepcopy(attrs_dict) + name = attrs_address.split(".")[-1] + component = attrs_address.split(".")[0] + # Check if attr dict gives a sufficient representation of a file + cls._check_config_syntax(attrs_dict, name) + kind = attrs_dict.get("kind") # Complete tree names if not defined by the user - self._complete_file_names() - - # possible paths for files: - location_keys = ["computer", "exp_tree", "run_tree", "work"] - # initialize the locations and complete paths for all possible locations - self.locations = dict.fromkeys(location_keys, None) - self._resolve_abs_paths() + names = cls._complete_file_names(attrs_dict) + paths = cls._resolve_abs_paths(full_config, component, attrs_dict, names, kind) + obj = cls(name=name, paths=paths, **attrs_dict) # Verbose set to true by default, for now at least - self._verbose = full_config.get("general", {}).get("verbose", True) + obj._verbose = full_config.get("general", {}).get("verbose", True) # Checks - self._check_path_in_computer_is_abs() + obj._check_path_in_computer_is_abs(paths, component, name) + return obj + + @classmethod + def _check_config_syntax(cls, cfg, name) -> None: + """ + Checks for missing variables: + - ``kind`` + - ``path_in_computer`` if the file it an input for the experiment + - ``name_in_computer`` if the file it an input for the experiment + - ``name_in_work`` if the file it an output of the experiment + + It also checks whether ``kind``'s value is correct. + + It notifies the user about these errors in the syntax using + ``esm_parser.error``. + """ + error_text = "" + missing_vars = "" + kinds_text = ", ".join(cls.all_model_filekinds) + this_filedict = copy.deepcopy(cfg) + + if "kind" not in cfg.keys(): + error_text = ( + f"{error_text}" + f"- the ``kind`` variable is missing. Please define a ``kind`` " + f"({kinds_text})\n" + ) + missing_vars = ( + f"{missing_vars} ``kind``: forcing/input/restart/outdata/...\n" + ) + elif cfg["kind"] not in cls.all_model_filekinds: + error_text = ( + f"{error_text}" + f"- ``{cfg['kind']}`` is not a supported ``kind`` " + f"(``files.{name}.kind``), please choose one of the following " + f"kinds: {kinds_text}\n" + ) + this_filedict["kind"] = f"``{this_filedict['kind']}``" + + if ( + "path_in_computer" not in cfg.keys() + and cfg.get("kind") in cls.input_file_kinds + ): + error_text = ( + f"{error_text}" + f"- the ``path_in_computer`` variable is missing. Please define a " + f"``path_in_computer`` (i.e. the path to the file excluding its name)." + f" NOTE: this is only required for {', '.join(cls.input_file_kinds)} file " + f"kinds\n" + ) + missing_vars = ( + f"{missing_vars} ``path_in_computer``: \n" + ) + + if ( + "name_in_computer" not in cfg.keys() + and cfg.get("kind") in cls.input_file_kinds + ): + error_text = ( + f"{error_text}" + f"- the ``name_in_computer`` variable is missing. Please define a ``name_in_computer`` " + f"(i.e. name of the file in the work folder). NOTE: this is only required for " + f"{', '.join(cls.input_file_kinds)} file kinds\n" + ) + missing_vars = f"{missing_vars} ``name_in_computer``: \n" + + if ( + "name_in_work" not in cfg.keys() + and cfg.get("kind") in cls.output_file_kinds + ): + error_text = ( + f"{error_text}" + f"- the ``name_in_work`` variable is missing. Please define a ``name_in_work`` " + f"(i.e. name of the file in the work folder). NOTE: this is only required for " + f"{', '.join(cls.output_file_kinds)} file kinds\n" + ) + missing_vars = ( + f"{missing_vars} ``name_in_work``: \n" + ) + + missing_vars = ( + f"Please, complete/correct the following vars for your file:\n\n" + f"{_pretty_filedict(name, this_filedict)}" + f"{missing_vars}" + ) + + if error_text: + error_text = ( + f"The file dictionary ``{name}`` is missing relevant information " + f"or is incorrect:\n{error_text}" + ) + user_error("File Dictionaries", f"{error_text}\n{missing_vars}") + + @classmethod + def _complete_file_names(cls, cfg): + """ + Complete missing names in the file with the default name, depending whether + the file is of kind ``input`` or ``output``. + """ + if cfg["kind"] in cls.input_file_kinds: + default_name = cfg["name_in_computer"] + elif cfg["kind"] in cls.output_file_kinds: + default_name = cfg["name_in_work"] + else: + raise TypeError(f"Unknown file kind: {cfg['kind']}") + names = {} + names["computer"] = cfg.get("name_in_computer", default_name) + names["run_tree"] = cfg.get("name_in_run_tree", default_name) + names["exp_tree"] = cfg.get("name_in_exp_tree", default_name) + names["work"] = cfg.get("name_in_work", default_name) + return names + + @staticmethod + def _resolve_abs_paths(config, component, attrs_dict, names, kind) -> Dict: + # NOTE(PG): I....hate this! :-( + """ + Builds the absolute paths of the file for the different locations + (``computer``, ``work``, ``exp_tree``, ``run_tree``) using the information + about the experiment paths in ``config`` and the + ``self["path_in_computer"]``. + + It defines these new variables in the ``SimulationFile`` dictionary: + - ``self["absolute_path_in_work"]`` + - ``self["absolute_path_in_computer"]`` + - ``self["absolute_path_in_run_tree"]`` + - ``self["absolute_path_in_exp_tree"]`` + """ + locations = { + "work": pathlib.Path(config["general"]["thisrun_work_dir"]), + "computer": pathlib.Path(attrs_dict.get("path_in_computer", "/dev/null")), + "exp_tree": pathlib.Path(config[component][f"experiment_{kind}_dir"]), + "run_tree": pathlib.Path(config[component][f"thisrun_{kind}_dir"]), + } + + return {key: path.joinpath(names[key]) for key, path in locations.items()} + + @staticmethod + def _check_path_in_computer_is_abs(paths, component, name): + if paths["computer"] is not None and not paths["computer"].is_absolute(): + user_error( + "File Dictionaries", + "The path defined for " + f"``{component}.files.{name}.path_in_computer`` is not " + f"absolute (``{paths['computer']}``). Please, always define an " + "absolute path for the ``path_in_computer`` variable.", + ) ############################################################################################## # Overrides of standard dict methods ############################################################################################## - def __str__(self): - address = " -> ".join(self.attrs_address.split(".")) - return address - def __setattr__(self, name: str, value: Any) -> None: """Checks when changing dot attributes for disallowed values""" if name == "datestamp_format": @@ -372,21 +588,9 @@ def update(self, *args, **kwargs): ############################################################################################## ############################################################################################## - # Object Properities + # Object Properties ############################################################################################## - # This part allows for dot-access to allowed_to_be_missing: - @property - def allowed_to_be_missing(self): - """ - Example - ------- - >>> sim_file = SimulationFile(config, 'echam.files.jan_surf') # doctest: +SKIP - >>> sim_file.allowed_to_be_missing # doctest: +SKIP - True - """ - return self.get("allowed_to_be_missing", False) - @property def datestamp_method(self): """ @@ -419,7 +623,7 @@ def datestamp_format(self): ############################################################################################## # Main Methods ############################################################################################## - @globbing + @_globbing @_allowed_to_be_missing def cp(self, source: str, target: str) -> None: """ @@ -446,9 +650,6 @@ def cp(self, source: str, target: str) -> None: source_path = self[f"absolute_path_in_{source}"] target_path = self[f"absolute_path_in_{target}"] - # Create subfolders contained in ``name_in_{target}`` - self._makedirs_in_name(target) - # Datestamps if self.datestamp_method == "always": target_path = self._always_datestamp(target_path) @@ -456,6 +657,7 @@ def cp(self, source: str, target: str) -> None: target_path = self._avoid_override_datestamp(target_path) # General Checks + # TODO (deniz): need to add higher level exception handler (eg. user_error) self._check_source_and_target(source_path, target_path) # Actual copy @@ -466,26 +668,40 @@ def cp(self, source: str, target: str) -> None: copy_func = shutil.copy2 try: copy_func(source_path, target_path) - logger.debug(f"Copied {source_path} --> {target_path}") + logger.success(f"Copied {source_path} --> {target_path}") except IOError as error: raise IOError( f"Unable to copy {source_path} to {target_path}\n\n" f"Exception details:\n{error}" ) - @globbing + @_globbing @_allowed_to_be_missing - def mv(self, source: str, target: str) -> None: - """ - Moves (renames) the SimulationFile from it's location in ``source`` to - it's location in ``target``. + def ln(self, source: AnyStr, target: AnyStr) -> None: + """creates symbolic links from the path retrieved by ``source`` to the one by ``target``. Parameters ---------- source : str - One of ``"computer"``, ``"work"``, ``"exp_tree"``, "``run_tree``" + key to retrieve the source from the file dictionary. Possible options: ``computer``, ``work``, ``exp_tree``, ``run_tree`` + target : str - One of ``"computer"``, ``"work"``, ``"exp_tree"``, "``run_tree``" + key to retrieve the target from the file dictionary. Possible options: ``computer``, ``work``, ``exp_tree``, ``run_tree`` + + Returns + ------- + None + + Raises + ------ + FileNotFoundError + - Source path does not exist + OSError + - Target path is a directory + - Symbolic link is trying to link to itself + - Target path does not exist + FileExistsError + - Target path already exists """ if source not in self.locations: raise ValueError( @@ -495,58 +711,40 @@ def mv(self, source: str, target: str) -> None: raise ValueError( f"Target is incorrectly defined, and needs to be in {self.locations}" ) + # full paths: directory path / file name source_path = self[f"absolute_path_in_{source}"] target_path = self[f"absolute_path_in_{target}"] - # Create subfolders contained in ``name_in_{target}`` - self._makedirs_in_name(target) - # Datestamps if self.datestamp_method == "always": target_path = self._always_datestamp(target_path) if self.datestamp_method == "avoid_overwrite": target_path = self._avoid_override_datestamp(target_path) - # General Checks + # TODO (deniz): need to add higher level exception handler (eg. user_error) self._check_source_and_target(source_path, target_path) - # Perform the movement: try: - source_path.rename(target_path) - logger.debug(f"Moved {source_path} --> {target_path}") + os.symlink(source_path, target_path) except IOError as error: raise IOError( - f"Unable to move {source_path} to {target_path}\n\n" + f"Unable to link {source_path} to {target_path}\n\n" f"Exception details:\n{error}" ) - @globbing + @_globbing @_allowed_to_be_missing - def ln(self, source: AnyStr, target: AnyStr) -> None: - """creates symbolic links from the path retrieved by ``source`` to the one by ``target``. + def mv(self, source: str, target: str) -> None: + """ + Moves (renames) the SimulationFile from it's location in ``source`` to + it's location in ``target``. Parameters ---------- source : str - key to retrieve the source from the file dictionary. Possible options: ``computer``, ``work``, ``exp_tree``, ``run_tree`` - + One of ``"computer"``, ``"work"``, ``"exp_tree"``, "``run_tree``" target : str - key to retrieve the target from the file dictionary. Possible options: ``computer``, ``work``, ``exp_tree``, ``run_tree`` - - Returns - ------- - None - - Raises - ------ - FileNotFoundError - - Source path does not exist - OSError - - Target path is a directory - - Symbolic link is trying to link to itself - - Target path does not exist - FileExistsError - - Target path already exists + One of ``"computer"``, ``"work"``, ``"exp_tree"``, "``run_tree``" """ if source not in self.locations: raise ValueError( @@ -556,53 +754,28 @@ def ln(self, source: AnyStr, target: AnyStr) -> None: raise ValueError( f"Target is incorrectly defined, and needs to be in {self.locations}" ) - # full paths: directory path / file name source_path = self[f"absolute_path_in_{source}"] target_path = self[f"absolute_path_in_{target}"] - # Create subfolders contained in ``name_in_{target}`` - self._makedirs_in_name(target) - # Datestamps if self.datestamp_method == "always": target_path = self._always_datestamp(target_path) if self.datestamp_method == "avoid_overwrite": target_path = self._avoid_override_datestamp(target_path) - # General Checks + # TODO (deniz): need to add higher level exception handler (eg. user_error) self._check_source_and_target(source_path, target_path) + # Perform the movement: try: - os.symlink(source_path, target_path) - logger.debug(f"Linked {source_path} --> {target_path}") + source_path.rename(target_path) + logger.success(f"Moved {source_path} --> {target_path}") except IOError as error: raise IOError( - f"Unable to link {source_path} to {target_path}\n\n" + f"Unable to move {source_path} to {target_path}\n\n" f"Exception details:\n{error}" ) - def pretty_filedict(self, filedict): - """ - Returns a string in yaml format of the given file dictionary. - - Parameters - ---------- - dict : - A file dictionary - - Returns - ------- - str : - A string in yaml format of the given file dictionary - """ - return yaml.dump({"files": {self.name: filedict}}) - - ############################################################################################## - - ############################################################################################## - # Private Methods, Attributes, and Class Variables - ############################################################################################## - _allowed_datestamp_methods = {"never", "always", "avoid_overwrite"} """ Set containing the allowed datestamp methods which can be chosen from. @@ -652,37 +825,7 @@ def _check_datestamp_format_is_allowed(self, datestamp_format): "The datestamp_format must be defined as one of check_from_filename or append" ) - def _resolve_abs_paths(self) -> None: - """ - Builds the absolute paths of the file for the different locations - (``computer``, ``work``, ``exp_tree``, ``run_tree``) using the information - about the experiment paths in ``self._config`` and the - ``self["path_in_computer"]``. - - It defines these new variables in the ``SimulationFile`` dictionary: - - ``self["absolute_path_in_work"]`` - - ``self["absolute_path_in_computer"]`` - - ``self["absolute_path_in_run_tree"]`` - - ``self["absolute_path_in_exp_tree"]`` - """ - self.locations = { - "work": pathlib.Path(self._config["general"]["thisrun_work_dir"]), - "computer": self.path_in_computer, # Already Path type from _init_ - "exp_tree": pathlib.Path( - self._config[self.component][f"experiment_{self['type']}_dir"] - ), - "run_tree": pathlib.Path( - self._config[self.component][f"thisrun_{self['type']}_dir"] - ), - } - - for key, path in self.locations.items(): - if key == "computer" and path is None: - self[f"absolute_path_in_{key}"] = None - else: - self[f"absolute_path_in_{key}"] = path.joinpath(self[f"name_in_{key}"]) - - def _path_type(self, path: pathlib.Path) -> FileTypes: + def _path_type(self, path: pathlib.Path) -> int: """ Checks if the given ``path`` exists. If it does returns it's type, if it doesn't, returns ``None``. @@ -709,16 +852,13 @@ def _path_type(self, path: pathlib.Path) -> FileTypes: f"Path ``{path}`` has an incompatible datatype ``{datatype}``. str or pathlib.Path is expected" ) - if isinstance(path, str): - path = pathlib.Path(path) + path = pathlib.Path(path) # NOTE: is_symlink() needs to come first because it is also a is_file() # NOTE: pathlib.Path().exists() also checks is the target of a symbolic link exists or not if path.is_symlink() and not path.exists(): - logger.warning(f"Broken link detected: {path}") return FileTypes.BROKEN_LINK elif not path.exists(): - logger.warning(f"File does not exist: {path}") return FileTypes.NOT_EXISTS elif path.is_symlink(): return FileTypes.LINK @@ -782,22 +922,8 @@ def _avoid_override_datestamp(self, target: pathlib.Path) -> pathlib.Path: # The other case ("check_from_filename") is meaningless? return target - def _complete_file_names(self): - """ - Complete missing names in the file with the default name, depending whether - the file is of type ``input`` or ``output``. - """ - if self["type"] in self.input_file_types: - default_name = self["name_in_computer"] - elif self["type"] in self.output_file_types: - default_name = self["name_in_work"] - self["name_in_computer"] = self.get("name_in_computer", default_name) - self["name_in_run_tree"] = self.get("name_in_run_tree", default_name) - self["name_in_exp_tree"] = self.get("name_in_exp_tree", default_name) - self["name_in_work"] = self.get("name_in_work", default_name) - @staticmethod - def _wild_card_check(source_pattern: list, target_pattern: list) -> bool: + def wild_card_check(source_pattern: list, target_pattern: list) -> True: """ Checks for syntax mistakes. If any were found, it notifies the user about these errors in the syntax using ``esm_parser.error``. @@ -821,14 +947,14 @@ def _wild_card_check(source_pattern: list, target_pattern: list) -> bool: "The wild card pattern of the source " + f"``{source_pattern}`` does not match with the " + f"target ``{target_pattern}``. Make sure the " - + "that the number of ``*`` are the same in both " - + "sources and targets." + + f"that the number of ``*`` are the same in both " + + f"sources and targets." ), ) return True - def _find_globbing_files(self, location: str) -> list: + def find_globbing_files(self, location: str) -> list: """ Lists the files matching the globbing path of the given ``location``, and notifies the user if none were found, via ``esm_parser.user_error``. @@ -856,158 +982,9 @@ def _find_globbing_files(self, location: str) -> list: return glob_paths - def _makedirs_in_name(self, name_type: str) -> None: - """ - Creates subdirectories included in the ``name_in_``, if any. - - Raises - ------ - FileNotFoundError - If ``self.locations[name_type]`` path does not exist - """ - # Are there any subdirectories in ``name_in_? - if "/" in self[f"name_in_{name_type}"]: - parent_path = self[f"absolute_path_in_{name_type}"].parent - # If the parent path does not exist check whether the file location - # exists - if not parent_path.exists(): - location = self.locations[name_type] - if location.exists(): - # The location exists therefore the remaining extra directories - # from the parent_path can be created - os.makedirs(parent_path) - else: - # The location does not exist, the role of this function is not - # to create it, therefore, raise an error - raise FileNotFoundError( - f"Unable to perform file operation. Path for ``{name_type}`` " - f"({location}) does not exist!" - ) - - def _check_file_syntax(self) -> None: - """ - Checks for missing variables: - - ``type`` - - ``path_in_computer`` if the file it an input for the experiment - - ``name_in_computer`` if the file it an input for the experiment - - ``name_in_work`` if the file it an output of the experiment - - It also checks whether ``type``'s value is correct. - - It notifies the user about these errors in the syntax using - ``esm_parser.error``. - """ - error_text = "" - missing_vars = "" - types_text = ", ".join(self.all_model_filetypes) - this_filedict = copy.deepcopy(self._original_filedict) - self.input_file_types = input_file_types = ["config", "forcing", "input"] - self.output_file_types = output_file_types = [ - "analysis", - "couple", - "log", - "mon", - "outdata", - "restart", - "viz", - "ignore", - ] - - if "type" not in self.keys(): - error_text = ( - f"{error_text}" - f"- the ``type`` variable is missing. Please define a ``type`` " - f"({types_text})\n" - ) - missing_vars = ( - f"{missing_vars} ``type``: forcing/input/restart/outdata/...\n" - ) - elif self["type"] not in self.all_model_filetypes: - error_text = ( - f"{error_text}" - f"- ``{self['type']}`` is not a supported ``type`` " - f"(``files.{self.name}.type``), please choose one of the following " - f"types: {types_text}\n" - ) - this_filedict["type"] = f"``{this_filedict['type']}``" - - if ( - "path_in_computer" not in self.keys() - and self.get("type") in input_file_types - ): - error_text = ( - f"{error_text}" - f"- the ``path_in_computer`` variable is missing. Please define a " - f"``path_in_computer`` (i.e. the path to the file excluding its name)." - f" NOTE: this is only required for {', '.join(input_file_types)} file " - f"types\n" - ) - missing_vars = ( - f"{missing_vars} ``path_in_computer``: \n" - ) - - if ( - "name_in_computer" not in self.keys() - and self.get("type") in input_file_types - ): - error_text = ( - f"{error_text}" - f"- the ``name_in_computer`` variable is missing. Please define a ``name_in_computer`` " - f"(i.e. name of the file in the work folder). NOTE: this is only required for " - f"{', '.join(input_file_types)} file types\n" - ) - missing_vars = f"{missing_vars} ``name_in_computer``: \n" - - if "name_in_work" not in self.keys() and self.get("type") in output_file_types: - error_text = ( - f"{error_text}" - f"- the ``name_in_work`` variable is missing. Please define a ``name_in_work`` " - f"(i.e. name of the file in the work folder). NOTE: this is only required for " - f"{', '.join(output_file_types)} file types\n" - ) - missing_vars = ( - f"{missing_vars} ``name_in_work``: \n" - ) - - missing_vars = ( - f"Please, complete/correct the following vars for your file:\n\n" - f"{self.pretty_filedict(this_filedict)}" - f"{missing_vars}" - ) - - if error_text: - error_text = ( - f"The file dictionary ``{self.name}`` is missing relevant information " - f"or is incorrect:\n{error_text}" - ) - user_error("File Dictionaries", f"{error_text}\n{missing_vars}") - - def _check_path_in_computer_is_abs(self): - """ - Determines if the path for files stored in the computer (rather than - the experiment tree or the work folder) is an absolute path. - - Raises - ------ - user_error : - The user_error function will raises a sys.exit with a user message if the path for - the computer is not an absolute path. - """ - if ( - self.path_in_computer is not None - and not self.path_in_computer.is_absolute() - ): - user_error( - "File Dictionaries", - "The path defined for " - f"``{self.component}.files.{self.name}.path_in_computer`` is not " - f"absolute (``{self.path_in_computer}``). Please, always define an " - "absolute path for the ``path_in_computer`` variable.", - ) - def _check_source_and_target( self, source_path: pathlib.Path, target_path: pathlib.Path - ) -> bool: + ) -> None: """ Performs common checks for file movements @@ -1063,34 +1040,50 @@ def _check_source_and_target( return True -class SimulationFiles(dict): - """ - Once instanciated, searches in the ``config`` dictionary for the ``files`` keys. - This class contains the methods to: 1) instanciate each of the files defined in - ``files`` as ``SimulationFile`` objects and 2) loop through these objects - triggering the desire file movement. - """ - def __init__(self, config): - # Loop through components? - # Loop through files? - pass +class DatedSimulationFile(SimulationFile): + """A SimultionFile which also needs to know about dates""" + def __init__( + self, + date=Date("2000-01-01"), + **kwargs, + ): + super().__init__(**kwargs) + self._sim_date = date -def resolve_file_movements(config: ConfigSetup) -> ConfigSetup: + @classmethod + def from_config(cls, full_config: dict, attrs_address: str, date: Date): + obj = super().from_config(full_config, attrs_address) + obj._sim_date = date + return obj + + +def _pretty_filedict(name, filedict): """ - Runs all methods required to get files into their correct locations. This will - instanciate the ``SimulationFiles`` class. It's called by the recipe manager. + Returns a string in yaml format of the given file dictionary. Parameters ---------- - config : ConfigSetup - The complete simulation configuration. + dict + A file dictionary Returns ------- - config : ConfigSetup - The complete simulation configuration, potentially modified. + str + A string in yaml format of the given file dictionary """ + return yaml.dump({"files": {name: filedict}}) + + +def copy_files(config): + """Copies files""" + # PG: No. We do not want this kind of general function. This is just to + # demonstrate how the test would work + return config + + +def resolve_file_movements(config: ConfigSetup) -> ConfigSetup: + """Replaces former assemble() function""" # TODO: to be filled with functions # DONE: type annotation # DONE: basic unit test: test_resolve_file_movements diff --git a/src/esm_runscripts/workflow.py b/src/esm_runscripts/workflow.py index c81aad75b..b677ea507 100644 --- a/src/esm_runscripts/workflow.py +++ b/src/esm_runscripts/workflow.py @@ -132,7 +132,7 @@ def order_clusters(config): not gw_config["subjob_clusters"][subjob_cluster]["run_after"] in gw_config["subjob_clusters"] ): - print(f"Unknown cluster {subjob_cluster['run_after']}.") + print(f"Unknown cluster {gw_config['subjob_clusters'][subjob_cluster]['run_after']}.") sys.exit(-1) calling_cluster = gw_config["subjob_clusters"][subjob_cluster]["run_after"] @@ -156,7 +156,7 @@ def order_clusters(config): not gw_config["subjob_clusters"][subjob_cluster]["run_before"] in gw_config["subjob_clusters"] ): - print(f"Unknown cluster {subjob_cluster['run_before']}.") + print(f"Unknown cluster {gw_config['subjob_clusters'][subjob_cluster]['run_before']}.") sys.exit(-1) called_cluster = gw_config["subjob_clusters"][subjob_cluster]["run_before"] @@ -330,7 +330,13 @@ def init_total_workflow(config): config["general"]["workflow"]["subjobs"] = prepcompute config["general"]["workflow"]["subjobs"].update(compute) config["general"]["workflow"]["subjobs"].update(tidy) - + else: + if not "prepcompute" in config["general"]["workflow"]["subjobs"]: + config["general"]["workflow"]["subjobs"].update(prepcompute) + if not "compute" in config["general"]["workflow"]["subjobs"]: + config["general"]["workflow"]["subjobs"].update(compute) + if not "tidy" in config["general"]["workflow"]["subjobs"]: + config["general"]["workflow"]["subjobs"].update(tidy) if not "last_task_in_queue" in config["general"]["workflow"]: config["general"]["workflow"]["last_task_in_queue"] = "tidy" if not "first_task_in_queue" in config["general"]["workflow"]: diff --git a/src/esm_tests/__init__.py b/src/esm_tests/__init__.py index 49cd73644..0f4e0ceba 100644 --- a/src/esm_tests/__init__.py +++ b/src/esm_tests/__init__.py @@ -2,7 +2,7 @@ __author__ = """Miguel Andres-Martinez""" __email__ = "miguel.andres-martinez@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from .initialization import * from .read_shipped_data import * diff --git a/src/esm_tests/resources b/src/esm_tests/resources index 4380094e7..aa7ec63fa 160000 --- a/src/esm_tests/resources +++ b/src/esm_tests/resources @@ -1 +1 @@ -Subproject commit 4380094e7d9f936ab8d909f520859c6f63dba4be +Subproject commit aa7ec63faaa10a355c5041eae4f106ce508c6be1 diff --git a/src/esm_tests/test_utilities.py b/src/esm_tests/test_utilities.py index c855ae70d..4be563ad2 100644 --- a/src/esm_tests/test_utilities.py +++ b/src/esm_tests/test_utilities.py @@ -208,7 +208,7 @@ def print_state_online(info={}): """ Returns the state of the tested models obtained directly from the repository online. This method is aimed to be used externally from ``esm_tests`` (i.e. throw the - ``esm_tools --test-state`` command). + ``esm_tools test-state`` command). Parameters ---------- diff --git a/src/esm_tools/__init__.py b/src/esm_tools/__init__.py index 844f0dcb5..7abc6cae0 100644 --- a/src/esm_tools/__init__.py +++ b/src/esm_tools/__init__.py @@ -23,7 +23,7 @@ __author__ = """Dirk Barbi, Paul Gierz""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" import functools import inspect diff --git a/src/esm_tools/cli.py b/src/esm_tools/cli.py index 488ec29e0..5c52351d8 100644 --- a/src/esm_tools/cli.py +++ b/src/esm_tools/cli.py @@ -1,24 +1,60 @@ """ Functionality for displaying the version number """ +import shutil import sys + import click -import esm_tools + import esm_tests +import esm_tools # click.version_option read the PKG_INFO which contains the wrong version # number. Get it directly from __init__.py version = esm_tools.__version__ + +@click.group() @click.version_option(version=version) -@click.option("--test-state", is_flag=True, help="Prints the state of the last tested experiments.") -@click.command() -def main(test_state): - """Console script for esm_tools""" +def main(): + pass + + +@main.command() +def test_state(): + """Prints the state of the last tested experiments.""" + + esm_tests.test_utilities.print_state_online() + + return 0 - if test_state: - esm_tests.test_utilities.print_state_online() +@main.command() +@click.option( + "-t", + "--type", + type=click.Choice(["component", "setup", "machine"], case_sensitive=False), + help="Creates either a new component (default) or a new setup", + default="component", + show_default=True, +) +@click.argument("name", nargs=1) +def create_new_config(name, type): + """Opens your $EDITOR and creates a new file for NAME""" + click.echo(f"Creating a new {type} configuration for {name}") + template_file = esm_tools.get_config_filepath( + config=f"templates/{type}_template.yaml" + ) + shutil.copy(template_file, f"{name}.yaml") + new_config_file = f"{name}.yaml" + # TODO(PG): Currently this lands in the current working directory. + # Would be nice if this landed already in an git-controlled + # editable version and prepared a commit for you: + click.edit(filename=new_config_file) + click.echo( + "Thank you! The new configuration has been saved. Please commit it (and get in touch with the" + ) + click.echo("esm-tools team if you need help)!") return 0 diff --git a/src/esm_utilities/__init__.py b/src/esm_utilities/__init__.py index 7cbc21772..4aa704f67 100644 --- a/src/esm_utilities/__init__.py +++ b/src/esm_utilities/__init__.py @@ -2,6 +2,6 @@ __author__ = """Paul Gierz""" __email__ = "pgierz@awi.de" -__version__ = "6.20.0" +__version__ = "6.21.11" from .utils import * diff --git a/tests/test_esm_runscripts/test_filedicts.py b/tests/test_esm_runscripts/test_filedicts.py index b5a6d563f..4002b21ca 100644 --- a/tests/test_esm_runscripts/test_filedicts.py +++ b/tests/test_esm_runscripts/test_filedicts.py @@ -58,7 +58,7 @@ def config_tuple(): echam: files: jan_surf: - type: input + kind: input allowed_to_be_missing: False name_in_computer: T63CORE2_jan_surf.nc name_in_work: unit.24 @@ -93,7 +93,7 @@ def simulation_file(fs, config_tuple): """ config = config_tuple.config attr_address = config_tuple.attr_address - fake_simulation_file = filedicts.SimulationFile(config, attr_address) + fake_simulation_file = filedicts.SimulationFile.from_config(config, attr_address) fs.create_dir(fake_simulation_file.locations["work"]) fs.create_dir(fake_simulation_file.locations["computer"]) @@ -104,6 +104,52 @@ def simulation_file(fs, config_tuple): yield fake_simulation_file +@pytest.fixture() +def dated_simulation_file(fs, config_tuple): + config = config_tuple.config + attr_address = config_tuple.attr_address + date = esm_calendar.Date("2000-01-01") + fake_simulation_file = filedicts.DatedSimulationFile.from_config( + config, attr_address, date + ) + + fs.create_dir(fake_simulation_file.locations["work"]) + fs.create_dir(fake_simulation_file.locations["computer"]) + fs.create_dir(fake_simulation_file.locations["exp_tree"]) + fs.create_dir(fake_simulation_file.locations["run_tree"]) + fs.create_file(fake_simulation_file["absolute_path_in_computer"]) + + yield fake_simulation_file + + +def test_example(fs): + # Make a fake config: + config = """ + general: + base_dir: /some/dummy/location/ + all_model_filetypes: [analysis, bin, config, forcing, input, couple, log, mon, outdata, restart, viz, ignore] + echam: + files: + jan_surf: + name: ECHAM Jan Surf File + path_in_computer: /work/ollie/pool/ECHAM + name_in_computer: T63CORE2_jan_surf.nc + name_in_work: unit.24 + kind: input + experiment_input_dir: /work/ollie/pgierz/some_exp/input/echam + """ + date = esm_calendar.Date("2000-01-01T00:00:00") + config = yaml.safe_load(config) + config["general"]["current_date"] = date + # Create some fake files and directories you might want in your test + fs.create_file("/work/ollie/pool/ECHAM/T63CORE2_jan_surf.nc") + fs.create_dir("/some/dummy/location/expid/run_18500101-18501231/work") + # This module also have functions for link files, globbing, etc. + esm_runscripts.filedicts.copy_files(config) + assert os.path.exists("/some/dummy/location/expid/run_18500101-18501231/work/") + assert os.path.exists("/work/ollie/pool/ECHAM/T63CORE2_jan_surf.nc") + + def test_filedicts_basics(fs): """Tests basic attribute behavior of filedicts""" @@ -122,7 +168,7 @@ def test_filedicts_basics(fs): name_in_work: unit.24 path_in_computer: /work/ollie/pool/ECHAM/T63 filetype: NetCDF - type: input + kind: input description: > Initial values used for the simulation, including properties such as geopotential, temperature, pressure @@ -134,12 +180,15 @@ def test_filedicts_basics(fs): config["general"]["current_date"] = date # Not needed for this test, just a demonstration: fs.create_file("/work/ollie/pool/ECHAM/T63/T63CORE2_jan_surf.nc") - sim_file = esm_runscripts.filedicts.SimulationFile(config, "echam.files.jan_surf") + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( + config, "echam.files.jan_surf" + ) assert sim_file["name_in_work"] == "unit.24" assert sim_file.locations["work"] == Path( "/work/ollie/pgierz/some_exp/run_20010101-20010101/work" ) - assert sim_file._config == config + # NOTE(PG): This check is removed, a SimulationFile no longer needs to know it was initialized from a config + # assert sim_file._config == config assert sim_file.locations["computer"] == Path("/work/ollie/pool/ECHAM/T63") @@ -242,21 +291,21 @@ def test_allowed_to_be_missing_attr(): allowed_to_be_missing: True path_in_computer: "/some/location/on/ollie" name_in_computer: "foo" - type: "input" + kind: "input" human_readable_tag_002: allowed_to_be_missing: False path_in_computer: "/some/location/on/ollie" name_in_computer: "bar" - type: "input" + kind: "input" """ date = esm_calendar.Date("2000-01-01T00:00:00") config = yaml.safe_load(dummy_config) config["general"]["current_date"] = date # Not needed for this test, just a demonstration: - sim_file_001 = esm_runscripts.filedicts.SimulationFile( + sim_file_001 = esm_runscripts.filedicts.SimulationFile.from_config( config, "echam.files.human_readable_tag_001" ) - sim_file_002 = esm_runscripts.filedicts.SimulationFile( + sim_file_002 = esm_runscripts.filedicts.SimulationFile.from_config( config, "echam.files.human_readable_tag_002" ) @@ -281,7 +330,7 @@ def test_allowed_to_be_missing_mv(fs): thisrun_input_dir: /work/ollie/pgierz/some_exp/run_20010101-20011231/input/echam files: human_readable_tag_001: - type: input + kind: input allowed_to_be_missing: True name_in_computer: foo path_in_computer: /work/data/pool @@ -294,7 +343,7 @@ def test_allowed_to_be_missing_mv(fs): config["general"]["current_date"] = date fs.create_dir("/work/data/pool") fs.create_file("/work/data/pool/not_foo_at_all") - sim_file = esm_runscripts.filedicts.SimulationFile( + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( config, "echam.files.human_readable_tag_001" ) sim_file.mv("computer", "work") @@ -319,7 +368,7 @@ def test_allowed_to_be_missing_mv_if_exists(fs): thisrun_input_dir: /work/ollie/pgierz/some_exp/run_20010101-20011231/input/echam files: human_readable_tag_001: - type: input + kind: input allowed_to_be_missing: True name_in_computer: foo path_in_computer: /work/data/pool @@ -333,7 +382,7 @@ def test_allowed_to_be_missing_mv_if_exists(fs): fs.create_dir("/work/data/pool") fs.create_file("/work/data/pool/foo") fs.create_dir("/work/ollie/pgierz/some_exp/run_20010101-20011231/work") - sim_file = esm_runscripts.filedicts.SimulationFile( + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( config, "echam.files.human_readable_tag_001" ) sim_file.mv("computer", "work") @@ -352,7 +401,7 @@ def test_cp_file(fs): jan_surf: name_in_computer: T63CORE2_jan_surf.nc name_in_work: unit.24 - type: input + kind: input path_in_computer: /work/ollie/pool/ECHAM/T63/ experiment_input_dir: /work/ollie/pgierz/some_exp/input/echam thisrun_input_dir: /work/ollie/pgierz/some_exp/run_20010101-20010101/input/echam @@ -376,7 +425,9 @@ def test_cp_file(fs): fs.create_dir(target_folder) # Test the method - sim_file = esm_runscripts.filedicts.SimulationFile(config, "echam.files.jan_surf") + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( + config, "echam.files.jan_surf" + ) sim_file.cp("computer", "work") assert os.path.exists(target) @@ -394,7 +445,7 @@ def test_cp_folder(fs): o3_data: name_in_computer: o3chem_l91 name_in_work: o3chem_l91 - type: input + kind: input path_in_computer: /work/ollie/pool/OIFS/159_4 datestamp_method: never experiment_input_dir: /work/ollie/pgierz/some_exp/input/oifs @@ -419,7 +470,9 @@ def test_cp_folder(fs): fs.create_dir(target_folder) # Test the method - sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.o3_data") + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( + config, "oifs.files.o3_data" + ) sim_file.cp("computer", "work") assert os.path.exists(target) @@ -429,7 +482,7 @@ def test_resolve_file_movements(config_tuple): # arrange config-in config = config_tuple.config attr_address = config_tuple.attr_address - simulation_file = filedicts.SimulationFile(config, attr_address) + simulation_file = filedicts.SimulationFile.from_config(config, attr_address) config = filedicts.resolve_file_movements(config) # check config-out @@ -451,7 +504,7 @@ def test_mv(fs): jan_surf: name_in_computer: T63CORE2_jan_surf.nc path_in_computer: /work/ollie/pool/ECHAM/T63/ - type: input + kind: input name_in_work: unit.24 path_in_work: . experiment_input_dir: /work/ollie/pgierz/some_exp/input/echam @@ -463,7 +516,9 @@ def test_mv(fs): fs.create_file("/work/ollie/pool/ECHAM/T63/T63CORE2_jan_surf.nc") fs.create_dir("/work/ollie/pgierz/some_exp/run_20010101-20010101/work") assert os.path.exists("/work/ollie/pool/ECHAM/T63/T63CORE2_jan_surf.nc") - sim_file = esm_runscripts.filedicts.SimulationFile(config, "echam.files.jan_surf") + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( + config, "echam.files.jan_surf" + ) sim_file.mv("computer", "work") assert not os.path.exists("/work/ollie/pool/ECHAM/T63/T63CORE2_jan_surf.nc") assert os.path.exists( @@ -506,23 +561,21 @@ def test_ln_raises_exception_when_source_is_a_broken_link(simulation_file, fs): simulation_file.ln("computer", "work") -def test_ln_raises_exception_when_target_is_a_directory_and_not_a_file( - simulation_file, fs -): +def test_ln_raises_exception_when_target_is_a_directory_and_not_a_file(simulation_file): # Since the simulation_file fixture is a fake_jan_surf, we need the work folder: simulation_file["absolute_path_in_work"] = simulation_file.locations["work"] with pytest.raises(OSError): simulation_file.ln("computer", "work") -def test_ln_raises_exception_when_target_path_exists(simulation_file, fs): - file_path_in_work = simulation_file["absolute_path_in_work"] +def test_ln_raises_exception_when_target_path_exists(dated_simulation_file, fs): + file_path_in_work = dated_simulation_file["absolute_path_in_work"] # create the target file so that it will raise an exception fs.create_file(file_path_in_work) - simulation_file.datestamp_method = "never" + dated_simulation_file._datestamp_method = "never" with pytest.raises(FileExistsError): - simulation_file.ln("computer", "work") + dated_simulation_file.ln("computer", "work") def test_ln_raises_exception_when_source_file_does_not_exist(simulation_file, fs): @@ -540,8 +593,8 @@ def test_ln_raises_exception_when_target_path_does_not_exist(simulation_file, fs # ========== end of ln() tests ========== -def test_check_file_syntax_type_missing(): - """Tests for ``type`` variable missing""" +def test_check_file_syntax_kind_missing(): + """Tests for ``kind`` variable missing""" dummy_config = """ general: thisrun_dir: "/work/ollie/pgierz/some_exp/run_20010101-20010101" @@ -561,16 +614,16 @@ def test_check_file_syntax_type_missing(): # Captures output (i.e. the user-friendly error) with Capturing() as output: with pytest.raises(SystemExit) as error: - sim_file = esm_runscripts.filedicts.SimulationFile( + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( config, "echam.files.jan_surf" ) - error_text = "the \x1b[31mtype\x1b[0m variable is missing" + error_text = "the \x1b[31mkind\x1b[0m variable is missing" assert any([error_text in line for line in output]) -def test_check_file_syntax_type_incorrect(): - """Tests for ``type`` variable being incorrectly defined""" +def test_check_file_syntax_kind_incorrect(): + """Tests for ``kind`` variable being incorrectly defined""" dummy_config = """ general: thisrun_dir: "/work/ollie/pgierz/some_exp/run_20010101-20010101" @@ -579,7 +632,7 @@ def test_check_file_syntax_type_incorrect(): echam: files: jan_surf: - type: is_wrong + kind: is_wrong experiment_input_dir: /work/ollie/pgierz/some_exp/input/echam thisrun_input_dir: /work/ollie/pgierz/some_exp/run_20010101-20010101/input/echam """ @@ -590,11 +643,11 @@ def test_check_file_syntax_type_incorrect(): # Captures output (i.e. the user-friendly error) with Capturing() as output: with pytest.raises(SystemExit) as error: - sim_file = esm_runscripts.filedicts.SimulationFile( + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( config, "echam.files.jan_surf" ) - error_text = "is_wrong\x1b[0m is not a supported \x1b[31mtype" + error_text = "is_wrong\x1b[0m is not a supported \x1b[31mkind" assert any([error_text in line for line in output]) @@ -610,7 +663,7 @@ def test_check_file_syntax_input(): echam: files: jan_surf: - type: input + kind: input experiment_input_dir: /work/ollie/pgierz/some_exp/input/echam thisrun_input_dir: /work/ollie/pgierz/some_exp/run_20010101-20010101/input/echam """ @@ -621,7 +674,7 @@ def test_check_file_syntax_input(): # Captures output (i.e. the user-friendly error) with Capturing() as output: with pytest.raises(SystemExit) as error: - sim_file = esm_runscripts.filedicts.SimulationFile( + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( config, "echam.files.jan_surf" ) @@ -641,7 +694,7 @@ def test_check_file_syntax_output(): echam: files: jan_surf: - type: outdata + kind: outdata experiment_input_dir: /work/ollie/pgierz/some_exp/input/echam thisrun_input_dir: /work/ollie/pgierz/some_exp/run_20010101-20010101/input/echam """ @@ -652,7 +705,7 @@ def test_check_file_syntax_output(): # Captures output (i.e. the user-friendly error) with Capturing() as output: with pytest.raises(SystemExit) as error: - sim_file = esm_runscripts.filedicts.SimulationFile( + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( config, "echam.files.jan_surf" ) @@ -660,17 +713,19 @@ def test_check_file_syntax_output(): assert any([error_text in line for line in output]) -def test_check_path_in_computer_is_abs(simulation_file, fs): +def test_check_path_in_computer_is_abs(simulation_file): """ Tests that ``esm_parser.user_error`` is used when the ``path_in_computer`` is not absolute """ - simulation_file.path_in_computer = Path("foo/bar") + simulation_file.paths["computer"] = Path("foo/bar") # Captures output (i.e. the user-friendly error) with Capturing() as output: - with pytest.raises(SystemExit) as error: - simulation_file._check_path_in_computer_is_abs() + with pytest.raises(SystemExit): + simulation_file._check_path_in_computer_is_abs( + simulation_file.paths, simulation_file.component, simulation_file.name + ) # error needs to occur as the path is not absolute assert any(["ERROR: File Dictionaries" in line for line in output]) @@ -689,7 +744,7 @@ def test_resolve_abs_paths(fs): echam: files: jan_surf: - type: input + kind: input name_in_computer: T63CORE2_jan_surf.nc name_in_work: unit.24 path_in_computer: /work/ollie/pool/ECHAM/T63/ @@ -700,7 +755,9 @@ def test_resolve_abs_paths(fs): config = yaml.safe_load(dummy_config) config["general"]["current_date"] = date - sim_file = esm_runscripts.filedicts.SimulationFile(config, "echam.files.jan_surf") + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( + config, "echam.files.jan_surf" + ) assert sim_file["absolute_path_in_work"] == Path( "/work/ollie/pgierz/some_exp/run_20010101-20010101/work/unit.24" @@ -729,12 +786,14 @@ def test_resolve_paths_old_config(): "o3_data": { "name_in_computer": "o3chem_l91", "name_in_work": "o3chem_l91", - "type": "input", + "kind": "input", "path_in_computer": "/work/ollie/jstreffi/input/oifs-43r3/43r3/climate/95_4", } } - sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.o3_data") + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( + config, "oifs.files.o3_data" + ) assert sim_file["absolute_path_in_work"] == Path( "/work/ollie/mandresm/testing/run/awicm3//awicm3-v3.1-TCO95L91-CORE2_initial/run_20000101-20000101/work/o3chem_l91" @@ -775,7 +834,7 @@ def test_datestamp_format_attr(simulation_file): @pytest.mark.parametrize("movement_type", ["mv", "ln", "cp"]) -def test_datestamp_added_by_default_mv_ln_cp(simulation_file, fs, movement_type): +def test_datestamp_added_by_default(dated_simulation_file, fs, movement_type): """ Checks that the simulation file gets a timestamp if a file already exists with that name @@ -784,8 +843,8 @@ def test_datestamp_added_by_default_mv_ln_cp(simulation_file, fs, movement_type) * datestamp_format: append * datestamp_method: avoid_overwrite """ - simulation_file_meth = getattr(simulation_file, movement_type) - simulation_file2 = simulation_file # Make a copy + simulation_file_meth = getattr(dated_simulation_file, movement_type) + simulation_file2 = dated_simulation_file # Make a copy simulation_file2_meth = getattr(simulation_file2, movement_type) simulation_file_meth("computer", "work") if movement_type == "mv": @@ -829,7 +888,7 @@ def test_wild_card_check(): source_pattern = source_name.split("*") target_pattern = target_name.split("*") - assert esm_runscripts.filedicts.SimulationFile._wild_card_check( + assert esm_runscripts.filedicts.SimulationFile.wild_card_check( source_pattern, target_pattern ) @@ -847,7 +906,7 @@ def test_wild_card_check_fails(): # Captures output (i.e. the user-friendly error) with Capturing() as output: with pytest.raises(SystemExit) as error: - esm_runscripts.filedicts.SimulationFile._wild_card_check( + esm_runscripts.filedicts.SimulationFile.wild_card_check( source_pattern, target_pattern ) @@ -869,7 +928,7 @@ def test_find_globbing_files(fs): oifsnc: name_in_work: input_expid_*_DATE_*.nc name_in_exp_tree: new_input_expid_*_NEW_DATE_*.nc - type: outdata + kind: outdata experiment_outdata_dir: /work/ollie/pgierz/some_exp/input/oifs thisrun_outdata_dir: /work/ollie/pgierz/some_exp/run_20010101-20010101/input/oifs """ @@ -887,7 +946,9 @@ def test_find_globbing_files(fs): fs.create_dir(config["oifs"]["experiment_outdata_dir"]) - sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.oifsnc") + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( + config, "oifs.files.oifsnc" + ) # Captures output (i.e. the user-friendly error) with Capturing() as output: @@ -910,7 +971,7 @@ def test_globbing_cp(fs): oifsnc: name_in_work: input_expid_*_DATE_*.nc name_in_exp_tree: new_input_expid_*_NEW_DATE_*.nc - type: outdata + kind: outdata experiment_outdata_dir: /work/ollie/pgierz/some_exp/input/oifs thisrun_outdata_dir: /work/ollie/pgierz/some_exp/run_20010101-20010101/input/oifs """ @@ -939,7 +1000,9 @@ def test_globbing_cp(fs): Path(config["oifs"]["experiment_outdata_dir"]).joinpath(f) ) - sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.oifsnc") + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( + config, "oifs.files.oifsnc" + ) sim_file.cp("work", "exp_tree") for nf in expected_new_paths: @@ -958,7 +1021,7 @@ def test_globbing_mv(fs): oifsnc: name_in_work: input_expid_*_DATE_*.nc name_in_exp_tree: new_input_expid_*_NEW_DATE_*.nc - type: outdata + kind: outdata experiment_outdata_dir: /work/ollie/pgierz/some_exp/input/oifs thisrun_outdata_dir: /work/ollie/pgierz/some_exp/run_20010101-20010101/input/oifs """ @@ -987,7 +1050,9 @@ def test_globbing_mv(fs): Path(config["oifs"]["experiment_outdata_dir"]).joinpath(f) ) - sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.oifsnc") + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( + config, "oifs.files.oifsnc" + ) sim_file.mv("work", "exp_tree") for nf in expected_new_paths: @@ -1006,7 +1071,7 @@ def test_globbing_ln(fs): oifsnc: name_in_work: input_expid_*_DATE_*.nc name_in_exp_tree: new_input_expid_*_NEW_DATE_*.nc - type: outdata + kind: outdata experiment_outdata_dir: /work/ollie/pgierz/some_exp/input/oifs thisrun_outdata_dir: /work/ollie/pgierz/some_exp/run_20010101-20010101/input/oifs """ @@ -1035,49 +1100,10 @@ def test_globbing_ln(fs): Path(config["oifs"]["experiment_outdata_dir"]).joinpath(f) ) - sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.oifsnc") + sim_file = esm_runscripts.filedicts.SimulationFile.from_config( + config, "oifs.files.oifsnc" + ) sim_file.ln("work", "exp_tree") for nf in expected_new_paths: assert os.path.exists(nf) - - -def test_makedirs_in_name(fs): - """Tests for creating the subfolders included in the target name""" - - dummy_config = """ - general: - thisrun_work_dir: /work/ollie/mandresm/awiesm/run_20010101-20010101/work/ - all_model_filetypes: [analysis, bin, config, forcing, input, couple, log, mon, outdata, restart, viz, ignore] - oifs: - files: - ICMGG: - name_in_work: ICMGG_input_expid+in_work - name_in_exp_tree: out/date/folder/ICMGG_input_expid - type: outdata - experiment_outdata_dir: /work/ollie/pgierz/some_exp/input/oifs - thisrun_outdata_dir: /work/ollie/pgierz/some_exp/run_20010101-20010101/input/oifs - """ - - date = esm_calendar.Date("2000-01-01T00:00:00") - config = yaml.safe_load(dummy_config) - config["general"]["current_date"] = date - - fs.create_file( - Path(config["general"]["thisrun_work_dir"]).joinpath( - f"ICMGG_input_expid+in_work" - ) - ) - - fs.create_dir(config["oifs"]["experiment_outdata_dir"]) - - # config["oifs"]["experiment_outdata_dir"] = "/work/ollie/pgierz/some_exp/input/oif" - - expected_new_path = Path(config["oifs"]["experiment_outdata_dir"]).joinpath( - "out/date/folder/ICMGG_input_expid" - ) - - sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.ICMGG") - sim_file.cp("work", "exp_tree") - - assert os.path.exists(expected_new_path) diff --git a/utils/fix_monorepo.py b/utils/fix_monorepo.py new file mode 100644 index 000000000..5372257d6 --- /dev/null +++ b/utils/fix_monorepo.py @@ -0,0 +1,278 @@ +""" +Script that cleans the multirepo packages of ESM-Tools (5.1.9 and some older packages:: + + python3 utils/fix_monorepo.py + +This script is based on the script used to upgrade from 5.1.9 yo 6.0.0: +https://github.com/esm-tools/esm_version_checker/blob/release/esm_version_checker/monorepo.py +""" + +import colorama +import glob +import os +import pkg_resources +import questionary +import shutil +import site +import subprocess +import sys + +import regex as re + + +def install_monorepo(esm_tools, version): + """ + Does all the magic for successfully installing the monorepo if the user has the + multirepo already installed. + """ + _, columns = os.popen("stty size", "r").read().split() + columns = int(columns) + + # Packages from multirepo + packages = [ + "esm_calendar", + "esm_database", + "esm_environment", + "esm_master", + "esm_motd", + "esm_parser", + "esm_plugin", + "esm_profile", + "esm_rcfile", + "esm_runscripts", + "esm_tools", + "esm_version", + ] + + tools_dir, bin_dir, lib_dirs = find_dir_to_remove(packages) + os.chdir(tools_dir) + + steps = { + "uninstall": "Uninstall packages (``pip uninstall esm_``)", + "esm_tools": f"Cleanup ``{tools_dir}/esm_tools`` folder", + "esm_tools.egg-info": f"Cleanup ``{tools_dir}/esm_tools.egg-info`` folder", + "rm_libs": ["Remove esm_ from python libraries in:"] + lib_dirs, + "rm_bins": f"Remove esm_ in the bin folder (``rm {bin_dir}/esm_``)", + "rm_easy": ["Remove ESM lines in the ``easy-install.pth`` files:"] + lib_dirs, + "install": "Install ``ESM-Tools`` again", + } + + # Printing and questionary + text = ( + "**Welcome to the monorepository version of ESM-Tools!**\n" + "\n" + f"You are trying to upgrade to the major version ``6`` which does " + "not use multiple repositories for different ``esm_`` anymore, " + "but instead all packages are contained in the ``esm_tools`` package (i.e. " + "esm_runscripts, esm_parser, esm_master, etc). You can find these packages " + "now in ``esm_tools/src/``.\n" + "\n" + "Also note that you won't be able to use ``esm_versions`` command from now " + "on, as this tool is not needed anymore for the monorepository, and it has " + "been consequently removed." + ) + + cprint() + cprint("**" + columns * "=" + "**") + cprint(text) + cprint("**" + columns * "=" + "**") + + cprint( + "The monorepository version needs a special installation. " + "ESM-Tools will perform the next steps:" + ) + + c = 1 + for key, value in steps.items(): + if isinstance(value, list): + cprint(f"``{c}`` - {value[0]}") + for substeps in value[1:]: + cprint(f"\t- {substeps}") + else: + cprint(f"``{c}`` - {value}") + c += 1 + + user_confirmed = False + while not user_confirmed: + response = questionary.select( + "Would you like to continue?", choices=(["[Quit] No, thank you...", "Yes!"]) + ).ask() # returns value of selection + if "[Quit]" in response and (version=="release" or version=="develop"): + # If the user refuses to install the monorepo bring back esm_tools to the + # last multirepo compatible version. + v = "v5.1.25" + if not version == "monorepo": + p = subprocess.check_call( + f"git reset {v}", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + ) + sys.exit(1) + elif "[Quit]" in response: + sys.exit(1) + user_confirmed = questionary.confirm("Are you sure?").ask() + + p = subprocess.check_call( + f"git checkout release", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + ) + p = subprocess.check_call( + f"git pull", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + ) + + # Dirty fix for installing the monorepo branch for testing previous version 6 is + # around. Not used when the monorepo is in release + if version == "monorepo": + subprocess.check_call(["git", "checkout", "monorepo"]) + subprocess.check_call(["git", "pull"]) + + cprint() + + # Uninstall packages + c = 1 + cprint(f"**{c}** - {steps['uninstall']}") + for package in packages: + uninstall(package) + + # Cleanup esm_tools folder + clean_folders = ["esm_tools", "esm_tools.egg-info"] + for cf in clean_folders: + if cf=="esm_tools": + cf = "esm_tools/__init__.py" + f = f"{tools_dir}/{cf}" + if os.path.isdir(f): + c += 1 + cprint(f"**{c}** - {steps[cf]}") + shutil.rmtree(f) + + # Remove libs + c += 1 + cprint(f"**{c}** - {steps['rm_libs'][0]}") + for lib_dir in lib_dirs: + for package in packages: + package_files = glob.glob(f"{lib_dir}/{package}*") + for f in package_files: + cprint(f"\tRemoving ``{f}``") + if os.path.isdir(f): + shutil.rmtree(f) + else: + os.remove(f) + + # Remove bins + c += 1 + cprint(f"**{c}** - {steps['rm_bins']}") + for package in packages: + bin_file = glob.glob(f"{bin_dir}/{package}*") + if bin_file: + bin_file = bin_file[0] + if os.path.isfile(bin_file): + cprint(f"\tRemoving ``{bin_file}``") + subprocess.run(["rm", "-f", bin_file]) + + # Clean ``easy-install.pth`` + c += 1 + cprint(f"**{c}** - {steps['rm_easy'][0]}") + clean_easy_install(lib_dirs, packages) + + # Install the tools + c += 1 + cprint(f"**{c}** - {steps['install']}") + p = subprocess.Popen( + f"cd {tools_dir} && pip install --user -e .", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) + out, err = p.communicate() + + if not err: + cprint(f"**Version 6 installed sucessfully!**") + sys.exit(0) + elif "ERROR" in err.decode("utf-8"): + print(out.decode("utf-8")) + print(err.decode("utf-8")) + cprint("--Installation failed!--") + sys.exit(1) + elif "WARNING" in err.decode("utf-8"): + print(err.decode("utf-8")) + cprint(f"**Version 6 installed sucessfully with warnings!**") + sys.exit(0) + else: + print(out.decode("utf-8")) + print(err.decode("utf-8")) + cprint("--Installation failed!--") + sys.exit(1) + + +def uninstall(package): + """ + Taken from https://stackoverflow.com/questions/35080207/how-to-pass-the-same-answer-to-subprocess-popen-automatically + """ + cprint(f"\tUninstalling ``{package}``") + process = subprocess.Popen( + [sys.executable, "-m", "pip", "uninstall", package], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + yes_proc = subprocess.Popen(["yes", "y"], stdout=process.stdin) + process_output = process.communicate()[0] + yes_proc.wait() + + +def find_dir_to_remove(packages): + path_to_dists = "/".join(site.getusersitepackages().split("/")[:-2]) + python_dist_libs = [x for x in os.listdir(f"{path_to_dists}/") if "python" in x] + + lib_dirs = [f"{path_to_dists}/{x}/site-packages" for x in python_dist_libs] + bin_dir = "/".join(path_to_dists.split("/")[:-1] + ["bin"]) + tools_dir = pkg_resources.get_distribution("esm_tools").location + if tools_dir.endswith("esm_tools/src"): + tools_dir = tools_dir.replace("esm_tools/src", "esm_tools") + + return tools_dir, bin_dir, lib_dirs + + +def clean_easy_install(lib_dirs, packages): + for ld in lib_dirs: + easy_install_file = f"{ld}/easy-install.pth" + if os.path.isfile(easy_install_file): + cprint(f"\tCleaning ``{easy_install_file}``") + with open(easy_install_file, "r+") as f: + lines = f.readlines() + f.seek(0) + for line in lines: + contains_package = False + for package in packages: + if package in line: + contains_package = True + if contains_package: + continue + else: + f.write(line) + f.truncate() + + +def cprint(text=""): + # Bold strings + bs = "\033[1m" + be = "\033[0m" + reset_s = colorama.Style.RESET_ALL + title_color = colorama.Fore.CYAN + error_color = colorama.Fore.RED + remarks_color = colorama.Fore.MAGENTA + + text = re.sub("\*\*([^*]*)\*\*", f"{bs}{title_color}\\1{reset_s}{be}", text) + text = re.sub("``([^`]*)``", f"{remarks_color}\\1{reset_s}", text) + text = re.sub("--([^-]*)--", f"{error_color}\\1{reset_s}", text) + print(text) + +if __name__ == "__main__": + install_monorepo("", "")