diff --git a/setup.cfg b/setup.cfg
index 8f1520709..66fb36488 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -68,7 +68,7 @@ universal = 1
[flake8]
exclude = docs
-max-line-length = 88
+max-line-length = 120
[aliases]
diff --git a/src/esm_runscripts/filedicts.py b/src/esm_runscripts/filedicts.py
index 154d251a4..1434f4776 100644
--- a/src/esm_runscripts/filedicts.py
+++ b/src/esm_runscripts/filedicts.py
@@ -11,12 +11,13 @@
import copy
import functools
import glob
+import inspect
import os
import pathlib
import shutil
import sys
from enum import Enum, auto
-from typing import Any, AnyStr
+from typing import Any, AnyStr, Iterator
import dpath.util
import yaml
@@ -26,13 +27,19 @@
logger.remove()
LEVEL = "ERROR" # "WARNING" # "INFO" # "DEBUG"
-LOGGING_FORMAT = "[{time:HH:mm:ss DD/MM/YYYY}] |{level}| [{file} -> {function}() line:{line: >3}] >> {message}"
+LOGGING_FORMAT = "[{time:HH:mm:ss DD/MM/YYYY}] |{level}| [{file} -> {function}() line:{line: >3}] >> {message}" #
logger.add(sys.stderr, level=LEVEL, format=LOGGING_FORMAT)
-# Enumeration of file types
-class FileTypes(Enum):
+
+def NameIterEnum(Enum) -> Iterator[str]:
+ def __iter__(self):
+ """Returns list of names of the iteration"""
+ return iter(str(name).lower() for name in self.__member_names__)
+
+
+class FileStatus(NameIterEnum):
"""
- Describes which type a particular file might have, e.g. ``FILE``,
+ Describes which status a particular file might have, e.g. ``FILE``,
``NOT_EXISTS``, ``BROKEN_LINK``.
"""
@@ -44,6 +51,35 @@ class FileTypes(Enum):
BROKEN_LINK = auto() # target of the symbolic link does not exist
+# FIXME(PG): This class belongs somewhere else, I think.
+class FileTypes(NameIterEnum):
+ """
+ Describes which type a file might belong to, e.g. input, outdata, forcing
+ """
+
+ ANALYSIS = auto()
+ CONFIG = auto()
+ COUPLE = auto()
+ FORCING = auto()
+ IGNORE = auto()
+ INPUT = auto()
+ LOG = auto()
+ MON = auto()
+ OUTDATA = auto()
+ RESTART = auto()
+ VIZ = auto()
+
+
+class FileLocations(Enum):
+ """Possible locations for a file"""
+
+ # TODO(PG): Figure out if this can have something like relative pathlib.Path...
+ COMPUTER = auto()
+ EXP_TREE = auto()
+ RUN_TREE = auto()
+ WORK = auto()
+
+
# NOTE(PG): Comment can be removed later. Here I prefix with an underscore as
# this decorator should **only** be used inside of this file.
def _allowed_to_be_missing(method):
@@ -52,17 +88,17 @@ def _allowed_to_be_missing(method):
If a method is decorated with ``@_allowed_to_be_missing``, it will return
``None`` instead of executing if the file has a attribute of
- ``allowed_to_be_missing`` set to ``True. You get a warning via the logger
+ ``allowed_to_be_missing`` set to ``True``. You get a warning via the logger
giving the full method name that was decorated and a representation of the
file that was trying to be moved, linked, or copied.
Usage Example
-------------
- Given you have an instanciated simulation file under ``sim_file`` with the following property::
+ Given you have an instantiated simulation file under ``sim_file`` with the following property::
>>> sim_file.allowed_to_be_missing # doctest: +SKIP
True
- And given that you have a decorated method foo, that would act on the file::
+ And given that you have a decorated method ``foo`` that would act on the file by moving, copying or linking it::
>>> rvalue = sim_file.foo(*args, **kwargs) # doctest: +SKIP
>>> rvalue is None # doctest: +SKIP
True
@@ -86,8 +122,10 @@ def foo(self, *args, **kwargs):
def inner_method(self, *args, **kwargs):
if self.allowed_to_be_missing:
try:
+ # Try to move/link/copy the file, since that is the requested operation:
return method(self, *args, **kwargs)
except (FileNotFoundError, IOError):
+ # Move/copy/link does not work, give a warning since this file is allowed to be missing!
logger.warning(
f"Skipping {method.__qualname__} as this file ({self}) is allowed to be missing!"
)
@@ -135,7 +173,7 @@ def _fname_has_date_stamp_info(fname, date, reqs=["%Y", "%m", "%d"]):
return fname.count("checked") == len(reqs)
-def globbing(method):
+def _globbing(method):
"""
Decorator method for ``SimulationFile``'s methods ``cp``, ``mv``, ``ln``, that
enables globbing. If a ``*`` is found on the ``source`` or ``target`` the globbing
@@ -198,7 +236,7 @@ def inner_method(self, source, target, *args, **kwargs):
)
glob_dict[f"name_in_{source}"] = glob_source_name
glob_dict[f"name_in_{target}"] = glob_target_name
- glob_file = SimulationFile(glob_config, self.attrs_address)
+ glob_file = SimulationFile.from_config(glob_config, self.attrs_address)
# Use method
this_method = getattr(glob_file, method_name)
this_method(source, target, *args, **kwargs)
@@ -276,15 +314,14 @@ class SimulationFile(dict):
- work: file in the current work directory. Eg. experiment/run_/work/
LOCATION_KEY is one of the strings defined in LOCATION_KEY list
- - name_in : file name (without path) in the LOCATION_KEY
+ - name_in_ : file name (without path) in the LOCATION_KEY
- eg. name_in_computer: T63CORE2_jan_surf.nc
- eg. name_in_work: unit.24
- absolute_path_in_ : absolute path in the LOCATION_KEY
- - eg. absolute_path_in_run_tree:
- - /work/ollie/pgierz/some_exp/run_20010101-20010101/input/echam/T63CORE2_jan_surf.nc
+ - eg. absolute_path_in_run_tree: /work/ollie/pgierz/some_exp/run_20010101-20010101/input/echam/T63CORE2_jan_surf.nc
"""
- def __init__(self, full_config: dict, attrs_address: str):
+ def __init__(self, attrs_dict: dict):
"""
- Initiates the properties of the object
- Triggers basic checks
@@ -296,19 +333,8 @@ def __init__(self, full_config: dict, attrs_address: str):
attrs_address : str
The address of this specific file in the full config, separated by dots.
"""
- attrs_dict = dpath.util.get(
- full_config, attrs_address, separator=".", default={}
- )
super().__init__(attrs_dict)
self._original_filedict = copy.deepcopy(attrs_dict)
- self._config = full_config
- self.attrs_address = attrs_address
- self._sim_date = full_config["general"][
- "current_date"
- ] # NOTE: we might have to change this in the future, depending on whether SimulationFile is access through tidy ("end_date") or prepcompute ("start_date")
- self.name = attrs_address.split(".")[-1]
- self.component = attrs_address.split(".")[0]
- self.all_model_filetypes = full_config["general"]["all_model_filetypes"]
self.path_in_computer = self.get("path_in_computer")
self._datestamp_method = self.get(
"datestamp_method", "avoid_overwrite"
@@ -322,24 +348,79 @@ def __init__(self, full_config: dict, attrs_address: str):
self._complete_file_names()
# possible paths for files:
- location_keys = ["computer", "exp_tree", "run_tree", "work"]
+ # TODO: Replace with enum ...?
+ location_keys = list(FileLocations)
# initialize the locations and complete paths for all possible locations
- self.locations = dict.fromkeys(location_keys, None)
+ self._locations = dict.fromkeys(location_keys, None)
self._resolve_abs_paths()
# Verbose set to true by default, for now at least
self._verbose = full_config.get("general", {}).get("verbose", True)
+ def _post_init(self):
# Checks
self._check_path_in_computer_is_abs()
+ @classmethod
+ def from_dict(cls, attrs_dict: dict):
+ """
+ Create a new instance of the SimulationFile class from a dictionary.
+
+ Parameters
+ ----------
+ attrs_dict : dict
+ A dictionary containing the attributes of the file.
+
+ Returns
+ -------
+ SimulationFile
+ A new instance of the SimulationFile class.
+ """
+ sim_file = cls(attrs_dict)
+ # I would rather have this be in the main __init__...
+ sim_file._post_init()
+ return sim_file
+
+ @classmethod
+ def from_config(cls, full_config: dict, attrs_address: str):
+ """
+ Create a new instance of the SimulationFile class from a full configuration and the address of the file within the configuration.
+
+ Parameters
+ ----------
+ full_config : dict
+ The full simulation configuration
+ attrs_address : str
+ The address of this specific file in the full config, separated by dots.
+
+ Returns
+ -------
+ SimulationFile
+ A new instance of the SimulationFile class.
+ """
+ attrs_dict = dpath.util.get(
+ full_config, attrs_address, separator=".", default={}
+ )
+ sim_file = cls(attrs_dict)
+ sim_file._config = full_config
+ sim_file.attrs_address = attrs_address
+ # NOTE: we might have to change this in the future, depending on whether SimulationFile is access through tidy ("end_date") or prepcompute ("start_date")
+ sim_file._sim_date = full_config["general"]["current_date"]
+ sim_file.name = attrs_address.split(".")[-1]
+ sim_file.component = attrs_address.split(".")[0]
+ # FIXME(PG): I would rathe just use our new little class for this, but there might be good reasons against it:
+ sim_file.all_model_filetypes = list(FileTypes)
+ # I would rather have this be in the main __init__...
+ sim_file._post_init()
+ return sim_file
+
##############################################################################################
# Overrides of standard dict methods
##############################################################################################
def __str__(self):
- address = " -> ".join(self.attrs_address.split("."))
- return address
+ """Nicely prints out the attribute address"""
+ return " -> ".join(getattr(self, "attrs_address", "").split("."))
def __setattr__(self, name: str, value: Any) -> None:
"""Checks when changing dot attributes for disallowed values"""
@@ -419,7 +500,7 @@ def datestamp_format(self):
##############################################################################################
# Main Methods
##############################################################################################
- @globbing
+ @_globbing
@_allowed_to_be_missing
def cp(self, source: str, target: str) -> None:
"""
@@ -435,13 +516,13 @@ def cp(self, source: str, target: str) -> None:
String specifying one of the following options: ``"computer"``, ``"work"``,
``"exp_tree"``, ``run_tree``
"""
- if source not in self.locations:
+ if source not in self._locations:
raise ValueError(
- f"Source is incorrectly defined, and needs to be in {self.locations}"
+ f"Source is incorrectly defined, and needs to be in {self._locations}"
)
- if target not in self.locations:
+ if target not in self._locations:
raise ValueError(
- f"Target is incorrectly defined, and needs to be in {self.locations}"
+ f"Target is incorrectly defined, and needs to be in {self._locations}"
)
source_path = self[f"absolute_path_in_{source}"]
target_path = self[f"absolute_path_in_{target}"]
@@ -460,7 +541,7 @@ def cp(self, source: str, target: str) -> None:
# Actual copy
source_path_type = self._path_type(source_path)
- if source_path_type == FileTypes.DIR:
+ if source_path_type == FileStatus.DIR:
copy_func = shutil.copytree
else:
copy_func = shutil.copy2
@@ -473,7 +554,7 @@ def cp(self, source: str, target: str) -> None:
f"Exception details:\n{error}"
)
- @globbing
+ @_globbing
@_allowed_to_be_missing
def mv(self, source: str, target: str) -> None:
"""
@@ -482,18 +563,18 @@ def mv(self, source: str, target: str) -> None:
Parameters
----------
- source : str
+ source : st r
One of ``"computer"``, ``"work"``, ``"exp_tree"``, "``run_tree``"
target : str
One of ``"computer"``, ``"work"``, ``"exp_tree"``, "``run_tree``"
"""
- if source not in self.locations:
+ if source not in self._locations:
raise ValueError(
- f"Source is incorrectly defined, and needs to be in {self.locations}"
+ f"Source is incorrectly defined, and needs to be in {self._locations}"
)
- if target not in self.locations:
+ if target not in self._locations:
raise ValueError(
- f"Target is incorrectly defined, and needs to be in {self.locations}"
+ f"Target is incorrectly defined, and needs to be in {self._locations}"
)
source_path = self[f"absolute_path_in_{source}"]
target_path = self[f"absolute_path_in_{target}"]
@@ -520,7 +601,7 @@ def mv(self, source: str, target: str) -> None:
f"Exception details:\n{error}"
)
- @globbing
+ @_globbing
@_allowed_to_be_missing
def ln(self, source: AnyStr, target: AnyStr) -> None:
"""creates symbolic links from the path retrieved by ``source`` to the one by ``target``.
@@ -528,10 +609,12 @@ def ln(self, source: AnyStr, target: AnyStr) -> None:
Parameters
----------
source : str
- key to retrieve the source from the file dictionary. Possible options: ``computer``, ``work``, ``exp_tree``, ``run_tree``
+ key to retrieve the source from the file dictionary. Possible options: ``computer``,
+ ``work``, ``exp_tree``, ``run_tree``
target : str
- key to retrieve the target from the file dictionary. Possible options: ``computer``, ``work``, ``exp_tree``, ``run_tree``
+ key to retrieve the target from the file dictionary. Possible options: ``computer``,
+ ``work``, ``exp_tree``, ``run_tree``
Returns
-------
@@ -548,13 +631,13 @@ def ln(self, source: AnyStr, target: AnyStr) -> None:
FileExistsError
- Target path already exists
"""
- if source not in self.locations:
+ if source not in self._locations:
raise ValueError(
- f"Source is incorrectly defined, and needs to be in {self.locations}"
+ f"Source is incorrectly defined, and needs to be in {self._locations}"
)
- if target not in self.locations:
+ if target not in self._locations:
raise ValueError(
- f"Target is incorrectly defined, and needs to be in {self.locations}"
+ f"Target is incorrectly defined, and needs to be in {self._locations}"
)
# full paths: directory path / file name
source_path = self[f"absolute_path_in_{source}"]
@@ -595,13 +678,13 @@ def pretty_filedict(self, filedict):
str :
A string in yaml format of the given file dictionary
"""
- return yaml.dump({"files": {self.name: filedict}})
+ return yaml.dump(filedict)
- ##############################################################################################
+ ####################################################################################
- ##############################################################################################
+ ####################################################################################
# Private Methods, Attributes, and Class Variables
- ##############################################################################################
+ ####################################################################################
_allowed_datestamp_methods = {"never", "always", "avoid_overwrite"}
"""
@@ -640,7 +723,7 @@ def _check_datestamp_method_is_allowed(self, datestamp_method):
"""
if datestamp_method not in self._allowed_datestamp_methods:
raise ValueError(
- "The datestamp_method must be defined as one of never, always, or avoid_overwrite"
+ f"The datestamp_method must be defined as one of {self._allowed_datestamp_methods}",
)
def _check_datestamp_format_is_allowed(self, datestamp_format):
@@ -649,7 +732,7 @@ def _check_datestamp_format_is_allowed(self, datestamp_format):
"""
if datestamp_format not in self._allowed_datestamp_formats:
raise ValueError(
- "The datestamp_format must be defined as one of check_from_filename or append"
+ f"The datestamp_format must be defined as one of {self._allowed_datestamp_formats}"
)
def _resolve_abs_paths(self) -> None:
@@ -665,7 +748,7 @@ def _resolve_abs_paths(self) -> None:
- ``self["absolute_path_in_run_tree"]``
- ``self["absolute_path_in_exp_tree"]``
"""
- self.locations = {
+ self._locations = {
"work": pathlib.Path(self._config["general"]["thisrun_work_dir"]),
"computer": self.path_in_computer, # Already Path type from _init_
"exp_tree": pathlib.Path(
@@ -676,15 +759,15 @@ def _resolve_abs_paths(self) -> None:
),
}
- for key, path in self.locations.items():
+ for key, path in self._locations.items():
if key == "computer" and path is None:
self[f"absolute_path_in_{key}"] = None
else:
self[f"absolute_path_in_{key}"] = path.joinpath(self[f"name_in_{key}"])
- def _path_type(self, path: pathlib.Path) -> FileTypes:
+ def _path_type(self, path: pathlib.Path) -> FileStatus:
"""
- Checks if the given ``path`` exists. If it does returns it's type, if it
+ Checks if the given ``path`` exists. If it does returns it's status, if it
doesn't, returns ``None``.
Parameters
@@ -695,7 +778,7 @@ def _path_type(self, path: pathlib.Path) -> FileTypes:
Returns
-------
Enum value
- One of the values from FileType enumeration
+ One of the values from FileStatus enumeration
Raises
------
@@ -716,23 +799,23 @@ def _path_type(self, path: pathlib.Path) -> FileTypes:
# NOTE: pathlib.Path().exists() also checks is the target of a symbolic link exists or not
if path.is_symlink() and not path.exists():
logger.warning(f"Broken link detected: {path}")
- return FileTypes.BROKEN_LINK
+ return FileStatus.BROKEN_LINK
elif not path.exists():
logger.warning(f"File does not exist: {path}")
- return FileTypes.NOT_EXISTS
+ return FileStatus.NOT_EXISTS
elif path.is_symlink():
- return FileTypes.LINK
+ return FileStatus.LINK
elif path.is_file():
- return FileTypes.FILE
+ return FileStatus.FILE
elif path.is_dir():
- return FileTypes.DIR
+ return FileStatus.DIR
else:
# probably, this will not happen
raise TypeError(f"{path} can not be identified")
- def _always_datestamp(self, fname):
+ def _always_datestamp(self, fname: pathlib.Path) -> pathlib.Path:
"""
- Method called when ``always`` is the ``datestamp_method.
+ Method called when ``always`` is the ``datestamp_method``.
Appends the datestamp in any case if ``datestamp_format`` is
``append``. Appends the datestamp only if it is not obviously in the
@@ -750,12 +833,12 @@ def _always_datestamp(self, fname):
A modified file with an added date stamp.
"""
if fname.is_dir():
- return fname
+ return pathlib.Path(fname)
if self.datestamp_format == "append":
return pathlib.Path(f"{fname}_{self._sim_date}")
if self.datestamp_format == "check_from_filename":
if _fname_has_date_stamp_info(fname, self._sim_date):
- return fname
+ return pathlib.Path(fname)
else:
return pathlib.Path(f"{fname}_{self._sim_date}")
@@ -789,7 +872,7 @@ def _complete_file_names(self):
"""
if self["type"] in self.input_file_types:
default_name = self["name_in_computer"]
- elif self["type"] in self.output_file_types:
+ else:
default_name = self["name_in_work"]
self["name_in_computer"] = self.get("name_in_computer", default_name)
self["name_in_run_tree"] = self.get("name_in_run_tree", default_name)
@@ -863,7 +946,7 @@ def _makedirs_in_name(self, name_type: str) -> None:
Raises
------
FileNotFoundError
- If ``self.locations[name_type]`` path does not exist
+ If ``self._locations[name_type]`` path does not exist
"""
# Are there any subdirectories in ``name_in_?
if "/" in self[f"name_in_{name_type}"]:
@@ -871,7 +954,7 @@ def _makedirs_in_name(self, name_type: str) -> None:
# If the parent path does not exist check whether the file location
# exists
if not parent_path.exists():
- location = self.locations[name_type]
+ location = self._locations[name_type]
if location.exists():
# The location exists therefore the remaining extra directories
# from the parent_path can be created
@@ -899,18 +982,23 @@ def _check_file_syntax(self) -> None:
"""
error_text = ""
missing_vars = ""
- types_text = ", ".join(self.all_model_filetypes)
+ all_model_filetypes = [name.lower() for name in FileTypes._member_names_]
+ types_text = ", ".join(all_model_filetypes)
this_filedict = copy.deepcopy(self._original_filedict)
- self.input_file_types = input_file_types = ["config", "forcing", "input"]
+ self.input_file_types = input_file_types = [
+ FileTypes.CONFIG,
+ FileTypes.FORCING,
+ FileTypes.INPUT,
+ ]
self.output_file_types = output_file_types = [
- "analysis",
- "couple",
- "log",
- "mon",
- "outdata",
- "restart",
- "viz",
- "ignore",
+ FileTypes.ANALYSIS,
+ FileTypes.COUPLE,
+ FileTypes.LOG,
+ FileTypes.MON,
+ FileTypes.OUTDATA,
+ FileTypes.RESTART,
+ FileTypes.VIZ,
+ FileTypes.IGNORE,
]
if "type" not in self.keys():
@@ -922,7 +1010,7 @@ def _check_file_syntax(self) -> None:
missing_vars = (
f"{missing_vars} ``type``: forcing/input/restart/outdata/...\n"
)
- elif self["type"] not in self.all_model_filetypes:
+ elif self["type"] not in all_model_filetypes:
error_text = (
f"{error_text}"
f"- ``{self['type']}`` is not a supported ``type`` "
@@ -1037,13 +1125,13 @@ def _check_source_and_target(
# Checks
# ------
# Source does not exist
- if source_path_type == FileTypes.NOT_EXISTS:
+ if source_path_type == FileStatus.NOT_EXISTS:
err_msg = f"Unable to perform file operation. Source ``{source_path}`` does not exist!"
raise FileNotFoundError(err_msg)
# Target already exists
target_exists = (
- os.path.exists(target_path) or target_path_type == FileTypes.LINK
+ os.path.exists(target_path) or target_path_type == FileStatus.LINK
)
if target_exists:
err_msg = f"Unable to perform file operation. Target ``{target_path}`` already exists"
@@ -1056,30 +1144,73 @@ def _check_source_and_target(
raise FileNotFoundError(err_msg)
# if source is a broken link. Ie. pointing to a non-existing file
- if source_path_type == FileTypes.BROKEN_LINK:
+ if source_path_type == FileStatus.BROKEN_LINK:
err_msg = f"Unable to create symbolic link: ``{source_path}`` points to a broken path: {source_path.resolve()}"
raise FileNotFoundError(err_msg)
return True
-class SimulationFiles(dict):
+class SimulationFileCollection(dict):
"""
Once instanciated, searches in the ``config`` dictionary for the ``files`` keys.
This class contains the methods to: 1) instanciate each of the files defined in
``files`` as ``SimulationFile`` objects and 2) loop through these objects
triggering the desire file movement.
"""
- def __init__(self, config):
- # Loop through components?
- # Loop through files?
+
+ def __init__(self):
pass
+ # PG: Not sure I need this...
+ @property
+ def _defined_from(self):
+ stack = inspect.stack()
+ caller_frame = stack[1] # Get the frame of the caller
+ caller_name = caller_frame.function
+ return caller_name
+
+ @classmethod
+ def from_config(cls, config: dict):
+ sim_files = cls()
+ for component in config['valid_model_names']:
+ config_address = f"{component}.files"
+ for file_key, file_spec in dpath.util.get(
+ config, config_address, separator="."
+ ).items():
+ sim_files[file_key] = SimulationFile.from_dict(file_spec)
+ return sim_files
+
+ def _gather_file_movements(self) -> None:
+ """Puts the methods for each file movement into the dictionary as callable values behind the `_filesystem_op` key""""
+ for sim_file_id, sim_file_obj in self.items():
+ if sim_file_obj["movement_type"] == "mv":
+ self[sim_file_id]["_filesystem_op"] = getattr(sim_file_obj, "mv")
+ elif sim_file_obj["movement_type"] == "cp":
+ self[sim_file_id]["_filesystem_op"] = getattr(sim_file_obj, "cp")
+ elif sim_file_obj["movement_type"] == "ln":
+ self[sim_file_id]["_filesystem_op"] = getattr(sim_file_obj, "ln")
+ else:
+ raise ValueError(f"Movement Type is not defined correctly, please use `mv`, `cp` or `ln` for {sim_file_id}")
+
+ def execute_filesystem_operation(self, config: ConfigSetup) -> ConfigSetup: #, from: pathlib.Path | str, to: pathlib.Path | str) -> None:
+ self._gather_file_movements()
+ for sim_file_id, sim_file_obj in self.items():
+ logger.info(f"Processing {sim_file_id}")
+ if config["general"]["jobtype"] == "prepexp":
+ from, to = "pool", "work"
+ elif config["general"]["jobtype"] == "tidy":
+ from, to = "work", "exp_tree"
+ else:
+ raise ValueError(f"Incorrect jobtype specified for {sim_file_obj}")
+ sim_file_obj["_filesystem_op"](from, to)
+ return config
+
def resolve_file_movements(config: ConfigSetup) -> ConfigSetup:
"""
Runs all methods required to get files into their correct locations. This will
- instanciate the ``SimulationFiles`` class. It's called by the recipe manager.
+ instantiate the ``SimulationFiles`` class. It's called by the recipe manager.
Parameters
----------
@@ -1091,7 +1222,6 @@ def resolve_file_movements(config: ConfigSetup) -> ConfigSetup:
config : ConfigSetup
The complete simulation configuration, potentially modified.
"""
- # TODO: to be filled with functions
- # DONE: type annotation
- # DONE: basic unit test: test_resolve_file_movements
+ sim_file_collection = SimulationFileCollection.from_config(config)
+ config = sim_file_collection.execute_filesystem_operation(config)
return config
diff --git a/tests/test_esm_runscripts/awicm3_config.yaml b/tests/test_esm_runscripts/example_configs/awicm3_config.yaml
similarity index 100%
rename from tests/test_esm_runscripts/awicm3_config.yaml
rename to tests/test_esm_runscripts/example_configs/awicm3_config.yaml
diff --git a/tests/test_esm_runscripts/example_configs/generic-config.yaml b/tests/test_esm_runscripts/example_configs/generic-config.yaml
new file mode 100644
index 000000000..dc7957318
--- /dev/null
+++ b/tests/test_esm_runscripts/example_configs/generic-config.yaml
@@ -0,0 +1,45 @@
+general:
+ thisrun_dir: "/work/ollie/pgierz/some_exp/run_20010101-20010101"
+ exp_dir: "/work/ollie/pgierz/some_exp"
+ thisrun_work_dir: "/work/ollie/pgierz/some_exp/run_20010101-20010101/work"
+ all_model_filetypes:
+ [
+ analysis,
+ bin,
+ config,
+ forcing,
+ input,
+ couple,
+ log,
+ mon,
+ outdata,
+ restart,
+ viz,
+ ignore,
+ ]
+computer:
+ pool_dir: "/work/ollie/pool"
+echam:
+ files:
+ jan_surf:
+ type: input
+ allowed_to_be_missing: False
+ name_in_computer: T63CORE2_jan_surf.nc
+ name_in_work: unit.24
+ path_in_computer: /work/ollie/pool/ECHAM/T63
+ filetype: NetCDF
+ description: >
+ Initial values used for the simulation, including
+ properties such as geopotential, temperature, pressure
+ vltclim:
+ type: input
+ allowed_to_be_missing: False
+ name_in_computer: T63CORE2_VLTCLIM.nc
+ name_in_work: unit.90
+ path_in_computer: /work/ollie/pool/ECHAM/T63
+ filetype: NetCDF
+ description: >
+ Initial values for the leaf area index, given as an annual cycle.
+ experiment_input_dir: "/work/ollie/pgierz/some_exp/input/echam"
+ thisrun_input_dir: "/work/ollie/pgierz/some_exp/run_20000101-20000101/input/echam"
+ thisrun_work_dir: "/work/ollie/pgierz/some_exp/run_20010101-20010101/work"
diff --git a/tests/test_esm_runscripts/test_filedicts.py b/tests/test_esm_runscripts/test_filedicts.py
index b5a6d563f..8c7f341c1 100644
--- a/tests/test_esm_runscripts/test_filedicts.py
+++ b/tests/test_esm_runscripts/test_filedicts.py
@@ -9,7 +9,7 @@
we want to translate later on.
* You _could_ use the config in each function to generate the fake files, to
avoid repeating yourself; however, that adds real programming logic into the
- unit test, which you don't really want.
+ unit test, which you don't really want.
"""
import os
import sys
@@ -40,6 +40,17 @@ def __exit__(self, *args):
sys.stdout = self._stdout
+@pytest.fixture()
+def fake_config():
+ """Provides a fake config for testing purposes"""
+ # FIXME(PG): Path should be not rely on location of test run...
+ cfg_path = Path("./tests/test_esm_runscripts/example_configs/generic-config.yaml")
+ date = esm_calendar.Date("2000-01-01T00:00:00")
+ config = yaml.safe_load(cfg_path.read_text())
+ config["general"]["current_date"] = date
+ return config
+
+
@pytest.fixture()
def config_tuple():
"""setup function
@@ -93,7 +104,7 @@ def simulation_file(fs, config_tuple):
"""
config = config_tuple.config
attr_address = config_tuple.attr_address
- fake_simulation_file = filedicts.SimulationFile(config, attr_address)
+ fake_simulation_file = filedicts.SimulationFile.from_config(config, attr_address)
fs.create_dir(fake_simulation_file.locations["work"])
fs.create_dir(fake_simulation_file.locations["computer"])
@@ -104,7 +115,7 @@ def simulation_file(fs, config_tuple):
yield fake_simulation_file
-def test_filedicts_basics(fs):
+def test_SimulationFile_basics(fs):
"""Tests basic attribute behavior of filedicts"""
dummy_config = """
@@ -134,7 +145,9 @@ def test_filedicts_basics(fs):
config["general"]["current_date"] = date
# Not needed for this test, just a demonstration:
fs.create_file("/work/ollie/pool/ECHAM/T63/T63CORE2_jan_surf.nc")
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "echam.files.jan_surf")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "echam.files.jan_surf"
+ )
assert sim_file["name_in_work"] == "unit.24"
assert sim_file.locations["work"] == Path(
"/work/ollie/pgierz/some_exp/run_20010101-20010101/work"
@@ -253,10 +266,10 @@ def test_allowed_to_be_missing_attr():
config = yaml.safe_load(dummy_config)
config["general"]["current_date"] = date
# Not needed for this test, just a demonstration:
- sim_file_001 = esm_runscripts.filedicts.SimulationFile(
+ sim_file_001 = esm_runscripts.filedicts.SimulationFile.from_config(
config, "echam.files.human_readable_tag_001"
)
- sim_file_002 = esm_runscripts.filedicts.SimulationFile(
+ sim_file_002 = esm_runscripts.filedicts.SimulationFile.from_config(
config, "echam.files.human_readable_tag_002"
)
@@ -294,7 +307,7 @@ def test_allowed_to_be_missing_mv(fs):
config["general"]["current_date"] = date
fs.create_dir("/work/data/pool")
fs.create_file("/work/data/pool/not_foo_at_all")
- sim_file = esm_runscripts.filedicts.SimulationFile(
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
config, "echam.files.human_readable_tag_001"
)
sim_file.mv("computer", "work")
@@ -333,7 +346,7 @@ def test_allowed_to_be_missing_mv_if_exists(fs):
fs.create_dir("/work/data/pool")
fs.create_file("/work/data/pool/foo")
fs.create_dir("/work/ollie/pgierz/some_exp/run_20010101-20011231/work")
- sim_file = esm_runscripts.filedicts.SimulationFile(
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
config, "echam.files.human_readable_tag_001"
)
sim_file.mv("computer", "work")
@@ -376,7 +389,9 @@ def test_cp_file(fs):
fs.create_dir(target_folder)
# Test the method
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "echam.files.jan_surf")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "echam.files.jan_surf"
+ )
sim_file.cp("computer", "work")
assert os.path.exists(target)
@@ -419,7 +434,9 @@ def test_cp_folder(fs):
fs.create_dir(target_folder)
# Test the method
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.o3_data")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "oifs.files.o3_data"
+ )
sim_file.cp("computer", "work")
assert os.path.exists(target)
@@ -429,7 +446,7 @@ def test_resolve_file_movements(config_tuple):
# arrange config-in
config = config_tuple.config
attr_address = config_tuple.attr_address
- simulation_file = filedicts.SimulationFile(config, attr_address)
+ simulation_file = filedicts.SimulationFile.from_config(config, attr_address)
config = filedicts.resolve_file_movements(config)
# check config-out
@@ -463,7 +480,9 @@ def test_mv(fs):
fs.create_file("/work/ollie/pool/ECHAM/T63/T63CORE2_jan_surf.nc")
fs.create_dir("/work/ollie/pgierz/some_exp/run_20010101-20010101/work")
assert os.path.exists("/work/ollie/pool/ECHAM/T63/T63CORE2_jan_surf.nc")
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "echam.files.jan_surf")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "echam.files.jan_surf"
+ )
sim_file.mv("computer", "work")
assert not os.path.exists("/work/ollie/pool/ECHAM/T63/T63CORE2_jan_surf.nc")
assert os.path.exists(
@@ -561,7 +580,7 @@ def test_check_file_syntax_type_missing():
# Captures output (i.e. the user-friendly error)
with Capturing() as output:
with pytest.raises(SystemExit) as error:
- sim_file = esm_runscripts.filedicts.SimulationFile(
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
config, "echam.files.jan_surf"
)
@@ -590,7 +609,7 @@ def test_check_file_syntax_type_incorrect():
# Captures output (i.e. the user-friendly error)
with Capturing() as output:
with pytest.raises(SystemExit) as error:
- sim_file = esm_runscripts.filedicts.SimulationFile(
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
config, "echam.files.jan_surf"
)
@@ -621,7 +640,7 @@ def test_check_file_syntax_input():
# Captures output (i.e. the user-friendly error)
with Capturing() as output:
with pytest.raises(SystemExit) as error:
- sim_file = esm_runscripts.filedicts.SimulationFile(
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
config, "echam.files.jan_surf"
)
@@ -652,7 +671,7 @@ def test_check_file_syntax_output():
# Captures output (i.e. the user-friendly error)
with Capturing() as output:
with pytest.raises(SystemExit) as error:
- sim_file = esm_runscripts.filedicts.SimulationFile(
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
config, "echam.files.jan_surf"
)
@@ -700,7 +719,9 @@ def test_resolve_abs_paths(fs):
config = yaml.safe_load(dummy_config)
config["general"]["current_date"] = date
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "echam.files.jan_surf")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "echam.files.jan_surf"
+ )
assert sim_file["absolute_path_in_work"] == Path(
"/work/ollie/pgierz/some_exp/run_20010101-20010101/work/unit.24"
@@ -722,7 +743,7 @@ def test_resolve_paths_old_config():
"""
# Load an old config
tests_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), ".")
- with open(f"{tests_path}/awicm3_config.yaml", "r") as f:
+ with open(f"{tests_path}/example_configs/awicm3_config.yaml", "r") as f:
config = yaml.safe_load(f)
# Add the new ``files`` dictionary
config["oifs"]["files"] = {
@@ -734,7 +755,9 @@ def test_resolve_paths_old_config():
}
}
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.o3_data")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "oifs.files.o3_data"
+ )
assert sim_file["absolute_path_in_work"] == Path(
"/work/ollie/mandresm/testing/run/awicm3//awicm3-v3.1-TCO95L91-CORE2_initial/run_20000101-20000101/work/o3chem_l91"
@@ -887,7 +910,9 @@ def test_find_globbing_files(fs):
fs.create_dir(config["oifs"]["experiment_outdata_dir"])
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.oifsnc")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "oifs.files.oifsnc"
+ )
# Captures output (i.e. the user-friendly error)
with Capturing() as output:
@@ -939,7 +964,9 @@ def test_globbing_cp(fs):
Path(config["oifs"]["experiment_outdata_dir"]).joinpath(f)
)
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.oifsnc")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "oifs.files.oifsnc"
+ )
sim_file.cp("work", "exp_tree")
for nf in expected_new_paths:
@@ -987,7 +1014,9 @@ def test_globbing_mv(fs):
Path(config["oifs"]["experiment_outdata_dir"]).joinpath(f)
)
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.oifsnc")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "oifs.files.oifsnc"
+ )
sim_file.mv("work", "exp_tree")
for nf in expected_new_paths:
@@ -1035,7 +1064,9 @@ def test_globbing_ln(fs):
Path(config["oifs"]["experiment_outdata_dir"]).joinpath(f)
)
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.oifsnc")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "oifs.files.oifsnc"
+ )
sim_file.ln("work", "exp_tree")
for nf in expected_new_paths:
@@ -1077,7 +1108,15 @@ def test_makedirs_in_name(fs):
"out/date/folder/ICMGG_input_expid"
)
- sim_file = esm_runscripts.filedicts.SimulationFile(config, "oifs.files.ICMGG")
+ sim_file = esm_runscripts.filedicts.SimulationFile.from_config(
+ config, "oifs.files.ICMGG"
+ )
sim_file.cp("work", "exp_tree")
assert os.path.exists(expected_new_path)
+
+
+def test_SimulationFiles_basics(fake_config):
+ sim_files = esm_runscripts.filedicts.SimulationFiles.from_config(
+ fake_config, "echam.files"
+ )