perturbations

ecml_tools/create/functions/__init__.py

@@ -6,83 +6,6 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 #
-import os
-
-from climetlab import load_source
-
-
-class Mockup:
-    write_directory = None
-    read_directory = None
-
-    def get_filename(self, r):
-        import hashlib
-
-        h = hashlib.md5(str(r).encode("utf8")).hexdigest()
-        return h + ".copy"
-
-    def write(self, ds, *args, **kwargs):
-        if self.write_directory is None:
-            return
-        if not hasattr(ds, "path"):
-            return
-
-        path = os.path.join(self.write_directory, self.get_filename([args, kwargs]))
-        print(f"Saving to {path} for {args}, {kwargs}")
-        import shutil
-
-        shutil.copy(ds.path, path)
-
-    def load_source(self, *args, **kwargs):
-        if self.read_directory is None:
-            return None
-        path = os.path.join(self.read_directory, self.get_filename([args, kwargs]))
-        if os.path.exists(path):
-            print(f"Loading path {path} for {args}, {kwargs}")
-            return load_source("file", path)
-        elif path.startswith("http"):
-            import requests
-
-            print(f"Loading url {path} for {args}, {kwargs}")
-            try:
-                return load_source("url", path)
-            except requests.exceptions.HTTPError:
-                pass
-        return None
-
-
-MOCKUP = Mockup()
-
-
-def enable_save_mars(d):
-    MOCKUP.write_directory = d
-
-
-def disable_save_mars():
-    MOCKUP.write_directory = None
-
-
-def enable_read_mars(d):
-    MOCKUP.read_directory = d
-
-
-def disable_read_mars():
-    MOCKUP.read_directory = None
-
-
-if os.environ.get("MOCKUP_MARS_SAVE_REQUESTS"):
-    enable_save_mars(os.environ.get("MOCKUP_MARS_SAVE_REQUESTS"))
-
-if os.environ.get("MOCKUP_MARS_READ_REQUESTS"):
-    enable_read_mars(os.environ.get("MOCKUP_MARS_READ_REQUESTS"))
-
-
-def _load_source(*args, **kwargs):
-    ds = MOCKUP.load_source(*args, **kwargs)
-    if ds is None:
-        ds = load_source(*args, **kwargs)
-    MOCKUP.write(ds, *args, **kwargs)
-    return ds
 
 
 def assert_is_fieldset(obj):

ecml_tools/create/functions/actions/accumulations.py

@@ -8,11 +8,11 @@
 #
 from copy import deepcopy
 
+from climetlab import load_source
+
 from ecml_tools.create.functions.actions.mars import factorise_requests
 from ecml_tools.create.utils import to_datetime_list
 
-from .. import _load_source as load_source
-
 DEBUG = True
 
 

ecml_tools/create/functions/actions/mars.py

@@ -9,12 +9,11 @@
 import datetime
 from copy import deepcopy
 
+from climetlab import load_source
 from climetlab.utils.availability import Availability
 
 from ecml_tools.create.utils import to_datetime_list
 
-from .. import _load_source as load_source
-
 DEBUG = False
 
 

tests/create/perturbations.yaml

@@ -1,27 +1,25 @@
 common:
-  sfc: &sfc
+  global: &global
     class: ea
     expver: "0001"
     grid: 20.0/20.0
+  sfc: &sfc
+    <<: *global
     levtype: sfc
     param: [2t]
     #levtype: sfc
     #param: [10u, 10v, 2d, 2t, lsm, msl, sdor, skt, slor, sp, tcw, z, cp, tp]
+  acc: &acc
+    <<: *global
+    levtype: sfc
+    param: [tp]
   pl: &pl
-    class: ea
-    expver: "0001"
-    grid: 20.0/20.0
+    <<: *global
     levtype: pl
     param: [q]
     level: [50]
     #param: [q, t, u, v, w, z]
     #level: [50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 850, 925, 1000]
-  acc: &acc
-    class: ea
-    expver: "0001"
-    grid: 20.0/20.0
-    levtype: sfc
-    param: [tp]
 
   ensembles: &ensembles
     stream: enda
@@ -31,17 +29,6 @@ common:
   center: &center
     stream: oper
     type: an
-  constants: &constants
-          param:
-            - cos_latitude
-            - cos_longitude
-            - sin_latitude
-            - sin_longitude
-            - cos_julian_day
-            - cos_local_time
-            - sin_julian_day
-            - sin_local_time
-            - insolation
 
 dates:
   start: 2020-12-30 00:00:00
@@ -63,9 +50,6 @@ data_sources:
       - accumulations:
           <<: *ensembles
           <<: *acc
-     # - constants:
-     #     <<: *constants
-     #     template: ${data_sources.ensembles.join.0.mars}
   center:
     join:
       - mars:
@@ -77,16 +61,26 @@ data_sources:
       - accumulations:
           <<: *center
           <<: *acc
-     # - constants:
-     #     <<: *constants
-     #     template: ${data_sources.center.join.0.mars}
-
+
 input:
-  ensemble_perturbations:
-    # the ensemble data which has one additional dimension
-    ensembles: ${data_sources.ensembles}
-    # the new center of the data
-    center: ${data_sources.center}
+  join:
+    - ensemble_perturbations:
+        # the ensemble data which has one additional dimension
+        ensembles: ${data_sources.ensembles}
+        # the new center of the data
+        center: ${data_sources.center}
+    - constants:
+        template: ${input.join.0.ensemble_perturbations}
+        param:
+          - cos_latitude
+          - cos_longitude
+          - sin_latitude
+          - sin_longitude
+          - cos_julian_day
+          - cos_local_time
+          - sin_julian_day
+          - sin_local_time
+          - insolation
 
 output:
   order_by: [valid_datetime, param_level, number]

tests/create/test_create.py

@@ -10,6 +10,7 @@
 
 import numpy as np
 import pytest
+from climetlab import load_source
 
 from ecml_tools.create import Creator
 from ecml_tools.create.functions import enable_read_mars
@@ -34,6 +35,80 @@
 enable_read_mars("https://object-store.os-api.cci1.ecmwf.int/ml-tests/test-data/anemoi-datasets/create")
 
 
+class Mockup:
+    write_directory = None
+    read_directory = None
+
+    def get_filename(self, r):
+        import hashlib
+
+        h = hashlib.md5(str(r).encode("utf8")).hexdigest()
+        return h + ".copy"
+
+    def write(self, ds, *args, **kwargs):
+        if self.write_directory is None:
+            return
+        if not hasattr(ds, "path"):
+            return
+
+        path = os.path.join(self.write_directory, self.get_filename([args, kwargs]))
+        print(f"Saving to {path} for {args}, {kwargs}")
+        import shutil
+
+        shutil.copy(ds.path, path)
+
+    def load_source(self, *args, **kwargs):
+        if self.read_directory is None:
+            return None
+        path = os.path.join(self.read_directory, self.get_filename([args, kwargs]))
+        if os.path.exists(path):
+            print(f"Loading path {path} for {args}, {kwargs}")
+            return load_source("file", path)
+        elif path.startswith("http"):
+            import requests
+
+            print(f"Loading url {path} for {args}, {kwargs}")
+            try:
+                return load_source("url", path)
+            except requests.exceptions.HTTPError:
+                pass
+        return None
+
+
+MOCKUP = Mockup()
+
+
+def enable_save_mars(d):
+    MOCKUP.write_directory = d
+
+
+def disable_save_mars():
+    MOCKUP.write_directory = None
+
+
+def enable_read_mars(d):
+    MOCKUP.read_directory = d
+
+
+def disable_read_mars():
+    MOCKUP.read_directory = None
+
+
+if os.environ.get("MOCKUP_MARS_SAVE_REQUESTS"):
+    enable_save_mars(os.environ.get("MOCKUP_MARS_SAVE_REQUESTS"))
+
+if os.environ.get("MOCKUP_MARS_READ_REQUESTS"):
+    enable_read_mars(os.environ.get("MOCKUP_MARS_READ_REQUESTS"))
+
+
+def _load_source(*args, **kwargs):
+    ds = MOCKUP.load_source(*args, **kwargs)
+    if ds is None:
+        ds = load_source(*args, **kwargs)
+    MOCKUP.write(ds, *args, **kwargs)
+    return ds
+
+
 def compare_dot_zattrs(a, b):
     if isinstance(a, dict):
         a_keys = list(a.keys())