Merge pull request #14 from ecmwf/feature/add_param_db

Feature/add param db

eccovjson/data/unit.json

@@ -0,0 +1 @@
+{"1": {"id": 1, "name": "m**2 s**-1"}, "2": {"id": 2, "name": "K"}, "3": {"id": 3, "name": "(0 - 1)"}, "4": {"id": 4, "name": "m"}, "5": {"id": 5, "name": "m s**-1"}, "6": {"id": 6, "name": "J m**-2"}, "7": {"id": 7, "name": "~"}, "8": {"id": 8, "name": "s**-1"}, "9": {"id": 9, "name": "kg m**-3"}, "10": {"id": 10, "name": "m**3 m**-3"}, "11": {"id": 11, "name": "10**-6 W m**-2 sr**-1 m**-1"}, "12": {"id": 12, "name": "s"}, "13": {"id": 13, "name": "mol mol**-1"}, "14": {"id": 14, "name": "N m**-2 s"}, "15": {"id": 15, "name": "m**2 s**-2"}, "16": {"id": 16, "name": "Pa"}, "17": {"id": 17, "name": "J kg**-1"}, "18": {"id": 18, "name": "K m**2 kg**-1 s**-1"}, "19": {"id": 19, "name": "m**2 m**-2"}, "20": {"id": 20, "name": "s m**-1"}, "21": {"id": 21, "name": "kg kg**-1"}, "22": {"id": 22, "name": "kg m**-2"}, "23": {"id": 23, "name": "dimensionless"}, "24": {"id": 24, "name": "m**-3"}, "25": {"id": 25, "name": "Various"}, "26": {"id": 26, "name": "Pa s**-1"}, "27": {"id": 27, "name": "m of water equivalent"}, "28": {"id": 28, "name": "gpm"}, "29": {"id": 29, "name": "%"}, "30": {"id": 30, "name": "radians"}, "31": {"id": 31, "name": "m**2"}, "32": {"id": 32, "name": "N m**-2"}, "33": {"id": 33, "name": "kg m**-2 s**-1"}, "34": {"id": 34, "name": "(-1 to 1)"}, "35": {"id": 35, "name": "Millimetres*100 + number of stations"}, "36": {"id": 36, "name": "kg m**-2 s"}, "37": {"id": 37, "name": "K s**-1"}, "38": {"id": 38, "name": "m**2 s**-3"}, "39": {"id": 39, "name": "kg kg**-1 s**-1"}, "40": {"id": 40, "name": "DU"}, "41": {"id": 41, "name": "degrees"}, "42": {"id": 42, "name": "m**2 s radian**-1"}, "43": {"id": 43, "name": "deg C"}, "44": {"id": 44, "name": "psu"}, "45": {"id": 45, "name": "kg m**-3 -1000"}, "46": {"id": 46, "name": "m s**-2"}, "47": {"id": 47, "name": "m s**-1 deg C"}, "48": {"id": 48, "name": "m s**-1 degC"}, "49": {"id": 49, "name": "m**3 s**-1"}, "50": {"id": 50, "name": "N m**-3"}, "51": {"id": 51, "name": "Nm**-3"}, "52": {"id": 52, "name": "degrees C"}, "53": {"id": 53, "name": "J m**-1 s**-1"}, "54": {"id": 54, "name": "m s**-1 per time step"}, "55": {"id": 55, "name": "psu per time step"}, "56": {"id": 56, "name": "deg C per time step"}, "57": {"id": 57, "name": "Pa m**-1"}, "58": {"id": 58, "name": "N m**-2 s**-1"}, "59": {"id": 59, "name": "K m**2 s**-2"}, "60": {"id": 60, "name": "m**3 s**-3"}, "61": {"id": 61, "name": "K m s**-1"}, "62": {"id": 62, "name": "Pa m**2 s**-3"}, "63": {"id": 63, "name": "K Pa s**-1"}, "64": {"id": 64, "name": "Pa m s**-2"}, "65": {"id": 65, "name": "K kg m**-2"}, "66": {"id": 66, "name": "kg m**-1 s**-1"}, "67": {"id": 67, "name": "m**4 s**-4"}, "68": {"id": 68, "name": "m**2 K s**-2"}, "69": {"id": 69, "name": "K**2"}, "70": {"id": 70, "name": "m s**-1 K"}, "71": {"id": 71, "name": "m**2 Pa s**-3"}, "72": {"id": 72, "name": "Pa s**-1 K"}, "73": {"id": 73, "name": "m Pa s**-2"}, "74": {"id": 74, "name": "Pa**2 s**-2"}, "75": {"id": 75, "name": "Pa**2"}, "76": {"id": 76, "name": "W m**-2"}, "77": {"id": 77, "name": "m of water s**-1"}, "78": {"id": 78, "name": "m of water equivalent s**-1"}, "79": {"id": 79, "name": "W m**-2 s**-1"}, "80": {"id": 80, "name": "kg C m**-2 s**-1"}, "81": {"id": 81, "name": "Code table 4.239"}, "82": {"id": 82, "name": "g kg**-1 m**-2 s**-1"}, "83": {"id": 83, "name": "g kg**-1 s**-1"}, "84": {"id": 84, "name": "W**-2 m**4"}, "85": {"id": 85, "name": "kg s**-1 m**-2"}, "86": {"id": 86, "name": "m**-1"}, "87": {"id": 87, "name": "m**3 m**-2"}, "88": {"id": 88, "name": "J kg**-1 s**-1"}, "89": {"id": 89, "name": "s**-2"}, "90": {"id": 90, "name": "kg s**2 m**-5"}, "91": {"id": 91, "name": "~"}, "92": {"id": 92, "name": "Degree true"}, "93": {"id": 93, "name": "W m**-1 sr**-1"}, "94": {"id": 94, "name": "W m**-3 sr**-1"}, "95": {"id": 95, "name": "J"}, "96": {"id": 96, "name": "m**-1 sr**-1"}, "97": {"id": 97, "name": "m**2/3 s**-1"}, "98": {"id": 98, "name": "day"}, "100": {"id": 100, "name": "code table (4.201)"}, "103": {"id": 103, "name": "code table (4.202)"}, "104": {"id": 104, "name": "(Code table 4.222)"}, "105": {"id": 105, "name": "Proportion"}, "106": {"id": 106, "name": "Numeric"}, "107": {"id": 107, "name": "km**-2 day**-1"}, "110": {"id": 110, "name": "code table (4.203)"}, "111": {"id": 111, "name": "code table (4.204)"}, "112": {"id": 112, "name": "(code table 4.234)"}, "113": {"id": 113, "name": "code table (4.205)"}, "114": {"id": 114, "name": "Dobson"}, "115": {"id": 115, "name": "dB"}, "116": {"id": 116, "name": "kg m**-1"}, "117": {"id": 117, "name": "Bq m**-3"}, "118": {"id": 118, "name": "Bq m**-2"}, "119": {"id": 119, "name": "Bq s m**-3"}, "120": {"id": 120, "name": "code table (4.206)"}, "121": {"id": 121, "name": "code table (4.207)"}, "122": {"id": 122, "name": "code table (4.208)"}, "123": {"id": 123, "name": "code table (4.209)"}, "124": {"id": 124, "name": "code table (4.210)"}, "125": {"id": 125, "name": "code table (4.211)"}, "126": {"id": 126, "name": "CCITTIA5"}, "127": {"id": 127, "name": "(code table 4.215)"}, "128": {"id": 128, "name": "(code table 4.216)"}, "129": {"id": 129, "name": "kg**-2 s**-1"}, "130": {"id": 130, "name": "code table (4.212)"}, "131": {"id": 131, "name": "Code Table JMA-252"}, "133": {"id": 133, "name": "Code table 4.219"}, "134": {"id": 134, "name": "Degree"}, "135": {"id": 135, "name": "W m**-2"}, "136": {"id": 136, "name": "K s**-1"}, "137": {"id": 137, "name": "J m**-2 K**-1"}, "139": {"id": 139, "name": "kg (m**2 s**-1)**-1"}, "140": {"id": 140, "name": "kg (kg s**-1)**-1"}, "141": {"id": 141, "name": "kg m**-2"}, "142": {"id": 142, "name": "cm per day"}, "143": {"id": 143, "name": "g kg**-1 m"}, "144": {"id": 144, "name": "kg m**-2"}, "145": {"id": 145, "name": "mm per day"}, "146": {"id": 146, "name": "m s**-1 per day"}, "147": {"id": 147, "name": "g kg**-1"}, "148": {"id": 148, "name": "deg"}, "149": {"id": 149, "name": "m**2 s**-1"}, "150": {"id": 150, "name": "m**2 s**-2"}, "153": {"id": 153, "name": "m s**-2"}, "154": {"id": 154, "name": "(s m**-1)**-1"}, "155": {"id": 155, "name": "Pa s**-1"}, "156": {"id": 156, "name": "m**-1"}, "157": {"id": 157, "name": "ln(kPa)"}, "158": {"id": 158, "name": "kg (m**2 s-1)**-1"}, "159": {"id": 159, "name": "10**-3"}, "161": {"id": 161, "name": "(10**-6 g) m**-3"}, "162": {"id": 162, "name": "log10((10**-6g) m**-3)"}, "163": {"id": 163, "name": "ppb"}, "164": {"id": 164, "name": "mm**6 m**-3"}, "165": {"id": 165, "name": "categorical"}, "166": {"id": 166, "name": "Integer"}, "167": {"id": 167, "name": "log10(kg m**-3)"}, "168": {"id": 168, "name": "Fraction"}, "169": {"id": 169, "name": "ppm"}, "170": {"id": 170, "name": "Integer(0-13)"}, "171": {"id": 171, "name": "10**-3 m**-2"}, "172": {"id": 172, "name": "W m**-2"}, "173": {"id": 173, "name": "Index"}, "174": {"id": 174, "name": "W m**-2"}, "175": {"id": 175, "name": "kg m**-3"}, "176": {"id": 176, "name": "degreeperday"}, "177": {"id": 177, "name": "psuperday"}, "178": {"id": 178, "name": "W m**-1"}, "179": {"id": 179, "name": "W"}, "180": {"id": 180, "name": "kg**2 m**-4"}, "181": {"id": 181, "name": "s**-1 m**-2"}, "182": {"id": 182, "name": "Degree N"}, "183": {"id": 183, "name": "Degree E"}, "184": {"id": 184, "name": "m**3 kg**-1 s**-1"}, "185": {"id": 185, "name": "m**2 kg**-1 s**-1"}, "186": {"id": 186, "name": "W m sr m**-2"}, "188": {"id": 188, "name": "10**5 s**-1"}, "189": {"id": 189, "name": "K s**-2"}, "190": {"id": 190, "name": "K m**-1"}, "191": {"id": 191, "name": "Pa hPa"}, "192": {"id": 192, "name": "Ws m**-2"}, "193": {"id": 193, "name": "cbar s**-1"}, "194": {"id": 194, "name": "cm"}, "195": {"id": 195, "name": "dam"}, "196": {"id": 196, "name": "hPa"}, "197": {"id": 197, "name": "image^data"}, "198": {"id": 198, "name": "kg m**-2/day"}, "199": {"id": 199, "name": "ln(cbar)"}, "200": {"id": 200, "name": "m cm"}, "201": {"id": 201, "name": "m s**-12"}, "202": {"id": 202, "name": "m**2 s**-12"}, "203": {"id": 203, "name": "mb"}, "204": {"id": 204, "name": "s s**-1"}, "205": {"id": 205, "name": "s**-12"}, "206": {"id": 206, "name": "-/+"}, "207": {"id": 207, "name": "m**3"}, "208": {"id": 208, "name": "m**3 s**-1 m**-1"}, "209": {"id": 209, "name": "Code table 4.217"}, "210": {"id": 210, "name": "Code table 4.218"}, "211": {"id": 211, "name": "Code table 4.223"}, "212": {"id": 212, "name": "K per day"}, "213": {"id": 213, "name": "kg kg**-1 per day"}, "214": {"id": 214, "name": "mg kg**-1"}, "215": {"id": 215, "name": "kg m**-3 s**-1"}, "216": {"id": 216, "name": "psu*m"}, "217": {"id": 217, "name": "hour"}, "218": {"id": 218, "name": "Number m**-2"}, "219": {"id": 219, "name": "Person m**-2"}, "220": {"id": 220, "name": "Bites per day per person"}, "221": {"id": 221, "name": "Byte"}, "222": {"id": 222, "name": "J m**-1"}, "223": {"id": 223, "name": "W s"}, "224": {"id": 224, "name": "km**-2"}, "225": {"id": 225, "name": "day**-1"}, "226": {"id": 226, "name": "% K"}, "227": {"id": 227, "name": "% m**3 m**-3"}, "228": {"id": 228, "name": "K K"}, "229": {"id": 229, "name": "K m**3 m**-3"}, "230": {"id": 230, "name": "m**3 m**-3 m**3 m**-3"}}

eccovjson/decoder/BoundingBox.py

@@ -54,7 +54,7 @@ def to_xarray(self):
             dataarray = xr.DataArray(self.get_values()[parameter][0], dims=dims)
             dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
             dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
-            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"]
+            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
             dataarraydict[dataarray.attrs["long_name"]] = dataarray
 
         ds = xr.Dataset(

eccovjson/decoder/Frame.py

@@ -51,7 +51,7 @@ def to_xarray(self):
             dataarray = xr.DataArray(self.get_values()[parameter][0], dims=dims)
             dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
             dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
-            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"]
+            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
             dataarraydict[dataarray.attrs["long_name"]] = dataarray
 
         ds = xr.Dataset(

eccovjson/decoder/Path.py

@@ -53,7 +53,7 @@ def to_xarray(self):
             dataarray = xr.DataArray(self.get_values()[parameter][0], dims=dims)
             dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
             dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
-            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"]
+            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
             dataarraydict[dataarray.attrs["long_name"]] = dataarray
 
         ds = xr.Dataset(

eccovjson/decoder/Shapefile.py

@@ -54,7 +54,7 @@ def to_xarray(self):
             dataarray = xr.DataArray(self.get_values()[parameter][0], dims=dims)
             dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
             dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
-            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"]
+            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
             dataarraydict[dataarray.attrs["long_name"]] = dataarray
 
         ds = xr.Dataset(

eccovjson/decoder/TimeSeries.py

@@ -86,7 +86,7 @@ def to_xarray(self):
 
                 dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
                 dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
-                dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"]
+                dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
                 dataarraydict[dataarray.attrs["long_name"]] = dataarray
 
         ds = xr.Dataset(dataarraydict)

eccovjson/decoder/VerticalProfile.py

@@ -81,7 +81,7 @@ def to_xarray(self):
                 )
                 dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
                 dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
-                dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"]
+                dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
                 dataarraydict[dataarray.attrs["long_name"]] = dataarray
 
         ds = xr.Dataset(dataarraydict)

eccovjson/decoder/Wkt.py

@@ -54,7 +54,7 @@ def to_xarray(self):
             dataarray = xr.DataArray(self.get_values()[parameter][0], dims=dims)
             dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
             dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
-            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"]
+            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
             dataarraydict[dataarray.attrs["long_name"]] = dataarray
 
         ds = xr.Dataset(

eccovjson/encoder/encoder.py

@@ -2,6 +2,7 @@
 
 from eccovjson.Coverage import Coverage
 from eccovjson.CoverageCollection import CoverageCollection
+from eccovjson.param_db import get_param_from_db, get_unit_from_db
 
 
 class Encoder(ABC):
@@ -24,77 +25,18 @@ def __init__(self, type, domaintype):
             raise TypeError("Type must be Coverage or CoverageCollection")
 
     def add_parameter(self, param):
-        param = self.convert_param_id_to_param(param)
-        if param == "t" or param == "167":
-            self.covjson["parameters"][param] = {
-                "type": "Parameter",
-                "description": "Temperature",
-                "unit": {"symbol": "K"},
-                "observedProperty": {
-                    "id": "t",
-                    "label": {"en": "Temperature"},
-                },
-            }
-        elif param == "tp" or param == "228":
-            self.covjson["parameters"][param] = {
-                "type": "Parameter",
-                "description": "Total Precipitation",
-                "unit": {"symbol": "m"},
-                "observedProperty": {
-                    "id": "tp",
-                    "label": {"en": "Total Precipitation"},
-                },
-            }
-        elif param == "10u" or param == "165":
-            self.covjson["parameters"][param] = {
-                "type": "Parameter",
-                "description": "10 metre U wind component",
-                "unit": {"symbol": "ms-1"},
-                "observedProperty": {
-                    "id": "10u",
-                    "label": {"en": "10 metre U wind component"},
-                },
-            }
-        elif param == "10v" or param == "166":
-            self.covjson["parameters"][param] = {
-                "type": "Parameter",
-                "description": "10 metre V wind component",
-                "unit": {"symbol": "ms-1"},
-                "observedProperty": {
-                    "id": "10v",
-                    "label": {"en": "10 metre V wind component"},
-                },
-            }
-        elif param == "10fg" or param == "49":
-            self.covjson["parameters"][param] = {
-                "type": "Parameter",
-                "description": "Maximum 10 metre wind gust since previous post-processing",
-                "unit": {"symbol": "ms-1"},
-                "observedProperty": {
-                    "id": "10fg",
-                    "label": {"en": "Maximum 10 metre wind gust since previous post-processing"},
-                },
-            }
-        elif param == "tcc" or param == "164":
-            self.covjson["parameters"][param] = {
-                "type": "Parameter",
-                "description": "Total cloud cover",
-                "unit": {"symbol": ""},
-                "observedProperty": {
-                    "id": "tcc",
-                    "label": {"en": "Total cloud cover"},
-                },
-            }
-        elif param == "2d" or param == "168":
-            self.covjson["parameters"][param] = {
-                "type": "Parameter",
-                "description": "2 metre dewpoint temperature",
-                "unit": {"symbol": "K"},
-                "observedProperty": {
-                    "id": "2d",
-                    "label": {"en": "2 metre dewpoint temperature"},
-                },
-            }
+        # param = self.convert_param_id_to_param(param)
+        param_dict = get_param_from_db(int(param))
+        unit = get_unit_from_db(param_dict["unit_id"])
+        self.covjson["parameters"][param_dict["shortname"]] = {
+            "type": "Parameter",
+            "description": param_dict["description"],
+            "unit": {"symbol": unit["name"]},
+            "observedProperty": {
+                "id": param_dict["shortname"],
+                "label": {"en": param_dict["name"]},
+            },
+        }
         self.parameters.append(param)
 
     def add_reference(self, reference):
@@ -105,12 +47,15 @@ def convert_param_id_to_param(self, paramid):
             param = int(paramid)
         except BaseException:
             return paramid
+        param_dict = get_param_from_db(int(param))
+        return param_dict["shortname"]
+        """
         if param == 165:
             return "10u"
         elif param == 166:
             return "10v"
         elif param == 167:
-            return "t"
+            return "2t"
         elif param == 228:
             return "tp"
         elif param == 49:
@@ -119,6 +64,7 @@ def convert_param_id_to_param(self, paramid):
             return "tcc"
         elif param == 168:
             return "2d"
+        """
 
     @abstractmethod
     def add_coverage(self, mars_metadata, coords, values):

eccovjson/param_db.py

@@ -0,0 +1,28 @@
+import json
+import os
+from os.path import dirname
+
+
+def get_param_from_db(param_id):
+    """
+    import requests
+    url = f"https://codes.ecmwf.int/parameter-database/api/v1/param/?format=json&searcg={param_id}"
+    response = requests.get(url)
+    if response.status_code == 200:
+        for param in response.json():
+            if param['id'] == param_id:
+                return param
+    else:
+        raise BaseException(f"Failed to get parameter from database: {response.status_code}")
+    """
+    param_path = os.path.join(dirname(__file__), "data/param.json")
+    with open(param_path) as f:
+        params = json.load(f)
+    return params[str(param_id)]
+
+
+def get_unit_from_db(unit_id):
+    unit_path = os.path.join(dirname(__file__), "data/unit.json")
+    with open(unit_path) as f:
+        units = json.load(f)
+    return units[str(unit_id)]

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.black]
 line-length = 120
 [build-system]
-requires = ["setuptools>=61.0"]
+requires = ["setuptools>=61.0","setuptools-scm>=8.0"]
 build-backend = "setuptools.build_meta"
 [tool.pytest.ini_options]
 markers = ["data: uses test data (deselect with '-m \"not data\"')",]

tests/test_decoder_vertical_profile.py

@@ -1,4 +1,3 @@
-import eccovjson.encoder.VerticalProfile
 from eccovjson.api import Eccovjson
 
 
@@ -242,9 +241,9 @@ def test_verticalprofile_values(self):
         }
         assert decoder.get_values() == values
 
-    def test_verticalprofile_to_xarray(self):
-        decoder = Eccovjson().decode(self.covjson)
-        dataset = decoder.to_xarray()
-        encoder = eccovjson.encoder.VerticalProfile.VerticalProfile("CoverageCollection", "VerticalProfile")
-        cov = encoder.from_xarray(dataset)
-        print(cov)
+    # def test_verticalprofile_to_xarray(self):
+    #    decoder = Eccovjson().decode(self.covjson)
+    #    dataset = decoder.to_xarray()
+    #    encoder = eccovjson.encoder.VerticalProfile.VerticalProfile("CoverageCollection", "VerticalProfile")
+    #    cov = encoder.from_xarray(dataset)
+    #    print(cov)