Merge pull request #16 from ecmwf/feature/add_covjson_pydantic_support

Feature/add covjson pydantic support

eccovjson/decoder/TimeSeries.py

@@ -74,7 +74,7 @@ def to_xarray(self):
                 try:
                     t = [dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") for coord in coords_fc]
                 except ValueError:
-                    t = [dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%S") for coord in coords_fc]
+                    t = [dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%SZ") for coord in coords_fc]
 
                 param_coords = {"x": x, "y": y, "z": z, "number": num, "t": t}
                 dataarray = xr.DataArray(

eccovjson/encoder/BoundingBox.py

@@ -1,4 +1,7 @@
+import json
+
 import pandas as pd
+from covjson_pydantic.coverage import Coverage
 
 from .encoder import Encoder
 
@@ -17,7 +20,8 @@ def add_coverage(self, mars_metadata, coords, values):
         self.add_mars_metadata(new_coverage, mars_metadata)
         self.add_domain(new_coverage, coords)
         self.add_range(new_coverage, values)
-        self.covjson["coverages"].append(new_coverage)
+        cov = Coverage.model_validate_json(json.dumps(new_coverage))
+        self.pydantic_coverage.coverages.append(cov)
 
     def add_domain(self, coverage, coords):
         coverage["domain"]["type"] = "Domain"
@@ -26,7 +30,7 @@ def add_domain(self, coverage, coords):
         coverage["domain"]["axes"]["t"]["values"] = coords["t"]
         coverage["domain"]["axes"]["composite"] = {}
         coverage["domain"]["axes"]["composite"]["dataType"] = "tuple"
-        coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson["referencing"][0]["coordinates"]
+        coverage["domain"]["axes"]["composite"]["coordinates"] = self.pydantic_coverage.referencing[0].coordinates
         coverage["domain"]["axes"]["composite"]["values"] = coords["composite"]
 
     def add_range(self, coverage, values):
@@ -120,7 +124,7 @@ def from_polytope(self, result):
         range_dict = {}
         coords = {}
         coords["composite"] = []
-        coords["t"] = df["date"].unique()[0]
+        coords["t"] = [df["date"].unique()[0] + "Z"]
 
         for param in params:
             df_param = df[df["param"] == param]
@@ -131,4 +135,4 @@ def from_polytope(self, result):
             coords["composite"].append([row[1]["latitude"], row[1]["longitude"]])
 
         self.add_coverage(mars_metadata, coords, range_dict)
-        return self.covjson
+        return json.loads(self.get_json())

eccovjson/encoder/Frame.py

@@ -1,4 +1,7 @@
+import json
+
 import pandas as pd
+from covjson_pydantic.coverage import Coverage
 
 from .encoder import Encoder
 
@@ -17,7 +20,8 @@ def add_coverage(self, mars_metadata, coords, values):
         self.add_mars_metadata(new_coverage, mars_metadata)
         self.add_domain(new_coverage, coords)
         self.add_range(new_coverage, values)
-        self.covjson["coverages"].append(new_coverage)
+        cov = Coverage.model_validate_json(json.dumps(new_coverage))
+        self.pydantic_coverage.coverages.append(cov)
 
     def add_domain(self, coverage, coords):
         coverage["domain"]["type"] = "Domain"
@@ -26,7 +30,7 @@ def add_domain(self, coverage, coords):
         coverage["domain"]["axes"]["t"]["values"] = coords["t"]
         coverage["domain"]["axes"]["composite"] = {}
         coverage["domain"]["axes"]["composite"]["dataType"] = "tuple"
-        coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson["referencing"][0]["coordinates"]
+        coverage["domain"]["axes"]["composite"]["coordinates"] = self.pydantic_coverage.referencing[0].coordinates
         coverage["domain"]["axes"]["composite"]["values"] = coords["composite"]
 
     def add_range(self, coverage, values):
@@ -120,7 +124,7 @@ def from_polytope(self, result):
         range_dict = {}
         coords = {}
         coords["composite"] = []
-        coords["t"] = df["date"].unique()[0]
+        coords["t"] = [df["date"].unique()[0] + "Z"]
 
         for param in params:
             df_param = df[df["param"] == param]
@@ -131,4 +135,4 @@ def from_polytope(self, result):
             coords["composite"].append([row[1]["latitude"], row[1]["longitude"]])
 
         self.add_coverage(mars_metadata, coords, range_dict)
-        return self.covjson
+        return json.loads(self.get_json())

eccovjson/encoder/Path.py

@@ -1,4 +1,7 @@
+import json
+
 import pandas as pd
+from covjson_pydantic.coverage import Coverage
 
 from .encoder import Encoder
 
@@ -17,14 +20,15 @@ def add_coverage(self, mars_metadata, coords, values):
         self.add_mars_metadata(new_coverage, mars_metadata)
         self.add_domain(new_coverage, coords)
         self.add_range(new_coverage, values)
-        self.covjson["coverages"].append(new_coverage)
+        cov = Coverage.model_validate_json(json.dumps(new_coverage))
+        self.pydantic_coverage.coverages.append(cov)
 
     def add_domain(self, coverage, coords):
         coverage["domain"]["type"] = "Domain"
         coverage["domain"]["axes"] = {}
         coverage["domain"]["axes"]["composite"] = {}
         coverage["domain"]["axes"]["composite"]["dataType"] = "tuple"
-        coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson["referencing"][0]["coordinates"]
+        coverage["domain"]["axes"]["composite"]["coordinates"] = self.referencing
         coverage["domain"]["axes"]["composite"]["values"] = coords["composite"]
 
     def add_range(self, coverage, values):
@@ -127,4 +131,4 @@ def from_polytope(self, result):
             coords["composite"].append([row[1]["date"], row[1]["latitude"], row[1]["longitude"]])
 
         self.add_coverage(mars_metadata, coords, range_dict)
-        return self.covjson
+        return json.loads(self.get_json())

eccovjson/encoder/Shapefile.py

@@ -1,4 +1,7 @@
+import json
+
 import pandas as pd
+from covjson_pydantic.coverage import Coverage
 
 from .encoder import Encoder
 
@@ -17,7 +20,8 @@ def add_coverage(self, mars_metadata, coords, values):
         self.add_mars_metadata(new_coverage, mars_metadata)
         self.add_domain(new_coverage, coords)
         self.add_range(new_coverage, values)
-        self.covjson["coverages"].append(new_coverage)
+        cov = Coverage.model_validate_json(json.dumps(new_coverage))
+        self.pydantic_coverage.coverages.append(cov)
 
     def add_domain(self, coverage, coords):
         coverage["domain"]["type"] = "Domain"
@@ -26,7 +30,7 @@ def add_domain(self, coverage, coords):
         coverage["domain"]["axes"]["t"]["values"] = coords["t"]
         coverage["domain"]["axes"]["composite"] = {}
         coverage["domain"]["axes"]["composite"]["dataType"] = "tuple"
-        coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson["referencing"][0]["coordinates"]
+        coverage["domain"]["axes"]["composite"]["coordinates"] = self.pydantic_coverage.referencing[0].coordinates
         coverage["domain"]["axes"]["composite"]["values"] = coords["composite"]
 
     def add_range(self, coverage, values):
@@ -120,7 +124,7 @@ def from_polytope(self, result):
         range_dict = {}
         coords = {}
         coords["composite"] = []
-        coords["t"] = df["date"].unique()[0]
+        coords["t"] = [df["date"].unique()[0] + "Z"]
 
         for param in params:
             df_param = df[df["param"] == param]
@@ -131,4 +135,4 @@ def from_polytope(self, result):
             coords["composite"].append([row[1]["latitude"], row[1]["longitude"]])
 
         self.add_coverage(mars_metadata, coords, range_dict)
-        return self.covjson
+        return json.loads(self.get_json())

eccovjson/encoder/TimeSeries.py

@@ -1,6 +1,8 @@
+import json
 from datetime import datetime, timedelta
 
 import pandas as pd
+from covjson_pydantic.coverage import Coverage
 
 from .encoder import Encoder
 
@@ -18,7 +20,8 @@ def add_coverage(self, mars_metadata, coords, values):
         self.add_mars_metadata(new_coverage, mars_metadata)
         self.add_domain(new_coverage, coords)
         self.add_range(new_coverage, values)
-        self.covjson["coverages"].append(new_coverage)
+        cov = Coverage.model_validate_json(json.dumps(new_coverage))
+        self.pydantic_coverage.coverages.append(cov)
 
     def add_domain(self, coverage, coords):
         coverage["domain"]["type"] = "Domain"
@@ -135,7 +138,7 @@ def from_polytope(self, result):
         for step in steps:
             # add current date to list by converting it to iso format
             stamp = start_time + timedelta(hours=int(step))
-            coords["t"].append(stamp.isoformat())
+            coords["t"].append(stamp.isoformat() + "Z")
             # increment start date by timedelta
 
         if "number" not in df.columns:
@@ -156,4 +159,4 @@ def from_polytope(self, result):
                     range_dict[param] = df_param["values"].values.tolist()
                 self.add_coverage(new_metadata, coords, range_dict)
 
-        return self.covjson
+        return json.loads(self.get_json())

eccovjson/encoder/VerticalProfile.py

@@ -1,3 +1,5 @@
+import json
+
 import pandas as pd
 
 from .encoder import Encoder
@@ -148,4 +150,4 @@ def from_polytope(self, result):
                     range_dict[param] = df_param["values"].values.tolist()
                 self.add_coverage(new_metadata, coords, range_dict)
 
-        return self.covjson
+        return json.loads(self.get_json())

eccovjson/encoder/Wkt.py

@@ -1,4 +1,7 @@
+import json
+
 import pandas as pd
+from covjson_pydantic.coverage import Coverage
 
 from .encoder import Encoder
 
@@ -17,7 +20,8 @@ def add_coverage(self, mars_metadata, coords, values):
         self.add_mars_metadata(new_coverage, mars_metadata)
         self.add_domain(new_coverage, coords)
         self.add_range(new_coverage, values)
-        self.covjson["coverages"].append(new_coverage)
+        cov = Coverage.model_validate_json(json.dumps(new_coverage))
+        self.pydantic_coverage.coverages.append(cov)
 
     def add_domain(self, coverage, coords):
         coverage["domain"]["type"] = "Domain"
@@ -26,7 +30,7 @@ def add_domain(self, coverage, coords):
         coverage["domain"]["axes"]["t"]["values"] = coords["t"]
         coverage["domain"]["axes"]["composite"] = {}
         coverage["domain"]["axes"]["composite"]["dataType"] = "tuple"
-        coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson["referencing"][0]["coordinates"]
+        coverage["domain"]["axes"]["composite"]["coordinates"] = self.pydantic_coverage.referencing[0].coordinates
         coverage["domain"]["axes"]["composite"]["values"] = coords["composite"]
 
     def add_range(self, coverage, values):
@@ -120,7 +124,7 @@ def from_polytope(self, result):
         range_dict = {}
         coords = {}
         coords["composite"] = []
-        coords["t"] = df["date"].unique()[0]
+        coords["t"] = [df["date"].unique()[0] + "Z"]
 
         for param in params:
             df_param = df[df["param"] == param]
@@ -131,4 +135,4 @@ def from_polytope(self, result):
             coords["composite"].append([row[1]["latitude"], row[1]["longitude"]])
 
         self.add_coverage(mars_metadata, coords, range_dict)
-        return self.covjson
+        return json.loads(self.get_json())

eccovjson/encoder/encoder.py

@@ -1,7 +1,12 @@
+import json
 from abc import ABC, abstractmethod
 
-from eccovjson.Coverage import Coverage
-from eccovjson.CoverageCollection import CoverageCollection
+from covjson_pydantic.coverage import CoverageCollection
+from covjson_pydantic.domain import DomainType
+from covjson_pydantic.parameter import Parameter
+from covjson_pydantic.reference_system import ReferenceSystemConnectionObject
+
+# from eccovjson.CoverageCollection import CoverageCollection
 from eccovjson.param_db import get_param_from_db, get_unit_from_db
 
 
@@ -10,36 +15,69 @@ def __init__(self, type, domaintype):
         self.covjson = {}
 
         self.type = type
+
+        self.referencing = []
+
+        domaintype = domaintype.lower()
+
+        if domaintype == "pointseries":
+            self.domaintype = DomainType.point_series
+        elif domaintype == "multipoint":
+            self.domaintype = DomainType.multi_point
+        elif domaintype == "wkt":
+            self.domaintype = DomainType.multi_point
+        elif domaintype == "boundingbox":
+            self.domaintype = DomainType.multi_point
+        elif domaintype == "shapefile":
+            self.domaintype = DomainType.multi_point
+        elif domaintype == "frame":
+            self.domaintype = DomainType.multi_point
+        elif domaintype == "path":
+            self.domaintype = DomainType.trajectory
+
+        self.pydantic_coverage = CoverageCollection(
+            type=type, coverages=[], domainType=self.domaintype, parameters={}, referencing=[]
+        )
+        # self.covjson = self.pydantic_coverage.model_dump_json(exclude_none=True)
         self.parameters = []
-        self.covjson["type"] = self.type
-        self.covjson["domainType"] = domaintype
-        self.covjson["coverages"] = []
-        self.covjson["parameters"] = {}
-        self.covjson["referencing"] = []
-
-        if type == "Coverage":
-            self.coverage = Coverage(self.covjson)
-        elif type == "CoverageCollection":
-            self.coverage = CoverageCollection(self.covjson)
-        else:
-            raise TypeError("Type must be Coverage or CoverageCollection")
+        # self.covjson["type"] = self.type
+        # self.covjson["domainType"] = domaintype
+        # self.covjson["coverages"] = []
+        # self.covjson["parameters"] = {}
+        # self.covjson["referencing"] = []
+
+        # if type == "Coverage":
+        #    self.coverage = Coverage(self.covjson)
+        # elif type == "CoverageCollection":
+        #    self.coverage = CoverageCollection(self.covjson)
+        # else:
+        #    raise TypeError("Type must be Coverage or CoverageCollection")
 
     def add_parameter(self, param):
         param_dict = get_param_from_db(param)
         unit = get_unit_from_db(param_dict["unit_id"])
-        self.covjson["parameters"][param_dict["shortname"]] = {
+        parameter = {
             "type": "Parameter",
-            "description": param_dict["description"],
+            "description": {"en": param_dict["description"]},
             "unit": {"symbol": unit["name"]},
             "observedProperty": {
                 "id": param_dict["shortname"],
                 "label": {"en": param_dict["name"]},
             },
         }
+        self.pydantic_coverage.parameters[param_dict["shortname"]] = Parameter.model_validate_json(
+            json.dumps(parameter)
+        )
         self.parameters.append(param)
 
     def add_reference(self, reference):
-        self.covjson["referencing"].append(reference)
+        self.pydantic_coverage.referencing.append(
+            ReferenceSystemConnectionObject.model_validate_json(json.dumps(reference))
+        )
+        # self.pydantic_coverage.referencing.append(reference)
+        for ref in reference["coordinates"]:
+            if ref not in self.referencing:
+                self.referencing.append(ref)
 
     def convert_param_id_to_param(self, paramid):
         try:
@@ -49,6 +87,10 @@ def convert_param_id_to_param(self, paramid):
         param_dict = get_param_from_db(int(param))
         return param_dict["shortname"]
 
+    def get_json(self):
+        self.covjson = self.pydantic_coverage.model_dump_json(exclude_none=True, indent=4)
+        return self.covjson
+
     @abstractmethod
     def add_coverage(self, mars_metadata, coords, values):
         pass

requirements.txt

@@ -3,3 +3,4 @@ datetime
 polytope
 xarray==2022.12.0
 pandas==1.5.2
+covjson-pydantic