From 6713199fe5d3ed3dfe3e18ebd5b04ee39cd37fd3 Mon Sep 17 00:00:00 2001 From: Adam Warde Date: Thu, 23 Nov 2023 14:49:04 +0000 Subject: [PATCH 01/58] Inital commit with standard python setup files and directory structure --- .gitignore | 12 +++ LICENSE | 201 +++++++++++++++++++++++++++++++++++++++++ Makefile | 23 +++++ covjson/__init__.py | 0 covjson/version.py | 1 + readme.md | 18 ---- requirements.txt | 2 + setup.py | 23 +++++ tests/requirements.txt | 3 + 9 files changed, 265 insertions(+), 18 deletions(-) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 covjson/__init__.py create mode 100644 covjson/version.py delete mode 100644 readme.md create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 tests/requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..19fea77 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +*.py[cod] +*.swp +**/__pycache__ +.vscode +polytope_mars.egg-info +.pytest_cache +*.prof +*.idx +.venv +PKG-info +SOURCES.txt +top_level.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..49c6fd8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 European Centre for Medium-Range Weather Forecasts + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..786ad99 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +init: + pip install -r requirements.txt + +testk: + python3 -m pytest -vsrA tests/* -k $(filter-out $@, $(MAKECMDGOALS)) -W ignore::DeprecationWarning -W ignore::FutureWarning --log-cli-level=DEBUG + +testx: + python3 -m pytest -vsx tests/* -W ignore::DeprecationWarning -W ignore::FutureWarning --log-cli-level=DEBUG + +test: + python3 -m pytest -vsrA tests/* -W ignore::DeprecationWarning -W ignore::FutureWarning --log-cli-level=DEBUG + +examples: + python3 -m pytest -vsrA examples/* -W ignore::DeprecationWarning -W ignore::FutureWarning --log-cli-level=DEBUG + +performance: + python3 -m pytest -vsrA performance/* -W ignore::DeprecationWarning -W ignore::FutureWarning --log-cli-level=DEBUG + +docs: + mkdocs build + mkdocs serve + +.PHONY: init test \ No newline at end of file diff --git a/covjson/__init__.py b/covjson/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/covjson/version.py b/covjson/version.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/covjson/version.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/readme.md b/readme.md deleted file mode 100644 index e2d2aeb..0000000 --- a/readme.md +++ /dev/null @@ -1,18 +0,0 @@ -# eccovjson - -* Encodes and decodes CoverageJSON objects - -* CoverageCollection(python dictionary) - * get_coverages() - -* Coverage() - * get_axes() - * get_metadata() - * get_ranges() - * get_parameters() - - -* TimeSeriesCoverage() - * set_axes() - * set_ranges() - * set_parameters() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c4dcbd9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +polytope +json diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..88c823e --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +import io +import re + +from setuptools import find_packages, setup + +__version__ = re.search( + r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', + io.open("covjson/version.py", encoding="utf_8_sig").read(), +).group(1) + + +setup( + name="eccovjson", + version=__version__, + description="ECMWF library for encoding and decoding coerageJSON files/objects of meteorlogical features such as vertical profiles and time series.", + long_description="", + url="https://github.com/ecmwf/eccovjson", + author="ECMWF", + author_email="James.Hawkes@ecmwf.int, Adam.Warde@ecmwf.int, Mathilde.Leuridan@ecmwf.int", + packages=find_packages(), + zip_safe=False, + include_package_data=True, +) diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..36491a6 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,3 @@ +-r ../requirements.txt +-e .. +pytest From 59b8fe5ebf11da1d9e8fffb38a00974354adcaa2 Mon Sep 17 00:00:00 2001 From: Adam Warde Date: Thu, 23 Nov 2023 14:51:10 +0000 Subject: [PATCH 02/58] Add readme --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..9b3a872 --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# eccovjson + +ECMWF library for encoding and decoding coerageJSON files/objects of meteorlogical features such as vertical profiles and time series. + +* Encodes and decodes CoverageJSON objects + +* CoverageCollection(python dictionary) + * get_coverages() + +* Coverage() + * get_axes() + * get_metadata() + * get_ranges() + * get_parameters() + + +* TimeSeriesCoverage() + * set_axes() + * set_ranges() + * set_parameters() From 08da37c2d0b1376b6ad9d5c704bf3e3c10a06efe Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 23 Nov 2023 14:56:45 +0000 Subject: [PATCH 03/58] Add inital decoder files for populating --- covjson/decoder.py | 0 tests/test_decoder.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 covjson/decoder.py create mode 100644 tests/test_decoder.py diff --git a/covjson/decoder.py b/covjson/decoder.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_decoder.py b/tests/test_decoder.py new file mode 100644 index 0000000..e69de29 From 797e852681247542c2f4e0f55a12770e08ce785e Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 23 Nov 2023 15:43:32 +0000 Subject: [PATCH 04/58] Added decoder, Coverage and CoverageCollection class as well as inital test --- covjson/Coverage.py | 15 ++++++ covjson/CoverageCollection.py | 17 ++++++ covjson/decoder.py | 0 covjson/decoder/decoder.py | 30 +++++++++++ tests/test_decoder.py | 98 +++++++++++++++++++++++++++++++++++ 5 files changed, 160 insertions(+) create mode 100644 covjson/Coverage.py create mode 100644 covjson/CoverageCollection.py delete mode 100644 covjson/decoder.py create mode 100644 covjson/decoder/decoder.py diff --git a/covjson/Coverage.py b/covjson/Coverage.py new file mode 100644 index 0000000..e57b2bb --- /dev/null +++ b/covjson/Coverage.py @@ -0,0 +1,15 @@ +import json + + +class Coverage: + def __init__(self, covjson): + if isinstance(covjson, dict): + print("Received Coverage") + self.coverage = covjson + + self.type = self.coverage.pop("type") + + if self.type == "Coverage": + print("Correct Type") + elif self.type == "CoverageCollection": + raise TypeError("Coverage class takes coverage not CoverageCollection") diff --git a/covjson/CoverageCollection.py b/covjson/CoverageCollection.py new file mode 100644 index 0000000..048cb8c --- /dev/null +++ b/covjson/CoverageCollection.py @@ -0,0 +1,17 @@ +import json + + +class CoverageCollection: + def __init__(self, covjson): + if isinstance(covjson, dict): + print("Received Coverage") + self.coverage = covjson + + self.type = self.coverage.pop("type") + + if self.type == "CoverageCollection": + print("Correct Type") + elif self.type == "Coverage": + raise TypeError( + "CoverageCollection class takes CoverageCollection not Coverage" + ) diff --git a/covjson/decoder.py b/covjson/decoder.py deleted file mode 100644 index e69de29..0000000 diff --git a/covjson/decoder/decoder.py b/covjson/decoder/decoder.py new file mode 100644 index 0000000..2b69e67 --- /dev/null +++ b/covjson/decoder/decoder.py @@ -0,0 +1,30 @@ +import os +import json +from ..Coverage import Coverage +from ..CoverageCollection import CoverageCollection + + +class Decoder: + def __init__(self, covjson): + # if python dictionary no need for loading, otherwise load json file + if isinstance(covjson, dict): + self.covjson = covjson + print("Dictionary") + elif isinstance(covjson, str): + with open(covjson) as json_file: + self.covjson = json.load(json_file) + print("Not dictionary") + else: + raise TypeError("Covjson must be dictionary or covjson file") + + self.type = self.covjson["type"] + + if self.type == "Coverage": + self.coverage = Coverage(self.covjson) + elif self.type == "CoverageCollection": + self.coverage = CoverageCollection(self.covjson) + else: + raise TypeError("Type must be Coverage or CoverageCollection") + + def get_type(self): + return self.type diff --git a/tests/test_decoder.py b/tests/test_decoder.py index e69de29..3353754 100644 --- a/tests/test_decoder.py +++ b/tests/test_decoder.py @@ -0,0 +1,98 @@ +import pytest +import json + +from covjson.decoder import decoder + + +class TestDecoder: + def setup_method(self, method): + self.covjson = { + "type": "CoverageCollection", + "domainType": "VerticalProfile", + "coverages": [ + { + "mars:metadata": { + "class": "ea", + "date": "2017-01-01 12:00:00", + "levtype": "pl", + "step": "0", + "stream": "enda", + }, + "domain": { + "type": "Domain", + "domainType": "VerticalProfile", + "axes": { + "x": {"values": ["0.0"]}, + "y": {"values": ["0.0"]}, + "z": {"values": ["500", "850"]}, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "axisNames": ["z"], + "shape": [2], + "values": [57517.77734375, 14814.95703125], + } + }, + }, + { + "mars:metadata": { + "class": "ea", + "date": "2017-01-02 12:00:00", + "levtype": "pl", + "step": "0", + "stream": "enda", + }, + "domain": { + "type": "Domain", + "domainType": "VerticalProfile", + "axes": { + "x": {"values": ["0.0"]}, + "y": {"values": ["0.0"]}, + "z": {"values": ["500", "850"]}, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "axisNames": ["z"], + "shape": [2], + "values": [57452.35546875, 14822.98046875], + } + }, + }, + ], + "referencing": [ + { + "coordinates": ["x", "y"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + }, + { + "coordinates": ["z"], + "system": { + "type": "VerticalCRS", + "cs": { + "csAxes": [{"name": {"en": "level"}, "direction": "down"}] + }, + }, + }, + ], + "parameters": { + "t": { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + } + }, + } + + def test_coveragecollection(self): + Decoder = decoder.Decoder(self.covjson) + assert Decoder.get_type() == "CoverageCollection" From a1061c1f736e9f7b8a8f46db7a903d8f866867af Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 28 Nov 2023 14:10:40 +0100 Subject: [PATCH 05/58] Made decoder abstract class with sub class TimeSeries and Vertical Profile, also implemented common functions between the two classes --- covjson/CoverageCollection.py | 2 ++ covjson/decoder/TimeSeries.py | 12 ++++++++ covjson/decoder/VerticalProfile.py | 12 ++++++++ covjson/decoder/decoder.py | 34 ++++++++++++++++++++-- tests/test_decoder.py | 45 ++++++++++++++++++++++++++---- 5 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 covjson/decoder/TimeSeries.py create mode 100644 covjson/decoder/VerticalProfile.py diff --git a/covjson/CoverageCollection.py b/covjson/CoverageCollection.py index 048cb8c..72d2cb4 100644 --- a/covjson/CoverageCollection.py +++ b/covjson/CoverageCollection.py @@ -15,3 +15,5 @@ def __init__(self, covjson): raise TypeError( "CoverageCollection class takes CoverageCollection not Coverage" ) + + self.coverages = self.coverage.pop("coverages") diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py new file mode 100644 index 0000000..e199d34 --- /dev/null +++ b/covjson/decoder/TimeSeries.py @@ -0,0 +1,12 @@ +from .Decoder import Decoder + + +class TimeSeries(Decoder): + def __init__(self, covjson): + super().__init__(covjson) + + def get_domain(self): + pass + + def get_ranges(self): + pass diff --git a/covjson/decoder/VerticalProfile.py b/covjson/decoder/VerticalProfile.py new file mode 100644 index 0000000..8df4917 --- /dev/null +++ b/covjson/decoder/VerticalProfile.py @@ -0,0 +1,12 @@ +from .Decoder import Decoder + + +class VerticalProfile(Decoder): + def __init__(self, covjson): + super().__init__(covjson) + + def get_domain(self): + pass + + def get_ranges(self): + pass diff --git a/covjson/decoder/decoder.py b/covjson/decoder/decoder.py index 2b69e67..8b8782f 100644 --- a/covjson/decoder/decoder.py +++ b/covjson/decoder/decoder.py @@ -1,10 +1,11 @@ import os import json +from abc import ABC, abstractmethod from ..Coverage import Coverage from ..CoverageCollection import CoverageCollection -class Decoder: +class Decoder(ABC): def __init__(self, covjson): # if python dictionary no need for loading, otherwise load json file if isinstance(covjson, dict): @@ -17,7 +18,7 @@ def __init__(self, covjson): else: raise TypeError("Covjson must be dictionary or covjson file") - self.type = self.covjson["type"] + self.type = self.get_type() if self.type == "Coverage": self.coverage = Coverage(self.covjson) @@ -26,5 +27,32 @@ def __init__(self, covjson): else: raise TypeError("Type must be Coverage or CoverageCollection") + self.parameters = self.get_parameters() + self.coordinates = self.get_referencing() + self.mars_metadata = self.get_mars_metadata() + def get_type(self): - return self.type + return self.covjson["type"] + + def get_parameters(self): + return list(self.covjson["parameters"].keys()) + + def get_referencing(self): + coordinates = [] + for coords in self.covjson["referencing"]: + coordinates.append(coords["coordinates"]) + return [coord for sublist in coordinates for coord in sublist] + + @abstractmethod + def get_ranges(self): + pass + + @abstractmethod + def get_domain(self): + pass + + def get_mars_metadata(self): + mars_metadata = [] + for coverage in self.coverage.coverages: + mars_metadata.append(coverage["mars:metadata"]) + return mars_metadata diff --git a/tests/test_decoder.py b/tests/test_decoder.py index 3353754..98f84a0 100644 --- a/tests/test_decoder.py +++ b/tests/test_decoder.py @@ -1,7 +1,9 @@ import pytest import json -from covjson.decoder import decoder +from covjson.decoder import Decoder +from covjson.decoder import VerticalProfile +from covjson.decoder import TimeSeries class TestDecoder: @@ -89,10 +91,43 @@ def setup_method(self, method): "description": "Temperature", "unit": {"symbol": "K"}, "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, - } + }, + "p": { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, }, } - def test_coveragecollection(self): - Decoder = decoder.Decoder(self.covjson) - assert Decoder.get_type() == "CoverageCollection" + def test_coveragecollection_type(self): + decoder = VerticalProfile.VerticalProfile(self.covjson) + assert decoder.type == "CoverageCollection" + + def test_coveragecollection_parameters(self): + decoder = VerticalProfile.VerticalProfile(self.covjson) + assert decoder.parameters == ["t", "p"] + + def test_coveragecollection_referencing(self): + decoder = VerticalProfile.VerticalProfile(self.covjson) + assert decoder.get_referencing() == ["x", "y", "z"] + + def test_coveragecollection_mars_metadata(self): + decoder = VerticalProfile.VerticalProfile(self.covjson) + metadata1 = { + "class": "ea", + "date": "2017-01-01 12:00:00", + "levtype": "pl", + "step": "0", + "stream": "enda", + } + assert decoder.mars_metadata[0] == metadata1 + metadata2 = { + "class": "ea", + "date": "2017-01-02 12:00:00", + "levtype": "pl", + "step": "0", + "stream": "enda", + } + assert decoder.mars_metadata[1] == metadata2 From e5beaefeb5997defcfe3116cb29cfc61092da156 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 28 Nov 2023 16:43:18 +0100 Subject: [PATCH 06/58] Added functions for extraction of values from covjson --- covjson/decoder/TimeSeries.py | 13 ++++- covjson/decoder/VerticalProfile.py | 22 ++++++-- covjson/decoder/decoder.py | 18 ++++--- tests/test_decoder.py | 86 +++++++++++++++++++++++++++++- 4 files changed, 124 insertions(+), 15 deletions(-) diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py index e199d34..21532b6 100644 --- a/covjson/decoder/TimeSeries.py +++ b/covjson/decoder/TimeSeries.py @@ -5,8 +5,17 @@ class TimeSeries(Decoder): def __init__(self, covjson): super().__init__(covjson) - def get_domain(self): - pass + def get_domains(self): + domains = [] + for coverage in self.coverage.coverages: + domains.append(coverage["domain"]) + return domains def get_ranges(self): + ranges = [] + for coverage in self.coverage.coverages: + ranges.append(coverage["ranges"]) + return ranges + + def get_values(self): pass diff --git a/covjson/decoder/VerticalProfile.py b/covjson/decoder/VerticalProfile.py index 8df4917..6cc9293 100644 --- a/covjson/decoder/VerticalProfile.py +++ b/covjson/decoder/VerticalProfile.py @@ -4,9 +4,25 @@ class VerticalProfile(Decoder): def __init__(self, covjson): super().__init__(covjson) + self.domains = self.get_domains() + self.ranges = self.get_ranges() - def get_domain(self): - pass + def get_domains(self): + domains = [] + for coverage in self.coverage.coverages: + domains.append(coverage["domain"]) + return domains def get_ranges(self): - pass + ranges = [] + for coverage in self.coverage.coverages: + ranges.append(coverage["ranges"]) + return ranges + + def get_values(self): + values = {} + for parameter in self.parameters: + values[parameter] = [] + for range in self.ranges: + values[parameter].append(range[parameter]["values"]) + return values diff --git a/covjson/decoder/decoder.py b/covjson/decoder/decoder.py index 8b8782f..02c997e 100644 --- a/covjson/decoder/decoder.py +++ b/covjson/decoder/decoder.py @@ -10,11 +10,9 @@ def __init__(self, covjson): # if python dictionary no need for loading, otherwise load json file if isinstance(covjson, dict): self.covjson = covjson - print("Dictionary") elif isinstance(covjson, str): with open(covjson) as json_file: self.covjson = json.load(json_file) - print("Not dictionary") else: raise TypeError("Covjson must be dictionary or covjson file") @@ -43,16 +41,20 @@ def get_referencing(self): coordinates.append(coords["coordinates"]) return [coord for sublist in coordinates for coord in sublist] + def get_mars_metadata(self): + mars_metadata = [] + for coverage in self.coverage.coverages: + mars_metadata.append(coverage["mars:metadata"]) + return mars_metadata + @abstractmethod def get_ranges(self): pass @abstractmethod - def get_domain(self): + def get_domains(self): pass - def get_mars_metadata(self): - mars_metadata = [] - for coverage in self.coverage.coverages: - mars_metadata.append(coverage["mars:metadata"]) - return mars_metadata + @abstractmethod + def get_values(self): + pass diff --git a/tests/test_decoder.py b/tests/test_decoder.py index 98f84a0..7ab892c 100644 --- a/tests/test_decoder.py +++ b/tests/test_decoder.py @@ -36,7 +36,14 @@ def setup_method(self, method): "axisNames": ["z"], "shape": [2], "values": [57517.77734375, 14814.95703125], - } + }, + "p": { + "type": "NdArray", + "dataType": "float", + "axisNames": ["z"], + "shape": [2], + "values": [16452.35546875, 44122.98046875], + }, }, }, { @@ -63,7 +70,14 @@ def setup_method(self, method): "axisNames": ["z"], "shape": [2], "values": [57452.35546875, 14822.98046875], - } + }, + "p": { + "type": "NdArray", + "dataType": "float", + "axisNames": ["z"], + "shape": [2], + "values": [56452.35546875, 14122.98046875], + }, }, }, ], @@ -131,3 +145,71 @@ def test_coveragecollection_mars_metadata(self): "stream": "enda", } assert decoder.mars_metadata[1] == metadata2 + + def test_coveragecollection_domain(self): + decoder = VerticalProfile.VerticalProfile(self.covjson) + domain1 = { + "type": "Domain", + "domainType": "VerticalProfile", + "axes": { + "x": {"values": ["0.0"]}, + "y": {"values": ["0.0"]}, + "z": {"values": ["500", "850"]}, + }, + } + assert decoder.domains[0] == domain1 + domain2 = { + "type": "Domain", + "domainType": "VerticalProfile", + "axes": { + "x": {"values": ["0.0"]}, + "y": {"values": ["0.0"]}, + "z": {"values": ["500", "850"]}, + }, + } + assert decoder.domains[1] == domain2 + + def test_coveragecollection_range(self): + decoder = VerticalProfile.VerticalProfile(self.covjson) + range1 = { + "t": { + "type": "NdArray", + "dataType": "float", + "axisNames": ["z"], + "shape": [2], + "values": [57517.77734375, 14814.95703125], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "axisNames": ["z"], + "shape": [2], + "values": [16452.35546875, 44122.98046875], + }, + } + assert decoder.ranges[0] == range1 + range2 = { + "t": { + "type": "NdArray", + "dataType": "float", + "axisNames": ["z"], + "shape": [2], + "values": [57452.35546875, 14822.98046875], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "axisNames": ["z"], + "shape": [2], + "values": [56452.35546875, 14122.98046875], + }, + } + assert decoder.ranges[1] == range2 + + def test_coveragecollection_values(self): + decoder = VerticalProfile.VerticalProfile(self.covjson) + values = { + "t": [[57517.77734375, 14814.95703125], [57452.35546875, 14822.98046875]], + "p": [[16452.35546875, 44122.98046875], [56452.35546875, 14122.98046875]], + } + assert decoder.get_values() == values From 6fe6f21ca19c569ac53a62a33f92d3dc725824ee Mon Sep 17 00:00:00 2001 From: awarde96 Date: Wed, 29 Nov 2023 10:14:56 +0100 Subject: [PATCH 07/58] Added functionaloty to vertical profile to get values and coordinates from covjson --- covjson/decoder/TimeSeries.py | 3 +++ covjson/decoder/VerticalProfile.py | 16 ++++++++++++++++ covjson/decoder/decoder.py | 9 +++++++++ tests/test_decoder.py | 14 ++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py index 21532b6..95ed0c3 100644 --- a/covjson/decoder/TimeSeries.py +++ b/covjson/decoder/TimeSeries.py @@ -19,3 +19,6 @@ def get_ranges(self): def get_values(self): pass + + def get_coordinates(self): + pass diff --git a/covjson/decoder/VerticalProfile.py b/covjson/decoder/VerticalProfile.py index 6cc9293..9b658ab 100644 --- a/covjson/decoder/VerticalProfile.py +++ b/covjson/decoder/VerticalProfile.py @@ -19,6 +19,19 @@ def get_ranges(self): ranges.append(coverage["ranges"]) return ranges + def get_coordinates(self): + coordinates = [] + # Get x,y,z coords and unpack z coords and match to x,y coords + for domain in self.domains: + x = domain["axes"]["x"]["values"][0] + y = domain["axes"]["y"]["values"][0] + zs = domain["axes"]["z"]["values"] + for z in zs: + # Have to replicate these coords for each parameter + for _ in self.parameters: + coordinates.append([x, y, z]) + return coordinates + def get_values(self): values = {} for parameter in self.parameters: @@ -26,3 +39,6 @@ def get_values(self): for range in self.ranges: values[parameter].append(range[parameter]["values"]) return values + + def to_geopandas(self): + pass diff --git a/covjson/decoder/decoder.py b/covjson/decoder/decoder.py index 02c997e..451fdaf 100644 --- a/covjson/decoder/decoder.py +++ b/covjson/decoder/decoder.py @@ -1,5 +1,6 @@ import os import json +import geopandas as gpd from abc import ABC, abstractmethod from ..Coverage import Coverage from ..CoverageCollection import CoverageCollection @@ -55,6 +56,14 @@ def get_ranges(self): def get_domains(self): pass + @abstractmethod + def get_coordinates(self): + pass + @abstractmethod def get_values(self): pass + + @abstractmethod + def to_geopandas(self): + pass diff --git a/tests/test_decoder.py b/tests/test_decoder.py index 7ab892c..ac38693 100644 --- a/tests/test_decoder.py +++ b/tests/test_decoder.py @@ -206,6 +206,20 @@ def test_coveragecollection_range(self): } assert decoder.ranges[1] == range2 + def test_coveragecollection_coordinates(self): + decoder = VerticalProfile.VerticalProfile(self.covjson) + coordinates = [ + ["0.0", "0.0", "500"], + ["0.0", "0.0", "500"], + ["0.0", "0.0", "850"], + ["0.0", "0.0", "850"], + ["0.0", "0.0", "500"], + ["0.0", "0.0", "500"], + ["0.0", "0.0", "850"], + ["0.0", "0.0", "850"], + ] + assert decoder.get_coordinates() == coordinates + def test_coveragecollection_values(self): decoder = VerticalProfile.VerticalProfile(self.covjson) values = { From 146305e9bc86fa6efc9e8afbe81630aa7acab0c3 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Wed, 29 Nov 2023 16:10:36 +0100 Subject: [PATCH 08/58] Added test file for timeseries decoder, implemented values and coordinate functions for decoder --- covjson/decoder/TimeSeries.py | 26 +- covjson/decoder/VerticalProfile.py | 6 +- covjson/decoder/decoder.py | 4 + tests/test_decoder_time_series.py | 271 ++++++++++++++++++ ...er.py => test_decoder_vertical_profile.py} | 44 +-- 5 files changed, 331 insertions(+), 20 deletions(-) create mode 100644 tests/test_decoder_time_series.py rename tests/{test_decoder.py => test_decoder_vertical_profile.py} (85%) diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py index 95ed0c3..b235729 100644 --- a/covjson/decoder/TimeSeries.py +++ b/covjson/decoder/TimeSeries.py @@ -4,6 +4,8 @@ class TimeSeries(Decoder): def __init__(self, covjson): super().__init__(covjson) + self.domains = self.get_domains() + self.ranges = self.get_ranges() def get_domains(self): domains = [] @@ -18,7 +20,29 @@ def get_ranges(self): return ranges def get_values(self): - pass + values = {} + for parameter in self.parameters: + values[parameter] = [] + for range in self.ranges: + values[parameter].append(range[parameter]["values"]) + return values def get_coordinates(self): + coordinates = [] + # Get x,y,z,t coords and unpack t coords and match to x,y,z coords + for domain in self.domains: + x = domain["axes"]["x"]["values"][0] + y = domain["axes"]["y"]["values"][0] + z = domain["axes"]["z"]["values"][0] + ts = domain["axes"]["t"]["values"] + for t in ts: + # Have to replicate these coords for each parameter + for _ in self.parameters: + coordinates.append([x, y, z, t]) + return coordinates + + def to_geopandas(self): + pass + + def to_xarray(self): pass diff --git a/covjson/decoder/VerticalProfile.py b/covjson/decoder/VerticalProfile.py index 9b658ab..8d71995 100644 --- a/covjson/decoder/VerticalProfile.py +++ b/covjson/decoder/VerticalProfile.py @@ -25,11 +25,12 @@ def get_coordinates(self): for domain in self.domains: x = domain["axes"]["x"]["values"][0] y = domain["axes"]["y"]["values"][0] + t = domain["axes"]["t"]["values"][0] zs = domain["axes"]["z"]["values"] for z in zs: # Have to replicate these coords for each parameter for _ in self.parameters: - coordinates.append([x, y, z]) + coordinates.append([x, y, z, t]) return coordinates def get_values(self): @@ -42,3 +43,6 @@ def get_values(self): def to_geopandas(self): pass + + def to_xarray(self): + pass diff --git a/covjson/decoder/decoder.py b/covjson/decoder/decoder.py index 451fdaf..64fc611 100644 --- a/covjson/decoder/decoder.py +++ b/covjson/decoder/decoder.py @@ -67,3 +67,7 @@ def get_values(self): @abstractmethod def to_geopandas(self): pass + + @abstractmethod + def to_xarray(self): + pass diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py new file mode 100644 index 0000000..2fb444e --- /dev/null +++ b/tests/test_decoder_time_series.py @@ -0,0 +1,271 @@ +import pytest +import json + +from covjson.decoder import Decoder +from covjson.decoder import VerticalProfile +from covjson.decoder import TimeSeries + + +class TestDecoder: + def setup_method(self, method): + self.covjson = { + "type": "CoverageCollection", + "domainType": "PointSeries", + "coverages": [ + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "x": {"values": [3]}, + "y": {"values": [7]}, + "z": {"values": [1]}, + "t": { + "values": [ + "2017-01-01 00:00:00", + "2017-01-01 06:00:00", + "2017-01-01 12:00:00", + ] + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [ + 264.93115234375, + 263.83115234375, + 265.12313132266, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [ + 9.93115234375, + 7.83115234375, + 14.12313132266, + ], + }, + }, + }, + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170102", + "step": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "x": {"values": [3]}, + "y": {"values": [7]}, + "z": {"values": [1]}, + "t": { + "values": [ + "2017-01-02 00:00:00", + "2017-01-02 06:00:00", + "2017-01-02 12:00:00", + ] + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [ + 263.83115234375, + 265.12313132266, + 264.93115234375, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [ + 13.83115234375, + 14.12313132266, + 7.93115234375, + ], + }, + }, + }, + ], + "referencing": [ + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ], + "parameters": { + "t": { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + "p": { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + }, + } + + def test_timeseries_type(self): + decoder = TimeSeries.TimeSeries(self.covjson) + assert decoder.type == "CoverageCollection" + + def test_timeseries_parameters(self): + decoder = TimeSeries.TimeSeries(self.covjson) + assert decoder.parameters == ["t", "p"] + + def test_timeseries_referencing(self): + decoder = TimeSeries.TimeSeries(self.covjson) + assert decoder.get_referencing() == ["x", "y", "z"] + + def test_timeseries_mars_metadata(self): + decoder = TimeSeries.TimeSeries(self.covjson) + metadata1 = { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + } + metadata2 = { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170102", + "step": "0", + } + assert decoder.mars_metadata == [metadata1, metadata2] + + def test_timeseries_domains(self): + decoder = TimeSeries.TimeSeries(self.covjson) + domain1 = { + "type": "Domain", + "axes": { + "x": {"values": [3]}, + "y": {"values": [7]}, + "z": {"values": [1]}, + "t": { + "values": [ + "2017-01-01 00:00:00", + "2017-01-01 06:00:00", + "2017-01-01 12:00:00", + ] + }, + }, + } + assert decoder.domains[0] == domain1 + domain2 = { + "type": "Domain", + "axes": { + "x": {"values": [3]}, + "y": {"values": [7]}, + "z": {"values": [1]}, + "t": { + "values": [ + "2017-01-02 00:00:00", + "2017-01-02 06:00:00", + "2017-01-02 12:00:00", + ] + }, + }, + } + assert decoder.domains[1] == domain2 + + def test_timeseries_ranges(self): + decoder = TimeSeries.TimeSeries(self.covjson) + range1 = { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [264.93115234375, 263.83115234375, 265.12313132266], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [9.93115234375, 7.83115234375, 14.12313132266], + }, + } + assert decoder.ranges[0] == range1 + range2 = { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [263.83115234375, 265.12313132266, 264.93115234375], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [13.83115234375, 14.12313132266, 7.93115234375], + }, + } + assert decoder.ranges[1] == range2 + + def test_timeseries_values(self): + decoder = TimeSeries.TimeSeries(self.covjson) + values = { + "t": [ + [264.93115234375, 263.83115234375, 265.12313132266], + [263.83115234375, 265.12313132266, 264.93115234375], + ], + "p": [ + [9.93115234375, 7.83115234375, 14.12313132266], + [13.83115234375, 14.12313132266, 7.93115234375], + ], + } + assert decoder.get_values() == values + + def test_timeseries_coordinates(self): + decoder = TimeSeries.TimeSeries(self.covjson) + coordinates = [ + [3, 7, 1, "2017-01-01 00:00:00"], + [3, 7, 1, "2017-01-01 00:00:00"], + [3, 7, 1, "2017-01-01 06:00:00"], + [3, 7, 1, "2017-01-01 06:00:00"], + [3, 7, 1, "2017-01-01 12:00:00"], + [3, 7, 1, "2017-01-01 12:00:00"], + [3, 7, 1, "2017-01-02 00:00:00"], + [3, 7, 1, "2017-01-02 00:00:00"], + [3, 7, 1, "2017-01-02 06:00:00"], + [3, 7, 1, "2017-01-02 06:00:00"], + [3, 7, 1, "2017-01-02 12:00:00"], + [3, 7, 1, "2017-01-02 12:00:00"], + ] + assert decoder.get_coordinates() == coordinates diff --git a/tests/test_decoder.py b/tests/test_decoder_vertical_profile.py similarity index 85% rename from tests/test_decoder.py rename to tests/test_decoder_vertical_profile.py index ac38693..fdbfce2 100644 --- a/tests/test_decoder.py +++ b/tests/test_decoder_vertical_profile.py @@ -19,6 +19,7 @@ def setup_method(self, method): "levtype": "pl", "step": "0", "stream": "enda", + "number": "0", }, "domain": { "type": "Domain", @@ -27,6 +28,7 @@ def setup_method(self, method): "x": {"values": ["0.0"]}, "y": {"values": ["0.0"]}, "z": {"values": ["500", "850"]}, + "t": {"values": ["2017-01-01 12:00:00"]}, }, }, "ranges": { @@ -49,10 +51,11 @@ def setup_method(self, method): { "mars:metadata": { "class": "ea", - "date": "2017-01-02 12:00:00", + "date": "2017-01-01 12:00:00", "levtype": "pl", "step": "0", "stream": "enda", + "number": "1", }, "domain": { "type": "Domain", @@ -61,6 +64,7 @@ def setup_method(self, method): "x": {"values": ["0.0"]}, "y": {"values": ["0.0"]}, "z": {"values": ["500", "850"]}, + "t": {"values": ["2017-01-01 12:00:00"]}, }, }, "ranges": { @@ -115,19 +119,19 @@ def setup_method(self, method): }, } - def test_coveragecollection_type(self): + def test_verticalprofile_type(self): decoder = VerticalProfile.VerticalProfile(self.covjson) assert decoder.type == "CoverageCollection" - def test_coveragecollection_parameters(self): + def test_verticalprofile_parameters(self): decoder = VerticalProfile.VerticalProfile(self.covjson) assert decoder.parameters == ["t", "p"] - def test_coveragecollection_referencing(self): + def test_verticalprofile_referencing(self): decoder = VerticalProfile.VerticalProfile(self.covjson) assert decoder.get_referencing() == ["x", "y", "z"] - def test_coveragecollection_mars_metadata(self): + def test_verticalprofile_mars_metadata(self): decoder = VerticalProfile.VerticalProfile(self.covjson) metadata1 = { "class": "ea", @@ -135,18 +139,20 @@ def test_coveragecollection_mars_metadata(self): "levtype": "pl", "step": "0", "stream": "enda", + "number": "0", } assert decoder.mars_metadata[0] == metadata1 metadata2 = { "class": "ea", - "date": "2017-01-02 12:00:00", + "date": "2017-01-01 12:00:00", "levtype": "pl", "step": "0", "stream": "enda", + "number": "1", } assert decoder.mars_metadata[1] == metadata2 - def test_coveragecollection_domain(self): + def test_verticalprofile_domains(self): decoder = VerticalProfile.VerticalProfile(self.covjson) domain1 = { "type": "Domain", @@ -155,6 +161,7 @@ def test_coveragecollection_domain(self): "x": {"values": ["0.0"]}, "y": {"values": ["0.0"]}, "z": {"values": ["500", "850"]}, + "t": {"values": ["2017-01-01 12:00:00"]}, }, } assert decoder.domains[0] == domain1 @@ -165,11 +172,12 @@ def test_coveragecollection_domain(self): "x": {"values": ["0.0"]}, "y": {"values": ["0.0"]}, "z": {"values": ["500", "850"]}, + "t": {"values": ["2017-01-01 12:00:00"]}, }, } assert decoder.domains[1] == domain2 - def test_coveragecollection_range(self): + def test_verticalprofile_ranges(self): decoder = VerticalProfile.VerticalProfile(self.covjson) range1 = { "t": { @@ -206,21 +214,21 @@ def test_coveragecollection_range(self): } assert decoder.ranges[1] == range2 - def test_coveragecollection_coordinates(self): + def test_verticalprofile_coordinates(self): decoder = VerticalProfile.VerticalProfile(self.covjson) coordinates = [ - ["0.0", "0.0", "500"], - ["0.0", "0.0", "500"], - ["0.0", "0.0", "850"], - ["0.0", "0.0", "850"], - ["0.0", "0.0", "500"], - ["0.0", "0.0", "500"], - ["0.0", "0.0", "850"], - ["0.0", "0.0", "850"], + ["0.0", "0.0", "500", "2017-01-01 12:00:00"], + ["0.0", "0.0", "500", "2017-01-01 12:00:00"], + ["0.0", "0.0", "850", "2017-01-01 12:00:00"], + ["0.0", "0.0", "850", "2017-01-01 12:00:00"], + ["0.0", "0.0", "500", "2017-01-01 12:00:00"], + ["0.0", "0.0", "500", "2017-01-01 12:00:00"], + ["0.0", "0.0", "850", "2017-01-01 12:00:00"], + ["0.0", "0.0", "850", "2017-01-01 12:00:00"], ] assert decoder.get_coordinates() == coordinates - def test_coveragecollection_values(self): + def test_verticalprofile_values(self): decoder = VerticalProfile.VerticalProfile(self.covjson) values = { "t": [[57517.77734375, 14814.95703125], [57452.35546875, 14822.98046875]], From c9f6f02c2ebd1846a1bec17c2f78408944035b2c Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 30 Nov 2023 09:50:48 +0100 Subject: [PATCH 09/58] Fixed broken imports from merge --- covjson/decoder/TimeSeries.py | 2 +- covjson/decoder/VerticalProfile.py | 2 +- tests/test_decoder_time_series.py | 2 +- tests/test_decoder_vertical_profile.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py index b235729..1735351 100644 --- a/covjson/decoder/TimeSeries.py +++ b/covjson/decoder/TimeSeries.py @@ -1,4 +1,4 @@ -from .Decoder import Decoder +from .decoder import Decoder class TimeSeries(Decoder): diff --git a/covjson/decoder/VerticalProfile.py b/covjson/decoder/VerticalProfile.py index 8d71995..f552535 100644 --- a/covjson/decoder/VerticalProfile.py +++ b/covjson/decoder/VerticalProfile.py @@ -1,4 +1,4 @@ -from .Decoder import Decoder +from .decoder import Decoder class VerticalProfile(Decoder): diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index 2fb444e..abb0eb1 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -1,7 +1,7 @@ import pytest import json -from covjson.decoder import Decoder +from covjson.decoder import decoder from covjson.decoder import VerticalProfile from covjson.decoder import TimeSeries diff --git a/tests/test_decoder_vertical_profile.py b/tests/test_decoder_vertical_profile.py index fdbfce2..eccd5a7 100644 --- a/tests/test_decoder_vertical_profile.py +++ b/tests/test_decoder_vertical_profile.py @@ -1,7 +1,7 @@ import pytest import json -from covjson.decoder import Decoder +from covjson.decoder import decoder from covjson.decoder import VerticalProfile from covjson.decoder import TimeSeries From b8b6489d94c3c25ddac1c4ddedb113a36f501745 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Fri, 1 Dec 2023 11:02:43 +0100 Subject: [PATCH 10/58] Added to xarray functionality to decoder fot time series --- covjson/decoder/TimeSeries.py | 52 +++++++++++++++++++++++++---- covjson/decoder/decoder.py | 5 +++ tests/test_decoder_time_series.py | 55 +++++++++++++++++++++---------- 3 files changed, 88 insertions(+), 24 deletions(-) diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py index 1735351..dad3cda 100644 --- a/covjson/decoder/TimeSeries.py +++ b/covjson/decoder/TimeSeries.py @@ -1,4 +1,6 @@ from .decoder import Decoder +import xarray as xr +import datetime as dt class TimeSeries(Decoder): @@ -25,24 +27,62 @@ def get_values(self): values[parameter] = [] for range in self.ranges: values[parameter].append(range[parameter]["values"]) + values[parameter] = [ + value for sublist in values[parameter] for value in sublist + ] return values def get_coordinates(self): coordinates = [] + coord_dict = {} + for param in self.parameters: + coord_dict[param] = [] # Get x,y,z,t coords and unpack t coords and match to x,y,z coords for domain in self.domains: x = domain["axes"]["x"]["values"][0] y = domain["axes"]["y"]["values"][0] z = domain["axes"]["z"]["values"][0] ts = domain["axes"]["t"]["values"] - for t in ts: - # Have to replicate these coords for each parameter - for _ in self.parameters: - coordinates.append([x, y, z, t]) - return coordinates + for param in self.parameters: + for t in ts: + # Have to replicate these coords for each parameter + # coordinates.append([x, y, z, t]) + coord_dict[param].append([x, y, z, t]) + return coord_dict def to_geopandas(self): pass + # function to convert covjson to xarray dataset def to_xarray(self): - pass + dims = ["x", "y", "z", "t"] + dataarrays = [] + + # Get coordinates + for parameter in self.parameters: + param_values = [[[self.get_values()[parameter]]]] + + coords = self.get_coordinates()[parameter] + x = [coords[0][0]] + y = [coords[0][1]] + z = [coords[0][2]] + t = [ + dt.datetime.strptime(coord[3], "%Y-%m-%d %H:%M:%S") for coord in coords + ] + + param_coords = {"x": x, "y": y, "z": z, "t": t} + dataarray = xr.DataArray( + param_values, + dims=dims, + coords=param_coords, + name=parameter, + ) + dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"][ + "symbol" + ] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ + "description" + ] + dataarrays.append(dataarray) + return dataarrays diff --git a/covjson/decoder/decoder.py b/covjson/decoder/decoder.py index 64fc611..5ffe5a9 100644 --- a/covjson/decoder/decoder.py +++ b/covjson/decoder/decoder.py @@ -1,5 +1,7 @@ import os import json +import xarray as xr +import datetime as dt import geopandas as gpd from abc import ABC, abstractmethod from ..Coverage import Coverage @@ -36,6 +38,9 @@ def get_type(self): def get_parameters(self): return list(self.covjson["parameters"].keys()) + def get_parameter_metadata(self, parameter): + return self.covjson["parameters"][parameter] + def get_referencing(self): coordinates = [] for coords in self.covjson["referencing"]: diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index abb0eb1..6433a15 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -242,30 +242,49 @@ def test_timeseries_values(self): decoder = TimeSeries.TimeSeries(self.covjson) values = { "t": [ - [264.93115234375, 263.83115234375, 265.12313132266], - [263.83115234375, 265.12313132266, 264.93115234375], + 264.93115234375, + 263.83115234375, + 265.12313132266, + 263.83115234375, + 265.12313132266, + 264.93115234375, ], "p": [ - [9.93115234375, 7.83115234375, 14.12313132266], - [13.83115234375, 14.12313132266, 7.93115234375], + 9.93115234375, + 7.83115234375, + 14.12313132266, + 13.83115234375, + 14.12313132266, + 7.93115234375, ], } + print(decoder.get_values()) assert decoder.get_values() == values def test_timeseries_coordinates(self): decoder = TimeSeries.TimeSeries(self.covjson) - coordinates = [ - [3, 7, 1, "2017-01-01 00:00:00"], - [3, 7, 1, "2017-01-01 00:00:00"], - [3, 7, 1, "2017-01-01 06:00:00"], - [3, 7, 1, "2017-01-01 06:00:00"], - [3, 7, 1, "2017-01-01 12:00:00"], - [3, 7, 1, "2017-01-01 12:00:00"], - [3, 7, 1, "2017-01-02 00:00:00"], - [3, 7, 1, "2017-01-02 00:00:00"], - [3, 7, 1, "2017-01-02 06:00:00"], - [3, 7, 1, "2017-01-02 06:00:00"], - [3, 7, 1, "2017-01-02 12:00:00"], - [3, 7, 1, "2017-01-02 12:00:00"], - ] + coordinates = { + "t": [ + [3, 7, 1, "2017-01-01 00:00:00"], + [3, 7, 1, "2017-01-01 06:00:00"], + [3, 7, 1, "2017-01-01 12:00:00"], + [3, 7, 1, "2017-01-02 00:00:00"], + [3, 7, 1, "2017-01-02 06:00:00"], + [3, 7, 1, "2017-01-02 12:00:00"], + ], + "p": [ + [3, 7, 1, "2017-01-01 00:00:00"], + [3, 7, 1, "2017-01-01 06:00:00"], + [3, 7, 1, "2017-01-01 12:00:00"], + [3, 7, 1, "2017-01-02 00:00:00"], + [3, 7, 1, "2017-01-02 06:00:00"], + [3, 7, 1, "2017-01-02 12:00:00"], + ], + } + print(decoder.get_coordinates()) assert decoder.get_coordinates() == coordinates + + def test_timeseries_to_xarray(self): + decoder = TimeSeries.TimeSeries(self.covjson) + ds = decoder.to_xarray() + print(ds) From 87940e02f4ea24458f3be793ed27304da8a405fb Mon Sep 17 00:00:00 2001 From: awarde96 Date: Fri, 1 Dec 2023 15:07:23 +0100 Subject: [PATCH 11/58] Make to_xarray create xarray dataset rather tahn list --- covjson/test.ipynb | 0 tests/test_decoder_time_series.py | 9 +++++++++ 2 files changed, 9 insertions(+) create mode 100644 covjson/test.ipynb diff --git a/covjson/test.ipynb b/covjson/test.ipynb new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index 6433a15..5de1d8d 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -4,6 +4,9 @@ from covjson.decoder import decoder from covjson.decoder import VerticalProfile from covjson.decoder import TimeSeries +from earthkit import data +import earthkit.data.readers.netcdf +import xarray as xr class TestDecoder: @@ -287,4 +290,10 @@ def test_timeseries_coordinates(self): def test_timeseries_to_xarray(self): decoder = TimeSeries.TimeSeries(self.covjson) ds = decoder.to_xarray() + # print(type(ekds)) print(ds) + xrds = xr.Dataset(ds) + print(xrds) + ekds = data.from_object(ds) + print(type(ekds)) + print(ekds.ls()) From 48615476bdca81f798fd8a2877d67ff45fcfc376 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Mon, 4 Dec 2023 09:22:03 +0100 Subject: [PATCH 12/58] Added mars metadata to attributes of xarray dataset, can now also read from earthkit data --- covjson/decoder/TimeSeries.py | 12 +++++++++--- tests/test_decoder_time_series.py | 15 ++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py index dad3cda..2db3885 100644 --- a/covjson/decoder/TimeSeries.py +++ b/covjson/decoder/TimeSeries.py @@ -56,7 +56,7 @@ def to_geopandas(self): # function to convert covjson to xarray dataset def to_xarray(self): dims = ["x", "y", "z", "t"] - dataarrays = [] + dataarraydict = {} # Get coordinates for parameter in self.parameters: @@ -77,6 +77,7 @@ def to_xarray(self): coords=param_coords, name=parameter, ) + dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"][ "symbol" @@ -84,5 +85,10 @@ def to_xarray(self): dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ "description" ] - dataarrays.append(dataarray) - return dataarrays + dataarraydict[dataarray.attrs["long_name"]] = dataarray + + ds = xr.Dataset(dataarraydict) + for mars_metadata in self.mars_metadata[0]: + ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata] + + return ds diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index 5de1d8d..6f2ae3e 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -5,7 +5,6 @@ from covjson.decoder import VerticalProfile from covjson.decoder import TimeSeries from earthkit import data -import earthkit.data.readers.netcdf import xarray as xr @@ -22,6 +21,7 @@ def setup_method(self, method): "levtype": "pl", "date": "20170101", "step": "0", + "number": "0", }, "type": "Coverage", "domain": { @@ -71,6 +71,7 @@ def setup_method(self, method): "levtype": "pl", "date": "20170102", "step": "0", + "number": "0", }, "type": "Coverage", "domain": { @@ -159,6 +160,7 @@ def test_timeseries_mars_metadata(self): "levtype": "pl", "date": "20170101", "step": "0", + "number": "0", } metadata2 = { "class": "od", @@ -166,6 +168,7 @@ def test_timeseries_mars_metadata(self): "levtype": "pl", "date": "20170102", "step": "0", + "number": "0", } assert decoder.mars_metadata == [metadata1, metadata2] @@ -261,7 +264,6 @@ def test_timeseries_values(self): 7.93115234375, ], } - print(decoder.get_values()) assert decoder.get_values() == values def test_timeseries_coordinates(self): @@ -284,16 +286,15 @@ def test_timeseries_coordinates(self): [3, 7, 1, "2017-01-02 12:00:00"], ], } - print(decoder.get_coordinates()) assert decoder.get_coordinates() == coordinates def test_timeseries_to_xarray(self): decoder = TimeSeries.TimeSeries(self.covjson) ds = decoder.to_xarray() - # print(type(ekds)) print(ds) - xrds = xr.Dataset(ds) - print(xrds) + # xrds.to_netcdf("timeseries.nc") + # ds = xr.open_dataset("timeseries.nc") ekds = data.from_object(ds) + print(ekds) print(type(ekds)) - print(ekds.ls()) + # print(ekds.ls()) From 0ef9be462d1c62af9d8fdc9571c525215a3c5d9b Mon Sep 17 00:00:00 2001 From: awarde96 Date: Mon, 4 Dec 2023 10:57:13 +0100 Subject: [PATCH 13/58] Updated gitignore file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 19fea77..97884ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.py[cod] +*.nc *.swp **/__pycache__ .vscode From 42fcf22551b10be678b6fb72491a988a2d3082f9 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Mon, 4 Dec 2023 11:20:54 +0100 Subject: [PATCH 14/58] Add initial setup files for encoder class and tests --- covjson/CoverageCollection.py | 4 +- covjson/encoder/encoder.py | 22 ++++++ tests/test_encoder.py | 141 ++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 covjson/encoder/encoder.py create mode 100644 tests/test_encoder.py diff --git a/covjson/CoverageCollection.py b/covjson/CoverageCollection.py index 72d2cb4..5743591 100644 --- a/covjson/CoverageCollection.py +++ b/covjson/CoverageCollection.py @@ -7,7 +7,7 @@ def __init__(self, covjson): print("Received Coverage") self.coverage = covjson - self.type = self.coverage.pop("type") + self.type = self.coverage["type"] if self.type == "CoverageCollection": print("Correct Type") @@ -16,4 +16,4 @@ def __init__(self, covjson): "CoverageCollection class takes CoverageCollection not Coverage" ) - self.coverages = self.coverage.pop("coverages") + self.coverages = self.coverage["coverages"] diff --git a/covjson/encoder/encoder.py b/covjson/encoder/encoder.py new file mode 100644 index 0000000..472b71e --- /dev/null +++ b/covjson/encoder/encoder.py @@ -0,0 +1,22 @@ +import os +import json +import xarray as xr +import datetime as dt +import geopandas as gpd +from abc import ABC, abstractmethod +from ..Coverage import Coverage +from ..CoverageCollection import CoverageCollection + + +class Encoder(ABC): + def __init__(self, type): + self.covjson = {} + self.type = type + self.covjson["type"] = self.type + self.covjson["coverages"] = [] + if self.type == "Coverage": + self.coverage = Coverage(self.covjson) + elif self.type == "CoverageCollection": + self.coverage = CoverageCollection(self.covjson) + else: + raise TypeError("Type must be Coverage or CoverageCollection") diff --git a/tests/test_encoder.py b/tests/test_encoder.py new file mode 100644 index 0000000..bff0fe7 --- /dev/null +++ b/tests/test_encoder.py @@ -0,0 +1,141 @@ +import pytest +import json + +from covjson.encoder import encoder + + +class TestDecoder: + def setup_method(self, method): + self.covjson = { + "type": "CoverageCollection", + "domainType": "PointSeries", + "coverages": [ + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "x": {"values": [3]}, + "y": {"values": [7]}, + "z": {"values": [1]}, + "t": { + "values": [ + "2017-01-01 00:00:00", + "2017-01-01 06:00:00", + "2017-01-01 12:00:00", + ] + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [ + 264.93115234375, + 263.83115234375, + 265.12313132266, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [ + 9.93115234375, + 7.83115234375, + 14.12313132266, + ], + }, + }, + }, + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170102", + "step": "0", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "x": {"values": [3]}, + "y": {"values": [7]}, + "z": {"values": [1]}, + "t": { + "values": [ + "2017-01-02 00:00:00", + "2017-01-02 06:00:00", + "2017-01-02 12:00:00", + ] + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [ + 263.83115234375, + 265.12313132266, + 264.93115234375, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["z"], + "values": [ + 13.83115234375, + 14.12313132266, + 7.93115234375, + ], + }, + }, + }, + ], + "referencing": [ + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ], + "parameters": { + "t": { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + "p": { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + }, + } + + def test_CoverageCollection(self): + encoder_obj = encoder.Encoder("CoverageCollection") + assert encoder_obj.type == "CoverageCollection" From 9029b855c96a63e931cb562073fa2ca028e57f6c Mon Sep 17 00:00:00 2001 From: awarde96 Date: Mon, 4 Dec 2023 14:37:15 +0100 Subject: [PATCH 15/58] Add base functions for encoder and tests --- covjson/encoder/encoder.py | 23 ++++++++-- tests/test_encoder.py | 86 +++++++++++++++++++++++++++++++++++++- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/covjson/encoder/encoder.py b/covjson/encoder/encoder.py index 472b71e..26cf49d 100644 --- a/covjson/encoder/encoder.py +++ b/covjson/encoder/encoder.py @@ -9,14 +9,31 @@ class Encoder(ABC): - def __init__(self, type): + def __init__(self, type, domaintype): self.covjson = {} + self.type = type self.covjson["type"] = self.type + self.covjson["domainType"] = domaintype self.covjson["coverages"] = [] - if self.type == "Coverage": + self.covjson["referencing"] = [] + self.covjson["parameters"] = {} + + if type == "Coverage": self.coverage = Coverage(self.covjson) - elif self.type == "CoverageCollection": + elif type == "CoverageCollection": self.coverage = CoverageCollection(self.covjson) else: raise TypeError("Type must be Coverage or CoverageCollection") + + if domaintype != "PointSeries" and domaintype != "VerticalProfile": + raise TypeError("DomainType must be PointSeries or VerticalProfile") + + def add_parameter(self, parameter, metadata): + self.covjson["parameters"][parameter] = metadata + + def add_reference(self, reference): + self.covjson["referencing"].append(reference) + + def add_coverage(self, coverage): + pass diff --git a/tests/test_encoder.py b/tests/test_encoder.py index bff0fe7..16b5d45 100644 --- a/tests/test_encoder.py +++ b/tests/test_encoder.py @@ -137,5 +137,89 @@ def setup_method(self, method): } def test_CoverageCollection(self): - encoder_obj = encoder.Encoder("CoverageCollection") + encoder_obj = encoder.Encoder("CoverageCollection", "PointSeries") assert encoder_obj.type == "CoverageCollection" + + def test_standard_Coverage(self): + encoder_obj = encoder.Encoder("CoverageCollection", "PointSeries") + covjson = { + "type": "CoverageCollection", + "domainType": "PointSeries", + "coverages": [], + "referencing": [], + "parameters": {}, + } + + assert encoder_obj.covjson == covjson + + def test_add_parameter(self): + encoder_obj = encoder.Encoder("CoverageCollection", "PointSeries") + encoder_obj.add_parameter( + "t", + { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + ) + encoder_obj.add_parameter( + "p", + { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + ) + covjson = { + "type": "CoverageCollection", + "domainType": "PointSeries", + "coverages": [], + "referencing": [], + "parameters": { + "t": { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + "p": { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + }, + } + + assert encoder_obj.covjson == covjson + + def test_add_reference(self): + encoder_obj = encoder.Encoder("CoverageCollection", "PointSeries") + encoder_obj.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + covjson = { + "type": "CoverageCollection", + "domainType": "PointSeries", + "coverages": [], + "referencing": [ + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ], + "parameters": {}, + } + + assert encoder_obj.covjson == covjson From e54671ed45a307f5a2bc939a0c5f296e9651194e Mon Sep 17 00:00:00 2001 From: awarde96 Date: Mon, 4 Dec 2023 17:16:46 +0100 Subject: [PATCH 16/58] Added TimeSeries class inheriting from encoder, added functionality to programatically create coverages --- covjson/encoder/TimeSeries.py | 46 ++++++++++++++++++++++ covjson/encoder/encoder.py | 23 ++++++++++- tests/test_decoder_time_series.py | 16 ++++---- tests/test_encoder.py | 63 +++++++++++++++++++++++++++---- 4 files changed, 130 insertions(+), 18 deletions(-) create mode 100644 covjson/encoder/TimeSeries.py diff --git a/covjson/encoder/TimeSeries.py b/covjson/encoder/TimeSeries.py new file mode 100644 index 0000000..cf68d62 --- /dev/null +++ b/covjson/encoder/TimeSeries.py @@ -0,0 +1,46 @@ +from .encoder import Encoder +import xarray as xr +import datetime as dt + + +class TimeSeries(Encoder): + def __init__(self, type, domaintype): + super().__init__(type, domaintype) + + def add_coverage(self, mars_metadata, coords, values): + new_coverage = {} + new_coverage["mars:metadata"] = {} + new_coverage["type"] = "Coverage" + new_coverage["domain"] = {} + new_coverage["ranges"] = {} + self.add_mars_metadata(new_coverage, mars_metadata) + self.add_domain(new_coverage, coords) + self.add_range(new_coverage, values) + self.covjson["coverages"].append(new_coverage) + + def add_domain(self, coverage, coords): + coverage["domain"]["type"] = "Domain" + coverage["domain"]["axes"] = {} + coverage["domain"]["axes"]["x"] = {} + coverage["domain"]["axes"]["y"] = {} + coverage["domain"]["axes"]["z"] = {} + coverage["domain"]["axes"]["t"] = {} + coverage["domain"]["axes"]["x"]["values"] = [] # [coords["x"]] + coverage["domain"]["axes"]["y"]["values"] = [] # [coords["y"]] + coverage["domain"]["axes"]["z"]["values"] = [] # [coords["z"]] + coverage["domain"]["axes"]["t"]["values"] = [] # [coords["t"]] + + def add_range(self, coverage, values): + for parameter in self.parameters: + coverage["ranges"][parameter] = {} + coverage["ranges"][parameter]["type"] = "NdArray" + coverage["ranges"][parameter]["dataType"] = "float" + coverage["ranges"][parameter]["shape"] = [] + coverage["ranges"][parameter]["axisNames"] = ["t"] + coverage["ranges"][parameter]["values"] = [] # [values[parameter]] + + def add_mars_metadata(self, coverage, metadata): + coverage["mars:metadata"] = metadata + + def from_xarray(self, dataset): + pass diff --git a/covjson/encoder/encoder.py b/covjson/encoder/encoder.py index 26cf49d..883765e 100644 --- a/covjson/encoder/encoder.py +++ b/covjson/encoder/encoder.py @@ -13,11 +13,12 @@ def __init__(self, type, domaintype): self.covjson = {} self.type = type + self.parameters = [] self.covjson["type"] = self.type self.covjson["domainType"] = domaintype self.covjson["coverages"] = [] - self.covjson["referencing"] = [] self.covjson["parameters"] = {} + self.covjson["referencing"] = [] if type == "Coverage": self.coverage = Coverage(self.covjson) @@ -31,9 +32,27 @@ def __init__(self, type, domaintype): def add_parameter(self, parameter, metadata): self.covjson["parameters"][parameter] = metadata + self.parameters.append(parameter) def add_reference(self, reference): self.covjson["referencing"].append(reference) - def add_coverage(self, coverage): + @abstractmethod + def add_coverage(self, mars_metadata, coords, values): + pass + + @abstractmethod + def add_domain(self, coverage, domain): + pass + + @abstractmethod + def add_range(self, coverage, range): + pass + + @abstractmethod + def add_mars_metadata(self, coverage, metadata): + pass + + @abstractmethod + def from_xarray(self, dataset): pass diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index 6f2ae3e..b3a35f6 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -44,7 +44,7 @@ def setup_method(self, method): "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [ 264.93115234375, 263.83115234375, @@ -55,7 +55,7 @@ def setup_method(self, method): "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [ 9.93115234375, 7.83115234375, @@ -94,7 +94,7 @@ def setup_method(self, method): "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [ 263.83115234375, 265.12313132266, @@ -105,7 +105,7 @@ def setup_method(self, method): "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [ 13.83115234375, 14.12313132266, @@ -214,14 +214,14 @@ def test_timeseries_ranges(self): "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [264.93115234375, 263.83115234375, 265.12313132266], }, "p": { "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [9.93115234375, 7.83115234375, 14.12313132266], }, } @@ -231,14 +231,14 @@ def test_timeseries_ranges(self): "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [263.83115234375, 265.12313132266, 264.93115234375], }, "p": { "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [13.83115234375, 14.12313132266, 7.93115234375], }, } diff --git a/tests/test_encoder.py b/tests/test_encoder.py index 16b5d45..2287a93 100644 --- a/tests/test_encoder.py +++ b/tests/test_encoder.py @@ -2,6 +2,7 @@ import json from covjson.encoder import encoder +from covjson.encoder import TimeSeries class TestDecoder: @@ -40,7 +41,7 @@ def setup_method(self, method): "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [ 264.93115234375, 263.83115234375, @@ -51,7 +52,7 @@ def setup_method(self, method): "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [ 9.93115234375, 7.83115234375, @@ -90,7 +91,7 @@ def setup_method(self, method): "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [ 263.83115234375, 265.12313132266, @@ -101,7 +102,7 @@ def setup_method(self, method): "type": "NdArray", "dataType": "float", "shape": [3], - "axisNames": ["z"], + "axisNames": ["t"], "values": [ 13.83115234375, 14.12313132266, @@ -137,11 +138,11 @@ def setup_method(self, method): } def test_CoverageCollection(self): - encoder_obj = encoder.Encoder("CoverageCollection", "PointSeries") + encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") assert encoder_obj.type == "CoverageCollection" def test_standard_Coverage(self): - encoder_obj = encoder.Encoder("CoverageCollection", "PointSeries") + encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") covjson = { "type": "CoverageCollection", "domainType": "PointSeries", @@ -153,7 +154,7 @@ def test_standard_Coverage(self): assert encoder_obj.covjson == covjson def test_add_parameter(self): - encoder_obj = encoder.Encoder("CoverageCollection", "PointSeries") + encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") encoder_obj.add_parameter( "t", { @@ -196,7 +197,7 @@ def test_add_parameter(self): assert encoder_obj.covjson == covjson def test_add_reference(self): - encoder_obj = encoder.Encoder("CoverageCollection", "PointSeries") + encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") encoder_obj.add_reference( { "coordinates": ["x", "y", "z"], @@ -223,3 +224,49 @@ def test_add_reference(self): } assert encoder_obj.covjson == covjson + + def test_add_coverage_marsmetadata(self): + encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") + encoder_obj.add_coverage( + { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": "0", + }, + {}, + {}, + ) + covjson = { + "type": "CoverageCollection", + "domainType": "PointSeries", + "coverages": [ + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "x": {"values": []}, + "y": {"values": []}, + "z": {"values": []}, + "t": {"values": []}, + }, + }, + "ranges": {}, + } + ], + "referencing": [], + "parameters": {}, + } + print(encoder_obj.covjson) + assert encoder_obj.covjson == covjson From 247108c3d6bf411f5d7de95368086d3ffffbfa55 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 5 Dec 2023 11:30:59 +0100 Subject: [PATCH 17/58] Fixed relative paths --- covjson/decoder/decoder.py | 4 ++-- covjson/encoder/encoder.py | 4 ++-- tests/test_decoder_time_series.py | 27 +++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/covjson/decoder/decoder.py b/covjson/decoder/decoder.py index 5ffe5a9..03b75e2 100644 --- a/covjson/decoder/decoder.py +++ b/covjson/decoder/decoder.py @@ -4,8 +4,8 @@ import datetime as dt import geopandas as gpd from abc import ABC, abstractmethod -from ..Coverage import Coverage -from ..CoverageCollection import CoverageCollection +from covjson.Coverage import Coverage +from covjson.CoverageCollection import CoverageCollection class Decoder(ABC): diff --git a/covjson/encoder/encoder.py b/covjson/encoder/encoder.py index 883765e..8fc23fa 100644 --- a/covjson/encoder/encoder.py +++ b/covjson/encoder/encoder.py @@ -4,8 +4,8 @@ import datetime as dt import geopandas as gpd from abc import ABC, abstractmethod -from ..Coverage import Coverage -from ..CoverageCollection import CoverageCollection +from covjson.Coverage import Coverage +from covjson.CoverageCollection import CoverageCollection class Encoder(ABC): diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index b3a35f6..d1f6ff8 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -298,3 +298,30 @@ def test_timeseries_to_xarray(self): print(ekds) print(type(ekds)) # print(ekds.ls()) + + +""" +[ +array([[[[264.93115234, 263.83115234, 265.12313132, 263.83115234, + 265.12313132, 264.93115234]]]]) +Coordinates: + * x (x) int64 3 + * y (y) int64 7 + * z (z) int64 1 + * t (t) datetime64[ns] 2017-01-01 ... 2017-01-02T12:00:00 +Attributes: + type: Parameter + units: K + long_name: Temperature, +array([[[[ 9.93115234, 7.83115234, 14.12313132, 13.83115234, + 14.12313132, 7.93115234]]]]) +Coordinates: + * x (x) int64 3 + * y (y) int64 7 + * z (z) int64 1 + * t (t) datetime64[ns] 2017-01-01 ... 2017-01-02T12:00:00 +Attributes: + type: Parameter + units: pa + long_name: Pressure] + """ From 7ff034f23765a24828d0eb34e0da78465fdaaf22 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 5 Dec 2023 13:48:55 +0100 Subject: [PATCH 18/58] Changed to_xarray function so now forecast time is also a dimension --- covjson/decoder/TimeSeries.py | 68 ++++++++++++++++++------------- tests/test_decoder_time_series.py | 50 ++++++++++++----------- 2 files changed, 65 insertions(+), 53 deletions(-) diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py index 2db3885..1206cca 100644 --- a/covjson/decoder/TimeSeries.py +++ b/covjson/decoder/TimeSeries.py @@ -27,9 +27,9 @@ def get_values(self): values[parameter] = [] for range in self.ranges: values[parameter].append(range[parameter]["values"]) - values[parameter] = [ - value for sublist in values[parameter] for value in sublist - ] + # values[parameter] = [ + # value for sublist in values[parameter] for value in sublist + # ] return values def get_coordinates(self): @@ -38,16 +38,19 @@ def get_coordinates(self): for param in self.parameters: coord_dict[param] = [] # Get x,y,z,t coords and unpack t coords and match to x,y,z coords - for domain in self.domains: + for ind, domain in enumerate(self.domains): x = domain["axes"]["x"]["values"][0] y = domain["axes"]["y"]["values"][0] z = domain["axes"]["z"]["values"][0] + fct = self.mars_metadata[ind]["date"] ts = domain["axes"]["t"]["values"] for param in self.parameters: + coords = [] for t in ts: # Have to replicate these coords for each parameter # coordinates.append([x, y, z, t]) - coord_dict[param].append([x, y, z, t]) + coords.append([x, y, z, fct, t]) + coord_dict[param].append(coords) return coord_dict def to_geopandas(self): @@ -55,40 +58,47 @@ def to_geopandas(self): # function to convert covjson to xarray dataset def to_xarray(self): - dims = ["x", "y", "z", "t"] + dims = ["x", "y", "z", "fct", "t"] dataarraydict = {} # Get coordinates for parameter in self.parameters: param_values = [[[self.get_values()[parameter]]]] + for ind, fc_time_vals in enumerate(self.get_values()[parameter]): + coords = self.get_coordinates()[parameter] + x = [coords[ind][0][0]] + y = [coords[ind][0][1]] + z = [coords[ind][0][2]] - coords = self.get_coordinates()[parameter] - x = [coords[0][0]] - y = [coords[0][1]] - z = [coords[0][2]] - t = [ - dt.datetime.strptime(coord[3], "%Y-%m-%d %H:%M:%S") for coord in coords - ] + fct = [ + dt.datetime.strptime((coord[0][3]), "%Y%m%d") for coord in coords + ] + coords_fc = coords[ind] + t = [ + dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") + for coord in coords_fc + ] - param_coords = {"x": x, "y": y, "z": z, "t": t} - dataarray = xr.DataArray( - param_values, - dims=dims, - coords=param_coords, - name=parameter, - ) + param_coords = {"x": x, "y": y, "z": z, "fct": fct, "t": t} + dataarray = xr.DataArray( + param_values, + dims=dims, + coords=param_coords, + name=parameter, + ) - dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] - dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"][ - "symbol" - ] - dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ - "description" - ] - dataarraydict[dataarray.attrs["long_name"]] = dataarray + dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)[ + "unit" + ]["symbol"] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ + "description" + ] + dataarraydict[dataarray.attrs["long_name"]] = dataarray ds = xr.Dataset(dataarraydict) for mars_metadata in self.mars_metadata[0]: - ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata] + if mars_metadata != "date" and mars_metadata != "step": + ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata] return ds diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index d1f6ff8..01478d4 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -248,20 +248,12 @@ def test_timeseries_values(self): decoder = TimeSeries.TimeSeries(self.covjson) values = { "t": [ - 264.93115234375, - 263.83115234375, - 265.12313132266, - 263.83115234375, - 265.12313132266, - 264.93115234375, + [264.93115234375, 263.83115234375, 265.12313132266], + [263.83115234375, 265.12313132266, 264.93115234375], ], "p": [ - 9.93115234375, - 7.83115234375, - 14.12313132266, - 13.83115234375, - 14.12313132266, - 7.93115234375, + [9.93115234375, 7.83115234375, 14.12313132266], + [13.83115234375, 14.12313132266, 7.93115234375], ], } assert decoder.get_values() == values @@ -270,28 +262,38 @@ def test_timeseries_coordinates(self): decoder = TimeSeries.TimeSeries(self.covjson) coordinates = { "t": [ - [3, 7, 1, "2017-01-01 00:00:00"], - [3, 7, 1, "2017-01-01 06:00:00"], - [3, 7, 1, "2017-01-01 12:00:00"], - [3, 7, 1, "2017-01-02 00:00:00"], - [3, 7, 1, "2017-01-02 06:00:00"], - [3, 7, 1, "2017-01-02 12:00:00"], + [ + [3, 7, 1, "20170101", "2017-01-01 00:00:00"], + [3, 7, 1, "20170101", "2017-01-01 06:00:00"], + [3, 7, 1, "20170101", "2017-01-01 12:00:00"], + ], + [ + [3, 7, 1, "20170102", "2017-01-02 00:00:00"], + [3, 7, 1, "20170102", "2017-01-02 06:00:00"], + [3, 7, 1, "20170102", "2017-01-02 12:00:00"], + ], ], "p": [ - [3, 7, 1, "2017-01-01 00:00:00"], - [3, 7, 1, "2017-01-01 06:00:00"], - [3, 7, 1, "2017-01-01 12:00:00"], - [3, 7, 1, "2017-01-02 00:00:00"], - [3, 7, 1, "2017-01-02 06:00:00"], - [3, 7, 1, "2017-01-02 12:00:00"], + [ + [3, 7, 1, "20170101", "2017-01-01 00:00:00"], + [3, 7, 1, "20170101", "2017-01-01 06:00:00"], + [3, 7, 1, "20170101", "2017-01-01 12:00:00"], + ], + [ + [3, 7, 1, "20170102", "2017-01-02 00:00:00"], + [3, 7, 1, "20170102", "2017-01-02 06:00:00"], + [3, 7, 1, "20170102", "2017-01-02 12:00:00"], + ], ], } + print(decoder.get_coordinates()) assert decoder.get_coordinates() == coordinates def test_timeseries_to_xarray(self): decoder = TimeSeries.TimeSeries(self.covjson) ds = decoder.to_xarray() print(ds) + print(ds["Temperature"]) # xrds.to_netcdf("timeseries.nc") # ds = xr.open_dataset("timeseries.nc") ekds = data.from_object(ds) From 9c9e4829893c3801b2656efdecc2427bc08d049e Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 5 Dec 2023 15:42:53 +0100 Subject: [PATCH 19/58] Added functionality to convert an xarray into a covjson --- covjson/encoder/TimeSeries.py | 89 ++++++++++++++++++++++++++++++++--- covjson/test.ipynb | 0 tests/test_encoder.py | 3 ++ 3 files changed, 85 insertions(+), 7 deletions(-) delete mode 100644 covjson/test.ipynb diff --git a/covjson/encoder/TimeSeries.py b/covjson/encoder/TimeSeries.py index cf68d62..7011c92 100644 --- a/covjson/encoder/TimeSeries.py +++ b/covjson/encoder/TimeSeries.py @@ -25,22 +25,97 @@ def add_domain(self, coverage, coords): coverage["domain"]["axes"]["y"] = {} coverage["domain"]["axes"]["z"] = {} coverage["domain"]["axes"]["t"] = {} - coverage["domain"]["axes"]["x"]["values"] = [] # [coords["x"]] - coverage["domain"]["axes"]["y"]["values"] = [] # [coords["y"]] - coverage["domain"]["axes"]["z"]["values"] = [] # [coords["z"]] - coverage["domain"]["axes"]["t"]["values"] = [] # [coords["t"]] + coverage["domain"]["axes"]["x"]["values"] = [coords["x"]] + coverage["domain"]["axes"]["y"]["values"] = [coords["y"]] + coverage["domain"]["axes"]["z"]["values"] = [coords["z"]] + coverage["domain"]["axes"]["t"]["values"] = [coords["t"]] def add_range(self, coverage, values): for parameter in self.parameters: coverage["ranges"][parameter] = {} coverage["ranges"][parameter]["type"] = "NdArray" coverage["ranges"][parameter]["dataType"] = "float" - coverage["ranges"][parameter]["shape"] = [] + coverage["ranges"][parameter]["shape"] = [values[parameter].shape[0]] coverage["ranges"][parameter]["axisNames"] = ["t"] - coverage["ranges"][parameter]["values"] = [] # [values[parameter]] + coverage["ranges"][parameter]["values"] = values[ + parameter + ] # [values[parameter]] def add_mars_metadata(self, coverage, metadata): coverage["mars:metadata"] = metadata def from_xarray(self, dataset): - pass + for parameter in dataset.data_vars: + if parameter == "Temperature": + self.add_parameter( + "t", + { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + ) + elif parameter == "Pressure": + self.add_parameter( + "p", + { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + ) + self.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + for fc_time in dataset["fct"]: + self.add_coverage( + { + "date": fc_time.values.astype("M8[ms]") + .astype("O") + .strftime("%m/%d/%Y"), + "type": "forecast", + "step": 0, + }, + { + "x": dataset["x"].values, + "y": dataset["y"].values, + "z": dataset["z"].values, + "t": dataset["t"].values, + }, + { + "t": dataset["Temperature"].sel(fct=fc_time).values[0][0][0], + "p": dataset["Pressure"].sel(fct=fc_time).values[0][0][0], + }, + ) + return self.covjson + + +""" + +Dimensions: (x: 1, y: 1, z: 1, fct: 2, t: 3) +Coordinates: + * x (x) int64 3 + * y (y) int64 7 + * z (z) int64 1 + * fct (fct) datetime64[ns] 2017-01-01 2017-01-02 + * t (t) datetime64[ns] 2017-01-02 ... 2017-01-02T12:00:00 +Data variables: + Temperature (x, y, z, fct, t) float64 264.9 263.8 265.1 263.8 265.1 264.9 + Pressure (x, y, z, fct, t) float64 9.931 7.831 14.12 13.83 14.12 7.931 +Attributes: + class: od + stream: oper + levtype: pl + number: 0 + +array([[[[[264.93115234, 263.83115234, 265.12313132], + [263.83115234, 265.12313132, 264.93115234]]]]]) + """ diff --git a/covjson/test.ipynb b/covjson/test.ipynb deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_encoder.py b/tests/test_encoder.py index 2287a93..40c31b6 100644 --- a/tests/test_encoder.py +++ b/tests/test_encoder.py @@ -225,6 +225,8 @@ def test_add_reference(self): assert encoder_obj.covjson == covjson + +""" def test_add_coverage_marsmetadata(self): encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") encoder_obj.add_coverage( @@ -270,3 +272,4 @@ def test_add_coverage_marsmetadata(self): } print(encoder_obj.covjson) assert encoder_obj.covjson == covjson +""" From ac36e5cd04e9ceab33050b54b41befce56f4e38a Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 7 Dec 2023 10:27:21 +0100 Subject: [PATCH 20/58] Changed to and from xarray functions to use ensemble members instead of forecast time as dim --- covjson/decoder/TimeSeries.py | 52 ++++++++++++++++++++++++- covjson/encoder/TimeSeries.py | 55 +++++++++------------------ covjson/encoder/encoder.py | 4 ++ tests/test_decoder_time_series.py | 24 ++++++------ tests/test_encoder.py | 63 +++++++++++++++++++++++++++++++ 5 files changed, 148 insertions(+), 50 deletions(-) diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py index 1206cca..8c06986 100644 --- a/covjson/decoder/TimeSeries.py +++ b/covjson/decoder/TimeSeries.py @@ -44,12 +44,13 @@ def get_coordinates(self): z = domain["axes"]["z"]["values"][0] fct = self.mars_metadata[ind]["date"] ts = domain["axes"]["t"]["values"] + num = self.mars_metadata[ind]["number"] for param in self.parameters: coords = [] for t in ts: # Have to replicate these coords for each parameter # coordinates.append([x, y, z, t]) - coords.append([x, y, z, fct, t]) + coords.append([x, y, z, fct, t, num]) coord_dict[param].append(coords) return coord_dict @@ -57,6 +58,54 @@ def to_geopandas(self): pass # function to convert covjson to xarray dataset + def to_xarray(self): + dims = ["x", "y", "z", "number", "t"] + dataarraydict = {} + + # Get coordinates + for parameter in self.parameters: + param_values = [[[self.get_values()[parameter]]]] + for ind, fc_time_vals in enumerate(self.get_values()[parameter]): + coords = self.get_coordinates()[parameter] + x = [coords[ind][0][0]] + y = [coords[ind][0][1]] + z = [coords[ind][0][2]] + # fct = [ + # dt.datetime.strptime((coord[0][3]), "%Y%m%d") for coord in coords + # ] + num = [int(coord[0][5]) for coord in coords] + coords_fc = coords[ind] + t = [ + dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") + for coord in coords_fc + ] + + param_coords = {"x": x, "y": y, "z": z, "number": num, "t": t} + dataarray = xr.DataArray( + param_values, + dims=dims, + coords=param_coords, + name=parameter, + ) + + dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)[ + "unit" + ]["symbol"] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ + "description" + ] + dataarraydict[dataarray.attrs["long_name"]] = dataarray + + ds = xr.Dataset(dataarraydict) + for mars_metadata in self.mars_metadata[0]: + if mars_metadata != "date" and mars_metadata != "step": + ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata] + + return ds + + +""" def to_xarray(self): dims = ["x", "y", "z", "fct", "t"] dataarraydict = {} @@ -102,3 +151,4 @@ def to_xarray(self): ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata] return ds +""" diff --git a/covjson/encoder/TimeSeries.py b/covjson/encoder/TimeSeries.py index 7011c92..ea04841 100644 --- a/covjson/encoder/TimeSeries.py +++ b/covjson/encoder/TimeSeries.py @@ -25,17 +25,17 @@ def add_domain(self, coverage, coords): coverage["domain"]["axes"]["y"] = {} coverage["domain"]["axes"]["z"] = {} coverage["domain"]["axes"]["t"] = {} - coverage["domain"]["axes"]["x"]["values"] = [coords["x"]] - coverage["domain"]["axes"]["y"]["values"] = [coords["y"]] - coverage["domain"]["axes"]["z"]["values"] = [coords["z"]] - coverage["domain"]["axes"]["t"]["values"] = [coords["t"]] + coverage["domain"]["axes"]["x"]["values"] = coords["x"] + coverage["domain"]["axes"]["y"]["values"] = coords["y"] + coverage["domain"]["axes"]["z"]["values"] = coords["z"] + coverage["domain"]["axes"]["t"]["values"] = coords["t"] def add_range(self, coverage, values): for parameter in self.parameters: coverage["ranges"][parameter] = {} coverage["ranges"][parameter]["type"] = "NdArray" coverage["ranges"][parameter]["dataType"] = "float" - coverage["ranges"][parameter]["shape"] = [values[parameter].shape[0]] + coverage["ranges"][parameter]["shape"] = [len(values[parameter])] coverage["ranges"][parameter]["axisNames"] = ["t"] coverage["ranges"][parameter]["values"] = values[ parameter @@ -75,47 +75,28 @@ def from_xarray(self, dataset): }, } ) - for fc_time in dataset["fct"]: + for num in dataset["number"].values: self.add_coverage( { - "date": fc_time.values.astype("M8[ms]") - .astype("O") - .strftime("%m/%d/%Y"), + # "date": fc_time.values.astype("M8[ms]") + # .astype("O") + # .strftime("%m/%d/%Y"), + "number": num, "type": "forecast", "step": 0, }, { - "x": dataset["x"].values, - "y": dataset["y"].values, - "z": dataset["z"].values, - "t": dataset["t"].values, + "x": list(dataset["x"].values), + "y": list(dataset["y"].values), + "z": list(dataset["z"].values), + "t": [str(x) for x in dataset["t"].values], }, { - "t": dataset["Temperature"].sel(fct=fc_time).values[0][0][0], - "p": dataset["Pressure"].sel(fct=fc_time).values[0][0][0], + "t": list(dataset["Temperature"].sel(number=num).values[0][0][0]), + # "p": dataset["Pressure"].sel(fct=fc_time).values[0][0][0], }, ) return self.covjson - -""" - -Dimensions: (x: 1, y: 1, z: 1, fct: 2, t: 3) -Coordinates: - * x (x) int64 3 - * y (y) int64 7 - * z (z) int64 1 - * fct (fct) datetime64[ns] 2017-01-01 2017-01-02 - * t (t) datetime64[ns] 2017-01-02 ... 2017-01-02T12:00:00 -Data variables: - Temperature (x, y, z, fct, t) float64 264.9 263.8 265.1 263.8 265.1 264.9 - Pressure (x, y, z, fct, t) float64 9.931 7.831 14.12 13.83 14.12 7.931 -Attributes: - class: od - stream: oper - levtype: pl - number: 0 - -array([[[[[264.93115234, 263.83115234, 265.12313132], - [263.83115234, 265.12313132, 264.93115234]]]]]) - """ + def from_polytope(self, result): + pass diff --git a/covjson/encoder/encoder.py b/covjson/encoder/encoder.py index 8fc23fa..7ed3e6b 100644 --- a/covjson/encoder/encoder.py +++ b/covjson/encoder/encoder.py @@ -56,3 +56,7 @@ def add_mars_metadata(self, coverage, metadata): @abstractmethod def from_xarray(self, dataset): pass + + @abstractmethod + def from_polytope(self, result): + pass diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index 01478d4..4b1c471 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -263,26 +263,26 @@ def test_timeseries_coordinates(self): coordinates = { "t": [ [ - [3, 7, 1, "20170101", "2017-01-01 00:00:00"], - [3, 7, 1, "20170101", "2017-01-01 06:00:00"], - [3, 7, 1, "20170101", "2017-01-01 12:00:00"], + [3, 7, 1, "20170101", "2017-01-01 00:00:00", "0"], + [3, 7, 1, "20170101", "2017-01-01 06:00:00", "0"], + [3, 7, 1, "20170101", "2017-01-01 12:00:00", "0"], ], [ - [3, 7, 1, "20170102", "2017-01-02 00:00:00"], - [3, 7, 1, "20170102", "2017-01-02 06:00:00"], - [3, 7, 1, "20170102", "2017-01-02 12:00:00"], + [3, 7, 1, "20170102", "2017-01-02 00:00:00", "0"], + [3, 7, 1, "20170102", "2017-01-02 06:00:00", "0"], + [3, 7, 1, "20170102", "2017-01-02 12:00:00", "0"], ], ], "p": [ [ - [3, 7, 1, "20170101", "2017-01-01 00:00:00"], - [3, 7, 1, "20170101", "2017-01-01 06:00:00"], - [3, 7, 1, "20170101", "2017-01-01 12:00:00"], + [3, 7, 1, "20170101", "2017-01-01 00:00:00", "0"], + [3, 7, 1, "20170101", "2017-01-01 06:00:00", "0"], + [3, 7, 1, "20170101", "2017-01-01 12:00:00", "0"], ], [ - [3, 7, 1, "20170102", "2017-01-02 00:00:00"], - [3, 7, 1, "20170102", "2017-01-02 06:00:00"], - [3, 7, 1, "20170102", "2017-01-02 12:00:00"], + [3, 7, 1, "20170102", "2017-01-02 00:00:00", "0"], + [3, 7, 1, "20170102", "2017-01-02 06:00:00", "0"], + [3, 7, 1, "20170102", "2017-01-02 12:00:00", "0"], ], ], } diff --git a/tests/test_encoder.py b/tests/test_encoder.py index 40c31b6..27def22 100644 --- a/tests/test_encoder.py +++ b/tests/test_encoder.py @@ -3,6 +3,19 @@ from covjson.encoder import encoder from covjson.encoder import TimeSeries +import covjson.decoder.TimeSeries +import random +from datetime import datetime, timedelta + + +def get_timestamps(start_dt, end_dt, delta): + dates = [] + while start_dt <= end_dt: + # add current date to list by converting it to iso format + dates.append(start_dt.isoformat().replace("T", " ")) + # increment start date by timedelta + start_dt += delta + return dates class TestDecoder: @@ -225,6 +238,56 @@ def test_add_reference(self): assert encoder_obj.covjson == covjson + def test_add_coverage(self): + encoder = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") + encoder.add_parameter( + "t", + { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + ) + encoder.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + + metadatas = [] + coords = [] + values = [] + for number in range(0, 50): + metadata = { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": str(number), + } + timestamps = get_timestamps( + datetime(2017, 1, 1, 0, 00), + datetime(2017, 1, 14, 0, 00), + timedelta(hours=6), + ) + coord = { + "x": [3], + "y": [7], + "z": [1], + "t": timestamps, + } + coords.append(coord) + value = {"t": [random.uniform(230, 270) for _ in range(0, len(timestamps))]} + values.append(value) + encoder.add_coverage(metadata, coord, value) + print(encoder.covjson) + """ def test_add_coverage_marsmetadata(self): From 6c443a0fcdbb330b0f1d8d80acac8bcd108455e2 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 7 Dec 2023 15:32:59 +0100 Subject: [PATCH 21/58] Added initial version of from_polytope converting polytope output to covjson --- .gitignore | 2 + covjson/encoder/TimeSeries.py | 72 +++++++++++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 97884ca..a8a2916 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ polytope_mars.egg-info PKG-info SOURCES.txt top_level.txt +*.ipynb +*.covjson diff --git a/covjson/encoder/TimeSeries.py b/covjson/encoder/TimeSeries.py index ea04841..454b30c 100644 --- a/covjson/encoder/TimeSeries.py +++ b/covjson/encoder/TimeSeries.py @@ -1,6 +1,7 @@ from .encoder import Encoder import xarray as xr -import datetime as dt +from datetime import timedelta, datetime +import datetime class TimeSeries(Encoder): @@ -98,5 +99,70 @@ def from_xarray(self, dataset): ) return self.covjson - def from_polytope(self, result): - pass + def from_polytope(self, result, request): + # ancestors = [val.get_ancestors() for val in result.leaves] + values = [val.result for val in result.leaves] + + mars_metadata = {} + coords = {} + for key in request.keys(): + if ( + key != "latitude" + and key != "longitude" + and key != "param" + and key != "number" + and key != "step" + ): + mars_metadata[key] = request[key] + elif key == "latitude": + coords["x"] = [request[key]] + elif key == "longitude": + coords["y"] = [request[key]] + + if request["param"] == "167": + self.add_parameter( + "t", + { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + ) + self.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + + coords["z"] = ["sfc"] + numbers = request["number"] + steps = request["step"] + + times = [] + date_format = "%Y%m%dT%H%M%S" + start_time = datetime.datetime.strptime(mars_metadata["date"], date_format) + for step in steps: + # add current date to list by converting it to iso format + stamp = start_time + timedelta(hours=step) + times.append(stamp.isoformat()) + # increment start date by timedelta + + coords["t"] = times + vals = [] + start = 0 + end = len(times) + new_metadata = mars_metadata.copy() + for num in numbers: + mars_metadata["number"] = num + new_metadata = mars_metadata.copy() + self.add_coverage(new_metadata, coords, {"t": values[start:end]}) + # vals.append(values[start:end]) + start = end + end += len(times) + + return self.covjson From 818f222d575e7f128c517640f9c8c7c49beac962 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 7 Dec 2023 16:04:13 +0100 Subject: [PATCH 22/58] Changed time dim parsing so different time formats work --- covjson/decoder/TimeSeries.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py index 8c06986..f5c0f31 100644 --- a/covjson/decoder/TimeSeries.py +++ b/covjson/decoder/TimeSeries.py @@ -75,10 +75,16 @@ def to_xarray(self): # ] num = [int(coord[0][5]) for coord in coords] coords_fc = coords[ind] - t = [ - dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") - for coord in coords_fc - ] + try: + t = [ + dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") + for coord in coords_fc + ] + except ValueError: + t = [ + dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%S") + for coord in coords_fc + ] param_coords = {"x": x, "y": y, "z": z, "number": num, "t": t} dataarray = xr.DataArray( From e9999b4201cc914866fc6ed868252ac00eee4b51 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Fri, 8 Dec 2023 11:20:15 +0100 Subject: [PATCH 23/58] Added to_xarray for vertical profiles --- covjson/decoder/TimeSeries.py | 1 - covjson/decoder/VerticalProfile.py | 64 ++++++++++++++++++++++---- tests/test_decoder_time_series.py | 8 ++-- tests/test_decoder_vertical_profile.py | 36 +++++++++++---- tests/test_encoder.py | 8 +++- 5 files changed, 93 insertions(+), 24 deletions(-) diff --git a/covjson/decoder/TimeSeries.py b/covjson/decoder/TimeSeries.py index f5c0f31..203c6d6 100644 --- a/covjson/decoder/TimeSeries.py +++ b/covjson/decoder/TimeSeries.py @@ -33,7 +33,6 @@ def get_values(self): return values def get_coordinates(self): - coordinates = [] coord_dict = {} for param in self.parameters: coord_dict[param] = [] diff --git a/covjson/decoder/VerticalProfile.py b/covjson/decoder/VerticalProfile.py index f552535..d8db709 100644 --- a/covjson/decoder/VerticalProfile.py +++ b/covjson/decoder/VerticalProfile.py @@ -1,4 +1,5 @@ from .decoder import Decoder +import xarray as xr class VerticalProfile(Decoder): @@ -20,18 +21,24 @@ def get_ranges(self): return ranges def get_coordinates(self): - coordinates = [] + coord_dict = {} + for param in self.parameters: + coord_dict[param] = [] # Get x,y,z coords and unpack z coords and match to x,y coords - for domain in self.domains: + for ind, domain in enumerate(self.domains): x = domain["axes"]["x"]["values"][0] y = domain["axes"]["y"]["values"][0] t = domain["axes"]["t"]["values"][0] zs = domain["axes"]["z"]["values"] - for z in zs: - # Have to replicate these coords for each parameter - for _ in self.parameters: - coordinates.append([x, y, z, t]) - return coordinates + num = self.mars_metadata[ind]["number"] + for param in self.parameters: + coords = [] + for z in zs: + # Have to replicate these coords for each parameter + # coordinates.append([x, y, z, t]) + coords.append([x, y, z, num, t]) + coord_dict[param].append(coords) + return coord_dict def get_values(self): values = {} @@ -45,4 +52,45 @@ def to_geopandas(self): pass def to_xarray(self): - pass + dims = ["x", "y", "t", "number", "z"] + dataarraydict = {} + + for parameter in self.parameters: + param_values = [[[self.get_values()[parameter]]]] + for ind, value in enumerate(self.get_values()[parameter]): + coords = self.get_coordinates()[parameter] + x = [coords[ind][0][0]] + y = [coords[ind][0][1]] + t = [coords[ind][0][4]] + num = [int(coord[0][3]) for coord in coords] + coords_z = coords[ind] + z = [int(coord[2]) for coord in coords_z] + param_coords = { + "x": x, + "y": y, + "t": t, + "number": num, + "z": z, + } + dataarray = xr.DataArray( + param_values, + dims=dims, + coords=param_coords, + name=parameter, + ) + dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)[ + "unit" + ]["symbol"] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ + "description" + ] + dataarraydict[dataarray.attrs["long_name"]] = dataarray + + ds = xr.Dataset(dataarraydict) + print(ds) + for mars_metadata in self.mars_metadata[0]: + if mars_metadata != "date" and mars_metadata != "step": + ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata] + + return ds diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index 4b1c471..50b7e0e 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -292,13 +292,13 @@ def test_timeseries_coordinates(self): def test_timeseries_to_xarray(self): decoder = TimeSeries.TimeSeries(self.covjson) ds = decoder.to_xarray() - print(ds) - print(ds["Temperature"]) + # print(ds) + # print(ds["Temperature"]) # xrds.to_netcdf("timeseries.nc") # ds = xr.open_dataset("timeseries.nc") ekds = data.from_object(ds) - print(ekds) - print(type(ekds)) + # print(ekds) + # print(type(ekds)) # print(ekds.ls()) diff --git a/tests/test_decoder_vertical_profile.py b/tests/test_decoder_vertical_profile.py index eccd5a7..97c5b2e 100644 --- a/tests/test_decoder_vertical_profile.py +++ b/tests/test_decoder_vertical_profile.py @@ -216,16 +216,28 @@ def test_verticalprofile_ranges(self): def test_verticalprofile_coordinates(self): decoder = VerticalProfile.VerticalProfile(self.covjson) - coordinates = [ - ["0.0", "0.0", "500", "2017-01-01 12:00:00"], - ["0.0", "0.0", "500", "2017-01-01 12:00:00"], - ["0.0", "0.0", "850", "2017-01-01 12:00:00"], - ["0.0", "0.0", "850", "2017-01-01 12:00:00"], - ["0.0", "0.0", "500", "2017-01-01 12:00:00"], - ["0.0", "0.0", "500", "2017-01-01 12:00:00"], - ["0.0", "0.0", "850", "2017-01-01 12:00:00"], - ["0.0", "0.0", "850", "2017-01-01 12:00:00"], - ] + coordinates = { + "t": [ + [ + ["0.0", "0.0", "500", "0", "2017-01-01 12:00:00"], + ["0.0", "0.0", "850", "0", "2017-01-01 12:00:00"], + ], + [ + ["0.0", "0.0", "500", "1", "2017-01-01 12:00:00"], + ["0.0", "0.0", "850", "1", "2017-01-01 12:00:00"], + ], + ], + "p": [ + [ + ["0.0", "0.0", "500", "0", "2017-01-01 12:00:00"], + ["0.0", "0.0", "850", "0", "2017-01-01 12:00:00"], + ], + [ + ["0.0", "0.0", "500", "1", "2017-01-01 12:00:00"], + ["0.0", "0.0", "850", "1", "2017-01-01 12:00:00"], + ], + ], + } assert decoder.get_coordinates() == coordinates def test_verticalprofile_values(self): @@ -235,3 +247,7 @@ def test_verticalprofile_values(self): "p": [[16452.35546875, 44122.98046875], [56452.35546875, 14122.98046875]], } assert decoder.get_values() == values + + def test_verticalprofile_to_xarray(self): + decoder = VerticalProfile.VerticalProfile(self.covjson) + dataset = decoder.to_xarray() diff --git a/tests/test_encoder.py b/tests/test_encoder.py index 27def22..e6011b7 100644 --- a/tests/test_encoder.py +++ b/tests/test_encoder.py @@ -6,6 +6,7 @@ import covjson.decoder.TimeSeries import random from datetime import datetime, timedelta +import xarray as xr def get_timestamps(start_dt, end_dt, delta): @@ -286,7 +287,12 @@ def test_add_coverage(self): value = {"t": [random.uniform(230, 270) for _ in range(0, len(timestamps))]} values.append(value) encoder.add_coverage(metadata, coord, value) - print(encoder.covjson) + # print(encoder.covjson) + + def test_from_xarray(self): + ds = xr.open_dataset("new_timeseries.nc") + encoder = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") + encoder.from_xarray(ds) """ From 2bc3ba3ed0bcf6a5e22e2eaf4a9bb216f08224bb Mon Sep 17 00:00:00 2001 From: awarde96 Date: Fri, 8 Dec 2023 13:40:18 +0100 Subject: [PATCH 24/58] Added timeseries mars request file, added from_xarray for vertical profile --- covjson/encoder/VerticalProfile.py | 103 +++++++++++++++++++++++++ ensemble-timeseries.req | 12 +++ tests/test_decoder_vertical_profile.py | 6 ++ 3 files changed, 121 insertions(+) create mode 100644 covjson/encoder/VerticalProfile.py create mode 100644 ensemble-timeseries.req diff --git a/covjson/encoder/VerticalProfile.py b/covjson/encoder/VerticalProfile.py new file mode 100644 index 0000000..7b14cdc --- /dev/null +++ b/covjson/encoder/VerticalProfile.py @@ -0,0 +1,103 @@ +from .encoder import Encoder +import xarray as xr +from datetime import timedelta, datetime +import datetime + + +class VerticalProfile(Encoder): + def __init__(self, type, domaintype): + super().__init__(type, domaintype) + + def add_coverage(self, mars_metadata, coords, values): + new_coverage = {} + new_coverage["mars:metadata"] = {} + new_coverage["type"] = "Coverage" + new_coverage["domain"] = {} + new_coverage["ranges"] = {} + self.add_mars_metadata(new_coverage, mars_metadata) + self.add_domain(new_coverage, coords) + self.add_range(new_coverage, values) + self.covjson["coverages"].append(new_coverage) + + def add_domain(self, coverage, coords): + coverage["domain"]["type"] = "Domain" + coverage["domain"]["axes"] = {} + coverage["domain"]["axes"]["x"] = {} + coverage["domain"]["axes"]["y"] = {} + coverage["domain"]["axes"]["z"] = {} + coverage["domain"]["axes"]["t"] = {} + coverage["domain"]["axes"]["x"]["values"] = coords["x"] + coverage["domain"]["axes"]["y"]["values"] = coords["y"] + coverage["domain"]["axes"]["z"]["values"] = coords["z"] + coverage["domain"]["axes"]["t"]["values"] = coords["t"] + + def add_range(self, coverage, values): + for parameter in self.parameters: + coverage["ranges"][parameter] = {} + coverage["ranges"][parameter]["type"] = "NdArray" + coverage["ranges"][parameter]["dataType"] = "float" + coverage["ranges"][parameter]["shape"] = [len(values[parameter])] + coverage["ranges"][parameter]["axisNames"] = ["z"] + coverage["ranges"][parameter]["values"] = values[ + parameter + ] # [values[parameter]] + + def add_mars_metadata(self, coverage, metadata): + coverage["mars:metadata"] = metadata + + def from_xarray(self, dataset): + for parameter in dataset.data_vars: + if parameter == "Temperature": + self.add_parameter( + "t", + { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + ) + elif parameter == "Pressure": + self.add_parameter( + "p", + { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + ) + self.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + for num in dataset["number"].values: + self.add_coverage( + { + # "date": fc_time.values.astype("M8[ms]") + # .astype("O") + # .strftime("%m/%d/%Y"), + "number": num, + "type": "forecast", + "step": 0, + }, + { + "x": list(dataset["x"].values), + "y": list(dataset["y"].values), + "z": list(dataset["z"].values), + "t": [str(x) for x in dataset["t"].values], + }, + { + "t": list(dataset["Temperature"].sel(number=num).values[0][0][0]), + "p": dataset["Pressure"].sel(number=num).values[0][0][0], + }, + ) + return self.covjson + + def from_polytope(self, result): + pass diff --git a/ensemble-timeseries.req b/ensemble-timeseries.req new file mode 100644 index 0000000..8e29822 --- /dev/null +++ b/ensemble-timeseries.req @@ -0,0 +1,12 @@ +retrieve, +class=od, +date=2023-12-05, +expver=1, +levtype=sfc, +number=1/2/3/4/5, +param=167.128, +step=0/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/22/23/24/25/26/27/28/29/30/31/32/33/34/35/36/37/38/39/40/41/42/43/44/45/46/47/48/49/50/51/52/53/54/55/56/57/58/59/60/61/62/63/64/65/66/67/68/69/70/71/72/73/74/75/76/77/78/79/80/81/82/83/84/85/86/87/88/89/90/93/96/99, +stream=enfo, +time=00:00:00, +type=pf, +target="output" diff --git a/tests/test_decoder_vertical_profile.py b/tests/test_decoder_vertical_profile.py index 97c5b2e..ed824a8 100644 --- a/tests/test_decoder_vertical_profile.py +++ b/tests/test_decoder_vertical_profile.py @@ -4,6 +4,7 @@ from covjson.decoder import decoder from covjson.decoder import VerticalProfile from covjson.decoder import TimeSeries +import covjson.encoder.VerticalProfile class TestDecoder: @@ -251,3 +252,8 @@ def test_verticalprofile_values(self): def test_verticalprofile_to_xarray(self): decoder = VerticalProfile.VerticalProfile(self.covjson) dataset = decoder.to_xarray() + encoder = covjson.encoder.VerticalProfile.VerticalProfile( + "CoverageCollection", "VerticalProfile" + ) + cov = encoder.from_xarray(dataset) + print(cov) From 5f491a636ef647644f96520cafe2d2683561baf0 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Mon, 11 Dec 2023 14:49:53 +0100 Subject: [PATCH 25/58] Renamed covjson directory to eccovjson --- .gitignore | 1 + covjson/__init__.py | 0 {covjson => eccovjson}/Coverage.py | 0 {covjson => eccovjson}/CoverageCollection.py | 0 eccovjson/__init__.py | 4 ++++ {covjson => eccovjson}/decoder/TimeSeries.py | 0 {covjson => eccovjson}/decoder/VerticalProfile.py | 0 {covjson => eccovjson}/decoder/decoder.py | 0 {covjson => eccovjson}/encoder/TimeSeries.py | 0 {covjson => eccovjson}/encoder/VerticalProfile.py | 0 {covjson => eccovjson}/encoder/encoder.py | 0 {covjson => eccovjson}/version.py | 0 12 files changed, 5 insertions(+) delete mode 100644 covjson/__init__.py rename {covjson => eccovjson}/Coverage.py (100%) rename {covjson => eccovjson}/CoverageCollection.py (100%) create mode 100644 eccovjson/__init__.py rename {covjson => eccovjson}/decoder/TimeSeries.py (100%) rename {covjson => eccovjson}/decoder/VerticalProfile.py (100%) rename {covjson => eccovjson}/decoder/decoder.py (100%) rename {covjson => eccovjson}/encoder/TimeSeries.py (100%) rename {covjson => eccovjson}/encoder/VerticalProfile.py (100%) rename {covjson => eccovjson}/encoder/encoder.py (100%) rename {covjson => eccovjson}/version.py (100%) diff --git a/.gitignore b/.gitignore index a8a2916..1d8525d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ SOURCES.txt top_level.txt *.ipynb *.covjson +*.egg-info diff --git a/covjson/__init__.py b/covjson/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/covjson/Coverage.py b/eccovjson/Coverage.py similarity index 100% rename from covjson/Coverage.py rename to eccovjson/Coverage.py diff --git a/covjson/CoverageCollection.py b/eccovjson/CoverageCollection.py similarity index 100% rename from covjson/CoverageCollection.py rename to eccovjson/CoverageCollection.py diff --git a/eccovjson/__init__.py b/eccovjson/__init__.py new file mode 100644 index 0000000..70f9bd7 --- /dev/null +++ b/eccovjson/__init__.py @@ -0,0 +1,4 @@ +import covjson.encoder.VerticalProfile +import covjson.encoder.TimeSeries +import covjson.decoder.VerticalProfile +import covjson.decoder.TimeSeries diff --git a/covjson/decoder/TimeSeries.py b/eccovjson/decoder/TimeSeries.py similarity index 100% rename from covjson/decoder/TimeSeries.py rename to eccovjson/decoder/TimeSeries.py diff --git a/covjson/decoder/VerticalProfile.py b/eccovjson/decoder/VerticalProfile.py similarity index 100% rename from covjson/decoder/VerticalProfile.py rename to eccovjson/decoder/VerticalProfile.py diff --git a/covjson/decoder/decoder.py b/eccovjson/decoder/decoder.py similarity index 100% rename from covjson/decoder/decoder.py rename to eccovjson/decoder/decoder.py diff --git a/covjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py similarity index 100% rename from covjson/encoder/TimeSeries.py rename to eccovjson/encoder/TimeSeries.py diff --git a/covjson/encoder/VerticalProfile.py b/eccovjson/encoder/VerticalProfile.py similarity index 100% rename from covjson/encoder/VerticalProfile.py rename to eccovjson/encoder/VerticalProfile.py diff --git a/covjson/encoder/encoder.py b/eccovjson/encoder/encoder.py similarity index 100% rename from covjson/encoder/encoder.py rename to eccovjson/encoder/encoder.py diff --git a/covjson/version.py b/eccovjson/version.py similarity index 100% rename from covjson/version.py rename to eccovjson/version.py From a5340d0184b9d944e84cbe38f03949da154527d4 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Mon, 11 Dec 2023 14:53:28 +0100 Subject: [PATCH 26/58] Updated imports with eccovjson from covjson --- eccovjson/__init__.py | 8 ++++---- eccovjson/decoder/decoder.py | 4 ++-- eccovjson/encoder/encoder.py | 4 ++-- tests/test_decoder_time_series.py | 6 +++--- tests/test_decoder_vertical_profile.py | 10 +++++----- tests/test_encoder.py | 6 +++--- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/eccovjson/__init__.py b/eccovjson/__init__.py index 70f9bd7..9b0b11c 100644 --- a/eccovjson/__init__.py +++ b/eccovjson/__init__.py @@ -1,4 +1,4 @@ -import covjson.encoder.VerticalProfile -import covjson.encoder.TimeSeries -import covjson.decoder.VerticalProfile -import covjson.decoder.TimeSeries +import eccovjson.encoder.VerticalProfile +import eccovjson.encoder.TimeSeries +import eccovjson.decoder.VerticalProfile +import eccovjson.decoder.TimeSeries diff --git a/eccovjson/decoder/decoder.py b/eccovjson/decoder/decoder.py index 03b75e2..01bb0ca 100644 --- a/eccovjson/decoder/decoder.py +++ b/eccovjson/decoder/decoder.py @@ -4,8 +4,8 @@ import datetime as dt import geopandas as gpd from abc import ABC, abstractmethod -from covjson.Coverage import Coverage -from covjson.CoverageCollection import CoverageCollection +from eccovjson.Coverage import Coverage +from eccovjson.CoverageCollection import CoverageCollection class Decoder(ABC): diff --git a/eccovjson/encoder/encoder.py b/eccovjson/encoder/encoder.py index 7ed3e6b..ed976fb 100644 --- a/eccovjson/encoder/encoder.py +++ b/eccovjson/encoder/encoder.py @@ -4,8 +4,8 @@ import datetime as dt import geopandas as gpd from abc import ABC, abstractmethod -from covjson.Coverage import Coverage -from covjson.CoverageCollection import CoverageCollection +from eccovjson.Coverage import Coverage +from eccovjson.CoverageCollection import CoverageCollection class Encoder(ABC): diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index 50b7e0e..d0b42f7 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -1,9 +1,9 @@ import pytest import json -from covjson.decoder import decoder -from covjson.decoder import VerticalProfile -from covjson.decoder import TimeSeries +from eccovjson.decoder import decoder +from eccovjson.decoder import VerticalProfile +from eccovjson.decoder import TimeSeries from earthkit import data import xarray as xr diff --git a/tests/test_decoder_vertical_profile.py b/tests/test_decoder_vertical_profile.py index ed824a8..7d93770 100644 --- a/tests/test_decoder_vertical_profile.py +++ b/tests/test_decoder_vertical_profile.py @@ -1,10 +1,10 @@ import pytest import json -from covjson.decoder import decoder -from covjson.decoder import VerticalProfile -from covjson.decoder import TimeSeries -import covjson.encoder.VerticalProfile +from eccovjson.decoder import decoder +from eccovjson.decoder import VerticalProfile +from eccovjson.decoder import TimeSeries +import eccovjson.encoder.VerticalProfile class TestDecoder: @@ -252,7 +252,7 @@ def test_verticalprofile_values(self): def test_verticalprofile_to_xarray(self): decoder = VerticalProfile.VerticalProfile(self.covjson) dataset = decoder.to_xarray() - encoder = covjson.encoder.VerticalProfile.VerticalProfile( + encoder = eccovjson.encoder.VerticalProfile.VerticalProfile( "CoverageCollection", "VerticalProfile" ) cov = encoder.from_xarray(dataset) diff --git a/tests/test_encoder.py b/tests/test_encoder.py index e6011b7..d789da7 100644 --- a/tests/test_encoder.py +++ b/tests/test_encoder.py @@ -1,9 +1,9 @@ import pytest import json -from covjson.encoder import encoder -from covjson.encoder import TimeSeries -import covjson.decoder.TimeSeries +from eccovjson.encoder import encoder +from eccovjson.encoder import TimeSeries +import eccovjson.decoder.TimeSeries import random from datetime import datetime, timedelta import xarray as xr From ca6b599cba4ffb05427fb93a577da7c735ba817e Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 12 Dec 2023 16:33:13 +0100 Subject: [PATCH 27/58] Parse ensemble number correctly even if contains /| --- eccovjson/encoder/TimeSeries.py | 5 ++++- setup.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index 454b30c..602c0e7 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -140,7 +140,10 @@ def from_polytope(self, result, request): ) coords["z"] = ["sfc"] - numbers = request["number"] + if "/" in request["number"]: + numbers = request["number"].split("/") + else: + numbers = request["number"] steps = request["step"] times = [] diff --git a/setup.py b/setup.py index 88c823e..ddbb22f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ __version__ = re.search( r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', - io.open("covjson/version.py", encoding="utf_8_sig").read(), + io.open("eccovjson/version.py", encoding="utf_8_sig").read(), ).group(1) From 0921b3f5a8f660aa59d9e291fc162971ed21c106 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 11 Jan 2024 13:31:51 +0100 Subject: [PATCH 28/58] Fixed problem where output of polytope with multiple paramas wasnt being converted to covjson correctly --- eccovjson/decoder/BoundingBox.py | 33 ++++++++++ eccovjson/encoder/TimeSeries.py | 76 +++++++++++++++++----- tests/test_decoder_bounding_box.py | 100 +++++++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 17 deletions(-) create mode 100644 eccovjson/decoder/BoundingBox.py create mode 100644 tests/test_decoder_bounding_box.py diff --git a/eccovjson/decoder/BoundingBox.py b/eccovjson/decoder/BoundingBox.py new file mode 100644 index 0000000..7146a52 --- /dev/null +++ b/eccovjson/decoder/BoundingBox.py @@ -0,0 +1,33 @@ +from .decoder import Decoder +import xarray as xr +import datetime as dt + + +class BoundingBox(Decoder): + def __init__(self, covjson): + super().__init__(covjson) + self.domains = self.get_domains() + self.ranges = self.get_ranges() + + def get_domains(self): + domains = [] + for coverage in self.coverage.coverages: + domains.append(coverage["domain"]) + return domains + + def get_ranges(self): + ranges = [] + for coverage in self.coverage.coverages: + ranges.append(coverage["ranges"]) + return ranges + + def get_values(self): + values = {} + for parameter in self.parameters: + values[parameter] = [] + for range in self.ranges: + values[parameter].append(range[parameter]["values"]) + # values[parameter] = [ + # value for sublist in values[parameter] for value in sublist + # ] + return values diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index 602c0e7..293a700 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -32,12 +32,12 @@ def add_domain(self, coverage, coords): coverage["domain"]["axes"]["t"]["values"] = coords["t"] def add_range(self, coverage, values): - for parameter in self.parameters: + for parameter in values.keys(): coverage["ranges"][parameter] = {} coverage["ranges"][parameter]["type"] = "NdArray" coverage["ranges"][parameter]["dataType"] = "float" coverage["ranges"][parameter]["shape"] = [len(values[parameter])] - coverage["ranges"][parameter]["axisNames"] = ["t"] + coverage["ranges"][parameter]["axisNames"] = [str(parameter)] coverage["ranges"][parameter]["values"] = values[ parameter ] # [values[parameter]] @@ -119,16 +119,56 @@ def from_polytope(self, result, request): elif key == "longitude": coords["y"] = [request[key]] - if request["param"] == "167": - self.add_parameter( - "t", - { - "type": "Parameter", - "description": "Temperature", - "unit": {"symbol": "K"}, - "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, - }, - ) + for param in request["param"].split("/"): + if param == "t": + self.add_parameter( + "t", + { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + ) + elif param == "tp": + self.add_parameter( + "tp", + { + "type": "Parameter", + "description": "Total Precipitation", + "unit": {"symbol": "m"}, + "observedProperty": { + "id": "tp", + "label": {"en": "Total Precipitation"}, + }, + }, + ) + elif param == "10u": + self.add_parameter( + "10u", + { + "type": "Parameter", + "description": "10 metre U wind component", + "unit": {"symbol": "ms-1"}, + "observedProperty": { + "id": "10u", + "label": {"en": "10 metre U wind component"}, + }, + }, + ) + elif param == "10v": + self.add_parameter( + "10v", + { + "type": "Parameter", + "description": "10 metre V wind component", + "unit": {"symbol": "ms-1"}, + "observedProperty": { + "id": "10v", + "label": {"en": "10 metre V wind component"}, + }, + }, + ) self.add_reference( { "coordinates": ["x", "y", "z"], @@ -163,9 +203,11 @@ def from_polytope(self, result, request): for num in numbers: mars_metadata["number"] = num new_metadata = mars_metadata.copy() - self.add_coverage(new_metadata, coords, {"t": values[start:end]}) - # vals.append(values[start:end]) - start = end - end += len(times) - + range_dict = {} + for param in request["param"].split("/"): + range_dict[param] = values[start:end] + # vals.append(values[start:end]) + start = end + end += len(times) + self.add_coverage(new_metadata, coords, range_dict) return self.covjson diff --git a/tests/test_decoder_bounding_box.py b/tests/test_decoder_bounding_box.py new file mode 100644 index 0000000..47f6453 --- /dev/null +++ b/tests/test_decoder_bounding_box.py @@ -0,0 +1,100 @@ +import pytest +import json + +from eccovjson.decoder import decoder +from eccovjson.decoder import VerticalProfile +from eccovjson.decoder import TimeSeries +from eccovjson.decoder import BoundingBox +from earthkit import data +import xarray as xr + + +class TestDecoder: + def setup_method(self, method): + self.covjson = { + "type": "CoverageCollection", + "domainType": "MultiPoint", + "coverages": [ + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T00:00:00Z"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 264.93115234375, + 263.83115234375, + 265.12313132266, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 9.93115234375, + 7.83115234375, + 14.12313132266, + ], + }, + }, + }, + ], + "referencing": [ + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ], + "parameters": { + "t": { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + "p": { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + }, + } + + def test_timeseries_type(self): + decoder = TimeSeries.TimeSeries(self.covjson) + assert decoder.type == "CoverageCollection" + + def test_timeseries_parameters(self): + decoder = TimeSeries.TimeSeries(self.covjson) + assert decoder.parameters == ["t", "p"] + + def test_timeseries_referencing(self): + decoder = TimeSeries.TimeSeries(self.covjson) + assert decoder.get_referencing() == ["x", "y", "z"] From 1c03e1bffdf08c962ec6b2490fd1339495df0f92 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 11 Jan 2024 16:04:10 +0100 Subject: [PATCH 29/58] Added most bounding box decoder functions --- eccovjson/decoder/BoundingBox.py | 76 +++++++++++ eccovjson/encoder/TimeSeries.py | 10 +- tests/test_decoder_bounding_box.py | 207 ++++++++++++++++++++++++++++- 3 files changed, 282 insertions(+), 11 deletions(-) diff --git a/eccovjson/decoder/BoundingBox.py b/eccovjson/decoder/BoundingBox.py index 7146a52..6afd05a 100644 --- a/eccovjson/decoder/BoundingBox.py +++ b/eccovjson/decoder/BoundingBox.py @@ -31,3 +31,79 @@ def get_values(self): # value for sublist in values[parameter] for value in sublist # ] return values + + def get_coordinates(self): + coord_dict = {} + for param in self.parameters: + coord_dict[param] = [] + # Get x,y,z,t coords and unpack t coords and match to x,y,z coords + for ind, domain in enumerate(self.domains): + t = domain["axes"]["t"]["values"][0] + num = self.mars_metadata[ind]["number"] + fct = self.mars_metadata[ind]["date"] + + for param in self.parameters: + coords = [] + for coord in domain["axes"]["composite"]["values"]: + x = coord[0] + y = coord[1] + z = coord[2] + coords.append([x, y, z, fct, t, num]) + coord_dict[param].append(coords) + return coord_dict + + def to_geopandas(self): + pass + + def to_xarray(self): + dims = ["x", "y", "z", "number", "t"] + dataarraydict = {} + + # Get coordinates + for parameter in self.parameters: + param_values = [[[self.get_values()[parameter]]]] + for ind, fc_time_vals in enumerate(self.get_values()[parameter]): + coords = self.get_coordinates()[parameter] + xs = [] + ys = [] + zs = [] + for coord in coords[ind]: + xs.append(coord[0]) + ys.append(coord[1]) + zs.append(coord[2]) + num = [int(coord[0][5]) for coord in coords] + coords_fc = coords[ind] + try: + t = [ + dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") + for coord in coords_fc + ] + except ValueError: + t = [ + dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%S") + for coord in coords_fc + ] + + param_coords = {"x": xs, "y": ys, "z": zs, "number": num, "t": t} + dataarray = xr.DataArray( + param_values, + dims=dims, + coords=param_coords, + name=parameter, + ) + + dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)[ + "unit" + ]["symbol"] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ + "description" + ] + dataarraydict[dataarray.attrs["long_name"]] = dataarray + + ds = xr.Dataset(dataarraydict) + for mars_metadata in self.mars_metadata[0]: + if mars_metadata != "date" and mars_metadata != "step": + ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata] + + return ds diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index 293a700..d07af54 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -77,6 +77,9 @@ def from_xarray(self, dataset): } ) for num in dataset["number"].values: + dv_dict = {} + for dv in dataset.data_vars: + dv_dict[dv] = list(dataset[dv].sel(number=num).values[0][0][0]) self.add_coverage( { # "date": fc_time.values.astype("M8[ms]") @@ -92,10 +95,9 @@ def from_xarray(self, dataset): "z": list(dataset["z"].values), "t": [str(x) for x in dataset["t"].values], }, - { - "t": list(dataset["Temperature"].sel(number=num).values[0][0][0]), - # "p": dataset["Pressure"].sel(fct=fc_time).values[0][0][0], - }, + # "t": list(dataset["Temperature"].sel(number=num).values[0][0][0]), + # "p": dataset["Pressure"].sel(fct=fc_time).values[0][0][0], + dv_dict, ) return self.covjson diff --git a/tests/test_decoder_bounding_box.py b/tests/test_decoder_bounding_box.py index 47f6453..59d273e 100644 --- a/tests/test_decoder_bounding_box.py +++ b/tests/test_decoder_bounding_box.py @@ -28,7 +28,7 @@ def setup_method(self, method): "domain": { "type": "Domain", "axes": { - "t": {"values": ["2017-01-01T00:00:00Z"]}, + "t": {"values": ["2017-01-01T00:00:00"]}, "composite": { "dataType": "tuple", "coordinates": ["x", "y", "z"], @@ -61,6 +61,52 @@ def setup_method(self, method): }, }, }, + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "1", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T01:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 266.93115234375, + 293.83115234375, + 165.12313132266, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 1.93115234375, + 22.83115234375, + 12.12313132266, + ], + }, + }, + }, ], "referencing": [ { @@ -87,14 +133,161 @@ def setup_method(self, method): }, } - def test_timeseries_type(self): - decoder = TimeSeries.TimeSeries(self.covjson) + def test_bounding_box_type(self): + decoder = BoundingBox.BoundingBox(self.covjson) assert decoder.type == "CoverageCollection" - def test_timeseries_parameters(self): - decoder = TimeSeries.TimeSeries(self.covjson) + def test_bounding_box_parameters(self): + decoder = BoundingBox.BoundingBox(self.covjson) assert decoder.parameters == ["t", "p"] - def test_timeseries_referencing(self): - decoder = TimeSeries.TimeSeries(self.covjson) + def test_bounding_box_referencing(self): + decoder = BoundingBox.BoundingBox(self.covjson) assert decoder.get_referencing() == ["x", "y", "z"] + + def test_bounding_box_get_parameter_metadata(self): + decoder = BoundingBox.BoundingBox(self.covjson) + assert decoder.get_parameter_metadata("t") == { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + } + assert decoder.get_parameter_metadata("p") == { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + } + + def test_bounding_box_mars_metadata(self): + decoder = BoundingBox.BoundingBox(self.covjson) + assert decoder.mars_metadata[0] == { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": "0", + } + assert decoder.mars_metadata[1] == { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "1", + "number": "0", + } + + def test_bounding_box_domains(self): + decoder = BoundingBox.BoundingBox(self.covjson) + domain1 = { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T00:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + } + assert decoder.domains[0] == domain1 + domain2 = { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T01:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + } + assert decoder.domains[1] == domain2 + + def test_bounding_box_ranges(self): + decoder = BoundingBox.BoundingBox(self.covjson) + range1 = { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [264.93115234375, 263.83115234375, 265.12313132266], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [9.93115234375, 7.83115234375, 14.12313132266], + }, + } + assert decoder.ranges[0] == range1 + range2 = { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [266.93115234375, 293.83115234375, 165.12313132266], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [1.93115234375, 22.83115234375, 12.12313132266], + }, + } + assert decoder.ranges[1] == range2 + + def test_bounding_box_get_coordinates(self): + decoder = BoundingBox.BoundingBox(self.covjson) + coordinates = { + "t": [ + [ + [1, 20, 1, "20170101", "2017-01-01T00:00:00", "0"], + [2, 21, 3, "20170101", "2017-01-01T00:00:00", "0"], + [3, 17, 7, "20170101", "2017-01-01T00:00:00", "0"], + ], + [ + [1, 20, 1, "20170101", "2017-01-01T01:00:00", "0"], + [2, 21, 3, "20170101", "2017-01-01T01:00:00", "0"], + [3, 17, 7, "20170101", "2017-01-01T01:00:00", "0"], + ], + ], + "p": [ + [ + [1, 20, 1, "20170101", "2017-01-01T00:00:00", "0"], + [2, 21, 3, "20170101", "2017-01-01T00:00:00", "0"], + [3, 17, 7, "20170101", "2017-01-01T00:00:00", "0"], + ], + [ + [1, 20, 1, "20170101", "2017-01-01T01:00:00", "0"], + [2, 21, 3, "20170101", "2017-01-01T01:00:00", "0"], + [3, 17, 7, "20170101", "2017-01-01T01:00:00", "0"], + ], + ], + } + assert decoder.get_coordinates() == coordinates + + def test_bounding_box_get_values(self): + decoder = BoundingBox.BoundingBox(self.covjson) + values = { + "t": [ + [264.93115234375, 263.83115234375, 265.12313132266], + [266.93115234375, 293.83115234375, 165.12313132266], + ], + "p": [ + [9.93115234375, 7.83115234375, 14.12313132266], + [1.93115234375, 22.83115234375, 12.12313132266], + ], + } + assert decoder.get_values() == values + + # def test_bounding_box_to_xarray(self): + # decoder = BoundingBox.BoundingBox(self.covjson) + # dataset = decoder.to_xarray() + # print(dataset) From 5ef0767cb4f4a8f144b0dc4f859862c2c68e3108 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Fri, 12 Jan 2024 11:50:39 +0100 Subject: [PATCH 30/58] Added init files to subfolders, created single point of entry for encoder and decoder in api.py --- eccovjson/__init__.py | 1 + eccovjson/api.py | 28 ++++++++++++++++++++++++++ eccovjson/decoder/__init__.py | 0 eccovjson/encoder/__init__.py | 0 tests/test_decoder_time_series.py | 21 ++++++++++--------- tests/test_decoder_vertical_profile.py | 19 ++++++++--------- tests/test_encoder.py | 22 ++++++++++++++------ 7 files changed, 67 insertions(+), 24 deletions(-) create mode 100644 eccovjson/api.py create mode 100644 eccovjson/decoder/__init__.py create mode 100644 eccovjson/encoder/__init__.py diff --git a/eccovjson/__init__.py b/eccovjson/__init__.py index 9b0b11c..d7f5c94 100644 --- a/eccovjson/__init__.py +++ b/eccovjson/__init__.py @@ -2,3 +2,4 @@ import eccovjson.encoder.TimeSeries import eccovjson.decoder.VerticalProfile import eccovjson.decoder.TimeSeries +import eccovjson.api diff --git a/eccovjson/api.py b/eccovjson/api.py new file mode 100644 index 0000000..170116d --- /dev/null +++ b/eccovjson/api.py @@ -0,0 +1,28 @@ +import json + +import eccovjson.decoder.TimeSeries +import eccovjson.decoder.VerticalProfile +import eccovjson.encoder.TimeSeries +import eccovjson.encoder.VerticalProfile + + +class Eccovjson: + def __init__(self): + # Initialise polytope + pass + + def encode(self, type, domaintype, requesttype): + if requesttype == "timeseries": + encoder_obj = eccovjson.encoder.TimeSeries.TimeSeries(type, domaintype) + elif requesttype == "VerticalProfile": + encoder_obj = eccovjson.encoder.VerticalProfile.VerticalProfile( + type, domaintype + ) + return encoder_obj + + def decode(self, covjson, requesttype): + if requesttype == "timeseries": + decoder_obj = eccovjson.decoder.TimeSeries.TimeSeries(covjson) + elif requesttype == "VerticalProfile": + decoder_obj = eccovjson.decoder.VerticalProfile.VerticalProfile(covjson) + return decoder_obj diff --git a/eccovjson/decoder/__init__.py b/eccovjson/decoder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eccovjson/encoder/__init__.py b/eccovjson/encoder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index d0b42f7..ea4c5ac 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -4,6 +4,7 @@ from eccovjson.decoder import decoder from eccovjson.decoder import VerticalProfile from eccovjson.decoder import TimeSeries +from eccovjson.api import Eccovjson from earthkit import data import xarray as xr @@ -141,19 +142,21 @@ def setup_method(self, method): } def test_timeseries_type(self): - decoder = TimeSeries.TimeSeries(self.covjson) + # decoder = TimeSeries.TimeSeries(self.covjson) + # assert decoder.type == "CoverageCollection" + decoder = Eccovjson().decode(self.covjson, "timeseries") assert decoder.type == "CoverageCollection" def test_timeseries_parameters(self): - decoder = TimeSeries.TimeSeries(self.covjson) + decoder = Eccovjson().decode(self.covjson, "timeseries") assert decoder.parameters == ["t", "p"] def test_timeseries_referencing(self): - decoder = TimeSeries.TimeSeries(self.covjson) + decoder = Eccovjson().decode(self.covjson, "timeseries") assert decoder.get_referencing() == ["x", "y", "z"] def test_timeseries_mars_metadata(self): - decoder = TimeSeries.TimeSeries(self.covjson) + decoder = Eccovjson().decode(self.covjson, "timeseries") metadata1 = { "class": "od", "stream": "oper", @@ -173,7 +176,7 @@ def test_timeseries_mars_metadata(self): assert decoder.mars_metadata == [metadata1, metadata2] def test_timeseries_domains(self): - decoder = TimeSeries.TimeSeries(self.covjson) + decoder = Eccovjson().decode(self.covjson, "timeseries") domain1 = { "type": "Domain", "axes": { @@ -208,7 +211,7 @@ def test_timeseries_domains(self): assert decoder.domains[1] == domain2 def test_timeseries_ranges(self): - decoder = TimeSeries.TimeSeries(self.covjson) + decoder = Eccovjson().decode(self.covjson, "timeseries") range1 = { "t": { "type": "NdArray", @@ -245,7 +248,7 @@ def test_timeseries_ranges(self): assert decoder.ranges[1] == range2 def test_timeseries_values(self): - decoder = TimeSeries.TimeSeries(self.covjson) + decoder = Eccovjson().decode(self.covjson, "timeseries") values = { "t": [ [264.93115234375, 263.83115234375, 265.12313132266], @@ -259,7 +262,7 @@ def test_timeseries_values(self): assert decoder.get_values() == values def test_timeseries_coordinates(self): - decoder = TimeSeries.TimeSeries(self.covjson) + decoder = Eccovjson().decode(self.covjson, "timeseries") coordinates = { "t": [ [ @@ -290,7 +293,7 @@ def test_timeseries_coordinates(self): assert decoder.get_coordinates() == coordinates def test_timeseries_to_xarray(self): - decoder = TimeSeries.TimeSeries(self.covjson) + decoder = Eccovjson().decode(self.covjson, "timeseries") ds = decoder.to_xarray() # print(ds) # print(ds["Temperature"]) diff --git a/tests/test_decoder_vertical_profile.py b/tests/test_decoder_vertical_profile.py index 7d93770..6a63a24 100644 --- a/tests/test_decoder_vertical_profile.py +++ b/tests/test_decoder_vertical_profile.py @@ -5,6 +5,7 @@ from eccovjson.decoder import VerticalProfile from eccovjson.decoder import TimeSeries import eccovjson.encoder.VerticalProfile +from eccovjson.api import Eccovjson class TestDecoder: @@ -121,19 +122,19 @@ def setup_method(self, method): } def test_verticalprofile_type(self): - decoder = VerticalProfile.VerticalProfile(self.covjson) + decoder = Eccovjson().decode(self.covjson, "VerticalProfile") assert decoder.type == "CoverageCollection" def test_verticalprofile_parameters(self): - decoder = VerticalProfile.VerticalProfile(self.covjson) + decoder = Eccovjson().decode(self.covjson, "VerticalProfile") assert decoder.parameters == ["t", "p"] def test_verticalprofile_referencing(self): - decoder = VerticalProfile.VerticalProfile(self.covjson) + decoder = Eccovjson().decode(self.covjson, "VerticalProfile") assert decoder.get_referencing() == ["x", "y", "z"] def test_verticalprofile_mars_metadata(self): - decoder = VerticalProfile.VerticalProfile(self.covjson) + decoder = Eccovjson().decode(self.covjson, "VerticalProfile") metadata1 = { "class": "ea", "date": "2017-01-01 12:00:00", @@ -154,7 +155,7 @@ def test_verticalprofile_mars_metadata(self): assert decoder.mars_metadata[1] == metadata2 def test_verticalprofile_domains(self): - decoder = VerticalProfile.VerticalProfile(self.covjson) + decoder = Eccovjson().decode(self.covjson, "VerticalProfile") domain1 = { "type": "Domain", "domainType": "VerticalProfile", @@ -179,7 +180,7 @@ def test_verticalprofile_domains(self): assert decoder.domains[1] == domain2 def test_verticalprofile_ranges(self): - decoder = VerticalProfile.VerticalProfile(self.covjson) + decoder = Eccovjson().decode(self.covjson, "VerticalProfile") range1 = { "t": { "type": "NdArray", @@ -216,7 +217,7 @@ def test_verticalprofile_ranges(self): assert decoder.ranges[1] == range2 def test_verticalprofile_coordinates(self): - decoder = VerticalProfile.VerticalProfile(self.covjson) + decoder = Eccovjson().decode(self.covjson, "VerticalProfile") coordinates = { "t": [ [ @@ -242,7 +243,7 @@ def test_verticalprofile_coordinates(self): assert decoder.get_coordinates() == coordinates def test_verticalprofile_values(self): - decoder = VerticalProfile.VerticalProfile(self.covjson) + decoder = Eccovjson().decode(self.covjson, "VerticalProfile") values = { "t": [[57517.77734375, 14814.95703125], [57452.35546875, 14822.98046875]], "p": [[16452.35546875, 44122.98046875], [56452.35546875, 14122.98046875]], @@ -250,7 +251,7 @@ def test_verticalprofile_values(self): assert decoder.get_values() == values def test_verticalprofile_to_xarray(self): - decoder = VerticalProfile.VerticalProfile(self.covjson) + decoder = Eccovjson().decode(self.covjson, "VerticalProfile") dataset = decoder.to_xarray() encoder = eccovjson.encoder.VerticalProfile.VerticalProfile( "CoverageCollection", "VerticalProfile" diff --git a/tests/test_encoder.py b/tests/test_encoder.py index d789da7..e3141b6 100644 --- a/tests/test_encoder.py +++ b/tests/test_encoder.py @@ -4,6 +4,7 @@ from eccovjson.encoder import encoder from eccovjson.encoder import TimeSeries import eccovjson.decoder.TimeSeries +from eccovjson.api import Eccovjson import random from datetime import datetime, timedelta import xarray as xr @@ -152,11 +153,16 @@ def setup_method(self, method): } def test_CoverageCollection(self): - encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") + # encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") + encoder_obj = Eccovjson().encode( + "CoverageCollection", "PointSeries", "timeseries" + ) assert encoder_obj.type == "CoverageCollection" def test_standard_Coverage(self): - encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") + encoder_obj = Eccovjson().encode( + "CoverageCollection", "PointSeries", "timeseries" + ) covjson = { "type": "CoverageCollection", "domainType": "PointSeries", @@ -168,7 +174,9 @@ def test_standard_Coverage(self): assert encoder_obj.covjson == covjson def test_add_parameter(self): - encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") + encoder_obj = Eccovjson().encode( + "CoverageCollection", "PointSeries", "timeseries" + ) encoder_obj.add_parameter( "t", { @@ -211,7 +219,9 @@ def test_add_parameter(self): assert encoder_obj.covjson == covjson def test_add_reference(self): - encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") + encoder_obj = Eccovjson().encode( + "CoverageCollection", "PointSeries", "timeseries" + ) encoder_obj.add_reference( { "coordinates": ["x", "y", "z"], @@ -240,7 +250,7 @@ def test_add_reference(self): assert encoder_obj.covjson == covjson def test_add_coverage(self): - encoder = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") + encoder = Eccovjson().encode("CoverageCollection", "PointSeries", "timeseries") encoder.add_parameter( "t", { @@ -291,7 +301,7 @@ def test_add_coverage(self): def test_from_xarray(self): ds = xr.open_dataset("new_timeseries.nc") - encoder = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") + encoder = Eccovjson().encode("CoverageCollection", "PointSeries", "timeseries") encoder.from_xarray(ds) From 09dd29d9cd2df6bc532ae447a52085e89af927a5 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Fri, 12 Jan 2024 13:53:13 +0100 Subject: [PATCH 31/58] Changed interface so now feature type is inferred and correct object called, timestamp now also inferred and converted to string --- eccovjson/api.py | 47 +++++++++++++++++++------- eccovjson/encoder/TimeSeries.py | 7 ++-- tests/test_decoder_time_series.py | 18 +++++----- tests/test_decoder_vertical_profile.py | 18 +++++----- tests/test_encoder.py | 20 ++++------- 5 files changed, 63 insertions(+), 47 deletions(-) diff --git a/eccovjson/api.py b/eccovjson/api.py index 170116d..59de716 100644 --- a/eccovjson/api.py +++ b/eccovjson/api.py @@ -5,24 +5,45 @@ import eccovjson.encoder.TimeSeries import eccovjson.encoder.VerticalProfile +features_encoder = { + "pointseries": eccovjson.encoder.TimeSeries.TimeSeries, + "verticalprofile": eccovjson.encoder.VerticalProfile.VerticalProfile, +} +features_decoder = { + "pointseries": eccovjson.decoder.TimeSeries.TimeSeries, + "verticalprofile": eccovjson.decoder.VerticalProfile.VerticalProfile, +} + class Eccovjson: def __init__(self): - # Initialise polytope pass - def encode(self, type, domaintype, requesttype): - if requesttype == "timeseries": - encoder_obj = eccovjson.encoder.TimeSeries.TimeSeries(type, domaintype) - elif requesttype == "VerticalProfile": - encoder_obj = eccovjson.encoder.VerticalProfile.VerticalProfile( - type, domaintype - ) - return encoder_obj + def encode(self, type, domaintype): + if domaintype == "timeseries": + domaintype = "PointSeries" + feature = self._feature_factory(domaintype.lower(), "encoder") + # if requesttype == "timeseries": + # encoder_obj = eccovjson.encoder.TimeSeries.TimeSeries(type, domaintype) + # elif requesttype == "VerticalProfile": + # encoder_obj = eccovjson.encoder.VerticalProfile.VerticalProfile( + # type, domaintype + # ) + return feature(type, domaintype) def decode(self, covjson, requesttype): if requesttype == "timeseries": - decoder_obj = eccovjson.decoder.TimeSeries.TimeSeries(covjson) - elif requesttype == "VerticalProfile": - decoder_obj = eccovjson.decoder.VerticalProfile.VerticalProfile(covjson) - return decoder_obj + requesttype = "PointSeries" + feature = self._feature_factory(requesttype.lower(), "decoder") + # if requesttype == "timeseries": + # decoder_obj = eccovjson.decoder.TimeSeries.TimeSeries(covjson) + # elif requesttype == "VerticalProfile": + # decoder_obj = eccovjson.decoder.VerticalProfile.VerticalProfile(covjson) + return feature(covjson) + + def _feature_factory(self, feature_type, encoder_decoder): + if encoder_decoder == "encoder": + features = features_encoder + elif encoder_decoder == "decoder": + features = features_decoder + return features[feature_type] diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index d07af54..6a43901 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -2,6 +2,8 @@ import xarray as xr from datetime import timedelta, datetime import datetime +import dateinfer +import pandas as pd class TimeSeries(Encoder): @@ -190,9 +192,10 @@ def from_polytope(self, result, request): times = [] date_format = "%Y%m%dT%H%M%S" - start_time = datetime.datetime.strptime(mars_metadata["date"], date_format) + date = pd.Timestamp(mars_metadata["date"]).strftime(date_format) + start_time = datetime.datetime.strptime(date, date_format) for step in steps: - # add current date to list by converting it to iso format + # add current date to list by converting it to iso format stamp = start_time + timedelta(hours=step) times.append(stamp.isoformat()) # increment start date by timedelta diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index ea4c5ac..ce5d45d 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -144,19 +144,19 @@ def setup_method(self, method): def test_timeseries_type(self): # decoder = TimeSeries.TimeSeries(self.covjson) # assert decoder.type == "CoverageCollection" - decoder = Eccovjson().decode(self.covjson, "timeseries") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) assert decoder.type == "CoverageCollection" def test_timeseries_parameters(self): - decoder = Eccovjson().decode(self.covjson, "timeseries") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) assert decoder.parameters == ["t", "p"] def test_timeseries_referencing(self): - decoder = Eccovjson().decode(self.covjson, "timeseries") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) assert decoder.get_referencing() == ["x", "y", "z"] def test_timeseries_mars_metadata(self): - decoder = Eccovjson().decode(self.covjson, "timeseries") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) metadata1 = { "class": "od", "stream": "oper", @@ -176,7 +176,7 @@ def test_timeseries_mars_metadata(self): assert decoder.mars_metadata == [metadata1, metadata2] def test_timeseries_domains(self): - decoder = Eccovjson().decode(self.covjson, "timeseries") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) domain1 = { "type": "Domain", "axes": { @@ -211,7 +211,7 @@ def test_timeseries_domains(self): assert decoder.domains[1] == domain2 def test_timeseries_ranges(self): - decoder = Eccovjson().decode(self.covjson, "timeseries") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) range1 = { "t": { "type": "NdArray", @@ -248,7 +248,7 @@ def test_timeseries_ranges(self): assert decoder.ranges[1] == range2 def test_timeseries_values(self): - decoder = Eccovjson().decode(self.covjson, "timeseries") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) values = { "t": [ [264.93115234375, 263.83115234375, 265.12313132266], @@ -262,7 +262,7 @@ def test_timeseries_values(self): assert decoder.get_values() == values def test_timeseries_coordinates(self): - decoder = Eccovjson().decode(self.covjson, "timeseries") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) coordinates = { "t": [ [ @@ -293,7 +293,7 @@ def test_timeseries_coordinates(self): assert decoder.get_coordinates() == coordinates def test_timeseries_to_xarray(self): - decoder = Eccovjson().decode(self.covjson, "timeseries") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) ds = decoder.to_xarray() # print(ds) # print(ds["Temperature"]) diff --git a/tests/test_decoder_vertical_profile.py b/tests/test_decoder_vertical_profile.py index 6a63a24..f6d73b9 100644 --- a/tests/test_decoder_vertical_profile.py +++ b/tests/test_decoder_vertical_profile.py @@ -122,19 +122,19 @@ def setup_method(self, method): } def test_verticalprofile_type(self): - decoder = Eccovjson().decode(self.covjson, "VerticalProfile") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) assert decoder.type == "CoverageCollection" def test_verticalprofile_parameters(self): - decoder = Eccovjson().decode(self.covjson, "VerticalProfile") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) assert decoder.parameters == ["t", "p"] def test_verticalprofile_referencing(self): - decoder = Eccovjson().decode(self.covjson, "VerticalProfile") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) assert decoder.get_referencing() == ["x", "y", "z"] def test_verticalprofile_mars_metadata(self): - decoder = Eccovjson().decode(self.covjson, "VerticalProfile") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) metadata1 = { "class": "ea", "date": "2017-01-01 12:00:00", @@ -155,7 +155,7 @@ def test_verticalprofile_mars_metadata(self): assert decoder.mars_metadata[1] == metadata2 def test_verticalprofile_domains(self): - decoder = Eccovjson().decode(self.covjson, "VerticalProfile") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) domain1 = { "type": "Domain", "domainType": "VerticalProfile", @@ -180,7 +180,7 @@ def test_verticalprofile_domains(self): assert decoder.domains[1] == domain2 def test_verticalprofile_ranges(self): - decoder = Eccovjson().decode(self.covjson, "VerticalProfile") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) range1 = { "t": { "type": "NdArray", @@ -217,7 +217,7 @@ def test_verticalprofile_ranges(self): assert decoder.ranges[1] == range2 def test_verticalprofile_coordinates(self): - decoder = Eccovjson().decode(self.covjson, "VerticalProfile") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) coordinates = { "t": [ [ @@ -243,7 +243,7 @@ def test_verticalprofile_coordinates(self): assert decoder.get_coordinates() == coordinates def test_verticalprofile_values(self): - decoder = Eccovjson().decode(self.covjson, "VerticalProfile") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) values = { "t": [[57517.77734375, 14814.95703125], [57452.35546875, 14822.98046875]], "p": [[16452.35546875, 44122.98046875], [56452.35546875, 14122.98046875]], @@ -251,7 +251,7 @@ def test_verticalprofile_values(self): assert decoder.get_values() == values def test_verticalprofile_to_xarray(self): - decoder = Eccovjson().decode(self.covjson, "VerticalProfile") + decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) dataset = decoder.to_xarray() encoder = eccovjson.encoder.VerticalProfile.VerticalProfile( "CoverageCollection", "VerticalProfile" diff --git a/tests/test_encoder.py b/tests/test_encoder.py index e3141b6..697efdb 100644 --- a/tests/test_encoder.py +++ b/tests/test_encoder.py @@ -154,15 +154,11 @@ def setup_method(self, method): def test_CoverageCollection(self): # encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") - encoder_obj = Eccovjson().encode( - "CoverageCollection", "PointSeries", "timeseries" - ) + encoder_obj = Eccovjson().encode("CoverageCollection", "PointSeries") assert encoder_obj.type == "CoverageCollection" def test_standard_Coverage(self): - encoder_obj = Eccovjson().encode( - "CoverageCollection", "PointSeries", "timeseries" - ) + encoder_obj = Eccovjson().encode("CoverageCollection", "PointSeries") covjson = { "type": "CoverageCollection", "domainType": "PointSeries", @@ -174,9 +170,7 @@ def test_standard_Coverage(self): assert encoder_obj.covjson == covjson def test_add_parameter(self): - encoder_obj = Eccovjson().encode( - "CoverageCollection", "PointSeries", "timeseries" - ) + encoder_obj = Eccovjson().encode("CoverageCollection", "PointSeries") encoder_obj.add_parameter( "t", { @@ -219,9 +213,7 @@ def test_add_parameter(self): assert encoder_obj.covjson == covjson def test_add_reference(self): - encoder_obj = Eccovjson().encode( - "CoverageCollection", "PointSeries", "timeseries" - ) + encoder_obj = Eccovjson().encode("CoverageCollection", "PointSeries") encoder_obj.add_reference( { "coordinates": ["x", "y", "z"], @@ -250,7 +242,7 @@ def test_add_reference(self): assert encoder_obj.covjson == covjson def test_add_coverage(self): - encoder = Eccovjson().encode("CoverageCollection", "PointSeries", "timeseries") + encoder = Eccovjson().encode("CoverageCollection", "PointSeries") encoder.add_parameter( "t", { @@ -301,7 +293,7 @@ def test_add_coverage(self): def test_from_xarray(self): ds = xr.open_dataset("new_timeseries.nc") - encoder = Eccovjson().encode("CoverageCollection", "PointSeries", "timeseries") + encoder = Eccovjson().encode("CoverageCollection", "PointSeries") encoder.from_xarray(ds) From 70d56d4e6822ecfb20d94c9d3e410c063d156492 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Mon, 15 Jan 2024 09:52:57 +0100 Subject: [PATCH 32/58] Changed interface to eccovjson.decode so no need to provide domain type --- eccovjson/api.py | 13 ++----------- tests/test_decoder_time_series.py | 18 +++++++++--------- tests/test_decoder_vertical_profile.py | 18 +++++++++--------- 3 files changed, 20 insertions(+), 29 deletions(-) diff --git a/eccovjson/api.py b/eccovjson/api.py index 59de716..9d8e4d1 100644 --- a/eccovjson/api.py +++ b/eccovjson/api.py @@ -23,22 +23,13 @@ def encode(self, type, domaintype): if domaintype == "timeseries": domaintype = "PointSeries" feature = self._feature_factory(domaintype.lower(), "encoder") - # if requesttype == "timeseries": - # encoder_obj = eccovjson.encoder.TimeSeries.TimeSeries(type, domaintype) - # elif requesttype == "VerticalProfile": - # encoder_obj = eccovjson.encoder.VerticalProfile.VerticalProfile( - # type, domaintype - # ) return feature(type, domaintype) - def decode(self, covjson, requesttype): + def decode(self, covjson): + requesttype = covjson["domainType"] if requesttype == "timeseries": requesttype = "PointSeries" feature = self._feature_factory(requesttype.lower(), "decoder") - # if requesttype == "timeseries": - # decoder_obj = eccovjson.decoder.TimeSeries.TimeSeries(covjson) - # elif requesttype == "VerticalProfile": - # decoder_obj = eccovjson.decoder.VerticalProfile.VerticalProfile(covjson) return feature(covjson) def _feature_factory(self, feature_type, encoder_decoder): diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index ce5d45d..0bbe230 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -144,19 +144,19 @@ def setup_method(self, method): def test_timeseries_type(self): # decoder = TimeSeries.TimeSeries(self.covjson) # assert decoder.type == "CoverageCollection" - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) assert decoder.type == "CoverageCollection" def test_timeseries_parameters(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) assert decoder.parameters == ["t", "p"] def test_timeseries_referencing(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) assert decoder.get_referencing() == ["x", "y", "z"] def test_timeseries_mars_metadata(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) metadata1 = { "class": "od", "stream": "oper", @@ -176,7 +176,7 @@ def test_timeseries_mars_metadata(self): assert decoder.mars_metadata == [metadata1, metadata2] def test_timeseries_domains(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) domain1 = { "type": "Domain", "axes": { @@ -211,7 +211,7 @@ def test_timeseries_domains(self): assert decoder.domains[1] == domain2 def test_timeseries_ranges(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) range1 = { "t": { "type": "NdArray", @@ -248,7 +248,7 @@ def test_timeseries_ranges(self): assert decoder.ranges[1] == range2 def test_timeseries_values(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) values = { "t": [ [264.93115234375, 263.83115234375, 265.12313132266], @@ -262,7 +262,7 @@ def test_timeseries_values(self): assert decoder.get_values() == values def test_timeseries_coordinates(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) coordinates = { "t": [ [ @@ -293,7 +293,7 @@ def test_timeseries_coordinates(self): assert decoder.get_coordinates() == coordinates def test_timeseries_to_xarray(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) ds = decoder.to_xarray() # print(ds) # print(ds["Temperature"]) diff --git a/tests/test_decoder_vertical_profile.py b/tests/test_decoder_vertical_profile.py index f6d73b9..43594f6 100644 --- a/tests/test_decoder_vertical_profile.py +++ b/tests/test_decoder_vertical_profile.py @@ -122,19 +122,19 @@ def setup_method(self, method): } def test_verticalprofile_type(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) assert decoder.type == "CoverageCollection" def test_verticalprofile_parameters(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) assert decoder.parameters == ["t", "p"] def test_verticalprofile_referencing(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) assert decoder.get_referencing() == ["x", "y", "z"] def test_verticalprofile_mars_metadata(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) metadata1 = { "class": "ea", "date": "2017-01-01 12:00:00", @@ -155,7 +155,7 @@ def test_verticalprofile_mars_metadata(self): assert decoder.mars_metadata[1] == metadata2 def test_verticalprofile_domains(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) domain1 = { "type": "Domain", "domainType": "VerticalProfile", @@ -180,7 +180,7 @@ def test_verticalprofile_domains(self): assert decoder.domains[1] == domain2 def test_verticalprofile_ranges(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) range1 = { "t": { "type": "NdArray", @@ -217,7 +217,7 @@ def test_verticalprofile_ranges(self): assert decoder.ranges[1] == range2 def test_verticalprofile_coordinates(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) coordinates = { "t": [ [ @@ -243,7 +243,7 @@ def test_verticalprofile_coordinates(self): assert decoder.get_coordinates() == coordinates def test_verticalprofile_values(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) values = { "t": [[57517.77734375, 14814.95703125], [57452.35546875, 14822.98046875]], "p": [[16452.35546875, 44122.98046875], [56452.35546875, 14122.98046875]], @@ -251,7 +251,7 @@ def test_verticalprofile_values(self): assert decoder.get_values() == values def test_verticalprofile_to_xarray(self): - decoder = Eccovjson().decode(self.covjson, self.covjson["domainType"]) + decoder = Eccovjson().decode(self.covjson) dataset = decoder.to_xarray() encoder = eccovjson.encoder.VerticalProfile.VerticalProfile( "CoverageCollection", "VerticalProfile" From b41246a3b2226e7586403f10475584b569752164 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Mon, 15 Jan 2024 10:00:29 +0100 Subject: [PATCH 33/58] Encoder can now accept param or param id --- eccovjson/encoder/TimeSeries.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index 6a43901..9b9c7e4 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -124,7 +124,7 @@ def from_polytope(self, result, request): coords["y"] = [request[key]] for param in request["param"].split("/"): - if param == "t": + if param == "t" or param == "167": self.add_parameter( "t", { @@ -134,7 +134,7 @@ def from_polytope(self, result, request): "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, }, ) - elif param == "tp": + elif param == "tp" or param == "228": self.add_parameter( "tp", { @@ -147,7 +147,7 @@ def from_polytope(self, result, request): }, }, ) - elif param == "10u": + elif param == "10u" or param == "165": self.add_parameter( "10u", { @@ -160,7 +160,7 @@ def from_polytope(self, result, request): }, }, ) - elif param == "10v": + elif param == "10v" or param == "166": self.add_parameter( "10v", { From 1b640d902c5980c09c052a0e4213391557ce314c Mon Sep 17 00:00:00 2001 From: awarde96 Date: Mon, 15 Jan 2024 15:37:34 +0100 Subject: [PATCH 34/58] Updated encoder so param names are shown in covjson not ids --- eccovjson/encoder/BoundingBox.py | 49 +++++++++++++++++++ eccovjson/encoder/TimeSeries.py | 13 ++--- eccovjson/encoder/encoder.py | 14 ++++++ ...encoder.py => test_encoder_time_series.py} | 2 +- 4 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 eccovjson/encoder/BoundingBox.py rename tests/{test_encoder.py => test_encoder_time_series.py} (99%) diff --git a/eccovjson/encoder/BoundingBox.py b/eccovjson/encoder/BoundingBox.py new file mode 100644 index 0000000..42b8dfb --- /dev/null +++ b/eccovjson/encoder/BoundingBox.py @@ -0,0 +1,49 @@ +from .encoder import Encoder +import xarray as xr +from datetime import timedelta, datetime +import datetime +import dateinfer +import pandas as pd + + +class BoundingBox(Encoder): + def __init__(self, type, domaintype): + super().__init__(type, domaintype) + + def add_coverage(self, mars_metadata, coords, values): + new_coverage = {} + new_coverage["mars:metadata"] = {} + new_coverage["type"] = "Coverage" + new_coverage["domain"] = {} + new_coverage["ranges"] = {} + self.add_mars_metadata(new_coverage, mars_metadata) + self.add_domain(new_coverage, coords) + self.add_range(new_coverage, values) + self.covjson["coverages"].append(new_coverage) + + def add_domain(self, coverage, coords): + coverage["domain"]["type"] = "Domain" + coverage["domain"]["axes"] = {} + coverage["domain"]["axes"]["x"] = {} + coverage["domain"]["axes"]["y"] = {} + coverage["domain"]["axes"]["z"] = {} + coverage["domain"]["axes"]["t"] = {} + coverage["domain"]["axes"]["x"]["values"] = coords["x"] + coverage["domain"]["axes"]["y"]["values"] = coords["y"] + coverage["domain"]["axes"]["z"]["values"] = coords["z"] + coverage["domain"]["axes"]["t"]["values"] = coords["t"] + + def add_range(self, coverage, values): + for parameter in values.keys(): + param = self.convert_param_id_to_param(parameter) + coverage["ranges"][param] = {} + coverage["ranges"][param]["type"] = "NdArray" + coverage["ranges"][param]["dataType"] = "float" + coverage["ranges"][param]["shape"] = [len(values[parameter])] + coverage["ranges"][param]["axisNames"] = [str(param)] + coverage["ranges"][param]["values"] = values[ + parameter + ] # [values[parameter]] + + def add_mars_metadata(self, coverage, metadata): + coverage["mars:metadata"] = metadata diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index 9b9c7e4..3d69667 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -35,12 +35,13 @@ def add_domain(self, coverage, coords): def add_range(self, coverage, values): for parameter in values.keys(): - coverage["ranges"][parameter] = {} - coverage["ranges"][parameter]["type"] = "NdArray" - coverage["ranges"][parameter]["dataType"] = "float" - coverage["ranges"][parameter]["shape"] = [len(values[parameter])] - coverage["ranges"][parameter]["axisNames"] = [str(parameter)] - coverage["ranges"][parameter]["values"] = values[ + param = self.convert_param_id_to_param(parameter) + coverage["ranges"][param] = {} + coverage["ranges"][param]["type"] = "NdArray" + coverage["ranges"][param]["dataType"] = "float" + coverage["ranges"][param]["shape"] = [len(values[parameter])] + coverage["ranges"][param]["axisNames"] = [str(param)] + coverage["ranges"][param]["values"] = values[ parameter ] # [values[parameter]] diff --git a/eccovjson/encoder/encoder.py b/eccovjson/encoder/encoder.py index ed976fb..b941f42 100644 --- a/eccovjson/encoder/encoder.py +++ b/eccovjson/encoder/encoder.py @@ -37,6 +37,20 @@ def add_parameter(self, parameter, metadata): def add_reference(self, reference): self.covjson["referencing"].append(reference) + def convert_param_id_to_param(self, paramid): + try: + param = int(paramid) + except: + return paramid + if param == 165: + return "10u" + elif param == 166: + return "10v" + elif param == 167: + return "t" + elif param == 228: + return "tp" + @abstractmethod def add_coverage(self, mars_metadata, coords, values): pass diff --git a/tests/test_encoder.py b/tests/test_encoder_time_series.py similarity index 99% rename from tests/test_encoder.py rename to tests/test_encoder_time_series.py index 697efdb..d075de6 100644 --- a/tests/test_encoder.py +++ b/tests/test_encoder_time_series.py @@ -20,7 +20,7 @@ def get_timestamps(start_dt, end_dt, delta): return dates -class TestDecoder: +class TestEecoder: def setup_method(self, method): self.covjson = { "type": "CoverageCollection", From c2429eef5d2608d50bdc5290d632fc9e1758312a Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 16 Jan 2024 11:48:30 +0100 Subject: [PATCH 35/58] Added bounding box as feature taht can be encoded to covjson from polytope --- eccovjson/api.py | 4 + eccovjson/encoder/BoundingBox.py | 128 ++++++++++++- eccovjson/encoder/encoder.py | 12 +- tests/test_encoder_bounding_box.py | 284 +++++++++++++++++++++++++++++ tests/test_encoder_time_series.py | 1 - 5 files changed, 420 insertions(+), 9 deletions(-) create mode 100644 tests/test_encoder_bounding_box.py diff --git a/eccovjson/api.py b/eccovjson/api.py index 9d8e4d1..db785d3 100644 --- a/eccovjson/api.py +++ b/eccovjson/api.py @@ -4,14 +4,18 @@ import eccovjson.decoder.VerticalProfile import eccovjson.encoder.TimeSeries import eccovjson.encoder.VerticalProfile +import eccovjson.encoder.BoundingBox +import eccovjson.decoder.BoundingBox features_encoder = { "pointseries": eccovjson.encoder.TimeSeries.TimeSeries, "verticalprofile": eccovjson.encoder.VerticalProfile.VerticalProfile, + "boundingbox": eccovjson.encoder.BoundingBox.BoundingBox, } features_decoder = { "pointseries": eccovjson.decoder.TimeSeries.TimeSeries, "verticalprofile": eccovjson.decoder.VerticalProfile.VerticalProfile, + "boundingbox": eccovjson.decoder.BoundingBox.BoundingBox, } diff --git a/eccovjson/encoder/BoundingBox.py b/eccovjson/encoder/BoundingBox.py index 42b8dfb..cb44888 100644 --- a/eccovjson/encoder/BoundingBox.py +++ b/eccovjson/encoder/BoundingBox.py @@ -9,6 +9,7 @@ class BoundingBox(Encoder): def __init__(self, type, domaintype): super().__init__(type, domaintype) + self.covjson["domainType"] = "MultiPoint" def add_coverage(self, mars_metadata, coords, values): new_coverage = {} @@ -24,14 +25,14 @@ def add_coverage(self, mars_metadata, coords, values): def add_domain(self, coverage, coords): coverage["domain"]["type"] = "Domain" coverage["domain"]["axes"] = {} - coverage["domain"]["axes"]["x"] = {} - coverage["domain"]["axes"]["y"] = {} - coverage["domain"]["axes"]["z"] = {} coverage["domain"]["axes"]["t"] = {} - coverage["domain"]["axes"]["x"]["values"] = coords["x"] - coverage["domain"]["axes"]["y"]["values"] = coords["y"] - coverage["domain"]["axes"]["z"]["values"] = coords["z"] coverage["domain"]["axes"]["t"]["values"] = coords["t"] + coverage["domain"]["axes"]["composite"] = {} + coverage["domain"]["axes"]["composite"]["dataType"] = ("tuple",) + coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson[ + "referencing" + ][0]["coordinates"] + coverage["domain"]["axes"]["composite"]["values"] = coords["composite"] def add_range(self, coverage, values): for parameter in values.keys(): @@ -47,3 +48,118 @@ def add_range(self, coverage, values): def add_mars_metadata(self, coverage, metadata): coverage["mars:metadata"] = metadata + + def from_xarray(self, dataset): + pass + + def from_polytope(self, result, request): + values = [val.result for val in result.leaves] + ancestors = [val.get_ancestors() for val in result.leaves] + + mars_metadata = {} + coords = {} + for key in request.keys(): + if ( + key != "latitude" + and key != "longitude" + and key != "param" + and key != "number" + and key != "step" + ): + mars_metadata[key] = request[key] + + for param in request["param"].split("/"): + if param == "t" or param == "167": + self.add_parameter( + "t", + { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + ) + elif param == "tp" or param == "228": + self.add_parameter( + "tp", + { + "type": "Parameter", + "description": "Total Precipitation", + "unit": {"symbol": "m"}, + "observedProperty": { + "id": "tp", + "label": {"en": "Total Precipitation"}, + }, + }, + ) + elif param == "10u" or param == "165": + self.add_parameter( + "10u", + { + "type": "Parameter", + "description": "10 metre U wind component", + "unit": {"symbol": "ms-1"}, + "observedProperty": { + "id": "10u", + "label": {"en": "10 metre U wind component"}, + }, + }, + ) + elif param == "10v" or param == "166": + self.add_parameter( + "10v", + { + "type": "Parameter", + "description": "10 metre V wind component", + "unit": {"symbol": "ms-1"}, + "observedProperty": { + "id": "10v", + "label": {"en": "10 metre V wind component"}, + }, + }, + ) + self.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + + new_metadata = mars_metadata.copy() + range_dict = {} + vals = {} + for param in request["param"].split("/"): + param = self.convert_param_id_to_param(param) + vals[param] = [] + coords = {} + coords["composite"] = [] + coords["t"] = str(ancestors[0][1]).split("=")[1] + + for ind, feature in enumerate(ancestors[0]): + if str(feature).split("=")[0] == "latitude": + lat = ind + elif str(feature).split("=")[0] == "longitude": + long = ind + elif str(feature).split("=")[0] == "param": + param = ind + param_id = str(feature).split("=")[1] + + for ind, ancestor in enumerate(ancestors): + coord = [] + coord.append(str(ancestor[lat]).split("=")[1]) + coord.append(str(ancestor[long]).split("=")[1]) + coord.append("sfc") + coords["composite"].append(coord) + param_id = self.convert_param_id_to_param( + str(ancestor[param]).split("=")[1] + ) + vals[param_id].append(values[ind]) + + param = self.convert_param_id_to_param(request["param"].split("/")[0]) + coords["composite"] = coords["composite"][0 : len(vals[param])] + + self.add_coverage(new_metadata, coords, vals) + return self.covjson diff --git a/eccovjson/encoder/encoder.py b/eccovjson/encoder/encoder.py index b941f42..04da753 100644 --- a/eccovjson/encoder/encoder.py +++ b/eccovjson/encoder/encoder.py @@ -27,8 +27,16 @@ def __init__(self, type, domaintype): else: raise TypeError("Type must be Coverage or CoverageCollection") - if domaintype != "PointSeries" and domaintype != "VerticalProfile": - raise TypeError("DomainType must be PointSeries or VerticalProfile") + """ + if ( + domaintype != "PointSeries" + and domaintype != "VerticalProfile" + and domaintype != "BoundingBox" + ): + raise TypeError( + "DomainType must be PointSeries or VerticalProfile or BoundingBox" + ) + """ def add_parameter(self, parameter, metadata): self.covjson["parameters"][parameter] = metadata diff --git a/tests/test_encoder_bounding_box.py b/tests/test_encoder_bounding_box.py new file mode 100644 index 0000000..8301ea9 --- /dev/null +++ b/tests/test_encoder_bounding_box.py @@ -0,0 +1,284 @@ +import pytest +import json + +from eccovjson.encoder import encoder +from eccovjson.encoder import TimeSeries +import eccovjson.decoder.TimeSeries +from eccovjson.api import Eccovjson +import random +from datetime import datetime, timedelta +import xarray as xr + + +def get_timestamps(start_dt, end_dt, delta): + dates = [] + while start_dt <= end_dt: + # add current date to list by converting it to iso format + dates.append(start_dt.isoformat().replace("T", " ")) + # increment start date by timedelta + start_dt += delta + return dates + + +class TestEncoder: + def setup_method(self, method): + self.covjson = { + "type": "CoverageCollection", + "domainType": "MultiPoint", + "coverages": [ + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T00:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 264.93115234375, + 263.83115234375, + 265.12313132266, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 9.93115234375, + 7.83115234375, + 14.12313132266, + ], + }, + }, + }, + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "1", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T01:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 266.93115234375, + 293.83115234375, + 165.12313132266, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 1.93115234375, + 22.83115234375, + 12.12313132266, + ], + }, + }, + }, + ], + "referencing": [ + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ], + "parameters": { + "t": { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + "p": { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + }, + } + + def test_CoverageCollection(self): + encoder_obj = Eccovjson().encode("CoverageCollection", "BoundingBox") + assert encoder_obj.type == "CoverageCollection" + + def test_standard_Coverage(self): + encoder_obj = Eccovjson().encode("CoverageCollection", "BoundingBox") + covjson = { + "type": "CoverageCollection", + "domainType": "MultiPoint", + "coverages": [], + "referencing": [], + "parameters": {}, + } + + assert encoder_obj.covjson == covjson + + def test_add_parameter(self): + encoder_obj = Eccovjson().encode("CoverageCollection", "BoundingBox") + encoder_obj.add_parameter( + "t", + { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + ) + encoder_obj.add_parameter( + "p", + { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + ) + covjson = { + "type": "CoverageCollection", + "domainType": "MultiPoint", + "coverages": [], + "referencing": [], + "parameters": { + "t": { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + "p": { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + }, + } + + assert encoder_obj.covjson == covjson + + def test_add_reference(self): + encoder_obj = Eccovjson().encode("CoverageCollection", "BoundingBox") + encoder_obj.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + covjson = { + "type": "CoverageCollection", + "domainType": "MultiPoint", + "coverages": [], + "referencing": [ + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ], + "parameters": {}, + } + + assert encoder_obj.covjson == covjson + + def test_add_coverage(self): + encoder = Eccovjson().encode("CoverageCollection", "BoundingBox") + encoder.add_parameter( + "t", + { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + ) + encoder.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + + metadatas = [] + coords = [] + values = [] + + metadata = { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": 0, + } + coords = {} + coords["t"] = ["2017-01-01T00:00:00"] + coords["composite"] = [[1, 20, 1], [2, 21, 3], [3, 17, 7]] + value = {"t": [111, 222, 333]} + encoder.add_coverage(metadata, coords, value) + # print(encoder.covjson) + + print(encoder.covjson) + + """ + + def test_from_xarray(self): + ds = xr.open_dataset("new_timeseries.nc") + encoder = Eccovjson().encode("CoverageCollection", "PointSeries") + encoder.from_xarray(ds) + + """ diff --git a/tests/test_encoder_time_series.py b/tests/test_encoder_time_series.py index d075de6..f92dbbf 100644 --- a/tests/test_encoder_time_series.py +++ b/tests/test_encoder_time_series.py @@ -153,7 +153,6 @@ def setup_method(self, method): } def test_CoverageCollection(self): - # encoder_obj = TimeSeries.TimeSeries("CoverageCollection", "PointSeries") encoder_obj = Eccovjson().encode("CoverageCollection", "PointSeries") assert encoder_obj.type == "CoverageCollection" From ec75bcf38993dbba3470d9a063eb65cb32f6bee5 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 16 Jan 2024 15:05:58 +0100 Subject: [PATCH 36/58] Adding bounding box to api --- eccovjson/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eccovjson/api.py b/eccovjson/api.py index db785d3..60b6572 100644 --- a/eccovjson/api.py +++ b/eccovjson/api.py @@ -33,6 +33,8 @@ def decode(self, covjson): requesttype = covjson["domainType"] if requesttype == "timeseries": requesttype = "PointSeries" + elif requesttype == "MultiPoint": + requesttype = "boundingbox" feature = self._feature_factory(requesttype.lower(), "decoder") return feature(covjson) From f2579a31da0398d79c12c33c088916e255acbd19 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Wed, 17 Jan 2024 09:17:17 +0100 Subject: [PATCH 37/58] Remove date infer from imports as not used --- eccovjson/encoder/BoundingBox.py | 2 +- eccovjson/encoder/TimeSeries.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eccovjson/encoder/BoundingBox.py b/eccovjson/encoder/BoundingBox.py index cb44888..26b5e25 100644 --- a/eccovjson/encoder/BoundingBox.py +++ b/eccovjson/encoder/BoundingBox.py @@ -2,7 +2,7 @@ import xarray as xr from datetime import timedelta, datetime import datetime -import dateinfer + import pandas as pd diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index 3d69667..1553de8 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -2,7 +2,7 @@ import xarray as xr from datetime import timedelta, datetime import datetime -import dateinfer + import pandas as pd From fe8b57f4890f590d8e0a899bbc2905bd5ba1d6ed Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 18 Jan 2024 15:35:30 +0100 Subject: [PATCH 38/58] Add more params that can be parsed --- eccovjson/encoder/TimeSeries.py | 28 ++++++++++++++++++++++++++++ eccovjson/encoder/encoder.py | 4 ++++ 2 files changed, 32 insertions(+) diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index 1553de8..c02784f 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -174,6 +174,34 @@ def from_polytope(self, result, request): }, }, ) + elif param == "10fg" or param == "49": + self.add_parameter( + "10fg", + { + "type": "Parameter", + "description": "Maximum 10 metre wind gust since previous post-processing", + "unit": {"symbol": "ms-1"}, + "observedProperty": { + "id": "10fg", + "label": { + "en": "Maximum 10 metre wind gust since previous post-processing" + }, + }, + }, + ) + elif param == "tcc" or param == "164": + self.add_parameter( + "10fg", + { + "type": "Parameter", + "description": "Total cloud cover", + "unit": {"symbol": "ms-1"}, + "observedProperty": { + "id": "tcc", + "label": {"en": "Total cloud cover"}, + }, + }, + ) self.add_reference( { "coordinates": ["x", "y", "z"], diff --git a/eccovjson/encoder/encoder.py b/eccovjson/encoder/encoder.py index 04da753..e9146f9 100644 --- a/eccovjson/encoder/encoder.py +++ b/eccovjson/encoder/encoder.py @@ -58,6 +58,10 @@ def convert_param_id_to_param(self, paramid): return "t" elif param == 228: return "tp" + elif param == 49: + return "10fg" + elif param == 164: + return "tcc" @abstractmethod def add_coverage(self, mars_metadata, coords, values): From 44b18b2d8845f15de54fa17954c4214a3372db30 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Thu, 18 Jan 2024 16:40:22 +0100 Subject: [PATCH 39/58] Fixed bug in parameter adding function --- eccovjson/encoder/TimeSeries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index c02784f..f26c88a 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -191,7 +191,7 @@ def from_polytope(self, result, request): ) elif param == "tcc" or param == "164": self.add_parameter( - "10fg", + "tcc", { "type": "Parameter", "description": "Total cloud cover", From 9331178de3a25b192359303307be842e5f145014 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Fri, 19 Jan 2024 11:47:27 +0100 Subject: [PATCH 40/58] Cleaned up the way parameters are added when converting from polytope --- eccovjson/encoder/BoundingBox.py | 51 +------------- eccovjson/encoder/TimeSeries.py | 99 ++-------------------------- eccovjson/encoder/VerticalProfile.py | 21 +----- eccovjson/encoder/encoder.py | 68 ++++++++++++++++++- tests/test_encoder_bounding_box.py | 41 +++--------- tests/test_encoder_time_series.py | 47 +++++-------- 6 files changed, 99 insertions(+), 228 deletions(-) diff --git a/eccovjson/encoder/BoundingBox.py b/eccovjson/encoder/BoundingBox.py index 26b5e25..9755f38 100644 --- a/eccovjson/encoder/BoundingBox.py +++ b/eccovjson/encoder/BoundingBox.py @@ -69,55 +69,8 @@ def from_polytope(self, result, request): mars_metadata[key] = request[key] for param in request["param"].split("/"): - if param == "t" or param == "167": - self.add_parameter( - "t", - { - "type": "Parameter", - "description": "Temperature", - "unit": {"symbol": "K"}, - "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, - }, - ) - elif param == "tp" or param == "228": - self.add_parameter( - "tp", - { - "type": "Parameter", - "description": "Total Precipitation", - "unit": {"symbol": "m"}, - "observedProperty": { - "id": "tp", - "label": {"en": "Total Precipitation"}, - }, - }, - ) - elif param == "10u" or param == "165": - self.add_parameter( - "10u", - { - "type": "Parameter", - "description": "10 metre U wind component", - "unit": {"symbol": "ms-1"}, - "observedProperty": { - "id": "10u", - "label": {"en": "10 metre U wind component"}, - }, - }, - ) - elif param == "10v" or param == "166": - self.add_parameter( - "10v", - { - "type": "Parameter", - "description": "10 metre V wind component", - "unit": {"symbol": "ms-1"}, - "observedProperty": { - "id": "10v", - "label": {"en": "10 metre V wind component"}, - }, - }, - ) + self.add_parameter(param) + self.add_reference( { "coordinates": ["x", "y", "z"], diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index f26c88a..58fe5dd 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -51,25 +51,9 @@ def add_mars_metadata(self, coverage, metadata): def from_xarray(self, dataset): for parameter in dataset.data_vars: if parameter == "Temperature": - self.add_parameter( - "t", - { - "type": "Parameter", - "description": "Temperature", - "unit": {"symbol": "K"}, - "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, - }, - ) + self.add_parameter("t") elif parameter == "Pressure": - self.add_parameter( - "p", - { - "type": "Parameter", - "description": "Pressure", - "unit": {"symbol": "pa"}, - "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, - }, - ) + self.add_parameter("p") self.add_reference( { "coordinates": ["x", "y", "z"], @@ -125,83 +109,8 @@ def from_polytope(self, result, request): coords["y"] = [request[key]] for param in request["param"].split("/"): - if param == "t" or param == "167": - self.add_parameter( - "t", - { - "type": "Parameter", - "description": "Temperature", - "unit": {"symbol": "K"}, - "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, - }, - ) - elif param == "tp" or param == "228": - self.add_parameter( - "tp", - { - "type": "Parameter", - "description": "Total Precipitation", - "unit": {"symbol": "m"}, - "observedProperty": { - "id": "tp", - "label": {"en": "Total Precipitation"}, - }, - }, - ) - elif param == "10u" or param == "165": - self.add_parameter( - "10u", - { - "type": "Parameter", - "description": "10 metre U wind component", - "unit": {"symbol": "ms-1"}, - "observedProperty": { - "id": "10u", - "label": {"en": "10 metre U wind component"}, - }, - }, - ) - elif param == "10v" or param == "166": - self.add_parameter( - "10v", - { - "type": "Parameter", - "description": "10 metre V wind component", - "unit": {"symbol": "ms-1"}, - "observedProperty": { - "id": "10v", - "label": {"en": "10 metre V wind component"}, - }, - }, - ) - elif param == "10fg" or param == "49": - self.add_parameter( - "10fg", - { - "type": "Parameter", - "description": "Maximum 10 metre wind gust since previous post-processing", - "unit": {"symbol": "ms-1"}, - "observedProperty": { - "id": "10fg", - "label": { - "en": "Maximum 10 metre wind gust since previous post-processing" - }, - }, - }, - ) - elif param == "tcc" or param == "164": - self.add_parameter( - "tcc", - { - "type": "Parameter", - "description": "Total cloud cover", - "unit": {"symbol": "ms-1"}, - "observedProperty": { - "id": "tcc", - "label": {"en": "Total cloud cover"}, - }, - }, - ) + self.add_parameter(param) + self.add_reference( { "coordinates": ["x", "y", "z"], diff --git a/eccovjson/encoder/VerticalProfile.py b/eccovjson/encoder/VerticalProfile.py index 7b14cdc..378ad76 100644 --- a/eccovjson/encoder/VerticalProfile.py +++ b/eccovjson/encoder/VerticalProfile.py @@ -48,25 +48,10 @@ def add_mars_metadata(self, coverage, metadata): def from_xarray(self, dataset): for parameter in dataset.data_vars: if parameter == "Temperature": - self.add_parameter( - "t", - { - "type": "Parameter", - "description": "Temperature", - "unit": {"symbol": "K"}, - "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, - }, - ) + self.add_parameter("t") elif parameter == "Pressure": - self.add_parameter( - "p", - { - "type": "Parameter", - "description": "Pressure", - "unit": {"symbol": "pa"}, - "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, - }, - ) + self.add_parameter("p") + self.add_reference( { "coordinates": ["x", "y", "z"], diff --git a/eccovjson/encoder/encoder.py b/eccovjson/encoder/encoder.py index e9146f9..fb986f2 100644 --- a/eccovjson/encoder/encoder.py +++ b/eccovjson/encoder/encoder.py @@ -38,9 +38,71 @@ def __init__(self, type, domaintype): ) """ - def add_parameter(self, parameter, metadata): - self.covjson["parameters"][parameter] = metadata - self.parameters.append(parameter) + def add_parameter(self, param): + param = self.convert_param_id_to_param(param) + if param == "t" or param == "167": + self.covjson["parameters"][param] = { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": { + "id": "t", + "label": {"en": "Temperature"}, + }, + } + elif param == "tp" or param == "228": + self.covjson["parameters"][param] = { + "type": "Parameter", + "description": "Total Precipitation", + "unit": {"symbol": "m"}, + "observedProperty": { + "id": "tp", + "label": {"en": "Total Precipitation"}, + }, + } + elif param == "10u" or param == "165": + self.covjson["parameters"][param] = { + "type": "Parameter", + "description": "10 metre U wind component", + "unit": {"symbol": "ms-1"}, + "observedProperty": { + "id": "10u", + "label": {"en": "10 metre U wind component"}, + }, + } + elif param == "10v" or param == "166": + self.covjson["parameters"][param] = { + "type": "Parameter", + "description": "10 metre V wind component", + "unit": {"symbol": "ms-1"}, + "observedProperty": { + "id": "10v", + "label": {"en": "10 metre V wind component"}, + }, + } + elif param == "10fg" or param == "49": + self.covjson["parameters"][param] = { + "type": "Parameter", + "description": "Maximum 10 metre wind gust since previous post-processing", + "unit": {"symbol": "ms-1"}, + "observedProperty": { + "id": "10fg", + "label": { + "en": "Maximum 10 metre wind gust since previous post-processing" + }, + }, + } + elif param == "tcc" or param == "164": + self.covjson["parameters"][param] = { + "type": "Parameter", + "description": "Total cloud cover", + "unit": {"symbol": "ms-1"}, + "observedProperty": { + "id": "tcc", + "label": {"en": "Total cloud cover"}, + }, + } + self.parameters.append(param) def add_reference(self, reference): self.covjson["referencing"].append(reference) diff --git a/tests/test_encoder_bounding_box.py b/tests/test_encoder_bounding_box.py index 8301ea9..7609a70 100644 --- a/tests/test_encoder_bounding_box.py +++ b/tests/test_encoder_bounding_box.py @@ -162,24 +162,8 @@ def test_standard_Coverage(self): def test_add_parameter(self): encoder_obj = Eccovjson().encode("CoverageCollection", "BoundingBox") - encoder_obj.add_parameter( - "t", - { - "type": "Parameter", - "description": "Temperature", - "unit": {"symbol": "K"}, - "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, - }, - ) - encoder_obj.add_parameter( - "p", - { - "type": "Parameter", - "description": "Pressure", - "unit": {"symbol": "pa"}, - "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, - }, - ) + encoder_obj.add_parameter("t") + encoder_obj.add_parameter("tp") covjson = { "type": "CoverageCollection", "domainType": "MultiPoint", @@ -192,11 +176,14 @@ def test_add_parameter(self): "unit": {"symbol": "K"}, "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, }, - "p": { + "tp": { "type": "Parameter", - "description": "Pressure", - "unit": {"symbol": "pa"}, - "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + "description": "Total Precipitation", + "unit": {"symbol": "m"}, + "observedProperty": { + "id": "tp", + "label": {"en": "Total Precipitation"}, + }, }, }, } @@ -234,15 +221,7 @@ def test_add_reference(self): def test_add_coverage(self): encoder = Eccovjson().encode("CoverageCollection", "BoundingBox") - encoder.add_parameter( - "t", - { - "type": "Parameter", - "description": "Temperature", - "unit": {"symbol": "K"}, - "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, - }, - ) + encoder.add_parameter("t") encoder.add_reference( { "coordinates": ["x", "y", "z"], diff --git a/tests/test_encoder_time_series.py b/tests/test_encoder_time_series.py index f92dbbf..539b271 100644 --- a/tests/test_encoder_time_series.py +++ b/tests/test_encoder_time_series.py @@ -170,24 +170,9 @@ def test_standard_Coverage(self): def test_add_parameter(self): encoder_obj = Eccovjson().encode("CoverageCollection", "PointSeries") - encoder_obj.add_parameter( - "t", - { - "type": "Parameter", - "description": "Temperature", - "unit": {"symbol": "K"}, - "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, - }, - ) - encoder_obj.add_parameter( - "p", - { - "type": "Parameter", - "description": "Pressure", - "unit": {"symbol": "pa"}, - "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, - }, - ) + encoder_obj.add_parameter("t") + encoder_obj.add_parameter("tp") + print(encoder_obj.covjson) covjson = { "type": "CoverageCollection", "domainType": "PointSeries", @@ -198,13 +183,19 @@ def test_add_parameter(self): "type": "Parameter", "description": "Temperature", "unit": {"symbol": "K"}, - "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + "observedProperty": { + "id": "t", + "label": {"en": "Temperature"}, + }, }, - "p": { + "tp": { "type": "Parameter", - "description": "Pressure", - "unit": {"symbol": "pa"}, - "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + "description": "Total Precipitation", + "unit": {"symbol": "m"}, + "observedProperty": { + "id": "tp", + "label": {"en": "Total Precipitation"}, + }, }, }, } @@ -242,15 +233,7 @@ def test_add_reference(self): def test_add_coverage(self): encoder = Eccovjson().encode("CoverageCollection", "PointSeries") - encoder.add_parameter( - "t", - { - "type": "Parameter", - "description": "Temperature", - "unit": {"symbol": "K"}, - "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, - }, - ) + encoder.add_parameter("t") encoder.add_reference( { "coordinates": ["x", "y", "z"], From dd604ca07337b75ea09759722c33e52123ee038a Mon Sep 17 00:00:00 2001 From: awarde96 Date: Fri, 19 Jan 2024 14:16:29 +0100 Subject: [PATCH 41/58] Add frame type for decoding and tests --- eccovjson/api.py | 4 + eccovjson/decoder/Frame.py | 106 +++++++++++++ eccovjson/encoder/Frame.py | 118 +++++++++++++++ tests/test_decoder_frame.py | 293 ++++++++++++++++++++++++++++++++++++ tests/test_encoder_frame.py | 263 ++++++++++++++++++++++++++++++++ 5 files changed, 784 insertions(+) create mode 100644 eccovjson/decoder/Frame.py create mode 100644 eccovjson/encoder/Frame.py create mode 100644 tests/test_decoder_frame.py create mode 100644 tests/test_encoder_frame.py diff --git a/eccovjson/api.py b/eccovjson/api.py index 60b6572..c7197d4 100644 --- a/eccovjson/api.py +++ b/eccovjson/api.py @@ -6,16 +6,20 @@ import eccovjson.encoder.VerticalProfile import eccovjson.encoder.BoundingBox import eccovjson.decoder.BoundingBox +import eccovjson.encoder.Frame +import eccovjson.decoder.Frame features_encoder = { "pointseries": eccovjson.encoder.TimeSeries.TimeSeries, "verticalprofile": eccovjson.encoder.VerticalProfile.VerticalProfile, "boundingbox": eccovjson.encoder.BoundingBox.BoundingBox, + "frame": eccovjson.encoder.Frame.Frame, } features_decoder = { "pointseries": eccovjson.decoder.TimeSeries.TimeSeries, "verticalprofile": eccovjson.decoder.VerticalProfile.VerticalProfile, "boundingbox": eccovjson.decoder.BoundingBox.BoundingBox, + "frame": eccovjson.decoder.Frame.Frame, } diff --git a/eccovjson/decoder/Frame.py b/eccovjson/decoder/Frame.py new file mode 100644 index 0000000..ab62c2f --- /dev/null +++ b/eccovjson/decoder/Frame.py @@ -0,0 +1,106 @@ +from .decoder import Decoder +import xarray as xr +import datetime as dt + + +class Frame(Decoder): + def __init__(self, covjson): + super().__init__(covjson) + self.domains = self.get_domains() + self.ranges = self.get_ranges() + + def get_domains(self): + domains = [] + for coverage in self.coverage.coverages: + domains.append(coverage["domain"]) + return domains + + def get_ranges(self): + ranges = [] + for coverage in self.coverage.coverages: + ranges.append(coverage["ranges"]) + return ranges + + def get_values(self): + values = {} + for parameter in self.parameters: + values[parameter] = [] + for range in self.ranges: + values[parameter].append(range[parameter]["values"]) + return values + + def get_coordinates(self): + coord_dict = {} + for param in self.parameters: + coord_dict[param] = [] + # Get x,y,z,t coords and unpack t coords and match to x,y,z coords + for ind, domain in enumerate(self.domains): + t = domain["axes"]["t"]["values"][0] + num = self.mars_metadata[ind]["number"] + fct = self.mars_metadata[ind]["date"] + + for param in self.parameters: + coords = [] + for coord in domain["axes"]["composite"]["values"]: + x = coord[0] + y = coord[1] + z = coord[2] + coords.append([x, y, z, fct, t, num]) + coord_dict[param].append(coords) + return coord_dict + + def to_geopandas(self): + pass + + def to_xarray(self): + dims = ["x", "y", "z", "number", "t"] + dataarraydict = {} + + # Get coordinates + for parameter in self.parameters: + param_values = [[[self.get_values()[parameter]]]] + for ind, fc_time_vals in enumerate(self.get_values()[parameter]): + coords = self.get_coordinates()[parameter] + xs = [] + ys = [] + zs = [] + for coord in coords[ind]: + xs.append(coord[0]) + ys.append(coord[1]) + zs.append(coord[2]) + num = [int(coord[0][5]) for coord in coords] + coords_fc = coords[ind] + try: + t = [ + dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") + for coord in coords_fc + ] + except ValueError: + t = [ + dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%S") + for coord in coords_fc + ] + + param_coords = {"x": xs, "y": ys, "z": zs, "number": num, "t": t} + dataarray = xr.DataArray( + param_values, + dims=dims, + coords=param_coords, + name=parameter, + ) + + dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)[ + "unit" + ]["symbol"] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ + "description" + ] + dataarraydict[dataarray.attrs["long_name"]] = dataarray + + ds = xr.Dataset(dataarraydict) + for mars_metadata in self.mars_metadata[0]: + if mars_metadata != "date" and mars_metadata != "step": + ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata] + + return ds diff --git a/eccovjson/encoder/Frame.py b/eccovjson/encoder/Frame.py new file mode 100644 index 0000000..3579a06 --- /dev/null +++ b/eccovjson/encoder/Frame.py @@ -0,0 +1,118 @@ +from .encoder import Encoder +import xarray as xr +from datetime import timedelta, datetime +import datetime + +import pandas as pd + + +class Frame(Encoder): + def __init__(self, type, domaintype): + super().__init__(type, domaintype) + self.covjson["domainType"] = "MultiPoint" + + def add_coverage(self, mars_metadata, coords, values): + new_coverage = {} + new_coverage["mars:metadata"] = {} + new_coverage["type"] = "Coverage" + new_coverage["domain"] = {} + new_coverage["ranges"] = {} + self.add_mars_metadata(new_coverage, mars_metadata) + self.add_domain(new_coverage, coords) + self.add_range(new_coverage, values) + self.covjson["coverages"].append(new_coverage) + + def add_domain(self, coverage, coords): + coverage["domain"]["type"] = "Domain" + coverage["domain"]["axes"] = {} + coverage["domain"]["axes"]["t"] = {} + coverage["domain"]["axes"]["t"]["values"] = coords["t"] + coverage["domain"]["axes"]["composite"] = {} + coverage["domain"]["axes"]["composite"]["dataType"] = ("tuple",) + coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson[ + "referencing" + ][0]["coordinates"] + coverage["domain"]["axes"]["composite"]["values"] = coords["composite"] + + def add_range(self, coverage, values): + for parameter in values.keys(): + param = self.convert_param_id_to_param(parameter) + coverage["ranges"][param] = {} + coverage["ranges"][param]["type"] = "NdArray" + coverage["ranges"][param]["dataType"] = "float" + coverage["ranges"][param]["shape"] = [len(values[parameter])] + coverage["ranges"][param]["axisNames"] = [str(param)] + coverage["ranges"][param]["values"] = values[ + parameter + ] # [values[parameter]] + + def add_mars_metadata(self, coverage, metadata): + coverage["mars:metadata"] = metadata + + def from_xarray(self, dataset): + pass + + def from_polytope(self, result, request): + values = [val.result for val in result.leaves] + ancestors = [val.get_ancestors() for val in result.leaves] + + mars_metadata = {} + coords = {} + for key in request.keys(): + if ( + key != "latitude" + and key != "longitude" + and key != "param" + and key != "number" + and key != "step" + ): + mars_metadata[key] = request[key] + + for param in request["param"].split("/"): + self.add_parameter(param) + + self.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + + new_metadata = mars_metadata.copy() + range_dict = {} + vals = {} + for param in request["param"].split("/"): + param = self.convert_param_id_to_param(param) + vals[param] = [] + coords = {} + coords["composite"] = [] + coords["t"] = str(ancestors[0][1]).split("=")[1] + + for ind, feature in enumerate(ancestors[0]): + if str(feature).split("=")[0] == "latitude": + lat = ind + elif str(feature).split("=")[0] == "longitude": + long = ind + elif str(feature).split("=")[0] == "param": + param = ind + param_id = str(feature).split("=")[1] + + for ind, ancestor in enumerate(ancestors): + coord = [] + coord.append(str(ancestor[lat]).split("=")[1]) + coord.append(str(ancestor[long]).split("=")[1]) + coord.append("sfc") + coords["composite"].append(coord) + param_id = self.convert_param_id_to_param( + str(ancestor[param]).split("=")[1] + ) + vals[param_id].append(values[ind]) + + param = self.convert_param_id_to_param(request["param"].split("/")[0]) + coords["composite"] = coords["composite"][0 : len(vals[param])] + + self.add_coverage(new_metadata, coords, vals) + return self.covjson diff --git a/tests/test_decoder_frame.py b/tests/test_decoder_frame.py new file mode 100644 index 0000000..d123d48 --- /dev/null +++ b/tests/test_decoder_frame.py @@ -0,0 +1,293 @@ +import pytest +import json + +from eccovjson.decoder import decoder +from eccovjson.decoder import VerticalProfile +from eccovjson.decoder import TimeSeries +from eccovjson.decoder import Frame +from earthkit import data +import xarray as xr + + +class TestDecoder: + def setup_method(self, method): + self.covjson = { + "type": "CoverageCollection", + "domainType": "MultiPoint", + "coverages": [ + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T00:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 264.93115234375, + 263.83115234375, + 265.12313132266, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 9.93115234375, + 7.83115234375, + 14.12313132266, + ], + }, + }, + }, + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "1", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T01:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 266.93115234375, + 293.83115234375, + 165.12313132266, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 1.93115234375, + 22.83115234375, + 12.12313132266, + ], + }, + }, + }, + ], + "referencing": [ + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ], + "parameters": { + "t": { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + "p": { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + }, + } + + def test_frame_type(self): + decoder = Frame.Frame(self.covjson) + assert decoder.type == "CoverageCollection" + + def test_frame_parameters(self): + decoder = Frame.Frame(self.covjson) + assert decoder.parameters == ["t", "p"] + + def test_frame_referencing(self): + decoder = Frame.Frame(self.covjson) + assert decoder.get_referencing() == ["x", "y", "z"] + + def test_frame_get_parameter_metadata(self): + decoder = Frame.Frame(self.covjson) + assert decoder.get_parameter_metadata("t") == { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + } + assert decoder.get_parameter_metadata("p") == { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + } + + def test_frame_mars_metadata(self): + decoder = Frame.Frame(self.covjson) + assert decoder.mars_metadata[0] == { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": "0", + } + assert decoder.mars_metadata[1] == { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "1", + "number": "0", + } + + def test_frame_domains(self): + decoder = Frame.Frame(self.covjson) + domain1 = { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T00:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + } + assert decoder.domains[0] == domain1 + domain2 = { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T01:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + } + assert decoder.domains[1] == domain2 + + def test_frame_ranges(self): + decoder = Frame.Frame(self.covjson) + range1 = { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [264.93115234375, 263.83115234375, 265.12313132266], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [9.93115234375, 7.83115234375, 14.12313132266], + }, + } + assert decoder.ranges[0] == range1 + range2 = { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [266.93115234375, 293.83115234375, 165.12313132266], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [1.93115234375, 22.83115234375, 12.12313132266], + }, + } + assert decoder.ranges[1] == range2 + + def test_frame_get_coordinates(self): + decoder = Frame.Frame(self.covjson) + coordinates = { + "t": [ + [ + [1, 20, 1, "20170101", "2017-01-01T00:00:00", "0"], + [2, 21, 3, "20170101", "2017-01-01T00:00:00", "0"], + [3, 17, 7, "20170101", "2017-01-01T00:00:00", "0"], + ], + [ + [1, 20, 1, "20170101", "2017-01-01T01:00:00", "0"], + [2, 21, 3, "20170101", "2017-01-01T01:00:00", "0"], + [3, 17, 7, "20170101", "2017-01-01T01:00:00", "0"], + ], + ], + "p": [ + [ + [1, 20, 1, "20170101", "2017-01-01T00:00:00", "0"], + [2, 21, 3, "20170101", "2017-01-01T00:00:00", "0"], + [3, 17, 7, "20170101", "2017-01-01T00:00:00", "0"], + ], + [ + [1, 20, 1, "20170101", "2017-01-01T01:00:00", "0"], + [2, 21, 3, "20170101", "2017-01-01T01:00:00", "0"], + [3, 17, 7, "20170101", "2017-01-01T01:00:00", "0"], + ], + ], + } + assert decoder.get_coordinates() == coordinates + + def test_frame_get_values(self): + decoder = Frame.Frame(self.covjson) + values = { + "t": [ + [264.93115234375, 263.83115234375, 265.12313132266], + [266.93115234375, 293.83115234375, 165.12313132266], + ], + "p": [ + [9.93115234375, 7.83115234375, 14.12313132266], + [1.93115234375, 22.83115234375, 12.12313132266], + ], + } + assert decoder.get_values() == values + + # def test_bounding_box_to_xarray(self): + # decoder = BoundingBox.BoundingBox(self.covjson) + # dataset = decoder.to_xarray() + # print(dataset) diff --git a/tests/test_encoder_frame.py b/tests/test_encoder_frame.py new file mode 100644 index 0000000..7609a70 --- /dev/null +++ b/tests/test_encoder_frame.py @@ -0,0 +1,263 @@ +import pytest +import json + +from eccovjson.encoder import encoder +from eccovjson.encoder import TimeSeries +import eccovjson.decoder.TimeSeries +from eccovjson.api import Eccovjson +import random +from datetime import datetime, timedelta +import xarray as xr + + +def get_timestamps(start_dt, end_dt, delta): + dates = [] + while start_dt <= end_dt: + # add current date to list by converting it to iso format + dates.append(start_dt.isoformat().replace("T", " ")) + # increment start date by timedelta + start_dt += delta + return dates + + +class TestEncoder: + def setup_method(self, method): + self.covjson = { + "type": "CoverageCollection", + "domainType": "MultiPoint", + "coverages": [ + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T00:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 264.93115234375, + 263.83115234375, + 265.12313132266, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 9.93115234375, + 7.83115234375, + 14.12313132266, + ], + }, + }, + }, + { + "mars:metadata": { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "1", + "number": "0", + }, + "type": "Coverage", + "domain": { + "type": "Domain", + "axes": { + "t": {"values": ["2017-01-01T01:00:00"]}, + "composite": { + "dataType": "tuple", + "coordinates": ["x", "y", "z"], + "values": [[1, 20, 1], [2, 21, 3], [3, 17, 7]], + }, + }, + }, + "ranges": { + "t": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 266.93115234375, + 293.83115234375, + 165.12313132266, + ], + }, + "p": { + "type": "NdArray", + "dataType": "float", + "shape": [3], + "axisNames": ["t"], + "values": [ + 1.93115234375, + 22.83115234375, + 12.12313132266, + ], + }, + }, + }, + ], + "referencing": [ + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ], + "parameters": { + "t": { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + "p": { + "type": "Parameter", + "description": "Pressure", + "unit": {"symbol": "pa"}, + "observedProperty": {"id": "p", "label": {"en": "Pressure"}}, + }, + }, + } + + def test_CoverageCollection(self): + encoder_obj = Eccovjson().encode("CoverageCollection", "BoundingBox") + assert encoder_obj.type == "CoverageCollection" + + def test_standard_Coverage(self): + encoder_obj = Eccovjson().encode("CoverageCollection", "BoundingBox") + covjson = { + "type": "CoverageCollection", + "domainType": "MultiPoint", + "coverages": [], + "referencing": [], + "parameters": {}, + } + + assert encoder_obj.covjson == covjson + + def test_add_parameter(self): + encoder_obj = Eccovjson().encode("CoverageCollection", "BoundingBox") + encoder_obj.add_parameter("t") + encoder_obj.add_parameter("tp") + covjson = { + "type": "CoverageCollection", + "domainType": "MultiPoint", + "coverages": [], + "referencing": [], + "parameters": { + "t": { + "type": "Parameter", + "description": "Temperature", + "unit": {"symbol": "K"}, + "observedProperty": {"id": "t", "label": {"en": "Temperature"}}, + }, + "tp": { + "type": "Parameter", + "description": "Total Precipitation", + "unit": {"symbol": "m"}, + "observedProperty": { + "id": "tp", + "label": {"en": "Total Precipitation"}, + }, + }, + }, + } + + assert encoder_obj.covjson == covjson + + def test_add_reference(self): + encoder_obj = Eccovjson().encode("CoverageCollection", "BoundingBox") + encoder_obj.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + covjson = { + "type": "CoverageCollection", + "domainType": "MultiPoint", + "coverages": [], + "referencing": [ + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ], + "parameters": {}, + } + + assert encoder_obj.covjson == covjson + + def test_add_coverage(self): + encoder = Eccovjson().encode("CoverageCollection", "BoundingBox") + encoder.add_parameter("t") + encoder.add_reference( + { + "coordinates": ["x", "y", "z"], + "system": { + "type": "GeographicCRS", + "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + }, + } + ) + + metadatas = [] + coords = [] + values = [] + + metadata = { + "class": "od", + "stream": "oper", + "levtype": "pl", + "date": "20170101", + "step": "0", + "number": 0, + } + coords = {} + coords["t"] = ["2017-01-01T00:00:00"] + coords["composite"] = [[1, 20, 1], [2, 21, 3], [3, 17, 7]] + value = {"t": [111, 222, 333]} + encoder.add_coverage(metadata, coords, value) + # print(encoder.covjson) + + print(encoder.covjson) + + """ + + def test_from_xarray(self): + ds = xr.open_dataset("new_timeseries.nc") + encoder = Eccovjson().encode("CoverageCollection", "PointSeries") + encoder.from_xarray(ds) + + """ From e1f4807e7efbe78e25740b0d7d6fbb941dda6b38 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Wed, 24 Jan 2024 09:21:23 +0100 Subject: [PATCH 42/58] Removed some comments --- eccovjson/encoder/encoder.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/eccovjson/encoder/encoder.py b/eccovjson/encoder/encoder.py index fb986f2..bbd944a 100644 --- a/eccovjson/encoder/encoder.py +++ b/eccovjson/encoder/encoder.py @@ -27,17 +27,6 @@ def __init__(self, type, domaintype): else: raise TypeError("Type must be Coverage or CoverageCollection") - """ - if ( - domaintype != "PointSeries" - and domaintype != "VerticalProfile" - and domaintype != "BoundingBox" - ): - raise TypeError( - "DomainType must be PointSeries or VerticalProfile or BoundingBox" - ) - """ - def add_parameter(self, param): param = self.convert_param_id_to_param(param) if param == "t" or param == "167": From 499fda1c52f0f0097ab4fa685f3dca1c5db3e3fa Mon Sep 17 00:00:00 2001 From: awarde96 Date: Wed, 24 Jan 2024 12:04:50 +0100 Subject: [PATCH 43/58] From_polytope no longer requires a request and uses the polytope output tree only for parsing --- eccovjson/encoder/TimeSeries.py | 88 ++++++++++++++++----------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index 58fe5dd..fb77e6e 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -69,9 +69,6 @@ def from_xarray(self, dataset): dv_dict[dv] = list(dataset[dv].sel(number=num).values[0][0][0]) self.add_coverage( { - # "date": fc_time.values.astype("M8[ms]") - # .astype("O") - # .strftime("%m/%d/%Y"), "number": num, "type": "forecast", "step": 0, @@ -82,33 +79,32 @@ def from_xarray(self, dataset): "z": list(dataset["z"].values), "t": [str(x) for x in dataset["t"].values], }, - # "t": list(dataset["Temperature"].sel(number=num).values[0][0][0]), - # "p": dataset["Pressure"].sel(fct=fc_time).values[0][0][0], dv_dict, ) return self.covjson - def from_polytope(self, result, request): - # ancestors = [val.get_ancestors() for val in result.leaves] + def from_polytope(self, result): + ancestors = [val.get_ancestors() for val in result.leaves] values = [val.result for val in result.leaves] - mars_metadata = {} - coords = {} - for key in request.keys(): - if ( - key != "latitude" - and key != "longitude" - and key != "param" - and key != "number" - and key != "step" - ): - mars_metadata[key] = request[key] - elif key == "latitude": - coords["x"] = [request[key]] - elif key == "longitude": - coords["y"] = [request[key]] - - for param in request["param"].split("/"): + columns = [] + df_dict = {} + # Create empty dataframe + for feature in ancestors[0]: + columns.append(str(feature).split("=")[0]) + df_dict[str(feature).split("=")[0]] = [] + + # populate dataframe + for ancestor in ancestors: + for feature in ancestor: + df_dict[str(feature).split("=")[0]].append(str(feature).split("=")[1]) + values = [val.result for val in result.leaves] + df_dict["values"] = values + df = pd.DataFrame(df_dict) + + params = df["param"].unique() + + for param in params: self.add_parameter(param) self.add_reference( @@ -120,37 +116,41 @@ def from_polytope(self, result, request): }, } ) + steps = df["step"].unique() + + mars_metadata = {} + mars_metadata["class"] = df["class"].unique()[0] + mars_metadata["expver"] = df["expver"].unique()[0] + mars_metadata["levtype"] = df["levtype"].unique()[0] + mars_metadata["type"] = df["type"].unique()[0] + mars_metadata["date"] = df["date"].unique()[0] + mars_metadata["domain"] = df["domain"].unique()[0] + mars_metadata["stream"] = df["stream"].unique()[0] + coords = {} + coords["x"] = list(df["latitude"].unique()) + coords["y"] = list(df["longitude"].unique()) coords["z"] = ["sfc"] - if "/" in request["number"]: - numbers = request["number"].split("/") - else: - numbers = request["number"] - steps = request["step"] + coords["t"] = [] - times = [] + # convert step into datetime date_format = "%Y%m%dT%H%M%S" date = pd.Timestamp(mars_metadata["date"]).strftime(date_format) start_time = datetime.datetime.strptime(date, date_format) for step in steps: # add current date to list by converting it to iso format - stamp = start_time + timedelta(hours=step) - times.append(stamp.isoformat()) + stamp = start_time + timedelta(hours=int(step)) + coords["t"].append(stamp.isoformat()) # increment start date by timedelta - coords["t"] = times - vals = [] - start = 0 - end = len(times) - new_metadata = mars_metadata.copy() - for num in numbers: - mars_metadata["number"] = num + for number in df["number"].unique(): new_metadata = mars_metadata.copy() + new_metadata["number"] = number + df_number = df[df["number"] == number] range_dict = {} - for param in request["param"].split("/"): - range_dict[param] = values[start:end] - # vals.append(values[start:end]) - start = end - end += len(times) + for param in params: + df_param = df_number[df_number["param"] == param] + range_dict[param] = df_param["values"].values.tolist() self.add_coverage(new_metadata, coords, range_dict) + return self.covjson From c7e858be4a8e771a83902ecbd39fbf0caef03be5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 12:34:53 +0100 Subject: [PATCH 44/58] add CI and warning to readme --- .github/workflows/ci.yaml | 152 ++++++++++++++++++++++++++++++++++++++ README.md | 15 +--- 2 files changed, 154 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..6bc9671 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,152 @@ +name: ci +on: + # Trigger the workflow on push to master or develop, except tag creation + push: + branches: + - 'main' + - 'develop' + # Trigger the workflow on pull request + pull_request: ~ + # Trigger the workflow manually + workflow_dispatch: ~ + # Trigger after public PR approved for CI + pull_request_target: + types: [labeled] + release: + types: [created] +jobs: + qa: + name: qa + runs-on: ubuntu-20.04 + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Install Python Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install black flake8 isort + - name: Check isort + run: isort --check . + + - name: Check black + run: black --check . + + - name: Check flake8 + run: flake8 . + setup: + name: setup + runs-on: ubuntu-20.04 + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + inputs: ${{ steps.prepare-inputs.outputs.inputs }} + inputs-for-ubuntu: ${{ steps.prepare-inputs.outputs.inputs-for-ubuntu }} + steps: + - name: Set Matrix + id: set-matrix + shell: bash -eux {0} + run: | + MATRIX=$(cat << 'EOS' + name: + - gnu-11@ubuntu-22.04 + - clang-14@ubuntu-22.04 + include: + - name: gnu-11@ubuntu-22.04 + os: ubuntu-22.04 + compiler: gnu-11 + compiler_cc: gcc-11 + compiler_cxx: g++-11 + compiler_fc: gfortran-11 + - name: clang-14@ubuntu-22.04 + os: ubuntu-22.04 + compiler: clang-14 + compiler_cc: clang-14 + compiler_cxx: clang++-14 + compiler_fc: gfortran-11 + # Xcode compiler requires empty environment variables, so we pass null (~) here + EOS + ) + SKIP_MATRIX_JOBS=$(cat << 'EOS' + ${{ inputs.skip_matrix_jobs }} + EOS + ) + SELECT_NAME_COND="1 != 1" + SELECT_INCLUDE_COND="1 != 1" + for skip_job in $SKIP_MATRIX_JOBS; do SELECT_NAME_COND="$SELECT_NAME_COND or . == \"$skip_job\""; SELECT_INCLUDE_COND="$SELECT_INCLUDE_COND or .name == \"$skip_job\""; done + echo matrix=$(echo "$MATRIX" | yq eval "del(.name[] | select($SELECT_NAME_COND)) | del(.include[] | select($SELECT_INCLUDE_COND))" --output-format json --indent 0 -) >> $GITHUB_OUTPUT + - name: Prepare build-package Inputs + id: prepare-inputs + shell: bash -eux {0} + run: | + echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT + echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-20.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT + test: + name: test + needs: + - qa + - setup + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Install Python Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest pytest-cov + python -m pip install -r requirements.txt + python -m pip install -r ./tests/requirements.txt + + - name: Verify Source Distribution + shell: bash -eux {0} + run: | + python setup.py sdist + python -m pip install dist/* + - name: Run Tests with Repository Code + env: + LD_LIBRARY_PATH: ${{ steps.install-dependencies.outputs.lib_path }} + shell: bash -eux {0} + run: | + DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest -m tests --cov=./ --cov-report=xml + python -m coverage report + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + files: coverage.xml + deploy: + needs: test + if: ${{ github.event_name == 'release' }} + name: Upload to Pypi + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist + twine upload dist/* diff --git a/README.md b/README.md index 9b3a872..196b114 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,6 @@ ECMWF library for encoding and decoding coerageJSON files/objects of meteorlogic * Encodes and decodes CoverageJSON objects -* CoverageCollection(python dictionary) - * get_coverages() - -* Coverage() - * get_axes() - * get_metadata() - * get_ranges() - * get_parameters() - -* TimeSeriesCoverage() - * set_axes() - * set_ranges() - * set_parameters() +| :warning: This project is BETA and will be experimental for the foreseeable future. Interfaces and functionality are likely to change. DO NOT use this software in any project/software that is operational. | +|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| From 25df4847b7f60f11f066f31bcbb9ac1fe44f6fe6 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 13:41:40 +0100 Subject: [PATCH 45/58] isort --- eccovjson/__init__.py | 8 ++++---- eccovjson/api.py | 8 ++++---- eccovjson/decoder/BoundingBox.py | 6 ++++-- eccovjson/decoder/Frame.py | 6 ++++-- eccovjson/decoder/TimeSeries.py | 6 ++++-- eccovjson/decoder/VerticalProfile.py | 3 ++- eccovjson/decoder/decoder.py | 10 ++++++---- eccovjson/encoder/BoundingBox.py | 7 ++++--- eccovjson/encoder/Frame.py | 7 ++++--- eccovjson/encoder/TimeSeries.py | 7 ++++--- eccovjson/encoder/VerticalProfile.py | 8 +++++--- eccovjson/encoder/encoder.py | 10 ++++++---- tests/test_decoder_bounding_box.py | 10 ++++------ tests/test_decoder_frame.py | 10 ++++------ tests/test_decoder_time_series.py | 11 +++++------ tests/test_decoder_vertical_profile.py | 7 +++---- tests/test_encoder_bounding_box.py | 12 ++++++------ tests/test_encoder_frame.py | 12 ++++++------ tests/test_encoder_time_series.py | 12 ++++++------ 19 files changed, 85 insertions(+), 75 deletions(-) diff --git a/eccovjson/__init__.py b/eccovjson/__init__.py index d7f5c94..e738b08 100644 --- a/eccovjson/__init__.py +++ b/eccovjson/__init__.py @@ -1,5 +1,5 @@ -import eccovjson.encoder.VerticalProfile -import eccovjson.encoder.TimeSeries -import eccovjson.decoder.VerticalProfile -import eccovjson.decoder.TimeSeries import eccovjson.api +import eccovjson.decoder.TimeSeries +import eccovjson.decoder.VerticalProfile +import eccovjson.encoder.TimeSeries +import eccovjson.encoder.VerticalProfile diff --git a/eccovjson/api.py b/eccovjson/api.py index c7197d4..2fb58ca 100644 --- a/eccovjson/api.py +++ b/eccovjson/api.py @@ -1,13 +1,13 @@ import json +import eccovjson.decoder.BoundingBox +import eccovjson.decoder.Frame import eccovjson.decoder.TimeSeries import eccovjson.decoder.VerticalProfile -import eccovjson.encoder.TimeSeries -import eccovjson.encoder.VerticalProfile import eccovjson.encoder.BoundingBox -import eccovjson.decoder.BoundingBox import eccovjson.encoder.Frame -import eccovjson.decoder.Frame +import eccovjson.encoder.TimeSeries +import eccovjson.encoder.VerticalProfile features_encoder = { "pointseries": eccovjson.encoder.TimeSeries.TimeSeries, diff --git a/eccovjson/decoder/BoundingBox.py b/eccovjson/decoder/BoundingBox.py index 6afd05a..63b2dbb 100644 --- a/eccovjson/decoder/BoundingBox.py +++ b/eccovjson/decoder/BoundingBox.py @@ -1,7 +1,9 @@ -from .decoder import Decoder -import xarray as xr import datetime as dt +import xarray as xr + +from .decoder import Decoder + class BoundingBox(Decoder): def __init__(self, covjson): diff --git a/eccovjson/decoder/Frame.py b/eccovjson/decoder/Frame.py index ab62c2f..58b0bda 100644 --- a/eccovjson/decoder/Frame.py +++ b/eccovjson/decoder/Frame.py @@ -1,7 +1,9 @@ -from .decoder import Decoder -import xarray as xr import datetime as dt +import xarray as xr + +from .decoder import Decoder + class Frame(Decoder): def __init__(self, covjson): diff --git a/eccovjson/decoder/TimeSeries.py b/eccovjson/decoder/TimeSeries.py index 203c6d6..c3de6ab 100644 --- a/eccovjson/decoder/TimeSeries.py +++ b/eccovjson/decoder/TimeSeries.py @@ -1,7 +1,9 @@ -from .decoder import Decoder -import xarray as xr import datetime as dt +import xarray as xr + +from .decoder import Decoder + class TimeSeries(Decoder): def __init__(self, covjson): diff --git a/eccovjson/decoder/VerticalProfile.py b/eccovjson/decoder/VerticalProfile.py index d8db709..b529e04 100644 --- a/eccovjson/decoder/VerticalProfile.py +++ b/eccovjson/decoder/VerticalProfile.py @@ -1,6 +1,7 @@ -from .decoder import Decoder import xarray as xr +from .decoder import Decoder + class VerticalProfile(Decoder): def __init__(self, covjson): diff --git a/eccovjson/decoder/decoder.py b/eccovjson/decoder/decoder.py index 01bb0ca..44d0a27 100644 --- a/eccovjson/decoder/decoder.py +++ b/eccovjson/decoder/decoder.py @@ -1,9 +1,11 @@ -import os -import json -import xarray as xr import datetime as dt -import geopandas as gpd +import json +import os from abc import ABC, abstractmethod + +import geopandas as gpd +import xarray as xr + from eccovjson.Coverage import Coverage from eccovjson.CoverageCollection import CoverageCollection diff --git a/eccovjson/encoder/BoundingBox.py b/eccovjson/encoder/BoundingBox.py index 9755f38..d75c952 100644 --- a/eccovjson/encoder/BoundingBox.py +++ b/eccovjson/encoder/BoundingBox.py @@ -1,9 +1,10 @@ -from .encoder import Encoder -import xarray as xr -from datetime import timedelta, datetime import datetime +from datetime import datetime, timedelta import pandas as pd +import xarray as xr + +from .encoder import Encoder class BoundingBox(Encoder): diff --git a/eccovjson/encoder/Frame.py b/eccovjson/encoder/Frame.py index 3579a06..b240ac6 100644 --- a/eccovjson/encoder/Frame.py +++ b/eccovjson/encoder/Frame.py @@ -1,9 +1,10 @@ -from .encoder import Encoder -import xarray as xr -from datetime import timedelta, datetime import datetime +from datetime import datetime, timedelta import pandas as pd +import xarray as xr + +from .encoder import Encoder class Frame(Encoder): diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index fb77e6e..a31b362 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -1,9 +1,10 @@ -from .encoder import Encoder -import xarray as xr -from datetime import timedelta, datetime import datetime +from datetime import datetime, timedelta import pandas as pd +import xarray as xr + +from .encoder import Encoder class TimeSeries(Encoder): diff --git a/eccovjson/encoder/VerticalProfile.py b/eccovjson/encoder/VerticalProfile.py index 378ad76..f32c7f0 100644 --- a/eccovjson/encoder/VerticalProfile.py +++ b/eccovjson/encoder/VerticalProfile.py @@ -1,7 +1,9 @@ -from .encoder import Encoder -import xarray as xr -from datetime import timedelta, datetime import datetime +from datetime import datetime, timedelta + +import xarray as xr + +from .encoder import Encoder class VerticalProfile(Encoder): diff --git a/eccovjson/encoder/encoder.py b/eccovjson/encoder/encoder.py index bbd944a..25213c3 100644 --- a/eccovjson/encoder/encoder.py +++ b/eccovjson/encoder/encoder.py @@ -1,9 +1,11 @@ -import os -import json -import xarray as xr import datetime as dt -import geopandas as gpd +import json +import os from abc import ABC, abstractmethod + +import geopandas as gpd +import xarray as xr + from eccovjson.Coverage import Coverage from eccovjson.CoverageCollection import CoverageCollection diff --git a/tests/test_decoder_bounding_box.py b/tests/test_decoder_bounding_box.py index 59d273e..b5a83a7 100644 --- a/tests/test_decoder_bounding_box.py +++ b/tests/test_decoder_bounding_box.py @@ -1,12 +1,10 @@ -import pytest import json -from eccovjson.decoder import decoder -from eccovjson.decoder import VerticalProfile -from eccovjson.decoder import TimeSeries -from eccovjson.decoder import BoundingBox -from earthkit import data +import pytest import xarray as xr +from earthkit import data + +from eccovjson.decoder import BoundingBox, TimeSeries, VerticalProfile, decoder class TestDecoder: diff --git a/tests/test_decoder_frame.py b/tests/test_decoder_frame.py index d123d48..9d07684 100644 --- a/tests/test_decoder_frame.py +++ b/tests/test_decoder_frame.py @@ -1,12 +1,10 @@ -import pytest import json -from eccovjson.decoder import decoder -from eccovjson.decoder import VerticalProfile -from eccovjson.decoder import TimeSeries -from eccovjson.decoder import Frame -from earthkit import data +import pytest import xarray as xr +from earthkit import data + +from eccovjson.decoder import Frame, TimeSeries, VerticalProfile, decoder class TestDecoder: diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index 0bbe230..5e4d5be 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -1,12 +1,11 @@ -import pytest import json -from eccovjson.decoder import decoder -from eccovjson.decoder import VerticalProfile -from eccovjson.decoder import TimeSeries -from eccovjson.api import Eccovjson -from earthkit import data +import pytest import xarray as xr +from earthkit import data + +from eccovjson.api import Eccovjson +from eccovjson.decoder import TimeSeries, VerticalProfile, decoder class TestDecoder: diff --git a/tests/test_decoder_vertical_profile.py b/tests/test_decoder_vertical_profile.py index 43594f6..f68246c 100644 --- a/tests/test_decoder_vertical_profile.py +++ b/tests/test_decoder_vertical_profile.py @@ -1,11 +1,10 @@ -import pytest import json -from eccovjson.decoder import decoder -from eccovjson.decoder import VerticalProfile -from eccovjson.decoder import TimeSeries +import pytest + import eccovjson.encoder.VerticalProfile from eccovjson.api import Eccovjson +from eccovjson.decoder import TimeSeries, VerticalProfile, decoder class TestDecoder: diff --git a/tests/test_encoder_bounding_box.py b/tests/test_encoder_bounding_box.py index 7609a70..f6e58ee 100644 --- a/tests/test_encoder_bounding_box.py +++ b/tests/test_encoder_bounding_box.py @@ -1,14 +1,14 @@ -import pytest import json - -from eccovjson.encoder import encoder -from eccovjson.encoder import TimeSeries -import eccovjson.decoder.TimeSeries -from eccovjson.api import Eccovjson import random from datetime import datetime, timedelta + +import pytest import xarray as xr +import eccovjson.decoder.TimeSeries +from eccovjson.api import Eccovjson +from eccovjson.encoder import TimeSeries, encoder + def get_timestamps(start_dt, end_dt, delta): dates = [] diff --git a/tests/test_encoder_frame.py b/tests/test_encoder_frame.py index 7609a70..f6e58ee 100644 --- a/tests/test_encoder_frame.py +++ b/tests/test_encoder_frame.py @@ -1,14 +1,14 @@ -import pytest import json - -from eccovjson.encoder import encoder -from eccovjson.encoder import TimeSeries -import eccovjson.decoder.TimeSeries -from eccovjson.api import Eccovjson import random from datetime import datetime, timedelta + +import pytest import xarray as xr +import eccovjson.decoder.TimeSeries +from eccovjson.api import Eccovjson +from eccovjson.encoder import TimeSeries, encoder + def get_timestamps(start_dt, end_dt, delta): dates = [] diff --git a/tests/test_encoder_time_series.py b/tests/test_encoder_time_series.py index 539b271..23ec50a 100644 --- a/tests/test_encoder_time_series.py +++ b/tests/test_encoder_time_series.py @@ -1,14 +1,14 @@ -import pytest import json - -from eccovjson.encoder import encoder -from eccovjson.encoder import TimeSeries -import eccovjson.decoder.TimeSeries -from eccovjson.api import Eccovjson import random from datetime import datetime, timedelta + +import pytest import xarray as xr +import eccovjson.decoder.TimeSeries +from eccovjson.api import Eccovjson +from eccovjson.encoder import TimeSeries, encoder + def get_timestamps(start_dt, end_dt, delta): dates = [] From 7482b9f4ded4e0a125b0b2dd19ce7ee8b3178907 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 13:43:39 +0100 Subject: [PATCH 46/58] add tox.ini --- tox.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..754e5f9 --- /dev/null +++ b/tox.ini @@ -0,0 +1,9 @@ +[flake8] +max-line-length = 120 +exclude = .* +extend-ignore = E203,W503 +per-file-ignores = + */__init__.py: F401,F403 +[isort] +profile=black +skip_glob=.* \ No newline at end of file From 3cf1c73825e79e19537cfa478137cfef2e6ae22d Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 13:45:31 +0100 Subject: [PATCH 47/58] add pyproject --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..89b8eb7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[tool.black] +line-length = 120 +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" \ No newline at end of file From 9fbbcb7c149f6b253e06c99be8bede195902b3ac Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 13:46:13 +0100 Subject: [PATCH 48/58] black --- eccovjson/CoverageCollection.py | 4 +--- eccovjson/decoder/BoundingBox.py | 18 ++++-------------- eccovjson/decoder/Frame.py | 18 ++++-------------- eccovjson/decoder/TimeSeries.py | 18 ++++-------------- eccovjson/decoder/VerticalProfile.py | 8 ++------ eccovjson/encoder/BoundingBox.py | 20 ++++---------------- eccovjson/encoder/Frame.py | 20 ++++---------------- eccovjson/encoder/TimeSeries.py | 4 +--- eccovjson/encoder/VerticalProfile.py | 4 +--- eccovjson/encoder/encoder.py | 4 +--- tests/test_decoder_vertical_profile.py | 8 ++------ 11 files changed, 28 insertions(+), 98 deletions(-) diff --git a/eccovjson/CoverageCollection.py b/eccovjson/CoverageCollection.py index 5743591..f6a623e 100644 --- a/eccovjson/CoverageCollection.py +++ b/eccovjson/CoverageCollection.py @@ -12,8 +12,6 @@ def __init__(self, covjson): if self.type == "CoverageCollection": print("Correct Type") elif self.type == "Coverage": - raise TypeError( - "CoverageCollection class takes CoverageCollection not Coverage" - ) + raise TypeError("CoverageCollection class takes CoverageCollection not Coverage") self.coverages = self.coverage["coverages"] diff --git a/eccovjson/decoder/BoundingBox.py b/eccovjson/decoder/BoundingBox.py index 63b2dbb..f1e0b7b 100644 --- a/eccovjson/decoder/BoundingBox.py +++ b/eccovjson/decoder/BoundingBox.py @@ -76,15 +76,9 @@ def to_xarray(self): num = [int(coord[0][5]) for coord in coords] coords_fc = coords[ind] try: - t = [ - dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") - for coord in coords_fc - ] + t = [dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") for coord in coords_fc] except ValueError: - t = [ - dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%S") - for coord in coords_fc - ] + t = [dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%S") for coord in coords_fc] param_coords = {"x": xs, "y": ys, "z": zs, "number": num, "t": t} dataarray = xr.DataArray( @@ -95,12 +89,8 @@ def to_xarray(self): ) dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] - dataarray.attrs["units"] = self.get_parameter_metadata(parameter)[ - "unit" - ]["symbol"] - dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ - "description" - ] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"] dataarraydict[dataarray.attrs["long_name"]] = dataarray ds = xr.Dataset(dataarraydict) diff --git a/eccovjson/decoder/Frame.py b/eccovjson/decoder/Frame.py index 58b0bda..750c32d 100644 --- a/eccovjson/decoder/Frame.py +++ b/eccovjson/decoder/Frame.py @@ -73,15 +73,9 @@ def to_xarray(self): num = [int(coord[0][5]) for coord in coords] coords_fc = coords[ind] try: - t = [ - dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") - for coord in coords_fc - ] + t = [dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") for coord in coords_fc] except ValueError: - t = [ - dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%S") - for coord in coords_fc - ] + t = [dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%S") for coord in coords_fc] param_coords = {"x": xs, "y": ys, "z": zs, "number": num, "t": t} dataarray = xr.DataArray( @@ -92,12 +86,8 @@ def to_xarray(self): ) dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] - dataarray.attrs["units"] = self.get_parameter_metadata(parameter)[ - "unit" - ]["symbol"] - dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ - "description" - ] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"] dataarraydict[dataarray.attrs["long_name"]] = dataarray ds = xr.Dataset(dataarraydict) diff --git a/eccovjson/decoder/TimeSeries.py b/eccovjson/decoder/TimeSeries.py index c3de6ab..217da94 100644 --- a/eccovjson/decoder/TimeSeries.py +++ b/eccovjson/decoder/TimeSeries.py @@ -77,15 +77,9 @@ def to_xarray(self): num = [int(coord[0][5]) for coord in coords] coords_fc = coords[ind] try: - t = [ - dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") - for coord in coords_fc - ] + t = [dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") for coord in coords_fc] except ValueError: - t = [ - dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%S") - for coord in coords_fc - ] + t = [dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%S") for coord in coords_fc] param_coords = {"x": x, "y": y, "z": z, "number": num, "t": t} dataarray = xr.DataArray( @@ -96,12 +90,8 @@ def to_xarray(self): ) dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] - dataarray.attrs["units"] = self.get_parameter_metadata(parameter)[ - "unit" - ]["symbol"] - dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ - "description" - ] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"] dataarraydict[dataarray.attrs["long_name"]] = dataarray ds = xr.Dataset(dataarraydict) diff --git a/eccovjson/decoder/VerticalProfile.py b/eccovjson/decoder/VerticalProfile.py index b529e04..e1c7a89 100644 --- a/eccovjson/decoder/VerticalProfile.py +++ b/eccovjson/decoder/VerticalProfile.py @@ -80,12 +80,8 @@ def to_xarray(self): name=parameter, ) dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] - dataarray.attrs["units"] = self.get_parameter_metadata(parameter)[ - "unit" - ]["symbol"] - dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)[ - "description" - ] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["description"] dataarraydict[dataarray.attrs["long_name"]] = dataarray ds = xr.Dataset(dataarraydict) diff --git a/eccovjson/encoder/BoundingBox.py b/eccovjson/encoder/BoundingBox.py index d75c952..e4172d7 100644 --- a/eccovjson/encoder/BoundingBox.py +++ b/eccovjson/encoder/BoundingBox.py @@ -30,9 +30,7 @@ def add_domain(self, coverage, coords): coverage["domain"]["axes"]["t"]["values"] = coords["t"] coverage["domain"]["axes"]["composite"] = {} coverage["domain"]["axes"]["composite"]["dataType"] = ("tuple",) - coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson[ - "referencing" - ][0]["coordinates"] + coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson["referencing"][0]["coordinates"] coverage["domain"]["axes"]["composite"]["values"] = coords["composite"] def add_range(self, coverage, values): @@ -43,9 +41,7 @@ def add_range(self, coverage, values): coverage["ranges"][param]["dataType"] = "float" coverage["ranges"][param]["shape"] = [len(values[parameter])] coverage["ranges"][param]["axisNames"] = [str(param)] - coverage["ranges"][param]["values"] = values[ - parameter - ] # [values[parameter]] + coverage["ranges"][param]["values"] = values[parameter] # [values[parameter]] def add_mars_metadata(self, coverage, metadata): coverage["mars:metadata"] = metadata @@ -60,13 +56,7 @@ def from_polytope(self, result, request): mars_metadata = {} coords = {} for key in request.keys(): - if ( - key != "latitude" - and key != "longitude" - and key != "param" - and key != "number" - and key != "step" - ): + if key != "latitude" and key != "longitude" and key != "param" and key != "number" and key != "step": mars_metadata[key] = request[key] for param in request["param"].split("/"): @@ -107,9 +97,7 @@ def from_polytope(self, result, request): coord.append(str(ancestor[long]).split("=")[1]) coord.append("sfc") coords["composite"].append(coord) - param_id = self.convert_param_id_to_param( - str(ancestor[param]).split("=")[1] - ) + param_id = self.convert_param_id_to_param(str(ancestor[param]).split("=")[1]) vals[param_id].append(values[ind]) param = self.convert_param_id_to_param(request["param"].split("/")[0]) diff --git a/eccovjson/encoder/Frame.py b/eccovjson/encoder/Frame.py index b240ac6..6d362a2 100644 --- a/eccovjson/encoder/Frame.py +++ b/eccovjson/encoder/Frame.py @@ -30,9 +30,7 @@ def add_domain(self, coverage, coords): coverage["domain"]["axes"]["t"]["values"] = coords["t"] coverage["domain"]["axes"]["composite"] = {} coverage["domain"]["axes"]["composite"]["dataType"] = ("tuple",) - coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson[ - "referencing" - ][0]["coordinates"] + coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson["referencing"][0]["coordinates"] coverage["domain"]["axes"]["composite"]["values"] = coords["composite"] def add_range(self, coverage, values): @@ -43,9 +41,7 @@ def add_range(self, coverage, values): coverage["ranges"][param]["dataType"] = "float" coverage["ranges"][param]["shape"] = [len(values[parameter])] coverage["ranges"][param]["axisNames"] = [str(param)] - coverage["ranges"][param]["values"] = values[ - parameter - ] # [values[parameter]] + coverage["ranges"][param]["values"] = values[parameter] # [values[parameter]] def add_mars_metadata(self, coverage, metadata): coverage["mars:metadata"] = metadata @@ -60,13 +56,7 @@ def from_polytope(self, result, request): mars_metadata = {} coords = {} for key in request.keys(): - if ( - key != "latitude" - and key != "longitude" - and key != "param" - and key != "number" - and key != "step" - ): + if key != "latitude" and key != "longitude" and key != "param" and key != "number" and key != "step": mars_metadata[key] = request[key] for param in request["param"].split("/"): @@ -107,9 +97,7 @@ def from_polytope(self, result, request): coord.append(str(ancestor[long]).split("=")[1]) coord.append("sfc") coords["composite"].append(coord) - param_id = self.convert_param_id_to_param( - str(ancestor[param]).split("=")[1] - ) + param_id = self.convert_param_id_to_param(str(ancestor[param]).split("=")[1]) vals[param_id].append(values[ind]) param = self.convert_param_id_to_param(request["param"].split("/")[0]) diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index a31b362..17785c9 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -42,9 +42,7 @@ def add_range(self, coverage, values): coverage["ranges"][param]["dataType"] = "float" coverage["ranges"][param]["shape"] = [len(values[parameter])] coverage["ranges"][param]["axisNames"] = [str(param)] - coverage["ranges"][param]["values"] = values[ - parameter - ] # [values[parameter]] + coverage["ranges"][param]["values"] = values[parameter] # [values[parameter]] def add_mars_metadata(self, coverage, metadata): coverage["mars:metadata"] = metadata diff --git a/eccovjson/encoder/VerticalProfile.py b/eccovjson/encoder/VerticalProfile.py index f32c7f0..765de48 100644 --- a/eccovjson/encoder/VerticalProfile.py +++ b/eccovjson/encoder/VerticalProfile.py @@ -40,9 +40,7 @@ def add_range(self, coverage, values): coverage["ranges"][parameter]["dataType"] = "float" coverage["ranges"][parameter]["shape"] = [len(values[parameter])] coverage["ranges"][parameter]["axisNames"] = ["z"] - coverage["ranges"][parameter]["values"] = values[ - parameter - ] # [values[parameter]] + coverage["ranges"][parameter]["values"] = values[parameter] # [values[parameter]] def add_mars_metadata(self, coverage, metadata): coverage["mars:metadata"] = metadata diff --git a/eccovjson/encoder/encoder.py b/eccovjson/encoder/encoder.py index 25213c3..691cb41 100644 --- a/eccovjson/encoder/encoder.py +++ b/eccovjson/encoder/encoder.py @@ -78,9 +78,7 @@ def add_parameter(self, param): "unit": {"symbol": "ms-1"}, "observedProperty": { "id": "10fg", - "label": { - "en": "Maximum 10 metre wind gust since previous post-processing" - }, + "label": {"en": "Maximum 10 metre wind gust since previous post-processing"}, }, } elif param == "tcc" or param == "164": diff --git a/tests/test_decoder_vertical_profile.py b/tests/test_decoder_vertical_profile.py index f68246c..e16f2c8 100644 --- a/tests/test_decoder_vertical_profile.py +++ b/tests/test_decoder_vertical_profile.py @@ -98,9 +98,7 @@ def setup_method(self, method): "coordinates": ["z"], "system": { "type": "VerticalCRS", - "cs": { - "csAxes": [{"name": {"en": "level"}, "direction": "down"}] - }, + "cs": {"csAxes": [{"name": {"en": "level"}, "direction": "down"}]}, }, }, ], @@ -252,8 +250,6 @@ def test_verticalprofile_values(self): def test_verticalprofile_to_xarray(self): decoder = Eccovjson().decode(self.covjson) dataset = decoder.to_xarray() - encoder = eccovjson.encoder.VerticalProfile.VerticalProfile( - "CoverageCollection", "VerticalProfile" - ) + encoder = eccovjson.encoder.VerticalProfile.VerticalProfile("CoverageCollection", "VerticalProfile") cov = encoder.from_xarray(dataset) print(cov) From 08fafdf25255359de1f26958f6cad3666061772c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 13:51:39 +0100 Subject: [PATCH 49/58] flake8 --- eccovjson/Coverage.py | 3 --- eccovjson/CoverageCollection.py | 3 --- eccovjson/api.py | 2 -- eccovjson/decoder/TimeSeries.py | 2 +- eccovjson/decoder/decoder.py | 5 ----- eccovjson/encoder/BoundingBox.py | 8 +------- eccovjson/encoder/Frame.py | 8 +------- eccovjson/encoder/TimeSeries.py | 2 -- eccovjson/encoder/VerticalProfile.py | 5 ----- eccovjson/encoder/encoder.py | 8 +------- setup.py | 3 ++- 11 files changed, 6 insertions(+), 43 deletions(-) diff --git a/eccovjson/Coverage.py b/eccovjson/Coverage.py index e57b2bb..495d107 100644 --- a/eccovjson/Coverage.py +++ b/eccovjson/Coverage.py @@ -1,6 +1,3 @@ -import json - - class Coverage: def __init__(self, covjson): if isinstance(covjson, dict): diff --git a/eccovjson/CoverageCollection.py b/eccovjson/CoverageCollection.py index f6a623e..99275b2 100644 --- a/eccovjson/CoverageCollection.py +++ b/eccovjson/CoverageCollection.py @@ -1,6 +1,3 @@ -import json - - class CoverageCollection: def __init__(self, covjson): if isinstance(covjson, dict): diff --git a/eccovjson/api.py b/eccovjson/api.py index 2fb58ca..aa58bcc 100644 --- a/eccovjson/api.py +++ b/eccovjson/api.py @@ -1,5 +1,3 @@ -import json - import eccovjson.decoder.BoundingBox import eccovjson.decoder.Frame import eccovjson.decoder.TimeSeries diff --git a/eccovjson/decoder/TimeSeries.py b/eccovjson/decoder/TimeSeries.py index 217da94..f176fa6 100644 --- a/eccovjson/decoder/TimeSeries.py +++ b/eccovjson/decoder/TimeSeries.py @@ -102,7 +102,7 @@ def to_xarray(self): return ds -""" +""" def to_xarray(self): dims = ["x", "y", "z", "fct", "t"] dataarraydict = {} diff --git a/eccovjson/decoder/decoder.py b/eccovjson/decoder/decoder.py index 44d0a27..1c27a24 100644 --- a/eccovjson/decoder/decoder.py +++ b/eccovjson/decoder/decoder.py @@ -1,11 +1,6 @@ -import datetime as dt import json -import os from abc import ABC, abstractmethod -import geopandas as gpd -import xarray as xr - from eccovjson.Coverage import Coverage from eccovjson.CoverageCollection import CoverageCollection diff --git a/eccovjson/encoder/BoundingBox.py b/eccovjson/encoder/BoundingBox.py index e4172d7..b347a23 100644 --- a/eccovjson/encoder/BoundingBox.py +++ b/eccovjson/encoder/BoundingBox.py @@ -1,9 +1,3 @@ -import datetime -from datetime import datetime, timedelta - -import pandas as pd -import xarray as xr - from .encoder import Encoder @@ -73,7 +67,7 @@ def from_polytope(self, result, request): ) new_metadata = mars_metadata.copy() - range_dict = {} + # range_dict = {} vals = {} for param in request["param"].split("/"): param = self.convert_param_id_to_param(param) diff --git a/eccovjson/encoder/Frame.py b/eccovjson/encoder/Frame.py index 6d362a2..2b7b24c 100644 --- a/eccovjson/encoder/Frame.py +++ b/eccovjson/encoder/Frame.py @@ -1,9 +1,3 @@ -import datetime -from datetime import datetime, timedelta - -import pandas as pd -import xarray as xr - from .encoder import Encoder @@ -73,7 +67,7 @@ def from_polytope(self, result, request): ) new_metadata = mars_metadata.copy() - range_dict = {} + # range_dict = {} vals = {} for param in request["param"].split("/"): param = self.convert_param_id_to_param(param) diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index 17785c9..71661d6 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -1,8 +1,6 @@ -import datetime from datetime import datetime, timedelta import pandas as pd -import xarray as xr from .encoder import Encoder diff --git a/eccovjson/encoder/VerticalProfile.py b/eccovjson/encoder/VerticalProfile.py index 765de48..6f698ac 100644 --- a/eccovjson/encoder/VerticalProfile.py +++ b/eccovjson/encoder/VerticalProfile.py @@ -1,8 +1,3 @@ -import datetime -from datetime import datetime, timedelta - -import xarray as xr - from .encoder import Encoder diff --git a/eccovjson/encoder/encoder.py b/eccovjson/encoder/encoder.py index 691cb41..b0a069d 100644 --- a/eccovjson/encoder/encoder.py +++ b/eccovjson/encoder/encoder.py @@ -1,11 +1,5 @@ -import datetime as dt -import json -import os from abc import ABC, abstractmethod -import geopandas as gpd -import xarray as xr - from eccovjson.Coverage import Coverage from eccovjson.CoverageCollection import CoverageCollection @@ -99,7 +93,7 @@ def add_reference(self, reference): def convert_param_id_to_param(self, paramid): try: param = int(paramid) - except: + except BaseException: return paramid if param == 165: return "10u" diff --git a/setup.py b/setup.py index ddbb22f..51bde8a 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,8 @@ setup( name="eccovjson", version=__version__, - description="ECMWF library for encoding and decoding coerageJSON files/objects of meteorlogical features such as vertical profiles and time series.", + description="""ECMWF library for encoding and decoding coerageJSON files/objects of meteorlogical features such as + vertical profiles and time series.""", long_description="", url="https://github.com/ecmwf/eccovjson", author="ECMWF", From ae1a16b5cfa934885f59ee23134953965fc49312 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 13:55:09 +0100 Subject: [PATCH 50/58] flake8 --- tests/test_decoder_bounding_box.py | 8 ++------ tests/test_decoder_frame.py | 10 +++++----- tests/test_decoder_time_series.py | 14 +++++--------- tests/test_decoder_vertical_profile.py | 5 ----- tests/test_encoder_bounding_box.py | 13 ++----------- tests/test_encoder_frame.py | 13 ++----------- tests/test_encoder_time_series.py | 6 +----- 7 files changed, 17 insertions(+), 52 deletions(-) diff --git a/tests/test_decoder_bounding_box.py b/tests/test_decoder_bounding_box.py index b5a83a7..3710b1c 100644 --- a/tests/test_decoder_bounding_box.py +++ b/tests/test_decoder_bounding_box.py @@ -1,10 +1,6 @@ -import json +# from earthkit import data -import pytest -import xarray as xr -from earthkit import data - -from eccovjson.decoder import BoundingBox, TimeSeries, VerticalProfile, decoder +from eccovjson.decoder import BoundingBox class TestDecoder: diff --git a/tests/test_decoder_frame.py b/tests/test_decoder_frame.py index 9d07684..072aab0 100644 --- a/tests/test_decoder_frame.py +++ b/tests/test_decoder_frame.py @@ -1,10 +1,10 @@ -import json +# import json -import pytest -import xarray as xr -from earthkit import data +# import pytest +# import xarray as xr +# from earthkit import data -from eccovjson.decoder import Frame, TimeSeries, VerticalProfile, decoder +from eccovjson.decoder import Frame class TestDecoder: diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index 5e4d5be..4861409 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -1,11 +1,6 @@ -import json - -import pytest -import xarray as xr -from earthkit import data +# from earthkit import data from eccovjson.api import Eccovjson -from eccovjson.decoder import TimeSeries, VerticalProfile, decoder class TestDecoder: @@ -292,16 +287,17 @@ def test_timeseries_coordinates(self): assert decoder.get_coordinates() == coordinates def test_timeseries_to_xarray(self): - decoder = Eccovjson().decode(self.covjson) - ds = decoder.to_xarray() + # decoder = Eccovjson().decode(self.covjson) + # ds = decoder.to_xarray() # print(ds) # print(ds["Temperature"]) # xrds.to_netcdf("timeseries.nc") # ds = xr.open_dataset("timeseries.nc") - ekds = data.from_object(ds) + # ekds = data.from_object(ds) # print(ekds) # print(type(ekds)) # print(ekds.ls()) + pass """ diff --git a/tests/test_decoder_vertical_profile.py b/tests/test_decoder_vertical_profile.py index e16f2c8..ebf21fa 100644 --- a/tests/test_decoder_vertical_profile.py +++ b/tests/test_decoder_vertical_profile.py @@ -1,10 +1,5 @@ -import json - -import pytest - import eccovjson.encoder.VerticalProfile from eccovjson.api import Eccovjson -from eccovjson.decoder import TimeSeries, VerticalProfile, decoder class TestDecoder: diff --git a/tests/test_encoder_bounding_box.py b/tests/test_encoder_bounding_box.py index f6e58ee..91f60b0 100644 --- a/tests/test_encoder_bounding_box.py +++ b/tests/test_encoder_bounding_box.py @@ -1,13 +1,4 @@ -import json -import random -from datetime import datetime, timedelta - -import pytest -import xarray as xr - -import eccovjson.decoder.TimeSeries from eccovjson.api import Eccovjson -from eccovjson.encoder import TimeSeries, encoder def get_timestamps(start_dt, end_dt, delta): @@ -232,9 +223,9 @@ def test_add_coverage(self): } ) - metadatas = [] + # metadatas = [] coords = [] - values = [] + # values = [] metadata = { "class": "od", diff --git a/tests/test_encoder_frame.py b/tests/test_encoder_frame.py index f6e58ee..91f60b0 100644 --- a/tests/test_encoder_frame.py +++ b/tests/test_encoder_frame.py @@ -1,13 +1,4 @@ -import json -import random -from datetime import datetime, timedelta - -import pytest -import xarray as xr - -import eccovjson.decoder.TimeSeries from eccovjson.api import Eccovjson -from eccovjson.encoder import TimeSeries, encoder def get_timestamps(start_dt, end_dt, delta): @@ -232,9 +223,9 @@ def test_add_coverage(self): } ) - metadatas = [] + # metadatas = [] coords = [] - values = [] + # values = [] metadata = { "class": "od", diff --git a/tests/test_encoder_time_series.py b/tests/test_encoder_time_series.py index 23ec50a..52b05f9 100644 --- a/tests/test_encoder_time_series.py +++ b/tests/test_encoder_time_series.py @@ -1,13 +1,9 @@ -import json import random from datetime import datetime, timedelta -import pytest import xarray as xr -import eccovjson.decoder.TimeSeries from eccovjson.api import Eccovjson -from eccovjson.encoder import TimeSeries, encoder def get_timestamps(start_dt, end_dt, delta): @@ -244,7 +240,7 @@ def test_add_coverage(self): } ) - metadatas = [] + # metadatas = [] coords = [] values = [] for number in range(0, 50): From f7027caf1ae81a011a432f8d075fdaef82d76abe Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 13:59:49 +0100 Subject: [PATCH 51/58] change CI for wheels --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6bc9671..9665794 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -107,6 +107,7 @@ jobs: - name: Install Python Dependencies run: | python -m pip install --upgrade pip + pip install --upgrade pip setuptools wheel python -m pip install pytest pytest-cov python -m pip install -r requirements.txt python -m pip install -r ./tests/requirements.txt From 1866809ff4548529c3bc3011a5f9f6ec952cb411 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 14:01:29 +0100 Subject: [PATCH 52/58] ignore built in json requirement --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c4dcbd9..503999b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ polytope -json +# json From b23586ea52287b2a390626b8b6c84a382af6adb3 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 14:03:28 +0100 Subject: [PATCH 53/58] change name of test requirements --- .github/workflows/ci.yaml | 2 +- tests/{requirements.txt => requirements_test.txt} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/{requirements.txt => requirements_test.txt} (100%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9665794..25aea0d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -110,7 +110,7 @@ jobs: pip install --upgrade pip setuptools wheel python -m pip install pytest pytest-cov python -m pip install -r requirements.txt - python -m pip install -r ./tests/requirements.txt + python -m pip install -r ./tests/requirements_test.txt - name: Verify Source Distribution shell: bash -eux {0} diff --git a/tests/requirements.txt b/tests/requirements_test.txt similarity index 100% rename from tests/requirements.txt rename to tests/requirements_test.txt From 0e90a80b6f0f79229ea6a66d338c6d0b13299126 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 14:05:45 +0100 Subject: [PATCH 54/58] change test requirements --- tests/requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index 36491a6..a152ac5 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -1,3 +1,3 @@ -r ../requirements.txt --e .. +# -e .. pytest From 111dded21121cc4b8cae87b0f2825f4aca65569a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 14:07:49 +0100 Subject: [PATCH 55/58] change test requirements --- tests/requirements_test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index a152ac5..41afafc 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -1,3 +1,4 @@ -r ../requirements.txt # -e .. pytest +xarray==2022.12.0 From 043f0796aa282508ddcef02cb9c57c70fc43eb13 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 14:10:26 +0100 Subject: [PATCH 56/58] change requirements --- requirements.txt | 2 ++ tests/requirements_test.txt | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 503999b..b873d8e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ polytope # json +xarray==2022.12.0 +pandas==1.5.2 diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index 41afafc..a152ac5 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -1,4 +1,3 @@ -r ../requirements.txt # -e .. pytest -xarray==2022.12.0 From fa24f5c3f5b28b1bb19162a281a676866ce8f8a7 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 14:18:34 +0100 Subject: [PATCH 57/58] fix CI tests --- .github/workflows/ci.yaml | 2 +- pyproject.toml | 4 +++- tests/test_encoder_time_series.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 25aea0d..6531672 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -122,7 +122,7 @@ jobs: LD_LIBRARY_PATH: ${{ steps.install-dependencies.outputs.lib_path }} shell: bash -eux {0} run: | - DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest -m tests --cov=./ --cov-report=xml + DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest -m "not data" tests --cov=./ --cov-report=xml python -m coverage report - name: Upload coverage to Codecov diff --git a/pyproject.toml b/pyproject.toml index 89b8eb7..146a802 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,4 +2,6 @@ line-length = 120 [build-system] requires = ["setuptools>=61.0"] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" +[tool.pytest.ini_options] +markers = ["data: uses test data (deselect with '-m \"not data\"')",] \ No newline at end of file diff --git a/tests/test_encoder_time_series.py b/tests/test_encoder_time_series.py index 52b05f9..f34614f 100644 --- a/tests/test_encoder_time_series.py +++ b/tests/test_encoder_time_series.py @@ -1,6 +1,7 @@ import random from datetime import datetime, timedelta +import pytest import xarray as xr from eccovjson.api import Eccovjson @@ -269,6 +270,7 @@ def test_add_coverage(self): encoder.add_coverage(metadata, coord, value) # print(encoder.covjson) + @pytest.mark.data def test_from_xarray(self): ds = xr.open_dataset("new_timeseries.nc") encoder = Eccovjson().encode("CoverageCollection", "PointSeries") From d8ac573c04d09f528f40823c53e0b36e93b24abd Mon Sep 17 00:00:00 2001 From: awarde96 Date: Wed, 24 Jan 2024 16:05:03 +0100 Subject: [PATCH 58/58] From_polytope no longer requires number to be in the tree --- README.md | 1 + eccovjson/encoder/TimeSeries.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 196b114..06ce793 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ ECMWF library for encoding and decoding coerageJSON files/objects of meteorlogical features such as vertical profiles and time series. * Encodes and decodes CoverageJSON objects +* Convert CoverageJSON files to and from xarray | :warning: This project is BETA and will be experimental for the foreseeable future. Interfaces and functionality are likely to change. DO NOT use this software in any project/software that is operational. | diff --git a/eccovjson/encoder/TimeSeries.py b/eccovjson/encoder/TimeSeries.py index 71661d6..0688c5f 100644 --- a/eccovjson/encoder/TimeSeries.py +++ b/eccovjson/encoder/TimeSeries.py @@ -40,7 +40,7 @@ def add_range(self, coverage, values): coverage["ranges"][param]["dataType"] = "float" coverage["ranges"][param]["shape"] = [len(values[parameter])] coverage["ranges"][param]["axisNames"] = [str(param)] - coverage["ranges"][param]["values"] = values[parameter] # [values[parameter]] + coverage["ranges"][param]["values"] = values[parameter] def add_mars_metadata(self, coverage, metadata): coverage["mars:metadata"] = metadata @@ -133,21 +133,29 @@ def from_polytope(self, result): # convert step into datetime date_format = "%Y%m%dT%H%M%S" date = pd.Timestamp(mars_metadata["date"]).strftime(date_format) - start_time = datetime.datetime.strptime(date, date_format) + start_time = datetime.strptime(date, date_format) for step in steps: # add current date to list by converting it to iso format stamp = start_time + timedelta(hours=int(step)) coords["t"].append(stamp.isoformat()) # increment start date by timedelta - for number in df["number"].unique(): + if "number" not in df.columns: new_metadata = mars_metadata.copy() - new_metadata["number"] = number - df_number = df[df["number"] == number] range_dict = {} for param in params: - df_param = df_number[df_number["param"] == param] + df_param = df[df["param"] == param] range_dict[param] = df_param["values"].values.tolist() self.add_coverage(new_metadata, coords, range_dict) + else: + for number in df["number"].unique(): + new_metadata = mars_metadata.copy() + new_metadata["number"] = number + df_number = df[df["number"] == number] + range_dict = {} + for param in params: + df_param = df_number[df_number["param"] == param] + range_dict[param] = df_param["values"].values.tolist() + self.add_coverage(new_metadata, coords, range_dict) return self.covjson