From e6bd0e43c14d24b07e92fb00a8d936acc1e46d51 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Thu, 20 Jan 2022 18:31:40 -0800 Subject: [PATCH 01/23] docs: update RTD versions discussion (latest > dev) [skip ci] (#533) I'll self merge --- docs/source/contributing.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 8c72fe8ea..38998bff0 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -35,7 +35,8 @@ This diagram depicts the complete workflow we use in the source GitHub repositor rel --> main - ``doc patch``: Updates to the documentation that refer to the current ``echopype`` - release can be pushed out immediately to the `echopype documentation site `_ + release can be pushed out immediately to the + `echopype documentation site `_ by contibuting patches (PRs) to the ``stable`` branch. See `Documentation development`_ below for more details. - ``code patch``: Code development is carried out as patches (PRs) to the ``dev`` @@ -131,7 +132,6 @@ and `S3 object-storage `_ sources, the latter via `minio `_. `.ci_helpers/run-test.py `_ - will execute all tests. The entire test suite can be a bit slow, taking up to 40 minutes or more. If your changes impact only some of the subpackages (``convert``, ``calibrate``, ``preprocess``, etc), you can run ``run-test.py`` with only a subset of tests by passing @@ -142,7 +142,6 @@ as an argument a comma-separated list of the modules that have changed. For exam python .ci_helpers/run-test.py --local --pytest-args="-vv" echopype/calibrate/calibrate_ek.py,echopype/preprocess/noise_est.py will run only tests associated with the ``calibrate`` and ``preprocess`` subpackages. - For ``run-test.py`` usage information, use the ``-h`` argument: ``python .ci_helpers/run-test.py -h`` @@ -224,12 +223,13 @@ Documentation versions ``_ redirects to the documentation ``stable`` version, ``_, which is built from the ``stable`` branch on the ``echopype`` GitHub repository. In addition, the ``latest`` version -(``_) is built from the ``main`` branch, -while the hidden `dev` version (``_) is built -from the ``dev`` branch. Finally, each new echopype release is built as a new release version -on ReadTheDocs. Merging pull requests into any of these three branches or issuing a -new tagged release will automatically result in a new ReadTheDocs build for the +(``_) is built from the ``dev`` branch and +therefore it reflects the bleeding edge development code (which may occasionally break +the documenation build). Finally, each new echopype release is built as a new release version +on ReadTheDocs. Merging pull requests into ``stable`` or ``dev`` or issuing a new +tagged release will automatically result in a new ReadTheDocs build for the corresponding version. We also maintain a test version of the documentation at ``_ for viewing and debugging larger, more experimental changes, typically from a separate fork. +This version is used to test one-off, major breaking changes. From a5912e77b28a5a5e312e5988d2b2743e4830b0bd Mon Sep 17 00:00:00 2001 From: b-reyes Date: Tue, 1 Feb 2022 14:34:22 -0800 Subject: [PATCH 02/23] add a period --- docs/source/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index ce223ac67..4c64ddb95 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,4 +1,4 @@ -.. echopype documentation master file, created by +OA.. echopype documentation master file, created by sphinx-quickstart on Wed Feb 13 15:33:27 2019. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. @@ -17,7 +17,7 @@ However, most of the new data remain under-utilized. echopype aims to address the root cause of this problem - the lack of interoperable data format and scalable analysis workflows that adapt well with increasing data volume - by providing open-source tools as entry points for -scientists to make discovery using these new data. +scientists to make discovery using these new data. . Documentation From 5aaa231f8fc0799780017589e25981ccedcf991b Mon Sep 17 00:00:00 2001 From: b-reyes Date: Wed, 2 Feb 2022 09:01:41 -0800 Subject: [PATCH 03/23] Change name of installation section and add an additional examples section --- docs/source/index.rst | 4 ++-- docs/source/installation.rst | 16 +++++++++++++++- docs/source/resources.rst | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 4c64ddb95..ce223ac67 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,4 +1,4 @@ -OA.. echopype documentation master file, created by +.. echopype documentation master file, created by sphinx-quickstart on Wed Feb 13 15:33:27 2019. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. @@ -17,7 +17,7 @@ However, most of the new data remain under-utilized. echopype aims to address the root cause of this problem - the lack of interoperable data format and scalable analysis workflows that adapt well with increasing data volume - by providing open-source tools as entry points for -scientists to make discovery using these new data. . +scientists to make discovery using these new data. Documentation diff --git a/docs/source/installation.rst b/docs/source/installation.rst index c1056acc7..996542804 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -1,5 +1,9 @@ +Installation and Examples +========================= + + Installation -============ +------------ Echopype is available and tested for Python>=3.7. The latest release can be installed from `PyPI `_: @@ -18,3 +22,13 @@ Previous releases are also available on PyPI and conda. For instructions on installing a development version of echopype, see the :doc:`contributing` page. + + +Examples +-------- + +Additional `Jupyter notebooks `_ +illustrating the workflow of Echopype are also made available to the public. These +examples include a quick tour of Echopype, a demonstration of how Echopype can be used +to explore ship echosounder data from a Pacific Hake survey, and using Echopype to +visualize the response of zooplankton to a solar eclipse. \ No newline at end of file diff --git a/docs/source/resources.rst b/docs/source/resources.rst index 8ef2ad7c2..02d677530 100644 --- a/docs/source/resources.rst +++ b/docs/source/resources.rst @@ -1,6 +1,7 @@ Other resources ================ + Software -------- From f9a74d254a357c3a58fd6c21338e82efb323ddd7 Mon Sep 17 00:00:00 2001 From: b-reyes Date: Wed, 2 Feb 2022 11:17:37 -0800 Subject: [PATCH 04/23] remove "to the public" --- docs/source/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 996542804..61b9e2c9b 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -28,7 +28,7 @@ Examples -------- Additional `Jupyter notebooks `_ -illustrating the workflow of Echopype are also made available to the public. These +illustrating the workflow of Echopype are also made available. These examples include a quick tour of Echopype, a demonstration of how Echopype can be used to explore ship echosounder data from a Pacific Hake survey, and using Echopype to visualize the response of zooplankton to a solar eclipse. \ No newline at end of file From e22c788768937a10808105ba011ed5ced83a0127 Mon Sep 17 00:00:00 2001 From: leewujung Date: Thu, 10 Feb 2022 19:56:31 -0800 Subject: [PATCH 05/23] add remnant of conflict resolution --- docs/source/contributing.rst | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index a29a104d2..9934d2433 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -34,16 +34,10 @@ This diagram depicts the complete workflow we use in the source GitHub repositor dev --> |dev merge| rel rel --> main -<<<<<<< HEAD - ``doc patch``: Updates to the documentation that refer to the current ``echopype`` release can be pushed out immediately to the `echopype documentation site `_ by contibuting patches (PRs) to the ``stable`` branch. See `Documentation development`_ -======= -- ``doc patch``: Updates to the documentation that refer to the current ``echopype`` - release can be pushed out immediately to the `echopype documentation site `_ - by contibuting patches (PRs) to the ``stable`` branch. See `Documentation development`_ ->>>>>>> main below for more details. - ``code patch``: Code development is carried out as patches (PRs) to the ``dev`` branch; changes in the documentation corresponding to changes in the code can be @@ -226,7 +220,6 @@ and adding a new section that documents a previously undocumented feature. Documentation versions ~~~~~~~~~~~~~~~~~~~~~~ -<<<<<<< HEAD ``_ redirects to the documentation ``stable`` version, ``_, which is built from the ``stable`` branch on the ``echopype`` GitHub repository. In addition, the ``latest`` version @@ -235,16 +228,6 @@ therefore it reflects the bleeding edge development code (which may occasionally the documenation build). Finally, each new echopype release is built as a new release version on ReadTheDocs. Merging pull requests into ``stable`` or ``dev`` or issuing a new tagged release will automatically result in a new ReadTheDocs build for the -======= -``_ redirects to the documentation ``stable`` version, -``_, which is built from the ``stable`` branch -on the ``echopype`` GitHub repository. In addition, the ``latest`` version -(``_) is built from the ``main`` branch, -while the hidden `dev` version (``_) is built -from the ``dev`` branch. Finally, each new echopype release is built as a new release version -on ReadTheDocs. Merging pull requests into any of these three branches or issuing a -new tagged release will automatically result in a new ReadTheDocs build for the ->>>>>>> main corresponding version. We also maintain a test version of the documentation at ``_ From 03a84d4fbd4f4fc999866397fa407f61855de2c0 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Sun, 13 Feb 2022 21:15:40 -0800 Subject: [PATCH 06/23] Add 0.5.6 to What's new (#564) * docs: update RTD versions discussion (latest > dev) [skip ci] * docs: Add 0.5.6 to What's new * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/source/contributing.rst | 22 ++++++++-------- docs/source/installation.rst | 2 +- docs/source/whats-new.rst | 50 ++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 9934d2433..8bb4b3a97 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -34,10 +34,10 @@ This diagram depicts the complete workflow we use in the source GitHub repositor dev --> |dev merge| rel rel --> main -- ``doc patch``: Updates to the documentation that refer to the current ``echopype`` - release can be pushed out immediately to the - `echopype documentation site `_ - by contibuting patches (PRs) to the ``stable`` branch. See `Documentation development`_ +- ``doc patch``: Updates to the documentation that refer to the current ``echopype`` + release can be pushed out immediately to the + `echopype documentation site `_ + by contibuting patches (PRs) to the ``stable`` branch. See `Documentation development`_ below for more details. - ``code patch``: Code development is carried out as patches (PRs) to the ``dev`` branch; changes in the documentation corresponding to changes in the code can be @@ -220,14 +220,14 @@ and adding a new section that documents a previously undocumented feature. Documentation versions ~~~~~~~~~~~~~~~~~~~~~~ -``_ redirects to the documentation ``stable`` version, -``_, which is built from the ``stable`` branch -on the ``echopype`` GitHub repository. In addition, the ``latest`` version -(``_) is built from the ``dev`` branch and +``_ redirects to the documentation ``stable`` version, +``_, which is built from the ``stable`` branch +on the ``echopype`` GitHub repository. In addition, the ``latest`` version +(``_) is built from the ``dev`` branch and therefore it reflects the bleeding edge development code (which may occasionally break -the documenation build). Finally, each new echopype release is built as a new release version -on ReadTheDocs. Merging pull requests into ``stable`` or ``dev`` or issuing a new -tagged release will automatically result in a new ReadTheDocs build for the +the documentation build). Finally, each new echopype release is built as a new release version +on ReadTheDocs. Merging pull requests into ``stable`` or ``dev`` or issuing a new +tagged release will automatically result in a new ReadTheDocs build for the corresponding version. We also maintain a test version of the documentation at ``_ diff --git a/docs/source/installation.rst b/docs/source/installation.rst index ee4cf1645..470e5ea3a 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -31,4 +31,4 @@ Additional `Jupyter notebooks `_ for the complete history. +v0.5.6 (2022 Feb 10) +-------------------- + +Overview +~~~~~~~~ + +This is a minor release that contains an experimental new feature and a number of enhancements, clean-up and bug fixes, which pave the way for the next major release. + +New feature +~~~~~~~~~~~ + +- (beta) Allow interpolating CTD data in calibration (#464) + + - Interpolation currently allowed along the ``ping_time`` dimension (the ``"stationary"`` case) and across ``latitude`` and ``longitude`` (the ``"mobile"`` case). + - This mechanism is enabled via a new ``EnvParams`` class at input of calibration functions. + +Enhancements +~~~~~~~~~~~~ + +- Make visualize module fully optional with ``matplotlib``, ``cmocean`` being optional dependency (#526, #559) +- Set range entries with no backscatter data to NaN in output of ``echodata.compute_range()`` (#547) and still allows quick visualization (#555) +- Add ``codespell`` GitHub action to ensure correct spellings of words (#557) +- Allow ``sonar_model="EA640"`` for ``open_raw`` (before it had to be "EK80") (#539) + +Bug fixes +~~~~~~~~~ + +- Allow using ``sonar_model="EA640"`` (#538, #539) +- Allow flexible and empty environment variables in EA640/EK80 files (#537) +- Docstring overhaul and fix bugs in ``utils.uwa`` (#525) + +Documentation +~~~~~~~~~~~~~ + +- Upgrade echopype docs to use jupyter book (#543) +- Change the RTD ``latest`` to point to the ``dev`` branch (#467) + +Testing +~~~~~~~ + +- Update convert tests to enable parallel testing (#556) +- Overhaul tests (#523, #498) + + - use ``pytest.fixture`` for testing + - add ES70/ES80/EA640 test files + - add new EK80 small test files with parameter combinations + - reduce size for a subset of large EK80 test data files + +- Add packaging testing for the ``dev`` branch (#554) + v0.5.5 (2021 Dec 10) -------------------- From 5b3167a42c12fe950393bd5f01fcb85a5e08b9b0 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Wed, 13 Apr 2022 18:57:23 -0700 Subject: [PATCH 07/23] docs: Update broken netcdf url [skip ci] (#627) --- docs/source/why.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/why.rst b/docs/source/why.rst index 73b2d6260..cd0c6cdb0 100644 --- a/docs/source/why.rst +++ b/docs/source/why.rst @@ -21,7 +21,7 @@ other climate and oceanographic data sets, facilitating the integration of ocean sonar data in interdisciplinary oceanographic research. .. _netCDF: - https://www.unidata.ucar.edu/software/netcdf/docs/netcdf_introduction.html + https://www.unidata.ucar.edu/software/netcdf/ .. _xarray: http://xarray.pydata.org/ .. _dask: http://dask.pydata.org/ .. _pandas: https://pandas.pydata.org/ From 17271a74deb2fc105a59395b3993dcb7c922e5c1 Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Tue, 19 Apr 2022 15:08:12 -0700 Subject: [PATCH 08/23] update creating conda dev env instructions (#633) --- docs/source/contributing.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 8bb4b3a97..13dd0e3e8 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -77,11 +77,18 @@ Create a `conda `_ environment for echopype development .. code-block:: bash - conda create -c conda-forge -n echopype --yes python=3.9 --file requirements.txt --file requirements-dev.txt + # create conda environment using the supplied requirements files + # note the last one docs/requirements.txt is only required for building docs + conda create -c conda-forge -n echopype --yes python=3.9 --file requirements.txt --file requirements-dev.txt --file docs/requirements.txt + + # switch to the newly built environment conda activate echopype + # ipykernel is recommended, in order to use with JupyterLab and IPython # to aid with development. We recommend you install JupyterLab separately conda install -c conda-forge ipykernel + + # install echopype in editable mode (setuptools "develop mode") pip install -e . See the :doc:`installation` page to simply install the latest echopype release from conda or PyPI. From a874b9315e4e73c27efcbd78e1760b7af3d44bd0 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Mon, 23 May 2022 15:03:38 -0700 Subject: [PATCH 09/23] Update contributors text [skip ci] (#703) * docs: update contributors text [skip ci] * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- README.md | 11 +++------ docs/source/_toc.yml | 2 +- docs/source/index.md | 42 +++++++++++++++++++++++++++++++ docs/source/index.rst | 57 ------------------------------------------- 4 files changed, 46 insertions(+), 66 deletions(-) create mode 100644 docs/source/index.md delete mode 100644 docs/source/index.rst diff --git a/README.md b/README.md index bf50746f2..5d1442a7f 100644 --- a/README.md +++ b/README.md @@ -67,19 +67,14 @@ Please report any bugs by [creating issues on GitHub](https://medium.com/nyc-pla Contributors ------------ -[Wu-Jung Lee](http://leewujung.github.io) (@leewujung) leads this project and together with -[Kavin Nguyen](https://github.com/ngkavin) (@ngkavin), [Landung "Don" Setiawan](https://github.com/lsetiawan) (@lsetiawan), and [Imran Majeed](https://github.com/imranmaj) (@imranmaj) are primary developers of this package. -[Emilio Mayorga](https://www.apl.washington.edu/people/profile.php?last_name=Mayorga&first_name=Emilio) (@emiliom) -and [Valentina Staneva](https://escience.washington.edu/people/valentina-staneva/) (@valentina-s) -are also part of the development team. +Wu-Jung Lee ([@leewujung](https://github.com/leewujung)) founded the echopype project in 2018. It is currently led by Wu-Jung Lee and Emilio Mayorga ([@emiliom](https://github.com/emiliom)), who are primary developers together with Brandon Reyes ([@b-reyes](https://github.com/b-reyes)), Landung "Don" Setiawan ([@lsetiawan](https://github.com/lsetiawan)), and previously Kavin Nguyen ([@ngkavin](https://github.com/ngkavin)) and Imran Majeed ([@imranmaj](https://github.com/imranmaj)). Valentina Staneva ([@valentina-s](https://github.com/valentina-s)) is also part of the development team. -Other contributors are listed in [echopype documentation](https://echopype.readthedocs.io). We thank Dave Billenness of ASL Environmental Sciences for providing the AZFP Matlab Toolbox as reference for our development of AZFP support in echopype. -We also thank [Rick Towler](https://github.com/rhtowler) (@rhtowler) -of the Alaska Fisheries Science Center +We also thank Rick Towler ([@rhtowler](https://github.com/rhtowler)) +of the NOAA Alaska Fisheries Science Center for providing low-level file parsing routines for Simrad EK60 and EK80 echosounders. diff --git a/docs/source/_toc.yml b/docs/source/_toc.yml index 328217844..535087490 100644 --- a/docs/source/_toc.yml +++ b/docs/source/_toc.yml @@ -2,7 +2,7 @@ # Learn more at https://jupyterbook.org/customize/toc.html format: jb-book -root: index.rst +root: index parts: - caption: Getting Started chapters: diff --git a/docs/source/index.md b/docs/source/index.md new file mode 100644 index 000000000..b58bea38f --- /dev/null +++ b/docs/source/index.md @@ -0,0 +1,42 @@ +# Welcome to echopype! + +**Echopype** is a package built to enable interoperability and scalability +in ocean sonar data processing. +These data are widely used for obtaining information about the distribution and +abundance of marine animals, such as fish and krill. +Our ability to collect large volumes of sonar data from a variety of +ocean platforms has grown significantly in the last decade. +However, most of the new data remain under-utilized. +echopype aims to address the root cause of this problem - the lack of +interoperable data format and scalable analysis workflows that adapt well +with increasing data volume - by providing open-source tools as entry points for +scientists to make discovery using these new data. + + +## Contributors + +Wu-Jung Lee ([@leewujung](https://github.com/leewujung)) founded the echopype project in 2018. It is currently led by Wu-Jung Lee and Emilio Mayorga ([@emiliom](https://github.com/emiliom)), who are primary developers together with Brandon Reyes ([@b-reyes](https://github.com/b-reyes)), Landung "Don" Setiawan ([@lsetiawan](https://github.com/lsetiawan)), and previously Kavin Nguyen ([@ngkavin](https://github.com/ngkavin)) and Imran Majeed ([@imranmaj](https://github.com/imranmaj)). Valentina Staneva ([@valentina-s](https://github.com/valentina-s)) is also part of the development team. + +Other contributors include: +Frederic Cyr ([@cyrf0006](https://github.com/cyrf0006)), +Paul Robinson ([@prarobinson](https://github.com/prarobinson)), +Sven Gastauer ([@SvenGastauer](https://github.com/SvenGastauer)), +Marian Peña ([@marianpena](https://github.com/marianpena)), +Mark Langhirt ([@bnwkeys](https://github.com/bnwkeys)), +Erin LaBrecque ([@erinann](https://github.com/erinann)), +Emma Ozanich ([@emma-ozanich](https://github.com/emma-ozanich)), +Aaron Marburg ([@amarburg](https://github.com/amarburg)). A complete list of direct contributors is on our [GitHub Contributors Page](https://github.com/OSOceanAcoustics/echopype/graphs/contributors). + +We thank Dave Billenness of ASL Environmental Sciences for +providing the AZFP Matlab Toolbox as reference for our +development of AZFP support in echopype. +We also thank Rick Towler ([@rhtowler](https://github.com/rhtowler)) +of the NOAA Alaska Fisheries Science Center +for providing low-level file parsing routines for +Simrad EK60 and EK80 echosounders. + + +## License + +Echopype is licensed under the open source +[Apache 2.0 license](https://opensource.org/licenses/Apache-2.0). diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index cdd5a5067..000000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. echopype documentation master file, created by - sphinx-quickstart on Wed Feb 13 15:33:27 2019. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - - -Welcome to echopype! -==================== - -**Echopype** is a package built to enable interoperability and scalability -in ocean sonar data processing. -These data are widely used for obtaining information about the distribution and -abundance of marine animals, such as fish and krill. -Our ability to collect large volumes of sonar data from a variety of -ocean platforms has grown significantly in the last decade. -However, most of the new data remain under-utilized. -echopype aims to address the root cause of this problem - the lack of -interoperable data format and scalable analysis workflows that adapt well -with increasing data volume - by providing open-source tools as entry points for -scientists to make discovery using these new data. - -Contributors ------------- - -`Wu-Jung Lee `_ (@leewujung) leads this project -and together with `Kavin Nguyen `_ (@ngkavin), -`Landung "Don" Setiawan `_ (@lsetiawan), -and `Imran Majeed `_ (@imranmaj) -are primary developers of this package. -`Emilio Mayorga `_ (@emiliom) -and `Valentina Staneva `_ (@valentina-s) -are also part of the development team. - -Other contributors include: -`Frederic Cyr `_ (@cyrf0006), -`Paul Robinson `_ (@prarobinson), -`Sven Gastauer `_ (@SvenGastauer), -`Marian Peña `_ (@marianpena), -`Mark Langhirt `_ (@bnwkeys), -`Erin LaBrecque `_ (@erinann), -`Emma Ozanich `_ (@emma-ozanich), -`Aaron Marburg `_ (@amarburg) - -We thank Dave Billenness of ASL Environmental Sciences for -providing the AZFP Matlab Toolbox as reference for our -development of AZFP support in echopype. -We also thank `Rick Towler `_ (@rhtowler) -of the Alaska Fisheries Science Center -for providing low-level file parsing routines for -Simrad EK60 and EK80 echosounders. - - -License -------- - -Echopype is licensed under the open source -`Apache 2.0 license `_. From 7ce7c5a8e5ba92de4ac8cdbe6293b8ab66a73e4a Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Mon, 23 May 2022 15:09:44 -0700 Subject: [PATCH 10/23] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5d1442a7f..c7d4fa31c 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ Contributors Wu-Jung Lee ([@leewujung](https://github.com/leewujung)) founded the echopype project in 2018. It is currently led by Wu-Jung Lee and Emilio Mayorga ([@emiliom](https://github.com/emiliom)), who are primary developers together with Brandon Reyes ([@b-reyes](https://github.com/b-reyes)), Landung "Don" Setiawan ([@lsetiawan](https://github.com/lsetiawan)), and previously Kavin Nguyen ([@ngkavin](https://github.com/ngkavin)) and Imran Majeed ([@imranmaj](https://github.com/imranmaj)). Valentina Staneva ([@valentina-s](https://github.com/valentina-s)) is also part of the development team. +Other contributors are listed in [echopype documentation](https://echopype.readthedocs.io). We thank Dave Billenness of ASL Environmental Sciences for providing the AZFP Matlab Toolbox as reference for our From 4eb435bd07e1d4789efe96402661eb2021a9d6d3 Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Wed, 29 Jun 2022 07:28:31 -0700 Subject: [PATCH 11/23] Fix python version requirements in docs (#744) * update required python version to 3.8 due to xarray requirements * add sphinx-panels to docs/requirements --- docs/requirements.txt | 1 + docs/source/installation.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 6e6afca68..a91018353 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ sphinx_rtd_theme sphinx-automodapi +sphinx-panels sphinxcontrib-mermaid jupyter-book numpydoc diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 470e5ea3a..2726009d5 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -5,7 +5,7 @@ Installation and Examples Installation ------------ -Echopype is available and tested for Python>=3.7. The latest release +Echopype is available and tested for Python>=3.8. The latest release can be installed from `PyPI `_: .. code-block:: console From a3a259fbba421b65fba7e33f77415bacee305e4b Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Wed, 13 Jul 2022 00:56:26 -0700 Subject: [PATCH 12/23] Updates to "Contributing to echopype" doc page [skip ci] (#764) * Update contributing page, CI flags and dev installation instructions * docs: CI actions flags now via PR title; mamba note; quote .[plot] for multi platform support * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/source/contributing.rst | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 576e3be8b..aa4542ac5 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -91,7 +91,14 @@ Create a `conda `_ environment for echopype development # install echopype in editable mode (setuptools "develop mode") # plot is an extra set of requirements that can be used for plotting. # the command will install all the dependencies along with plotting dependencies. - pip install -e .[plot] + pip install -e ".[plot]" + +.. note:: + + Try using `mamba `_ instead of ``conda`` + if the ``conda create`` and ``conda install`` step fail or take too long. + ``Mamba`` is a drop-in replacement for conda environment creation and package + installation that is typically faster than conda. See the :doc:`installation` page to simply install the latest echopype release from conda or PyPI. @@ -102,13 +109,6 @@ Tests and test infrastructure Test data files ~~~~~~~~~~~~~~~ -.. attention:: - - Echopype previously used Git LFS for managing and accessing large test data files. - We have deprecated its use starting with echopype version 0.5.0. The files - in https://github.com/OSOceanAcoustics/echopype/tree/main/echopype/test_data - are also being deprecated. - Test echosounder data files are managed in a private Google Drive folder and made available via the `cormorack/http `_ Docker image on Docker hub; the image is rebuilt daily when new test data are added @@ -181,14 +181,12 @@ The entire test suite can be a bit slow, taking up to 40 minutes or more. To mitigate this, the CI default is to run tests only for subpackages that were modified in the PR; this is done via ``.ci_helpers/run-test.py`` (see the `Running the tests`_ section). To have the CI execute the -entire test suite, add the GitHub label ``Needs Complete Testing`` to the -PR before submitting it. - +entire test suite, add the string "[all tests ci]" to the PR title. Under special circumstances, when the submitted changes have a very limited scope (such as contributions to the documentation) or you know exactly what you're doing (you're a seasoned echopype contributor), the CI can be skipped. -This is done by including the string "[skip ci]" in your last commit's message. +This is done by adding the string "[skip ci]" to the PR title. Documentation development From 72909babcfbc567943072c2d7293fcf7665810c4 Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Thu, 4 Aug 2022 16:39:23 -0400 Subject: [PATCH 13/23] Add function to interpolate location to calibrated dataset (#749) * add first prototype of add_location * add simple test * use test_path directly * add typing for echodata * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * move add_location to preprocess * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix init * move add_latlon to subpackage consolidate * fix test * Added test for missing and all-nan lon & lat variables. Added support for propagating fixed-location (mooring) lat-lon coordinate * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Emilio Mayorga --- echopype/__init__.py | 12 +++- echopype/consolidate/__init__.py | 3 + echopype/consolidate/api.py | 67 +++++++++++++++++++ .../tests/consolidate/test_consolidate.py | 20 ++++++ 4 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 echopype/consolidate/__init__.py create mode 100644 echopype/consolidate/api.py create mode 100644 echopype/tests/consolidate/test_consolidate.py diff --git a/echopype/__init__.py b/echopype/__init__.py index ca1953a3b..03adaafe2 100644 --- a/echopype/__init__.py +++ b/echopype/__init__.py @@ -2,9 +2,17 @@ from _echopype_version import version as __version__ # noqa -from . import calibrate, preprocess, utils +from . import calibrate, consolidate, preprocess, utils from .convert.api import open_raw from .echodata.api import open_converted from .echodata.combine import combine_echodata -__all__ = ["open_raw", "open_converted", "combine_echodata", "calibrate", "preprocess", "utils"] +__all__ = [ + "open_raw", + "open_converted", + "combine_echodata", + "calibrate", + "consolidate", + "preprocess", + "utils", +] diff --git a/echopype/consolidate/__init__.py b/echopype/consolidate/__init__.py new file mode 100644 index 000000000..ec7ca9dec --- /dev/null +++ b/echopype/consolidate/__init__.py @@ -0,0 +1,3 @@ +from .api import add_location + +__all__ = ["add_location"] diff --git a/echopype/consolidate/api.py b/echopype/consolidate/api.py new file mode 100644 index 000000000..090d3a108 --- /dev/null +++ b/echopype/consolidate/api.py @@ -0,0 +1,67 @@ +import datetime +from typing import Optional + +import numpy as np +import xarray as xr + +from ..echodata import EchoData + + +def add_location(ds: xr.Dataset, echodata: EchoData = None, nmea_sentence: Optional[str] = None): + """ + Add geographical location (latitude/longitude) to the Sv dataset. + + This function interpolates the location from the Platform group in the original data file + based on the time when the latitude/longitude data are recorded and the time the acoustic + data are recorded (`ping_time`). + + Parameters + ---------- + ds : xr.Dataset + An Sv or MVBS dataset for which the geographical locations will be added to + echodata + An `EchoData` object holding the raw data + nmea_sentence + NMEA sentence to select a subset of location data (optional) + + Returns + ------- + The input dataset with the the location data added + """ + + def sel_interp(var): + # NMEA sentence selection + if nmea_sentence: + coord_var = echodata["Platform"][var][ + echodata["Platform"]["sentence_type"] == nmea_sentence + ] + else: + coord_var = echodata["Platform"][var] + + if len(coord_var) == 1: + # Propagate single, fixed-location coordinate + return xr.DataArray( + data=coord_var.values[0] * np.ones(len(ds["ping_time"]), dtype=np.float64), + dims=["ping_time"], + attrs=coord_var.attrs, + ) + else: + # Interpolation. time1 is always associated with location data + return coord_var.interp(time1=ds["ping_time"]) + + if "longitude" not in echodata["Platform"] or echodata["Platform"]["longitude"].isnull().all(): + raise ValueError("Coordinate variables not present or all nan") + + interp_ds = ds.copy() + interp_ds["latitude"] = sel_interp("latitude") + interp_ds["longitude"] = sel_interp("longitude") + # Most attributes are attached automatically via interpolation + # here we add the history + history = ( + f"{datetime.datetime.utcnow()} +00:00. " + "Interpolated or propagated from Platform latitude/longitude." # noqa + ) + interp_ds["latitude"] = interp_ds["latitude"].assign_attrs({"history": history}) + interp_ds["longitude"] = interp_ds["longitude"].assign_attrs({"history": history}) + + return interp_ds.drop_vars("time1") diff --git a/echopype/tests/consolidate/test_consolidate.py b/echopype/tests/consolidate/test_consolidate.py new file mode 100644 index 000000000..e1cfdbcf6 --- /dev/null +++ b/echopype/tests/consolidate/test_consolidate.py @@ -0,0 +1,20 @@ +import echopype as ep + + +def test_add_location(test_path): + ed = ep.open_raw( + test_path["EK60"] / "Winter2017-D20170115-T150122.raw", + sonar_model="EK60" + ) + ds = ep.calibrate.compute_Sv(ed) + + def _check_var(ds_test): + assert "latitude" in ds_test + assert "longitude" in ds_test + assert "time1" not in ds_test + + ds_all = ep.consolidate.add_location(ds=ds, echodata=ed) + _check_var(ds_all) + + ds_sel = ep.consolidate.add_location(ds=ds, echodata=ed, nmea_sentence="GGA") + _check_var(ds_sel) From ad509db895a80bf435969919fa56cece73972c68 Mon Sep 17 00:00:00 2001 From: b-reyes <53541061+b-reyes@users.noreply.github.com> Date: Fri, 5 Aug 2022 11:56:59 -0700 Subject: [PATCH 14/23] change long_name in ds_power for EK80 (#771) --- echopype/convert/set_groups_ek80.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index 9b279072c..e8ce4f047 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -648,7 +648,7 @@ def _assemble_ds_power(self, ch): "backscatter_r": ( ["ping_time", "range_sample"], self.parser_obj.ping_data_dict["power"][ch], - {"long_name": "Backscattering power", "units": "dB"}, + {"long_name": "Backscatter power", "units": "dB"}, ), }, coords={ From a020d71dde5978c98d1f36c69e733d5047d6af9c Mon Sep 17 00:00:00 2001 From: Don Setiawan Date: Tue, 9 Aug 2022 12:10:19 -0700 Subject: [PATCH 15/23] Try pinning xarray to previous version (#775) --- .ci_helpers/py3.10.yaml | 2 +- .ci_helpers/py3.8.yaml | 2 +- .ci_helpers/py3.9.yaml | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci_helpers/py3.10.yaml b/.ci_helpers/py3.10.yaml index 5b7e99d61..fd2585b91 100644 --- a/.ci_helpers/py3.10.yaml +++ b/.ci_helpers/py3.10.yaml @@ -8,7 +8,7 @@ dependencies: - pynmea2 - pytz - scipy - - xarray + - xarray==2022.3.0 - zarr - fsspec - s3fs==2022.5.0 diff --git a/.ci_helpers/py3.8.yaml b/.ci_helpers/py3.8.yaml index 2597b6a3c..3623c47d3 100644 --- a/.ci_helpers/py3.8.yaml +++ b/.ci_helpers/py3.8.yaml @@ -8,7 +8,7 @@ dependencies: - pynmea2 - pytz - scipy - - xarray + - xarray==2022.3.0 - zarr - fsspec - s3fs==2022.5.0 diff --git a/.ci_helpers/py3.9.yaml b/.ci_helpers/py3.9.yaml index 2a0090659..41ffe9f7d 100644 --- a/.ci_helpers/py3.9.yaml +++ b/.ci_helpers/py3.9.yaml @@ -8,7 +8,7 @@ dependencies: - pynmea2 - pytz - scipy - - xarray + - xarray==2022.3.0 - zarr - fsspec - s3fs==2022.5.0 diff --git a/requirements.txt b/requirements.txt index 2b4a3a6db..772214e73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ numpy pynmea2 pytz scipy -xarray +xarray==2022.3.0 zarr fsspec s3fs From de13aeae82053872f8e210f2cc3d6b57b49c88fe Mon Sep 17 00:00:00 2001 From: Don Setiawan Date: Wed, 10 Aug 2022 14:36:36 -0700 Subject: [PATCH 16/23] Overhaul access pattern [all tests ci] (#762) * Update access patterns for convert api * Update access pattern for 'Environment' * Update access pattern for 'Platform' * Update access pattern for 'Provenance' * Update access pattern for 'Vendor_specific' * Update access pattern for 'Sonar/Beam_group2' * Update access pattern for 'Sonar/Beam_group1' and others * Remove getattrs * Update echopype/tests/echodata/test_echodata.py Co-authored-by: Don Setiawan * Update echopype/tests/echodata/test_echodata_combine.py Co-authored-by: Don Setiawan * Modify getitem to return None and remove try/except * Add spec comment * Update echopype/tests/echodata/test_echodata_combine.py Co-authored-by: Wu-Jung Lee Co-authored-by: Wu-Jung Lee --- echopype/calibrate/api.py | 4 +- echopype/calibrate/calibrate_azfp.py | 22 ++- echopype/calibrate/calibrate_ek.py | 154 +++++++++++------- echopype/calibrate/env_params.py | 16 +- echopype/convert/api.py | 31 ++-- echopype/echodata/combine.py | 37 +++-- echopype/echodata/echodata.py | 76 ++++----- echopype/tests/calibrate/test_calibrate.py | 6 +- echopype/tests/convert/test_convert_azfp.py | 20 +-- echopype/tests/convert/test_convert_ek60.py | 12 +- echopype/tests/convert/test_convert_ek80.py | 34 ++-- .../test_convert_source_target_locs.py | 58 +++---- echopype/tests/echodata/test_echodata.py | 93 +++++------ .../tests/echodata/test_echodata_combine.py | 66 +++++--- echopype/tests/preprocess/test_preprocess.py | 2 +- echopype/tests/visualize/test_plot.py | 18 +- echopype/visualize/api.py | 4 +- 17 files changed, 336 insertions(+), 317 deletions(-) diff --git a/echopype/calibrate/api.py b/echopype/calibrate/api.py index 4136e8134..6c5805ad7 100644 --- a/echopype/calibrate/api.py +++ b/echopype/calibrate/api.py @@ -104,9 +104,9 @@ def add_attrs(cal_type, ds): prov_dict["processing_function"] = f"calibrate.compute_{cal_type}" cal_ds = cal_ds.assign_attrs(prov_dict) - if "water_level" in echodata.platform.data_vars.keys(): + if "water_level" in echodata["Platform"].data_vars.keys(): # add water_level to the created xr.Dataset - cal_ds["water_level"] = echodata.platform.water_level + cal_ds["water_level"] = echodata["Platform"].water_level return cal_ds diff --git a/echopype/calibrate/calibrate_azfp.py b/echopype/calibrate/calibrate_azfp.py index ca01beb35..b61c79c7f 100644 --- a/echopype/calibrate/calibrate_azfp.py +++ b/echopype/calibrate/calibrate_azfp.py @@ -34,13 +34,15 @@ def get_cal_params(self, cal_params): self.cal_params["equivalent_beam_angle"] = ( cal_params["equivalent_beam_angle"] if "equivalent_beam_angle" in cal_params - else self.echodata.beam["equivalent_beam_angle"] + else self.echodata["Sonar/Beam_group1"]["equivalent_beam_angle"] ) # Get params from the Vendor_specific group for p in ["EL", "DS", "TVR", "VTX", "Sv_offset"]: # substitute if None in user input - self.cal_params[p] = cal_params[p] if p in cal_params else self.echodata.vendor[p] + self.cal_params[p] = ( + cal_params[p] if p in cal_params else self.echodata["Vendor_specific"][p] + ) def get_env_params(self): """Get env params using user inputs or values from data file. @@ -53,7 +55,7 @@ def get_env_params(self): self.env_params["temperature"] = ( self.env_params["temperature"] if "temperature" in self.env_params - else self.echodata.environment["temperature"] + else self.echodata["Environment"]["temperature"] ) # Salinity and pressure always come from user input @@ -71,7 +73,7 @@ def get_env_params(self): formula_source="AZFP", ) self.env_params["sound_absorption"] = uwa.calc_absorption( - frequency=self.echodata.beam["frequency_nominal"], + frequency=self.echodata["Sonar/Beam_group1"]["frequency_nominal"], temperature=self.env_params["temperature"], salinity=self.env_params["salinity"], pressure=self.env_params["pressure"], @@ -108,10 +110,10 @@ def _cal_power(self, cal_type, **kwargs): # Compute derived params # Harmonize time coordinate between Beam_groupX data and env_params - # Use self.echodata.beam because complex sample is always in Beam_group1 + # Use self.echodata["Sonar/Beam_group1"] because complex sample is always in Beam_group1 for p in self.env_params.keys(): self.env_params[p] = self.echodata._harmonize_env_param_time( - self.env_params[p], ping_time=self.echodata.beam.ping_time + self.env_params[p], ping_time=self.echodata["Sonar/Beam_group1"].ping_time ) # TODO: take care of dividing by zero encountered in log10 @@ -122,7 +124,9 @@ def _cal_power(self, cal_type, **kwargs): # scaling factor (slope) in Fig.G-1, units Volts/dB], see p.84 a = self.cal_params["DS"] EL = ( - self.cal_params["EL"] - 2.5 / a + self.echodata.beam.backscatter_r / (26214 * a) + self.cal_params["EL"] + - 2.5 / a + + self.echodata["Sonar/Beam_group1"].backscatter_r / (26214 * a) ) # eq.(5) # has beam dim due to backscatter_r if cal_type == "Sv": @@ -136,7 +140,7 @@ def _cal_power(self, cal_type, **kwargs): * np.log10( 0.5 * self.env_params["sound_speed"] - * self.echodata.beam["transmit_duration_nominal"] + * self.echodata["Sonar/Beam_group1"]["transmit_duration_nominal"] * self.cal_params["equivalent_beam_angle"] ) + self.cal_params["Sv_offset"] @@ -155,7 +159,7 @@ def _cal_power(self, cal_type, **kwargs): out = out.merge(self.range_meter) # Add frequency_nominal to data set - out["frequency_nominal"] = self.echodata.beam["frequency_nominal"] + out["frequency_nominal"] = self.echodata["Sonar/Beam_group1"]["frequency_nominal"] # Add env and cal parameters out = self._add_params_to_output(out) diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index a08b7fe04..42eacf3ba 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -54,7 +54,7 @@ def _get_vend_cal_params_power(self, param, waveform_mode): param : str {"sa_correction", "gain_correction"} name of parameter to retrieve """ - ds_vend = self.echodata.vendor + ds_vend = self.echodata["Vendor_specific"] if ds_vend is None or param not in ds_vend: return None @@ -62,10 +62,10 @@ def _get_vend_cal_params_power(self, param, waveform_mode): if param not in ["sa_correction", "gain_correction"]: raise ValueError(f"Unknown parameter {param}") - if waveform_mode == "CW" and self.echodata.beam_power is not None: - beam = self.echodata.beam_power + if waveform_mode == "CW" and self.echodata["Sonar/Beam_group2"] is not None: + beam = self.echodata["Sonar/Beam_group2"] else: - beam = self.echodata.beam + beam = self.echodata["Sonar/Beam_group1"] # indexes of frequencies that are for power, not complex relevant_indexes = np.where( @@ -105,11 +105,11 @@ def get_cal_params(self, cal_params, waveform_mode, encode_mode): if ( encode_mode == "power" and waveform_mode == "CW" - and self.echodata.beam_power is not None + and self.echodata["Sonar/Beam_group2"] is not None ): - beam = self.echodata.beam_power + beam = self.echodata["Sonar/Beam_group2"] else: - beam = self.echodata.beam + beam = self.echodata["Sonar/Beam_group1"] # Params from the Vendor_specific group @@ -141,8 +141,9 @@ def _cal_power(self, cal_type, use_beam_power=False) -> xr.Dataset: 'TS' for calculating target strength use_beam_power : bool whether to use beam_power. - If ``True`` use ``echodata.beam_power``; if ``False`` use ``echodata.beam``. - Note ``echodata.beam_power`` could only exist for EK80 data. + If ``True`` use ``echodata["Sonar/Beam_group2"]``; + if ``False`` use ``echodata["Sonar/Beam_group1"]``. + Note ``echodata["Sonar/Beam_group2"]`` could only exist for EK80 data. Returns ------- @@ -151,9 +152,9 @@ def _cal_power(self, cal_type, use_beam_power=False) -> xr.Dataset: """ # Select source of backscatter data if use_beam_power: - beam = self.echodata.beam_power + beam = self.echodata["Sonar/Beam_group2"] else: - beam = self.echodata.beam + beam = self.echodata["Sonar/Beam_group1"] # Harmonize time coordinate between Beam_groupX data and env_params for p in self.env_params.keys(): @@ -261,7 +262,7 @@ def get_env_params(self, **kwargs): pressure=self.env_params["pressure"], ) self.env_params["sound_absorption"] = uwa.calc_absorption( - frequency=self.echodata.beam["frequency_nominal"], + frequency=self.echodata["Sonar/Beam_group1"]["frequency_nominal"], temperature=self.env_params["temperature"], salinity=self.env_params["salinity"], pressure=self.env_params["pressure"], @@ -271,12 +272,12 @@ def get_env_params(self, **kwargs): self.env_params["sound_speed"] = ( self.env_params["sound_speed"] if "sound_speed" in self.env_params - else self.echodata.environment["sound_speed_indicative"] + else self.echodata["Environment"]["sound_speed_indicative"] ) self.env_params["sound_absorption"] = ( self.env_params["sound_absorption"] if "sound_absorption" in self.env_params - else self.echodata.environment["absorption_indicative"] + else self.echodata["Environment"]["absorption_indicative"] ) def compute_Sv(self, **kwargs): @@ -340,11 +341,11 @@ def get_env_params(self, waveform_mode=None, encode_mode="complex"): if ( encode_mode == "power" and waveform_mode == "CW" - and self.echodata.beam_power is not None + and self.echodata["Sonar/Beam_group2"] is not None ): - beam = self.echodata.beam_power + beam = self.echodata["Sonar/Beam_group2"] else: - beam = self.echodata.beam + beam = self.echodata["Sonar/Beam_group1"] # Use center frequency if in BB mode, else use nominal channel frequency if waveform_mode == "BB": @@ -380,12 +381,14 @@ def get_env_params(self, waveform_mode=None, encode_mode="complex"): ["temperature", "salinity", "depth"], ): self.env_params[p1] = ( - self.env_params[p1] if p1 in self.env_params else self.echodata.environment[p2] + self.env_params[p1] + if p1 in self.env_params + else self.echodata["Environment"][p2] ) self.env_params["sound_speed"] = ( self.env_params["sound_speed"] if "sound_speed" in self.env_params - else self.echodata.environment["sound_speed_indicative"] + else self.echodata["Environment"]["sound_speed_indicative"] ) self.env_params["sound_absorption"] = ( self.env_params["sound_absorption"] @@ -411,16 +414,18 @@ def _get_vend_cal_params_complex(self, channel_id, filter_name, param_type): 'coeff' or 'decimation' """ if param_type == "coeff": - v = self.echodata.vendor.attrs[ + v = self.echodata["Vendor_specific"].attrs[ "%s %s filter_r" % (channel_id, filter_name) ] + 1j * np.array( - self.echodata.vendor.attrs["%s %s filter_i" % (channel_id, filter_name)] + self.echodata["Vendor_specific"].attrs["%s %s filter_i" % (channel_id, filter_name)] ) if v.size == 1: v = np.expand_dims(v, axis=0) # expand dims for convolution return v else: - return self.echodata.vendor.attrs["%s %s decimation" % (channel_id, filter_name)] + return self.echodata["Vendor_specific"].attrs[ + "%s %s decimation" % (channel_id, filter_name) + ] def _tapered_chirp( self, @@ -513,15 +518,15 @@ def get_transmit_chirp(self, waveform_mode): """ # Make sure it is BB mode data if waveform_mode == "BB" and ( - ("frequency_start" not in self.echodata.beam) - or ("frequency_end" not in self.echodata.beam) + ("frequency_start" not in self.echodata["Sonar/Beam_group1"]) + or ("frequency_end" not in self.echodata["Sonar/Beam_group1"]) ): raise TypeError("File does not contain BB mode complex samples!") y_all = {} y_time_all = {} tau_effective = {} - for chan in self.echodata.beam.channel.values: + for chan in self.echodata["Sonar/Beam_group1"].channel.values: # TODO: currently only deal with the case with # a fixed tx key param values within a channel if waveform_mode == "BB": @@ -541,13 +546,15 @@ def get_transmit_chirp(self, waveform_mode): ] tx_params = {} for p in tx_param_names: - tx_params[p] = np.unique(self.echodata.beam[p].sel(channel=chan)) + tx_params[p] = np.unique(self.echodata["Sonar/Beam_group1"][p].sel(channel=chan)) if tx_params[p].size != 1: raise TypeError("File contains changing %s!" % p) y_tmp, _ = self._tapered_chirp(**tx_params) # Filter and decimate chirp template - fs_deci = 1 / self.echodata.beam.sel(channel=chan)["sample_interval"].values + fs_deci = ( + 1 / self.echodata["Sonar/Beam_group1"].sel(channel=chan)["sample_interval"].values + ) y_tmp, y_tmp_time = self._filter_decimate_chirp(y_tmp, chan) # Compute effective pulse length @@ -570,9 +577,9 @@ def compress_pulse(self, chirp, chan_BB=None): channels that transmit in BB mode (since CW mode can be in mixed in complex samples too) """ - backscatter = self.echodata.beam["backscatter_r"].sel( + backscatter = self.echodata["Sonar/Beam_group1"]["backscatter_r"].sel( channel=chan_BB - ) + 1j * self.echodata.beam["backscatter_i"].sel(channel=chan_BB) + ) + 1j * self.echodata["Sonar/Beam_group1"]["backscatter_i"].sel(channel=chan_BB) pc_all = [] for chan in chan_BB: @@ -622,26 +629,28 @@ def _get_gain_for_complex(self, waveform_mode, chan_sel) -> xr.DataArray: "gain_correction", waveform_mode=waveform_mode ) gain = [] - if "gain" in self.echodata.vendor.data_vars: + if "gain" in self.echodata["Vendor_specific"].data_vars: # index using channel_id as order of frequency across channel can be arbitrary # reference to freq_center in case some channels are CW complex samples # (already dropped when computing freq_center in the calling function) for ch_id in chan_sel: # if channel gain exists in data - if ch_id in self.echodata.vendor.cal_channel_id: - gain_vec = self.echodata.vendor.gain.sel(cal_channel_id=ch_id) + if ch_id in self.echodata["Vendor_specific"].cal_channel_id: + gain_vec = self.echodata["Vendor_specific"].gain.sel(cal_channel_id=ch_id) gain_temp = ( gain_vec.interp( - cal_frequency=self.echodata.vendor.frequency_nominal.sel( - channel=ch_id - ) + cal_frequency=self.echodata[ + "Vendor_specific" + ].frequency_nominal.sel(channel=ch_id) ).drop(["cal_channel_id", "cal_frequency"]) ).expand_dims("channel") # if no freq-dependent gain use CW gain else: gain_temp = ( gain_single.sel(channel=ch_id) - .reindex_like(self.echodata.beam.backscatter_r, method="nearest") + .reindex_like( + self.echodata["Sonar/Beam_group1"].backscatter_r, method="nearest" + ) .expand_dims("channel") ) gain_temp.name = "gain" @@ -682,9 +691,13 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: # use center frequency for each ping to select BB or CW channels # when all samples are encoded as complex samples - if "frequency_start" in self.echodata.beam and "frequency_end" in self.echodata.beam: + if ( + "frequency_start" in self.echodata["Sonar/Beam_group1"] + and "frequency_end" in self.echodata["Sonar/Beam_group1"] + ): freq_center = ( - self.echodata.beam["frequency_start"] + self.echodata.beam["frequency_end"] + self.echodata["Sonar/Beam_group1"]["frequency_start"] + + self.echodata["Sonar/Beam_group1"]["frequency_end"] ) / 2 # has beam dim else: freq_center = None @@ -702,7 +715,7 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: # backscatter data pc = self.compress_pulse(chirp, chan_BB=chan_sel) # has beam dim prx = ( - self.echodata.beam.beam.size + self.echodata["Sonar/Beam_group1"].beam.size * np.abs(pc.mean(dim="beam")) ** 2 / (2 * np.sqrt(2)) ** 2 * (np.abs(self.z_er + self.z_et) / self.z_er) ** 2 @@ -711,7 +724,7 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: else: if freq_center is None: # when only have CW complex samples - chan_sel = self.echodata.beam.channel + chan_sel = self.echodata["Sonar/Beam_group1"].channel else: # if BB and CW complex samples co-exist # drop those that contain BB samples (not nan in freq start/end) @@ -719,10 +732,11 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: # backscatter data backscatter_cw = ( - self.echodata.beam["backscatter_r"] + 1j * self.echodata.beam["backscatter_i"] + self.echodata["Sonar/Beam_group1"]["backscatter_r"] + + 1j * self.echodata["Sonar/Beam_group1"]["backscatter_i"] ) prx = ( - self.echodata.beam.beam.size + self.echodata["Sonar/Beam_group1"].beam.size * np.abs(backscatter_cw.mean(dim="beam")) ** 2 / (2 * np.sqrt(2)) ** 2 * (np.abs(self.z_er + self.z_et) / self.z_er) ** 2 @@ -734,10 +748,10 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: # Compute derived params # Harmonize time coordinate between Beam_groupX data and env_params - # Use self.echodata.beam because complex sample is always in Beam_group1 + # Use self.echodata["Sonar/Beam_group1"] because complex sample is always in Beam_group1 for p in self.env_params.keys(): self.env_params[p] = self.echodata._harmonize_env_param_time( - self.env_params[p], ping_time=self.echodata.beam.ping_time + self.env_params[p], ping_time=self.echodata["Sonar/Beam_group1"].ping_time ) sound_speed = self.env_params["sound_speed"] @@ -745,14 +759,18 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: range_meter = self.range_meter.sel(channel=chan_sel) if waveform_mode == "BB": # use true center frequency for BB pulse - wavelength = sound_speed / self.echodata.beam.frequency_nominal.sel(channel=chan_sel) + wavelength = sound_speed / self.echodata["Sonar/Beam_group1"].frequency_nominal.sel( + channel=chan_sel + ) # use true center frequency to interpolate for gain factor gain = self._get_gain_for_complex(waveform_mode=waveform_mode, chan_sel=chan_sel) else: # use nominal channel frequency for CW pulse - wavelength = sound_speed / self.echodata.beam.frequency_nominal.sel(channel=chan_sel) + wavelength = sound_speed / self.echodata["Sonar/Beam_group1"].frequency_nominal.sel( + channel=chan_sel + ) # use nominal channel frequency to select gain factor gain = self._get_gain_for_complex(waveform_mode=waveform_mode, chan_sel=chan_sel) @@ -767,21 +785,29 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: # effective pulse length tau_effective = xr.DataArray( data=list(tau_effective.values()), - coords=[self.echodata.beam.channel, self.echodata.beam.ping_time], + coords=[ + self.echodata["Sonar/Beam_group1"].channel, + self.echodata["Sonar/Beam_group1"].ping_time, + ], dims=["channel", "ping_time"], ).sel(channel=chan_sel) # other params - transmit_power = self.echodata.beam["transmit_power"].sel(channel=chan_sel) + transmit_power = self.echodata["Sonar/Beam_group1"]["transmit_power"].sel( + channel=chan_sel + ) # equivalent_beam_angle has beam dim if waveform_mode == "BB": - psifc = self.echodata.beam["equivalent_beam_angle"].sel( + psifc = self.echodata["Sonar/Beam_group1"]["equivalent_beam_angle"].sel( channel=chan_sel ) + 10 * np.log10( - self.echodata.vendor.frequency_nominal.sel(channel=chan_sel) / freq_center + self.echodata["Vendor_specific"].frequency_nominal.sel(channel=chan_sel) + / freq_center ) elif waveform_mode == "CW": - psifc = self.echodata.beam["equivalent_beam_angle"].sel(channel=chan_sel) + psifc = self.echodata["Sonar/Beam_group1"]["equivalent_beam_angle"].sel( + channel=chan_sel + ) out = ( 10 * np.log10(prx) @@ -795,7 +821,9 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: out = out.rename_vars({list(out.data_vars.keys())[0]: "Sv"}) elif cal_type == "TS": - transmit_power = self.echodata.beam["transmit_power"].sel(channel=chan_sel) + transmit_power = self.echodata["Sonar/Beam_group1"]["transmit_power"].sel( + channel=chan_sel + ) out = ( 10 * np.log10(prx) @@ -810,7 +838,7 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: out = out.merge(range_meter) # Add frequency_nominal to data set - out["frequency_nominal"] = self.echodata.beam["frequency_nominal"] + out["frequency_nominal"] = self.echodata["Sonar/Beam_group1"]["frequency_nominal"] # Add env and cal parameters out = self._add_params_to_output(out) @@ -879,22 +907,22 @@ def _compute_cal(self, cal_type, waveform_mode, encode_mode) -> xr.Dataset: # Raise error when waveform_mode and actual recording mode do not match # This simple check is only possible for BB-only data, # since for data with both BB and CW complex samples, - # frequency_start will exist in echodata.beam for the BB channels - if waveform_mode == "BB" and "frequency_start" not in self.echodata.beam: + # frequency_start will exist in echodata["Sonar/Beam_group1"] for the BB channels + if waveform_mode == "BB" and "frequency_start" not in self.echodata["Sonar/Beam_group1"]: raise ValueError("waveform_mode='BB' but broadband data not found!") # Set use_beam_power - # - True: use self.echodata.beam_power for cal - # - False: use self.echodata.beam for cal + # - True: use self.echodata["Sonar/Beam_group2"] for cal + # - False: use self.echodata["Sonar/Beam_group1"] for cal use_beam_power = False # Warn user about additional data in the raw file if another type exists # When both power and complex samples exist: - # complex samples will be stored in echodata.beam - # power samples will be stored in echodata.beam_power + # complex samples will be stored in echodata["Sonar/Beam_group1"] + # power samples will be stored in echodata["Sonar/Beam_group2"] # When only one type of samples exist, - # all samples with be stored in echodata.beam - if self.echodata.beam_power is not None: # both power and complex samples exist + # all samples with be stored in echodata["Sonar/Beam_group1"] + if self.echodata["Sonar/Beam_group2"] is not None: # both power and complex samples exist # If both beam and beam_power groups exist, # this means that CW data are encoded as power samples and in beam_power group if waveform_mode == "CW" and encode_mode == "complex": @@ -910,7 +938,9 @@ def _compute_cal(self, cal_type, waveform_mode, encode_mode) -> xr.Dataset: "Only complex samples are calibrated, but power samples also exist in the raw data file!" # noqa ) else: # only power OR complex samples exist - if "backscatter_i" in self.echodata.beam.variables: # data contain only complex samples + if ( + "backscatter_i" in self.echodata["Sonar/Beam_group1"].variables + ): # data contain only complex samples if encode_mode == "power": raise TypeError( "File does not contain power samples! Use encode_mode='complex'" diff --git a/echopype/calibrate/env_params.py b/echopype/calibrate/env_params.py index 5856ccc76..ba6f2d588 100644 --- a/echopype/calibrate/env_params.py +++ b/echopype/calibrate/env_params.py @@ -95,7 +95,7 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]: raise ValueError("invalid data_kind") for dim in dims: - if dim not in echodata.platform: + if dim not in echodata["Platform"]: raise ValueError( f"could not interpolate env_params; EchoData is missing dimension {dim}" ) @@ -103,10 +103,12 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]: env_params = self.env_params if self.data_kind == "mobile": - if np.isnan(echodata.platform["time1"]).all(): + if np.isnan(echodata["Platform"]["time1"]).all(): raise ValueError("cannot perform mobile interpolation without time1") # compute_range needs indexing by ping_time - interp_plat = echodata.platform.interp({"time1": echodata.beam["ping_time"]}) + interp_plat = echodata["Platform"].interp( + {"time1": echodata["Sonar/Beam_group1"]["ping_time"]} + ) result = {} for var, values in env_params.data_vars.items(): @@ -134,7 +136,7 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]: } extrap = env_params.interp( - {dim: echodata.platform[dim].data for dim in dims}, + {dim: echodata["Platform"][dim].data for dim in dims}, method=self.extrap_method, # scipy interp uses "extrapolate" but scipy interpn uses None kwargs={"fill_value": "extrapolate" if len(dims) == 1 else None}, @@ -143,7 +145,7 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]: extrap_unique_idx = {dim: np.unique(extrap[dim], return_index=True)[1] for dim in dims} extrap = extrap.isel(**extrap_unique_idx) interp = env_params.interp( - {dim: echodata.platform[dim].data for dim in dims}, + {dim: echodata["Platform"][dim].data for dim in dims}, method=self.interp_method, ) interp_unique_idx = {dim: np.unique(interp[dim], return_index=True)[1] for dim in dims} @@ -179,8 +181,8 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]: # if self.data_kind == "organized": # # get platform latitude and longitude indexed by ping_time - # interp_plat = echodata.platform.interp( - # {"time": echodata.platform["ping_time"]} + # interp_plat = echodata["Platform"].interp( + # {"time": echodata["Platform"]["ping_time"]} # ) # # get env_params latitude and longitude indexed by ping_time # env_params = env_params.interp( diff --git a/echopype/convert/api.py b/echopype/convert/api.py index c9d9fe9be..3e04e31e5 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -105,11 +105,11 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): # TODO: in terms of chunking, would using rechunker at the end be faster and more convenient? # Top-level group - io.save_file(echodata.top, path=output_path, mode="w", engine=engine) + io.save_file(echodata["Top-level"], path=output_path, mode="w", engine=engine) # Provenance group io.save_file( - echodata.provenance, + echodata["Provenance"], path=output_path, group="Provenance", mode="a", @@ -117,9 +117,9 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): ) # Environment group - if "time1" in echodata.environment: + if "time1" in echodata["Environment"]: io.save_file( - echodata.environment.chunk( + echodata["Environment"].chunk( {"time1": DEFAULT_CHUNK_SIZE["ping_time"]} ), # TODO: chunking necessary? path=output_path, @@ -129,7 +129,7 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): ) else: io.save_file( - echodata.environment, + echodata["Environment"], path=output_path, mode="a", engine=engine, @@ -138,7 +138,7 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): # Sonar group io.save_file( - echodata.sonar, + echodata["Sonar"], path=output_path, group="Sonar", mode="a", @@ -162,7 +162,7 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): ) else: io.save_file( - echodata.beam.chunk( + echodata[f"Sonar/{BEAM_SUBGROUP_DEFAULT}"].chunk( { "range_sample": DEFAULT_CHUNK_SIZE["range_sample"], "ping_time": DEFAULT_CHUNK_SIZE["ping_time"], @@ -174,9 +174,10 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): group=f"Sonar/{BEAM_SUBGROUP_DEFAULT}", compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, ) - if echodata.beam_power is not None: + if echodata["Sonar/Beam_group2"] is not None: + # some sonar model does not produce Sonar/Beam_group2 io.save_file( - echodata.beam_power.chunk( + echodata["Sonar/Beam_group2"].chunk( { "range_sample": DEFAULT_CHUNK_SIZE["range_sample"], "ping_time": DEFAULT_CHUNK_SIZE["ping_time"], @@ -191,7 +192,7 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): # Platform group io.save_file( - echodata.platform, # TODO: chunking necessary? time1 and time2 (EK80) only + echodata["Platform"], # TODO: chunking necessary? time1 and time2 (EK80) only path=output_path, mode="a", engine=engine, @@ -200,9 +201,9 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): ) # Platform/NMEA group: some sonar model does not produce NMEA data - if echodata.nmea is not None: + if echodata["Platform/NMEA"] is not None: io.save_file( - echodata.nmea, # TODO: chunking necessary? + echodata["Platform/NMEA"], # TODO: chunking necessary? path=output_path, mode="a", engine=engine, @@ -211,9 +212,9 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): ) # Vendor_specific group - if "ping_time" in echodata.vendor: + if "ping_time" in echodata["Vendor_specific"]: io.save_file( - echodata.vendor.chunk( + echodata["Vendor_specific"].chunk( {"ping_time": DEFAULT_CHUNK_SIZE["ping_time"]} ), # TODO: chunking necessary? path=output_path, @@ -224,7 +225,7 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): ) else: io.save_file( - echodata.vendor, # TODO: chunking necessary? + echodata["Vendor_specific"], # TODO: chunking necessary? path=output_path, mode="a", engine=engine, diff --git a/echopype/echodata/combine.py b/echopype/echodata/combine.py index a9b679ddb..244a9da5f 100644 --- a/echopype/echodata/combine.py +++ b/echopype/echodata/combine.py @@ -176,14 +176,20 @@ def combine_echodata(echodatas: List[EchoData], combine_attrs="override") -> Ech # { group1: [echodata1 attrs, echodata2 attrs, ...], ... } old_attrs: Dict[str, List[Dict[str, Any]]] = dict() + # Specification for Echodata.group_map can be found in + # echopype/echodata/convention/1.0.yml for group, value in EchoData.group_map.items(): - group_datasets = [ - getattr(echodata, group) - for echodata in echodatas - if getattr(echodata, group) is not None - ] + group_datasets = [] + group_path = value["ep_group"] + if group_path is None: + group_path = "Top-level" + + for echodata in echodatas: + if echodata[group_path] is not None: + group_datasets.append(echodata[group_path]) + if group in ("top", "sonar"): - combined_group = getattr(echodatas[0], group) + combined_group = echodatas[0][group_path] elif group == "provenance": combined_group = assemble_combined_provenance( [ @@ -195,7 +201,6 @@ def combine_echodata(echodatas: List[EchoData], combine_attrs="override") -> Ech ) else: if len(group_datasets) == 0: - setattr(result, group, None) continue concat_dim = SONAR_MODELS[sonar_model]["concat_dims"].get( @@ -265,20 +270,20 @@ def combine_echodata(echodatas: List[EchoData], combine_attrs="override") -> Ech # save ping time before reversal correction if old_ping_time is not None: - result.provenance["old_ping_time"] = old_ping_time - result.provenance.attrs["reversed_ping_times"] = 1 + result["Provenance"]["old_ping_time"] = old_ping_time + result["Provenance"].attrs["reversed_ping_times"] = 1 # save location time before reversal correction if old_time1 is not None: - result.provenance["old_time1"] = old_time1 - result.provenance.attrs["reversed_ping_times"] = 1 + result["Provenance"]["old_time1"] = old_time1 + result["Provenance"].attrs["reversed_ping_times"] = 1 # save mru time before reversal correction if old_time2 is not None: - result.provenance["old_time2"] = old_time2 - result.provenance.attrs["reversed_ping_times"] = 1 + result["Provenance"]["old_time2"] = old_time2 + result["Provenance"].attrs["reversed_ping_times"] = 1 # save time3 before reversal correction if old_time3 is not None: - result.provenance["old_time3"] = old_time3 - result.provenance.attrs["reversed_ping_times"] = 1 + result["Provenance"]["old_time3"] = old_time3 + result["Provenance"].attrs["reversed_ping_times"] = 1 # TODO: possible parameter to disable original attributes and original ping_time storage # in provenance group? # save attrs from before combination @@ -311,7 +316,7 @@ def combine_echodata(echodatas: List[EchoData], combine_attrs="override") -> Ech }, dims=["echodata_filename", f"{group}_attr_key"], ) - result.provenance = result.provenance.assign({f"{group}_attrs": attrs}) + result["Provenance"] = result["Provenance"].assign({f"{group}_attrs": attrs}) # Add back sonar model result.sonar_model = sonar_model diff --git a/echopype/echodata/echodata.py b/echopype/echodata/echodata.py index 07f84526b..ce07064c6 100644 --- a/echopype/echodata/echodata.py +++ b/echopype/echodata/echodata.py @@ -204,7 +204,7 @@ def __getitem__(self, __key: Optional[str]) -> Optional[xr.Dataset]: node = self.__get_node(__key) return self.__get_dataset(node) except KeyError: - raise GroupNotFoundError(__key) + return None else: raise ValueError("Datatree not found!") @@ -219,30 +219,6 @@ def __setitem__(self, __key: Optional[str], __newvalue: Any) -> Optional[xr.Data else: raise ValueError("Datatree not found!") - # NOTE: Temporary for now until the attribute access pattern is deprecated - def __getattribute__(self, __name: str) -> Any: - attr_value = super().__getattribute__(__name) - group_map = sonarnetcdf_1.yaml_dict["groups"] - if __name in group_map: - group = group_map.get(__name) - group_path = group["ep_group"] - if __name == "top": - group_path = "Top-level" - msg_list = ["This access pattern will be deprecated in future releases."] - if attr_value is not None: - msg_list.append(f"Access the group directly by doing echodata['{group_path}']") - if self._tree: - if group_path == "Top-level": - node = self._tree - else: - node = self._tree[group_path] - attr_value = self.__get_dataset(node) - else: - msg_list.append(f"No group path exists for '{self.__class__.__name__}.{__name}'") - msg = " ".join(msg_list) - warnings.warn(message=msg, category=DeprecationWarning, stacklevel=2) - return attr_value - def __setattr__(self, __name: str, __value: Any) -> None: attr_value = __value if isinstance(__value, DataTree) and __name != "_tree": @@ -368,7 +344,7 @@ def compute_range( - When `sonar_model` is `"AZFP"` and `env_params` does not contain either `"sound_speed"` or all of `"temperature"`, `"salinity"`, and `"pressure"`. - When `sonar_model` is `"EK60"` or `"EK80"`, - EchoData.environment.sound_speed_indicative does not exist, + EchoData["Environment"].sound_speed_indicative does not exist, and `env_params` does not contain either `"sound_speed"` or all of `"temperature"`, `"salinity"`, and `"pressure"`. - When `sonar_model` is not `"AZFP"`, `"EK60"`, or `"EK80"`. @@ -395,8 +371,10 @@ def compute_range( if "sound_speed" in env_params: sound_speed = env_params["sound_speed"] - elif self.sonar_model in ("EK60", "EK80") and "sound_speed_indicative" in self.environment: - sound_speed = self.environment["sound_speed_indicative"] + elif ( + self.sonar_model in ("EK60", "EK80") and "sound_speed_indicative" in self["Environment"] + ): + sound_speed = self["Environment"]["sound_speed_indicative"] elif all([param in env_params for param in ("temperature", "salinity", "pressure")]): sound_speed = calc_sound_speed( env_params["temperature"], @@ -409,7 +387,8 @@ def compute_range( "sound speed must be specified in env_params, " "with temperature, salinity, and pressure all specified in env_params " "for sound speed to be calculated, " - "or in EchoData.environment.sound_speed_indicative for EK60 and EK80 sonar models" + "or in EchoData['Environment'].sound_speed_indicative " + "for EK60 and EK80 sonar models" ) # AZFP @@ -419,9 +398,9 @@ def compute_range( raise ValueError("azfp_cal_type must be specified when sonar_model is AZFP") # Notation below follows p.86 of user manual - N = self.vendor["number_of_samples_per_average_bin"] # samples per bin - f = self.vendor["digitization_rate"] # digitization rate - L = self.vendor["lockout_index"] # number of lockout samples + N = self["Vendor_specific"]["number_of_samples_per_average_bin"] # samples per bin + f = self["Vendor_specific"]["digitization_rate"] # digitization rate + L = self["Vendor_specific"]["lockout_index"] # number of lockout samples # keep this in ref of AZFP matlab code, # set to 1 since we want to calculate from raw data @@ -430,7 +409,7 @@ def compute_range( # Harmonize sound_speed time1 and Beam_group1 ping_time sound_speed = self._harmonize_env_param_time( p=sound_speed, - ping_time=self.beam.ping_time, + ping_time=self["Sonar/Beam_group1"].ping_time, ) # Calculate range using parameters for each freq @@ -440,14 +419,15 @@ def compute_range( range_offset = 0 else: range_offset = ( - sound_speed * self.beam["transmit_duration_nominal"] / 4 + sound_speed * self["Sonar/Beam_group1"]["transmit_duration_nominal"] / 4 ) # from matlab code range_meter = ( sound_speed * L / (2 * f) + (sound_speed / 4) * ( - ((2 * (self.beam.range_sample + 1) - 1) * N * bins_to_avg - 1) / f - + self.beam["transmit_duration_nominal"] + ((2 * (self["Sonar/Beam_group1"].range_sample + 1) - 1) * N * bins_to_avg - 1) + / f + + self["Sonar/Beam_group1"]["transmit_duration_nominal"] ) - range_offset ) @@ -481,13 +461,13 @@ def compute_range( if ( self.sonar_model == "EK80" and encode_mode == "power" - and self.beam_power is not None + and self["Sonar/Beam_group2"] is not None ): # if both CW and BB exist and beam_power group is not empty # this means that CW is recorded in power/angle mode - beam = self.beam_power + beam = self["Sonar/Beam_group2"] else: - beam = self.beam + beam = self["Sonar/Beam_group1"] # Harmonize sound_speed time1 and Beam_groupX ping_time sound_speed = self._harmonize_env_param_time( @@ -501,7 +481,7 @@ def compute_range( beam.range_sample - tvg_correction_factor ) * sample_thickness # [frequency x range_sample] elif waveform_mode == "BB": - beam = self.beam # always use the Beam group + beam = self["Sonar/Beam_group1"] # always use the Beam group # TODO: bug: right now only first ping_time has non-nan range shift = beam["transmit_duration_nominal"] # based on Lar Anderson's Matlab code @@ -555,7 +535,7 @@ def update_platform( extra_platform_data_file_name=None, ): """ - Updates the `EchoData.platform` group with additional external platform data. + Updates the `EchoData["Platform"]` group with additional external platform data. `extra_platform_data` must be an xarray Dataset. The name of the time dimension in `extra_platform_data` is specified by the @@ -575,7 +555,7 @@ def update_platform( ---------- extra_platform_data : xr.Dataset An `xr.Dataset` containing the additional platform data to be added - to the `EchoData.platform` group. + to the `EchoData["Platform"]` group. time_dim: str, default="time" The name of the time dimension in `extra_platform_data`; used for extracting data from `extra_platform_data`. @@ -611,8 +591,8 @@ def update_platform( extra_platform_data = extra_platform_data.drop_vars(trajectory_var) extra_platform_data = extra_platform_data.swap_dims({"obs": time_dim}) - # clip incoming time to 1 less than min of EchoData.beam["ping_time"] and - # 1 greater than max of EchoData.beam["ping_time"] + # clip incoming time to 1 less than min of EchoData["Sonar/Beam_group1"]["ping_time"] and + # 1 greater than max of EchoData["Sonar/Beam_group1"]["ping_time"] # account for unsorted external time by checking whether each time value is between # min and max ping_time instead of finding the 2 external times corresponding to the # min and max ping_time and taking all the times between those indices @@ -621,7 +601,7 @@ def update_platform( # fmt: off min_index = max( np.searchsorted( - sorted_external_time, self.beam["ping_time"].min(), side="left" + sorted_external_time, self["Sonar/Beam_group1"]["ping_time"].min(), side="left" ) - 1, 0, ) @@ -629,7 +609,7 @@ def update_platform( max_index = min( np.searchsorted( sorted_external_time, - self.beam["ping_time"].max(), + self["Sonar/Beam_group1"]["ping_time"].max(), side="right", ), len(sorted_external_time) - 1, @@ -643,7 +623,7 @@ def update_platform( } ) - platform = self.platform + platform = self["Platform"] platform = platform.drop_dims(["time1"], errors="ignore") # drop_dims is also dropping latitude, longitude and sentence_type why? platform = platform.assign_coords(time1=extra_platform_data[time_dim].values) @@ -707,7 +687,7 @@ def mapping_search_variable(mapping, keys, default=None): var_attrs["history"] = history_attr platform[var] = platform[var].assign_attrs(**var_attrs) - self.platform = set_encodings(platform) + self["Platform"] = set_encodings(platform) @classmethod def _load_convert(cls, convert_obj): diff --git a/echopype/tests/calibrate/test_calibrate.py b/echopype/tests/calibrate/test_calibrate.py index 4cab3c77f..db3fd3b87 100644 --- a/echopype/tests/calibrate/test_calibrate.py +++ b/echopype/tests/calibrate/test_calibrate.py @@ -37,7 +37,7 @@ def test_compute_Sv_returns_water_level(ek60_path): # make sure the returned Dataset has water_level and throw an assertion error if the # EchoData object does not have water_level (just in case we remove it from the file # used in the future) - assert 'water_level' in ed.platform.data_vars.keys() + assert 'water_level' in ed["Platform"].data_vars.keys() assert 'water_level' in ds_Sv.data_vars @@ -147,7 +147,7 @@ def test_compute_Sv_azfp(azfp_path): # Calibrate using identical env params as in Matlab ParametersAZFP.m # AZFP Matlab code uses average temperature avg_temperature = ( - echodata.environment['temperature'].mean('time1').values + echodata["Environment"]['temperature'].mean('time1').values ) env_params = { 'temperature': avg_temperature, @@ -236,7 +236,7 @@ def test_compute_Sv_ek80_pc_echoview(ek80_path): ) # compute range [m] chirp, _, tau_effective = cal_obj.get_transmit_chirp(waveform_mode="BB") freq_center = ( - echodata.beam["frequency_start"] + echodata.beam["frequency_end"] + echodata["Sonar/Beam_group1"]["frequency_start"] + echodata["Sonar/Beam_group1"]["frequency_end"] ).dropna( dim="channel" ) / 2 # drop those that contain CW samples (nan in freq start/end) diff --git a/echopype/tests/convert/test_convert_azfp.py b/echopype/tests/convert/test_convert_azfp.py index 4e52bdbea..ddad4da3b 100644 --- a/echopype/tests/convert/test_convert_azfp.py +++ b/echopype/tests/convert/test_convert_azfp.py @@ -62,40 +62,40 @@ def test_convert_azfp_01a_matlab_raw(azfp_path): # frequency assert np.array_equal( ds_matlab['Data']['Freq'][0][0].squeeze(), - echodata.beam.frequency_nominal / 1000, + echodata["Sonar/Beam_group1"].frequency_nominal / 1000, ) # matlab file in kHz # backscatter count assert np.array_equal( np.array( [ds_matlab_output['Output'][0]['N'][fidx] for fidx in range(4)] ), - echodata.beam.backscatter_r.isel(beam=0).drop('beam').values, + echodata["Sonar/Beam_group1"].backscatter_r.isel(beam=0).drop('beam').values, ) # Test vendor group # Test temperature assert np.array_equal( np.array([d[4] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), - echodata.vendor.ancillary.isel(ancillary_len=4).values, + echodata["Vendor_specific"].ancillary.isel(ancillary_len=4).values, ) assert np.array_equal( np.array([d[0] for d in ds_matlab['Data']['BatteryTx'][0]]).squeeze(), - echodata.vendor.battery_tx, + echodata["Vendor_specific"].battery_tx, ) assert np.array_equal( np.array( [d[0] for d in ds_matlab['Data']['BatteryMain'][0]] ).squeeze(), - echodata.vendor.battery_main, + echodata["Vendor_specific"].battery_main, ) # tilt x-y assert np.array_equal( np.array([d[0] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), - echodata.vendor.tilt_x_count, + echodata["Vendor_specific"].tilt_x_count, ) assert np.array_equal( np.array([d[1] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), - echodata.vendor.tilt_y_count, + echodata["Vendor_specific"].tilt_y_count, ) # check convention-required variables in the Platform group @@ -137,7 +137,7 @@ def test_convert_azfp_01a_raw_echoview(azfp_path): echodata = open_raw( raw_file=azfp_01a_path, sonar_model='AZFP', xml_path=azfp_xml_path ) - assert np.array_equal(test_power, echodata.beam.backscatter_r.isel(beam=0).drop('beam')) + assert np.array_equal(test_power, echodata["Sonar/Beam_group1"].backscatter_r.isel(beam=0).drop('beam')) # check convention-required variables in the Platform group check_platform_required_vars(echodata) @@ -152,10 +152,10 @@ def test_convert_azfp_01a_different_ranges(azfp_path): echodata = open_raw( raw_file=azfp_01a_path, sonar_model='AZFP', xml_path=azfp_xml_path ) - assert echodata.beam.backscatter_r.sel(channel='55030-125-1').dropna( + assert echodata["Sonar/Beam_group1"].backscatter_r.sel(channel='55030-125-1').dropna( 'range_sample' ).shape == (360, 438, 1) - assert echodata.beam.backscatter_r.sel(channel='55030-769-4').dropna( + assert echodata["Sonar/Beam_group1"].backscatter_r.sel(channel='55030-769-4').dropna( 'range_sample' ).shape == (360, 135, 1) diff --git a/echopype/tests/convert/test_convert_ek60.py b/echopype/tests/convert/test_convert_ek60.py index 75d0c7ecf..59de58201 100644 --- a/echopype/tests/convert/test_convert_ek60.py +++ b/echopype/tests/convert/test_convert_ek60.py @@ -69,7 +69,7 @@ def test_convert_ek60_matlab_raw(ek60_path): ds_matlab['rawData'][0]['pings'][0]['power'][0][fidx] for fidx in range(5) ], - echodata.beam.backscatter_r.isel(beam=0).transpose( + echodata["Sonar/Beam_group1"].backscatter_r.isel(beam=0).transpose( 'channel', 'range_sample', 'ping_time' ), rtol=0, @@ -82,7 +82,7 @@ def test_convert_ek60_matlab_raw(ek60_path): ds_matlab['rawData'][0]['pings'][0][angle][0][fidx] for fidx in range(5) ], - echodata.beam['angle_' + angle].isel(beam=0).transpose( + echodata["Sonar/Beam_group1"]['angle_' + angle].isel(beam=0).transpose( 'channel', 'range_sample', 'ping_time' ), ) @@ -113,12 +113,12 @@ def test_convert_ek60_echoview_raw(ek60_path): # get indices of sorted frequency_nominal values. This is necessary # because the frequency_nominal values are not always in ascending order. - sorted_freq_ind = np.argsort(echodata.beam.frequency_nominal) + sorted_freq_ind = np.argsort(echodata["Sonar/Beam_group1"].frequency_nominal) for fidx, atol in zip(range(5), [1e-5, 1.1e-5, 1.1e-5, 1e-5, 1e-5]): assert np.allclose( test_power[fidx, :, :], - echodata.beam.backscatter_r.isel( + echodata["Sonar/Beam_group1"].backscatter_r.isel( channel=sorted_freq_ind[fidx], ping_time=slice(None, 10), range_sample=slice(1, None), @@ -166,8 +166,8 @@ def test_convert_ek60_duplicate_ping_times(ek60_path): ) ed = open_raw(raw_path, "EK60") - assert "duplicate_ping_times" in ed.provenance.attrs - assert "old_ping_time" in ed.provenance + assert "duplicate_ping_times" in ed["Provenance"].attrs + assert "old_ping_time" in ed["Provenance"] def test_convert_ek60_duplicate_frequencies(ek60_path): diff --git a/echopype/tests/convert/test_convert_ek80.py b/echopype/tests/convert/test_convert_ek80.py index e6fef9e85..fbba98328 100644 --- a/echopype/tests/convert/test_convert_ek80.py +++ b/echopype/tests/convert/test_convert_ek80.py @@ -105,7 +105,7 @@ def test_convert_ek80_complex_matlab(ek80_path): # Test complex parsed data ds_matlab = loadmat(ek80_matlab_path_bb) assert np.array_equal( - echodata.beam.backscatter_r.sel(channel='WBT 549762-15 ES70-7C', + echodata["Sonar/Beam_group1"].backscatter_r.sel(channel='WBT 549762-15 ES70-7C', ping_time='2017-09-12T23:49:10.722999808') .dropna('range_sample') .values[1:, :], @@ -114,7 +114,7 @@ def test_convert_ek80_complex_matlab(ek80_path): ), # real part ) assert np.array_equal( - echodata.beam.backscatter_i.sel(channel='WBT 549762-15 ES70-7C', + echodata["Sonar/Beam_group1"].backscatter_i.sel(channel='WBT 549762-15 ES70-7C', ping_time='2017-09-12T23:49:10.722999808') .dropna('range_sample') .values[1:, :], @@ -173,22 +173,22 @@ def test_convert_ek80_cw_power_angle_echoview(ek80_path): # get indices of sorted frequency_nominal values. This is necessary # because the frequency_nominal values are not always in ascending order. - sorted_freq_ind = np.argsort(echodata.beam.frequency_nominal) + sorted_freq_ind = np.argsort(echodata["Sonar/Beam_group1"].frequency_nominal) # get sorted channel list based on frequency_nominal values - channel_list = echodata.beam.channel[sorted_freq_ind.values] + channel_list = echodata["Sonar/Beam_group1"].channel[sorted_freq_ind.values] # check water_level assert (echodata["Platform"]["water_level"] == 0).all() # Test power # single point error in original raw data. Read as -2000 by echopype and -999 by EchoView - echodata.beam.backscatter_r[sorted_freq_ind.values[3], 4, 13174] = -999 + echodata["Sonar/Beam_group1"].backscatter_r[sorted_freq_ind.values[3], 4, 13174] = -999 for file, chan in zip(ek80_echoview_power_csv, channel_list): test_power = pd.read_csv(file, delimiter=';').iloc[:, 13:].values assert np.allclose( test_power, - echodata.beam.backscatter_r.sel(channel=chan, + echodata["Sonar/Beam_group1"].backscatter_r.sel(channel=chan, beam='1').dropna('range_sample'), rtol=0, atol=1.1e-5, @@ -196,16 +196,16 @@ def test_convert_ek80_cw_power_angle_echoview(ek80_path): # Convert from electrical angles to physical angle [deg] major = ( - echodata.beam['angle_athwartship'] + echodata["Sonar/Beam_group1"]['angle_athwartship'] * 1.40625 - / echodata.beam['angle_sensitivity_athwartship'] - - echodata.beam['angle_offset_athwartship'] + / echodata["Sonar/Beam_group1"]['angle_sensitivity_athwartship'] + - echodata["Sonar/Beam_group1"]['angle_offset_athwartship'] ) minor = ( - echodata.beam['angle_alongship'] + echodata["Sonar/Beam_group1"]['angle_alongship'] * 1.40625 - / echodata.beam['angle_sensitivity_alongship'] - - echodata.beam['angle_offset_alongship'] + / echodata["Sonar/Beam_group1"]['angle_sensitivity_alongship'] + - echodata["Sonar/Beam_group1"]['angle_offset_alongship'] ) for chan, file in zip(channel_list, ek80_echoview_angle_csv): df_angle = pd.read_csv(file) @@ -275,7 +275,7 @@ def test_convert_ek80_complex_echoview(ek80_path): ek80_echoview_bb_power_csv, header=None, skiprows=[0] ) # averaged across beams assert np.allclose( - echodata.beam.backscatter_r.sel(channel='WBT 549762-15 ES70-7C') + echodata["Sonar/Beam_group1"].backscatter_r.sel(channel='WBT 549762-15 ES70-7C') .dropna('range_sample') .mean(dim='beam'), df_bb.iloc[::2, 14:], # real rows @@ -283,7 +283,7 @@ def test_convert_ek80_complex_echoview(ek80_path): atol=8e-6, ) assert np.allclose( - echodata.beam.backscatter_i.sel(channel='WBT 549762-15 ES70-7C') + echodata["Sonar/Beam_group1"].backscatter_i.sel(channel='WBT 549762-15 ES70-7C') .dropna('range_sample') .mean(dim='beam'), df_bb.iloc[1::2, 14:], # imag rows @@ -325,8 +325,8 @@ def test_convert_ek80_cw_bb_in_single_file(ek80_path): echodata = open_raw(raw_file=ek80_raw_path_bb_cw, sonar_model='EK80') # Check there are both Sonar/Beam_group1 and /Sonar/Beam_power groups in the converted file - assert echodata.beam_power is not None - assert echodata.beam is not None + assert echodata["Sonar/Beam_group2"] + assert echodata["Sonar/Beam_group1"] # check platform nan_plat_vars = [ @@ -366,7 +366,7 @@ def test_convert_ek80_freq_subset(ek80_path): echodata = open_raw(raw_file=ek80_raw_path_freq_subset, sonar_model='EK80') # Check if converted output has only 2 frequency channels - assert echodata.beam.channel.size == 2 + assert echodata["Sonar/Beam_group1"].channel.size == 2 # check platform nan_plat_vars = [ diff --git a/echopype/tests/convert/test_convert_source_target_locs.py b/echopype/tests/convert/test_convert_source_target_locs.py index d0e96802a..c4f22f73f 100644 --- a/echopype/tests/convert/test_convert_source_target_locs.py +++ b/echopype/tests/convert/test_convert_source_target_locs.py @@ -239,34 +239,36 @@ def test_convert_time_encodings(sonar_model, raw_file, xml_path, test_path): ) ed.to_netcdf(overwrite=True) for group, details in ed.group_map.items(): - if hasattr(ed, group): - group_ds = getattr(ed, group) - if isinstance(group_ds, xr.Dataset): - for var, encoding in DEFAULT_ENCODINGS.items(): - if var in group_ds: - da = group_ds[var] - assert da.encoding == encoding - - # Combine encoding and attributes since this - # is what is shown when using decode_cf=False - # without dtype attribute - total_attrs = dict(**da.attrs, **da.encoding) - total_attrs.pop('dtype') - - # Read converted file back in - file_da = xr.open_dataset( - ed.converted_raw_path, - group=details['ep_group'], - decode_cf=False, - )[var] - assert file_da.dtype == encoding['dtype'] - - # Read converted file back in - decoded_da = xr.open_dataset( - ed.converted_raw_path, - group=details['ep_group'], - )[var] - assert da.equals(decoded_da) is True + group_path = details['ep_group'] + if group_path is None: + group_path = 'Top-level' + group_ds = ed[group_path] + if isinstance(group_ds, xr.Dataset): + for var, encoding in DEFAULT_ENCODINGS.items(): + if var in group_ds: + da = group_ds[var] + assert da.encoding == encoding + + # Combine encoding and attributes since this + # is what is shown when using decode_cf=False + # without dtype attribute + total_attrs = dict(**da.attrs, **da.encoding) + total_attrs.pop('dtype') + + # Read converted file back in + file_da = xr.open_dataset( + ed.converted_raw_path, + group=details['ep_group'], + decode_cf=False, + )[var] + assert file_da.dtype == encoding['dtype'] + + # Read converted file back in + decoded_da = xr.open_dataset( + ed.converted_raw_path, + group=details['ep_group'], + )[var] + assert da.equals(decoded_da) is True os.unlink(ed.converted_raw_path) diff --git a/echopype/tests/echodata/test_echodata.py b/echopype/tests/echodata/test_echodata.py index 4a4c3dcec..f22b38724 100644 --- a/echopype/tests/echodata/test_echodata.py +++ b/echopype/tests/echodata/test_echodata.py @@ -198,20 +198,20 @@ def converted_zarr(self, single_ek60_zarr): def test_constructor(self, converted_zarr): ed = EchoData.from_file(converted_raw_path=converted_zarr) expected_groups = [ - 'top', - 'environment', - 'platform', - 'provenance', - 'sonar', - 'beam', - 'vendor', + 'Top-level', + 'Environment', + 'Platform', + 'Provenance', + 'Sonar', + 'Sonar/Beam_group1', + 'Vendor_specific', ] assert ed.sonar_model == 'EK60' assert ed.converted_raw_path == converted_zarr assert ed.storage_options == {} for group in expected_groups: - assert isinstance(getattr(ed, group), xr.Dataset) + assert isinstance(ed[group], xr.Dataset) def test_repr(self, converted_zarr): zarr_path_string = str(converted_zarr.absolute()) @@ -252,27 +252,22 @@ def test_setattr(self, converted_zarr): sample_data = xr.Dataset({"x": [0, 0, 0]}) sample_data2 = xr.Dataset({"y": [0, 0, 0]}) ed = EchoData.from_file(converted_raw_path=converted_zarr) - current_ed_beam = ed.beam - current_ed_top = ed.top - ed.beam = sample_data - ed.top = sample_data2 + current_ed_beam = ed["Sonar/Beam_group1"] + current_ed_top = ed['Top-level'] + ed["Sonar/Beam_group1"] = sample_data + ed['Top-level'] = sample_data2 - assert ed.beam.equals(sample_data) is True - assert ed.beam.equals(ed['Sonar/Beam_group1']) is True - assert ed.beam.equals(current_ed_beam) is False + assert ed["Sonar/Beam_group1"].equals(sample_data) is True + assert ed["Sonar/Beam_group1"].equals(current_ed_beam) is False - assert ed.top.equals(sample_data2) is True - assert ed.top.equals(ed['Top-level']) is True - assert ed.top.equals(current_ed_top) is False + assert ed['Top-level'].equals(sample_data2) is True + assert ed['Top-level'].equals(current_ed_top) is False def test_getitem(self, converted_zarr): ed = EchoData.from_file(converted_raw_path=converted_zarr) beam = ed['Sonar/Beam_group1'] assert isinstance(beam, xr.Dataset) - try: - ed['MyGroup'] - except Exception as e: - assert isinstance(e, GroupNotFoundError) + assert ed['MyGroup'] is None ed._tree = None try: @@ -280,28 +275,17 @@ def test_getitem(self, converted_zarr): except Exception as e: assert isinstance(e, ValueError) - def test_getattr(self, converted_zarr): - ed = EchoData.from_file(converted_raw_path=converted_zarr) - expected_groups = { - 'top': 'Top-level', - 'environment': 'Environment', - 'platform': 'Platform', - 'nmea': 'Platform/NMEA', - 'provenance': 'Provenance', - 'sonar': 'Sonar', - 'beam': 'Sonar/Beam_group1', - 'vendor': 'Vendor_specific', - } - for group, path in expected_groups.items(): - ds = getattr(ed, group) - assert ds.equals(ed[path]) - def test_setitem(self, converted_zarr): ed = EchoData.from_file(converted_raw_path=converted_zarr) ed['Sonar/Beam_group1'] = ed['Sonar/Beam_group1'].rename({'beam': 'beam_newname'}) assert sorted(ed['Sonar/Beam_group1'].dims.keys()) == ['beam_newname', 'channel', 'ping_time', 'range_sample'] + try: + ed['SomeRandomGroup'] = 'Testing value' + except Exception as e: + assert isinstance(e, GroupNotFoundError) + def test_get_dataset(self, converted_zarr): ed = EchoData.from_file(converted_raw_path=converted_zarr) node = DataTree() @@ -352,7 +336,6 @@ def test_compute_range(compute_range_samples): ek_encode_mode, ) = compute_range_samples ed = echopype.open_raw(filepath, sonar_model, azfp_xml_path) - print(ed.platform) rng = np.random.default_rng(0) stationary_env_params = EnvParams( xr.Dataset( @@ -367,7 +350,7 @@ def test_compute_range(compute_range_samples): ), data_kind="stationary" ) - if "time3" in ed.platform and sonar_model != "AD2CP": + if "time3" in ed["Platform"] and sonar_model != "AD2CP": ed.compute_range(stationary_env_params, azfp_cal_type, ek_waveform_mode) else: try: @@ -392,7 +375,7 @@ def test_compute_range(compute_range_samples): ), data_kind="mobile" ) - if "latitude" in ed.platform and "longitude" in ed.platform and sonar_model != "AD2CP" and not np.isnan(ed.platform["time1"]).all(): + if "latitude" in ed["Platform"] and "longitude" in ed["Platform"] and sonar_model != "AD2CP" and not np.isnan(ed["Platform"]["time1"]).all(): ed.compute_range(mobile_env_params, azfp_cal_type, ek_waveform_mode) else: try: @@ -427,11 +410,11 @@ def test_nan_range_entries(range_check_files): if sonar_model == "EK80": ds_Sv = echopype.calibrate.compute_Sv(echodata, waveform_mode='BB', encode_mode='complex') range_output = echodata.compute_range(env_params=[], ek_waveform_mode='BB') - nan_locs_backscatter_r = ~echodata.beam.backscatter_r.isel(beam=0).drop("beam").isnull() + nan_locs_backscatter_r = ~echodata["Sonar/Beam_group1"].backscatter_r.isel(beam=0).drop("beam").isnull() else: ds_Sv = echopype.calibrate.compute_Sv(echodata) range_output = echodata.compute_range(env_params=[]) - nan_locs_backscatter_r = ~echodata.beam.backscatter_r.isel(beam=0).drop("beam").isnull() + nan_locs_backscatter_r = ~echodata["Sonar/Beam_group1"].backscatter_r.isel(beam=0).drop("beam").isnull() nan_locs_Sv_range = ~ds_Sv.echo_range.isnull() nan_locs_range = ~range_output.isnull() @@ -482,7 +465,7 @@ def test_update_platform( ed = echopype.open_raw(raw_file, sonar_model=sonar_model) for variable in updated: - assert np.isnan(ed.platform[variable].values).all() + assert np.isnan(ed["Platform"][variable].values).all() if ext_type == "external-trajectory": extra_platform_data_file_name = platform_data[1] @@ -507,30 +490,30 @@ def test_update_platform( ) for variable in updated: - assert not np.isnan(ed.platform[variable].values).all() + assert not np.isnan(ed["Platform"][variable].values).all() # times have max interval of 2s - # check times are > min(ed.beam["ping_time"]) - 2s + # check times are > min(ed["Sonar/Beam_group1"]["ping_time"]) - 2s assert ( - ed.platform["time1"] - > ed.beam["ping_time"].min() - np.timedelta64(2, "s") + ed["Platform"]["time1"] + > ed["Sonar/Beam_group1"]["ping_time"].min() - np.timedelta64(2, "s") ).all() - # check there is only 1 time < min(ed.beam["ping_time"]) + # check there is only 1 time < min(ed["Sonar/Beam_group1"]["ping_time"]) assert ( np.count_nonzero( - ed.platform["time1"] < ed.beam["ping_time"].min() + ed["Platform"]["time1"] < ed["Sonar/Beam_group1"]["ping_time"].min() ) <= 1 ) - # check times are < max(ed.beam["ping_time"]) + 2s + # check times are < max(ed["Sonar/Beam_group1"]["ping_time"]) + 2s assert ( - ed.platform["time1"] - < ed.beam["ping_time"].max() + np.timedelta64(2, "s") + ed["Platform"]["time1"] + < ed["Sonar/Beam_group1"]["ping_time"].max() + np.timedelta64(2, "s") ).all() - # check there is only 1 time > max(ed.beam["ping_time"]) + # check there is only 1 time > max(ed["Sonar/Beam_group1"]["ping_time"]) assert ( np.count_nonzero( - ed.platform["time1"] > ed.beam["ping_time"].max() + ed["Platform"]["time1"] > ed["Sonar/Beam_group1"]["ping_time"].max() ) <= 1 ) diff --git a/echopype/tests/echodata/test_echodata_combine.py b/echopype/tests/echodata/test_echodata_combine.py index 754bf4df9..229e3178e 100644 --- a/echopype/tests/echodata/test_echodata_combine.py +++ b/echopype/tests/echodata/test_echodata_combine.py @@ -107,14 +107,14 @@ def test_combine_echodata(raw_datasets): eds = [echopype.open_raw(file, sonar_model, xml_file) for file in files] combined = echopype.combine_echodata(eds, "overwrite_conflicts") # type: ignore - for group_name in combined.group_map: + for group_name, value in combined.group_map.items(): if group_name in ("top", "sonar", "provenance"): continue - combined_group: xr.Dataset = getattr(combined, group_name) + combined_group: xr.Dataset = combined[value['ep_group']] eds_groups = [ - getattr(ed, group_name) + ed[value['ep_group']] for ed in eds - if getattr(ed, group_name) is not None + if ed[value['ep_group']] is not None ] def union_attrs(datasets: List[xr.Dataset]) -> Dict[str, Any]: @@ -140,6 +140,7 @@ def union_attrs(datasets: List[xr.Dataset]) -> Dict[str, Any]: test_ds.attrs.update(union_attrs(eds_groups)) test_ds = test_ds.drop_dims( [ + # xarray inserts "concat_dim" when concatenating along multiple dimensions "concat_dim", "old_ping_time", "ping_time", @@ -174,8 +175,11 @@ def test_ping_time_reversal(ek60_reversed_ping_time_test_data): ] combined = echopype.combine_echodata(eds, "overwrite_conflicts") # type: ignore - for group_name in combined.group_map: - combined_group: xr.Dataset = getattr(combined, group_name) + for group_name, value in combined.group_map.items(): + if value['ep_group'] is None: + combined_group: xr.Dataset = combined['Top-level'] + else: + combined_group: xr.Dataset = combined[value['ep_group']] if combined_group is not None: if "ping_time" in combined_group and group_name != "provenance": @@ -199,11 +203,15 @@ def test_attr_storage(ek60_test_data): # check storage of attributes before combination in provenance group eds = [echopype.open_raw(file, "EK60") for file in ek60_test_data] combined = echopype.combine_echodata(eds, "overwrite_conflicts") # type: ignore - for group in combined.group_map: - if f"{group}_attrs" in combined.provenance: - group_attrs = combined.provenance[f"{group}_attrs"] + for group, value in combined.group_map.items(): + if value['ep_group'] is None: + group_path = 'Top-level' + else: + group_path = value['ep_group'] + if f"{group}_attrs" in combined["Provenance"]: + group_attrs = combined["Provenance"][f"{group}_attrs"] for i, ed in enumerate(eds): - for attr, value in getattr(ed, group).attrs.items(): + for attr, value in ed[group_path].attrs.items(): assert str( group_attrs.isel(echodata_filename=i) .sel({f"{group}_attr_key": attr}) @@ -212,10 +220,10 @@ def test_attr_storage(ek60_test_data): # check selection by echodata_filename for file in ek60_test_data: - assert Path(file).name in combined.provenance["echodata_filename"] + assert Path(file).name in combined["Provenance"]["echodata_filename"] for group in combined.group_map: - if f"{group}_attrs" in combined.provenance: - group_attrs = combined.provenance[f"{group}_attrs"] + if f"{group}_attrs" in combined["Provenance"]: + group_attrs = combined["Provenance"][f"{group}_attrs"] assert np.array_equal( group_attrs.sel( echodata_filename=Path(ek60_test_data[0]).name @@ -227,15 +235,15 @@ def test_attr_storage(ek60_test_data): def test_combine_attrs(ek60_test_data): # check parameter passed to combine_echodata that controls behavior of attribute combination eds = [echopype.open_raw(file, "EK60") for file in ek60_test_data] - eds[0].beam.attrs.update({"foo": 1}) - eds[1].beam.attrs.update({"foo": 2}) - eds[2].beam.attrs.update({"foo": 3}) + eds[0]["Sonar/Beam_group1"].attrs.update({"foo": 1}) + eds[1]["Sonar/Beam_group1"].attrs.update({"foo": 2}) + eds[2]["Sonar/Beam_group1"].attrs.update({"foo": 3}) combined = echopype.combine_echodata(eds, "override") # type: ignore - assert combined.beam.attrs["foo"] == 1 + assert combined["Sonar/Beam_group1"].attrs["foo"] == 1 combined = echopype.combine_echodata(eds, "drop") # type: ignore - assert "foo" not in combined.beam.attrs + assert "foo" not in combined["Sonar/Beam_group1"].attrs try: combined = echopype.combine_echodata(eds, "identical") # type: ignore @@ -252,17 +260,17 @@ def test_combine_attrs(ek60_test_data): raise AssertionError combined = echopype.combine_echodata(eds, "overwrite_conflicts") # type: ignore - assert combined.beam.attrs["foo"] == 3 + assert combined["Sonar/Beam_group1"].attrs["foo"] == 3 - eds[0].beam.attrs.update({"foo": 1}) - eds[1].beam.attrs.update({"foo": 1}) - eds[2].beam.attrs.update({"foo": 1}) + eds[0]["Sonar/Beam_group1"].attrs.update({"foo": 1}) + eds[1]["Sonar/Beam_group1"].attrs.update({"foo": 1}) + eds[2]["Sonar/Beam_group1"].attrs.update({"foo": 1}) combined = echopype.combine_echodata(eds, "identical") # type: ignore - assert combined.beam.attrs["foo"] == 1 + assert combined["Sonar/Beam_group1"].attrs["foo"] == 1 combined = echopype.combine_echodata(eds, "no_conflicts") # type: ignore - assert combined.beam.attrs["foo"] == 1 + assert combined["Sonar/Beam_group1"].attrs["foo"] == 1 def test_combined_encodings(ek60_test_data): @@ -270,15 +278,19 @@ def test_combined_encodings(ek60_test_data): combined = echopype.combine_echodata(eds, "overwrite_conflicts") # type: ignore group_checks = [] - for group in combined.group_map: - ds = getattr(combined, group) + for group, value in combined.group_map.items(): + if value['ep_group'] is None: + ds = combined['Top-level'] + else: + ds = combined[value['ep_group']] + if ds is not None: for k, v in ds.variables.items(): if k in DEFAULT_ENCODINGS: encoding = ds[k].encoding if encoding != DEFAULT_ENCODINGS[k]: group_checks.append( - f" {combined.group_map[group]['name']}::{k}" + f" {value['name']}::{k}" ) if len(group_checks) > 0: diff --git a/echopype/tests/preprocess/test_preprocess.py b/echopype/tests/preprocess/test_preprocess.py index 2a0ecc3c4..58be7cff0 100644 --- a/echopype/tests/preprocess/test_preprocess.py +++ b/echopype/tests/preprocess/test_preprocess.py @@ -495,7 +495,7 @@ def test_preprocess_mvbs(test_data_samples): ed = ep.open_raw(filepath, sonar_model, azfp_xml_path) if ed.sonar_model.lower() == 'azfp': avg_temperature = ( - ed.environment['temperature'].mean('time1').values + ed["Environment"]['temperature'].mean('time1').values ) env_params = { 'temperature': avg_temperature, diff --git a/echopype/tests/visualize/test_plot.py b/echopype/tests/visualize/test_plot.py index be6ed34db..caaa32550 100644 --- a/echopype/tests/visualize/test_plot.py +++ b/echopype/tests/visualize/test_plot.py @@ -88,7 +88,7 @@ def test_plot_single( # TODO: Need to figure out how to compare the actual rendered plots ed = echopype.open_raw(filepath, sonar_model, azfp_xml_path) plots = echopype.visualize.create_echogram( - ed, channel=ed.beam.channel[0].values + ed, channel=ed["Sonar/Beam_group1"].channel[0].values ) assert isinstance(plots, list) is True if ( @@ -111,7 +111,7 @@ def test_plot_multi_get_range( ed = echopype.open_raw(filepath, sonar_model, azfp_xml_path) if ed.sonar_model.lower() == 'azfp': avg_temperature = ( - ed.environment['temperature'].mean('time1').values + ed["Environment"]['temperature'].mean('time1').values ) env_params = { 'temperature': avg_temperature, @@ -135,7 +135,7 @@ def test_plot_multi_get_range( assert plots[0].axes.shape[-1] == 1 # Channel shape check - assert ed.beam.channel.shape[0] == len(plots) + assert ed["Sonar/Beam_group1"].channel.shape[0] == len(plots) @pytest.mark.parametrize(param_args, param_testdata) @@ -149,7 +149,7 @@ def test_plot_Sv( ed = echopype.open_raw(filepath, sonar_model, azfp_xml_path) if ed.sonar_model.lower() == 'azfp': avg_temperature = ( - ed.environment['temperature'].mean('time1').values + ed["Environment"]['temperature'].mean('time1').values ) env_params = { 'temperature': avg_temperature, @@ -176,7 +176,7 @@ def test_plot_mvbs( ed = echopype.open_raw(filepath, sonar_model, azfp_xml_path) if ed.sonar_model.lower() == 'azfp': avg_temperature = ( - ed.environment['temperature'].mean('time1').values + ed["Environment"]['temperature'].mean('time1').values ) env_params = { 'temperature': avg_temperature, @@ -237,7 +237,7 @@ def test_water_level_echodata(water_level, expect_warning): if isinstance(water_level, list): water_level = water_level[0] - echodata.platform = echodata.platform.drop_vars('water_level') + echodata["Platform"] = echodata["Platform"].drop_vars('water_level') no_input_water_level = True if isinstance(water_level, xr.DataArray): @@ -247,7 +247,7 @@ def test_water_level_echodata(water_level, expect_warning): if no_input_water_level is False: original_array = ( single_array - + echodata.platform.water_level.sel(channel='GPT 18 kHz 009072058c8d 1-1 ES18-11', + + echodata["Platform"].water_level.sel(channel='GPT 18 kHz 009072058c8d 1-1 ES18-11', time3='2017-07-19T21:13:47.984999936').values ) else: @@ -265,14 +265,14 @@ def test_water_level_echodata(water_level, expect_warning): range_in_meter=range_in_meter, water_level=water_level, data_type=EchoData, - platform_data=echodata.platform, + platform_data=echodata["Platform"], ) else: results = _add_water_level( range_in_meter=range_in_meter, water_level=water_level, data_type=EchoData, - platform_data=echodata.platform, + platform_data=echodata["Platform"], ) except Exception as e: assert isinstance(e, ValueError) diff --git a/echopype/visualize/api.py b/echopype/visualize/api.py index 544a32579..992aa323d 100644 --- a/echopype/visualize/api.py +++ b/echopype/visualize/api.py @@ -84,7 +84,7 @@ def create_echogram( ) yaxis = 'range_sample' variable = 'backscatter_r' - ds = data.beam + ds = data["Sonar/Beam_group1"] if 'ping_time' in ds: _check_ping_time(ds.ping_time) if get_range is True: @@ -147,7 +147,7 @@ def create_echogram( range_in_meter=range_in_meter, water_level=water_level, data_type=EchoData, - platform_data=data.platform, + platform_data=data["Platform"], ) ds = ds.assign_coords({'echo_range': range_in_meter}) ds.echo_range.attrs = range_attrs From 96ed2fd9388a631192e410f5eec05379a5ecc083 Mon Sep 17 00:00:00 2001 From: b-reyes <53541061+b-reyes@users.noreply.github.com> Date: Thu, 11 Aug 2022 08:26:08 -0700 Subject: [PATCH 17/23] change the order in _save_groups_to_file so they are the same as the EchoData structure (#779) --- echopype/convert/api.py | 60 ++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/echopype/convert/api.py b/echopype/convert/api.py index 3e04e31e5..26fbb3e4f 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -107,15 +107,6 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): # Top-level group io.save_file(echodata["Top-level"], path=output_path, mode="w", engine=engine) - # Provenance group - io.save_file( - echodata["Provenance"], - path=output_path, - group="Provenance", - mode="a", - engine=engine, - ) - # Environment group if "time1" in echodata["Environment"]: io.save_file( @@ -136,6 +127,36 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): group="Environment", ) + # Platform group + io.save_file( + echodata["Platform"], # TODO: chunking necessary? time1 and time2 (EK80) only + path=output_path, + mode="a", + engine=engine, + group="Platform", + compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, + ) + + # Platform/NMEA group: some sonar model does not produce NMEA data + if echodata["Platform/NMEA"] is not None: + io.save_file( + echodata["Platform/NMEA"], # TODO: chunking necessary? + path=output_path, + mode="a", + engine=engine, + group="Platform/NMEA", + compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, + ) + + # Provenance group + io.save_file( + echodata["Provenance"], + path=output_path, + group="Provenance", + mode="a", + engine=engine, + ) + # Sonar group io.save_file( echodata["Sonar"], @@ -190,27 +211,6 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, ) - # Platform group - io.save_file( - echodata["Platform"], # TODO: chunking necessary? time1 and time2 (EK80) only - path=output_path, - mode="a", - engine=engine, - group="Platform", - compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, - ) - - # Platform/NMEA group: some sonar model does not produce NMEA data - if echodata["Platform/NMEA"] is not None: - io.save_file( - echodata["Platform/NMEA"], # TODO: chunking necessary? - path=output_path, - mode="a", - engine=engine, - group="Platform/NMEA", - compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, - ) - # Vendor_specific group if "ping_time" in echodata["Vendor_specific"]: io.save_file( From 0e226338744577af9f49d2171d4f3a15197d88fa Mon Sep 17 00:00:00 2001 From: b-reyes <53541061+b-reyes@users.noreply.github.com> Date: Thu, 11 Aug 2022 08:48:50 -0700 Subject: [PATCH 18/23] Remove the user option to select NMEA sentences (#778) * remove the user option to select NMEA sentences * change _parse_NMEA function name to _extract_NMEA_latlon --- echopype/convert/api.py | 6 ------ echopype/convert/set_groups_base.py | 6 ++++-- echopype/convert/set_groups_ek60.py | 2 +- echopype/convert/set_groups_ek80.py | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/echopype/convert/api.py b/echopype/convert/api.py index 26fbb3e4f..9400d7019 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -25,8 +25,6 @@ DEFAULT_CHUNK_SIZE = {"range_sample": 25000, "ping_time": 2500} -NMEA_SENTENCE_DEFAULT = ["GGA", "GLL", "RMC"] - BEAM_SUBGROUP_DEFAULT = "Beam_group1" @@ -239,9 +237,6 @@ def _set_convert_params(param_dict: Dict[str, str]) -> Dict[str, str]: The default set of parameters include: - Platform group: ``platform_name``, ``platform_type``, ``platform_code_ICES``, ``water_level`` - - Platform/NMEA: ``nmea_gps_sentence``, - for selecting specific NMEA sentences, - with default values ['GGA', 'GLL', 'RMC']. - Top-level group: ``survey_name`` Other parameters will be saved to the top level. @@ -262,7 +257,6 @@ def _set_convert_params(param_dict: Dict[str, str]) -> Dict[str, str]: out_params["platform_code_ICES"] = param_dict.get("platform_code_ICES", "") out_params["platform_type"] = param_dict.get("platform_type", "") out_params["water_level"] = param_dict.get("water_level", None) - out_params["nmea_gps_sentence"] = param_dict.get("nmea_gps_sentence", NMEA_SENTENCE_DEFAULT) # Parameters for the Top-level group out_params["survey_name"] = param_dict.get("survey_name", "") diff --git a/echopype/convert/set_groups_base.py b/echopype/convert/set_groups_base.py index 13088c877..5f0e118dd 100644 --- a/echopype/convert/set_groups_base.py +++ b/echopype/convert/set_groups_base.py @@ -11,6 +11,8 @@ DEFAULT_CHUNK_SIZE = {"range_sample": 25000, "ping_time": 2500} +NMEA_SENTENCE_DEFAULT = ["GGA", "GLL", "RMC"] + class SetGroupsBase(abc.ABC): """Base class for saving groups to netcdf or zarr from echosounder data files.""" @@ -143,10 +145,10 @@ def set_vendor(self) -> xr.Dataset: raise NotImplementedError # TODO: move this to be part of parser as it is not a "set" operation - def _parse_NMEA(self): + def _extract_NMEA_latlon(self): """Get the lat and lon values from the raw nmea data""" messages = [string[3:6] for string in self.parser_obj.nmea["nmea_string"]] - idx_loc = np.argwhere(np.isin(messages, self.ui_param["nmea_gps_sentence"])).squeeze() + idx_loc = np.argwhere(np.isin(messages, NMEA_SENTENCE_DEFAULT)).squeeze() if idx_loc.size == 1: # in case of only 1 matching message idx_loc = np.expand_dims(idx_loc, axis=0) nmea_msg = [] diff --git a/echopype/convert/set_groups_ek60.py b/echopype/convert/set_groups_ek60.py index d03bac109..5e8df4b9f 100644 --- a/echopype/convert/set_groups_ek60.py +++ b/echopype/convert/set_groups_ek60.py @@ -218,7 +218,7 @@ def set_platform(self, NMEA_only=False) -> xr.Dataset: # Collect variables # Read lat/long from NMEA datagram - time1, msg_type, lat, lon = self._parse_NMEA() + time1, msg_type, lat, lon = self._extract_NMEA_latlon() # NMEA dataset: variables filled with nan if do not exist ds = xr.Dataset( diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index e8ce4f047..c8a95fad5 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -228,7 +228,7 @@ def set_platform(self) -> xr.Dataset: water_level = np.nan print("WARNING: The water_level_draft was not in the file. " "Value set to NaN.") - time1, msg_type, lat, lon = self._parse_NMEA() + time1, msg_type, lat, lon = self._extract_NMEA_latlon() time2 = self.parser_obj.mru.get("timestamp", None) time2 = np.array(time2) if time2 is not None else [np.nan] From ad6dbc7f79458255ba93e68fbdd5db91bceb2780 Mon Sep 17 00:00:00 2001 From: Don Setiawan Date: Thu, 11 Aug 2022 12:43:43 -0700 Subject: [PATCH 19/23] Add logging and optional printouts (#772) * Add logging and optional printouts * Fix autoimport * Tweak propagation and warning check on plot water_level * Make _init_logger consistent * Missed _init_logger * Fix missed access w/in test * Update echopype/tests/visualize/test_plot.py Co-authored-by: Don Setiawan * Modify logic based on @emiliom suggestions Co-authored-by: Emilio Mayorga --- echopype/__init__.py | 4 + echopype/calibrate/api.py | 9 +- echopype/calibrate/calibrate_ek.py | 7 +- echopype/convert/api.py | 17 +-- echopype/convert/parse_azfp.py | 10 +- echopype/convert/parse_base.py | 18 +-- echopype/convert/set_groups_ek60.py | 8 +- echopype/convert/set_groups_ek80.py | 5 +- echopype/convert/utils/ek_raw_io.py | 32 ++--- echopype/convert/utils/ek_raw_parsers.py | 26 ++-- echopype/echodata/combine.py | 6 +- echopype/echodata/echodata.py | 5 +- .../sensor_ep_version_mapping/v05x_to_v06x.py | 5 +- echopype/tests/utils/test_utils_log.py | 105 ++++++++++++++++ echopype/tests/visualize/test_plot.py | 47 +++----- echopype/utils/io.py | 13 +- echopype/utils/log.py | 114 ++++++++++++++++++ echopype/visualize/api.py | 12 +- echopype/visualize/plot.py | 6 +- 19 files changed, 345 insertions(+), 104 deletions(-) create mode 100644 echopype/tests/utils/test_utils_log.py create mode 100644 echopype/utils/log.py diff --git a/echopype/__init__.py b/echopype/__init__.py index 03adaafe2..31bd5df62 100644 --- a/echopype/__init__.py +++ b/echopype/__init__.py @@ -6,6 +6,9 @@ from .convert.api import open_raw from .echodata.api import open_converted from .echodata.combine import combine_echodata +from .utils.log import verbose + +verbose(override=True) __all__ = [ "open_raw", @@ -15,4 +18,5 @@ "consolidate", "preprocess", "utils", + "verbose", ] diff --git a/echopype/calibrate/api.py b/echopype/calibrate/api.py index 6c5805ad7..979091e32 100644 --- a/echopype/calibrate/api.py +++ b/echopype/calibrate/api.py @@ -1,8 +1,7 @@ -import warnings - import xarray as xr from ..echodata import EchoData +from ..utils.log import _init_logger from ..utils.prov import echopype_prov_attrs, source_files_vars from .calibrate_azfp import CalibrateAZFP from .calibrate_ek import CalibrateEK60, CalibrateEK80 @@ -16,6 +15,8 @@ "EA640": CalibrateEK80, } +logger = _init_logger(__name__) + def _compute_cal( cal_type, @@ -39,12 +40,12 @@ def _compute_cal( ) elif echodata.sonar_model in ("EK60", "AZFP"): if waveform_mode is not None and waveform_mode != "CW": - warnings.warn( + logger.warning( "This sonar model transmits only narrowband signals (waveform_mode='CW'). " "Calibration will be in CW mode", ) if encode_mode is not None and encode_mode != "power": - warnings.warn( + logger.warning( "This sonar model only record data as power or power/angle samples " "(encode_mode='power'). Calibration will be done on the power samples.", ) diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index 42eacf3ba..706be6d34 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -4,8 +4,11 @@ from ..echodata import EchoData from ..utils import uwa +from ..utils.log import _init_logger from .calibrate_base import CAL_PARAMS, CalibrateBase +logger = _init_logger(__name__) + class CalibrateEK(CalibrateBase): def __init__(self, echodata: EchoData, env_params): @@ -930,11 +933,11 @@ def _compute_cal(self, cal_type, waveform_mode, encode_mode) -> xr.Dataset: if encode_mode == "power": use_beam_power = True # switch source of backscatter data - print( + logger.info( "Only power samples are calibrated, but complex samples also exist in the raw data file!" # noqa ) else: - print( + logger.info( "Only complex samples are calibrated, but power samples also exist in the raw data file!" # noqa ) else: # only power OR complex samples exist diff --git a/echopype/convert/api.py b/echopype/convert/api.py index 9400d7019..5f28e771f 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -1,5 +1,4 @@ import warnings -from datetime import datetime as dt from pathlib import Path from typing import TYPE_CHECKING, Dict, Optional, Tuple @@ -17,6 +16,7 @@ # fmt: on from ..echodata.echodata import XARRAY_ENGINE_MAP, EchoData from ..utils import io +from ..utils.log import _init_logger COMPRESSION_SETTINGS = { "netcdf4": {"zlib": True, "complevel": 4}, @@ -27,6 +27,9 @@ BEAM_SUBGROUP_DEFAULT = "Beam_group1" +# Logging setup +logger = _init_logger(__name__) + def to_file( echodata: EchoData, @@ -76,15 +79,15 @@ def to_file( # Sequential or parallel conversion if exists and not overwrite: - print( - f"{dt.now().strftime('%H:%M:%S')} {echodata.source_file} has already been converted to {engine}. " # noqa + logger.info( + f"{echodata.source_file} has already been converted to {engine}. " # noqa f"File saving not executed." ) else: if exists: - print(f"{dt.now().strftime('%H:%M:%S')} overwriting {output_file}") + logger.info(f"overwriting {output_file}") else: - print(f"{dt.now().strftime('%H:%M:%S')} saving {output_file}") + logger.info(f"saving {output_file}") _save_groups_to_file( echodata, output_path=io.sanitize_file_path( @@ -362,7 +365,7 @@ def open_raw( EchoData object """ if (sonar_model is None) and (raw_file is None): - print("Please specify the path to the raw data file and the sonar model.") + logger.warning("Please specify the path to the raw data file and the sonar model.") return # Check inputs @@ -371,7 +374,7 @@ def open_raw( storage_options = storage_options if storage_options is not None else {} if sonar_model is None: - print("Please specify the sonar model.") + logger.warning("Please specify the sonar model.") if xml_path is None: sonar_model = "EK60" diff --git a/echopype/convert/parse_azfp.py b/echopype/convert/parse_azfp.py index 0bffb380e..35371b84a 100644 --- a/echopype/convert/parse_azfp.py +++ b/echopype/convert/parse_azfp.py @@ -8,10 +8,13 @@ import fsspec import numpy as np +from ..utils.log import _init_logger from .parse_base import ParseBase FILENAME_DATETIME_AZFP = "\\w+.01A" +logger = _init_logger(__name__) + class ParseAZFP(ParseBase): """Class for converting data from ASL Environmental Sciences AZFP echosounder.""" @@ -273,10 +276,7 @@ def _print_status(self): ) timestr = timestamp.strftime("%Y-%b-%d %H:%M:%S") pathstr, xml_name = os.path.split(self.xml_path) - print( - f"{dt.now().strftime('%H:%M:%S')} parsing file {filename} with {xml_name}, " - f"time of first ping: {timestr}" - ) + logger.info(f"parsing file {filename} with {xml_name}, " f"time of first ping: {timestr}") def _split_header(self, raw, header_unpacked): """Splits the header information into a dictionary. @@ -300,7 +300,7 @@ def _split_header(self, raw, header_unpacked): ): # first field should match hard-coded FILE_TYPE from manufacturer check_eof = raw.read(1) if check_eof: - print("Error: Unknown file type") + logger.error("Unknown file type") return False header_byte_cnt = 0 diff --git a/echopype/convert/parse_base.py b/echopype/convert/parse_base.py index c7dbafe46..cd2f19d3d 100644 --- a/echopype/convert/parse_base.py +++ b/echopype/convert/parse_base.py @@ -4,12 +4,15 @@ import numpy as np +from ..utils.log import _init_logger from .utils.ek_raw_io import RawSimradFile, SimradEOF FILENAME_DATETIME_EK60 = ( "(?P.+)?-?D(?P\\w{1,8})-T(?P