Merge pull request #432 from OSOceanAcoustics/release/v0.5.3

Release/v0.5.3
OSOceanAcoustics · Aug 20, 2021 · 55f8e33 · 55f8e33
2 parents 1edc95c + 2eb47cb
commit 55f8e33
Show file tree

Hide file tree

Showing 46 changed files with 2,092 additions and 1,030 deletions.
diff --git a/.ci_helpers/py3.7.yaml b/.ci_helpers/py3.7.yaml
@@ -10,10 +10,10 @@ dependencies:
   - pynmea2
   - pytz
   - scipy
-  - xarray<0.18
+  - xarray
   - zarr
-  - fsspec==0.8.7
+  - fsspec
   - requests
   - aiohttp
-  - s3fs==0.5.2
+  - s3fs
   - mamba
diff --git a/.ci_helpers/py3.8.yaml b/.ci_helpers/py3.8.yaml
@@ -10,10 +10,10 @@ dependencies:
   - pynmea2
   - pytz
   - scipy
-  - xarray<0.18
+  - xarray
   - zarr
-  - fsspec==0.8.7
+  - fsspec
   - requests
   - aiohttp
-  - s3fs==0.5.2
+  - s3fs
   - mamba
diff --git a/.ci_helpers/py3.9.yaml b/.ci_helpers/py3.9.yaml
@@ -10,10 +10,10 @@ dependencies:
   - pynmea2
   - pytz
   - scipy
-  - xarray<0.18
+  - xarray
   - zarr
-  - fsspec==0.8.7
+  - fsspec
   - requests
   - aiohttp
-  - s3fs==0.5.2
+  - s3fs
   - mamba
diff --git a/.ci_helpers/run-test.py b/.ci_helpers/run-test.py
@@ -6,6 +6,7 @@
 import argparse
 import glob
 import os
+import re
 import shutil
 import sys
 from pathlib import Path
@@ -22,6 +23,16 @@
     ExitCode.NO_TESTS_COLLECTED: 5,
 }
 
+MODULES_TO_TEST = {
+    "root": {},  # This is to test the root folder.
+    "convert": {},
+    "calibrate": {},
+    "echodata": {},
+    "preprocess": {},
+    "utils": {},
+    "old": {"extra_globs": ["echopype/convert/convert.py", "echopype/process/*"]},
+}
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Run tests listed.")
     parser.add_argument(
@@ -40,6 +51,11 @@
         action="store_true",
         help="Optional flag for running tests locally, not in continuous integration.",
     )
+    parser.add_argument(
+        "--include-cov",
+        action="store_true",
+        help="Optional flag for including coverage. Exports to coverage.xml by default.",
+    )
     args = parser.parse_args()
     if args.local:
         temp_path = Path("temp_echopype_output")
@@ -50,69 +66,66 @@
         if dump_path.exists():
             shutil.rmtree(dump_path)
         echopype_folder = Path("echopype")
-        file_list = glob.glob(str(echopype_folder / "**" / "*.py"))
+        file_list = glob.glob(str(echopype_folder / "**" / "*.py"), recursive=True)
     else:
         file_list = args.touchedfiles.split(",")
     pytest_args = []
     if args.pytest_args:
         pytest_args = args.pytest_args.split(",")
+        if args.include_cov:
+            # Checks for cov in pytest_args
+            for arg in pytest_args:
+                if re.match("--cov", arg) is not None:
+                    raise ValueError(
+                        "pytest args may not have any cov arguments if --include-cov is set."
+                    )
+            pytest_args = pytest_args + [
+                "--cov-report=xml",
+                "--cov-append",
+            ]
     test_to_run = {}
-    for f in file_list:
-        file_path = Path(f)
-        file_name, file_ext = os.path.splitext(os.path.basename(f))
-        if file_ext == ".py":
-            if any(
-                [
-                    (file_path.match("echopype/convert/*")),
-                    (file_path.match("echopype/tests/test_convert*")),
-                ]
-            ):
-                if "convert" not in test_to_run:
-                    test_to_run["convert"] = []
-                test_to_run["convert"].append(file_path)
-            elif any(
-                [
-                    (file_path.match("echopype/calibrate/*")),
-                    (file_path.match("echopype/tests/test_calibrate*")),
-                ]
-            ):
-                if "calibrate" not in test_to_run:
-                    test_to_run["calibrate"] = []
-                test_to_run["calibrate"].append(file_path)
-            elif any(
-                [
-                    (file_path.match("echopype/echodata/*")),
-                    (file_path.match("echopype/tests/test_echodata*")),
-                ]
-            ):
-                if "echodata" not in test_to_run:
-                    test_to_run["echodata"] = []
-                test_to_run["echodata"].append(file_path)
-            elif any(
-                [
-                    (file_path.match("echopype/preprocess/*")),
-                    (file_path.match("echopype/tests/test_preprocess*")),
-                ]
-            ):
-                if "preprocess" not in test_to_run:
-                    test_to_run["preprocess"] = []
-                test_to_run["preprocess"].append(file_path)
-            elif any(
-                [
-                    (file_path.match("echopype/convert/convert.py")),
-                    (file_path.match("echopype/process/*")),
-                    (file_path.match("echopype/tests/test_old.py")),
-                ]
-            ):
-                if "old" not in test_to_run:
-                    test_to_run["old"] = []
-                test_to_run["old"].append(file_path)
+    for module, mod_extras in MODULES_TO_TEST.items():
+        if module == "root":
+            file_globs = [
+                "echopype/*",
+                "echopype/tests/*",
+            ]
+        else:
+            file_globs = [
+                f"echopype/{module}/*",
+                f"echopype/tests/{module}/*",
+            ]
+        if "extra_globs" in mod_extras:
+            file_globs = file_globs + mod_extras["extra_globs"]
+        for f in file_list:
+            file_path = Path(f)
+            file_name, file_ext = os.path.splitext(os.path.basename(f))
+            if file_ext == ".py":
+                if any(((file_path.match(fg)) for fg in file_globs)):
+                    if module not in test_to_run:
+                        test_to_run[module] = []
+                    test_to_run[module].append(file_path)
 
+    original_pytest_args = pytest_args.copy()
     total_exit_codes = []
     for k, v in test_to_run.items():
         print(f"=== RUNNING {k.upper()} TESTS===")
         print(f"Touched files: {','.join([os.path.basename(p) for p in v])}")
-        test_files = glob.glob(f"echopype/tests/test_{k}*.py")
+        if k == "root":
+            file_glob_str = "echopype/tests/test_*.py"
+            cov_mod_arg = ["--cov=echopype"]
+        else:
+            file_glob_str = f"echopype/tests/{k}/*.py"
+            cov_mod_arg = [f"--cov=echopype/{k}"]
+        if args.include_cov:
+            if k == "old":
+                pytest_args = original_pytest_args + [
+                    "--cov=echopype/convert",
+                    "--cov=echopype/process",
+                ]
+            else:
+                pytest_args = original_pytest_args + cov_mod_arg
+        test_files = glob.glob(file_glob_str)
         final_args = pytest_args + test_files
         print(f"Pytest args: {final_args}")
         exit_code = pytest.main(final_args)
@@ -121,7 +134,7 @@
     if len(total_exit_codes) == 0:
         print("No test(s) were run.")
         sys.exit(0)
-    if all([True if e == 0 else False for e in total_exit_codes]):
+    if all(True if e == 0 else False for e in total_exit_codes):
         print("All test run successful")
         sys.exit(0)
     else:

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -56,11 +56,11 @@ jobs:
       - name: Cache conda
         uses: actions/cache@v2
         env:
-          # Increase this value to reset cache if .ci_helpers/py${{ matrix.python-version }}.yaml has not changed
+          # Increase this value to reset cache if '.ci_helpers/py{0}.yaml' has not changed
           CACHE_NUMBER: 0
         with:
           path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('.ci_helpers/py${{ matrix.python-version }}.yaml') }}
+          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('.ci_helpers/py{0}.yaml', matrix.python-version)) }}
       - name: Setup miniconda
         uses: conda-incubator/setup-miniconda@v2
         with:
@@ -87,7 +87,7 @@ jobs:
       - name: Running All Tests
         shell: bash -l {0}
         run: |
-          python .ci_helpers/run-test.py --local --pytest-args="--cov=echopype,--cov-report=xml,--log-cli-level=WARNING,-vv,--disable-warnings" |& tee ci_test_log.log
+          pytest -vv -rx --cov=echopype --cov-report=xml --log-cli-level=WARNING --disable-warnings |& tee ci_${{ matrix.python-version }}_test_log.log
       - name: Upload ci test log
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -52,11 +52,11 @@ jobs:
       - name: Cache conda
         uses: actions/cache@v2
         env:
-          # Increase this value to reset cache if .ci_helpers/py${{ matrix.python-version }}.yaml has not changed
+          # Increase this value to reset cache if '.ci_helpers/py{0}.yaml' has not changed
           CACHE_NUMBER: 0
         with:
           path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('.ci_helpers/py${{ matrix.python-version }}.yaml') }}
+          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('.ci_helpers/py{0}.yaml', matrix.python-version)) }}
       - name: Setup miniconda
         uses: conda-incubator/setup-miniconda@v2
         with:
@@ -87,17 +87,17 @@ jobs:
           format: 'csv'
       - name: Print Changed files
         shell: bash -l {0}
-        run: echo "${{ steps.files.outputs.added_modified }}"
+        run: echo "${{ steps.files.outputs.added_modified_renamed }}"
       - name: Running all Tests
         if: contains(github.event.pull_request.labels.*.name, 'Needs Complete Testing')
         shell: bash -l {0}
         run: |
-          python .ci_helpers/run-test.py --local --pytest-args="--cov=echopype,--cov-report=xml,--log-cli-level=WARNING,-vv,--disable-warnings" |& tee ci_test_log.log
+          pytest -vv -rx --cov=echopype --cov-report=xml --log-cli-level=WARNING --disable-warnings |& tee ci_${{ matrix.python-version }}_test_log.log
       - name: Running Tests
         if: "!contains(github.event.pull_request.labels.*.name, 'Needs Complete Testing')"
         shell: bash -l {0}
         run: |
-          python .ci_helpers/run-test.py --pytest-args="--cov=echopype,--cov-report=xml,--log-cli-level=WARNING,-vv,--disable-warnings" ${{ steps.files.outputs.added_modified }} |& tee ci_test_log.log
+          python .ci_helpers/run-test.py --pytest-args="--log-cli-level=WARNING,-vv,-rx,--disable-warnings" --include-cov ${{ steps.files.outputs.added_modified_renamed }} |& tee ci_test_log.log
       - name: Upload ci test log
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2

diff --git a/README.md b/README.md
@@ -1,36 +1,46 @@
 <div align="center">
-<img src="https://raw.githubusercontent.com/OSOceanAcoustics/echopype/master/docs/source/_static/echopype_logo_banner.png" width="400">
+  <img src="https://raw.githubusercontent.com/OSOceanAcoustics/echopype/master/docs/source/_static/echopype_logo_banner.png" width="400">
 </div>
 
 # Echopype
 
-<a href="https://github.com/OSOceanAcoustics/echopype/actions/workflows/ci.yaml">
-  <img src="https://github.com/OSOceanAcoustics/echopype/actions/workflows/ci.yaml/badge.svg"/>
-</a>
+<div>
+  <a href="https://doi.org/10.5281/zenodo.4066742">
+    <img src="https://zenodo.org/badge/DOI/10.5281/zenodo.4066742.svg" alt="DOI">
+  </a>
 
-<a href="https://echopype.readthedocs.io/en/latest/?badge=latest">
-  <img src="https://readthedocs.org/projects/echopype/badge/?version=latest"/>
-</a>
+  <a href="https://raw.githubusercontent.com/OSOceanAcoustics/echopype/master/LICENSE">
+    <img alt="GitHub License" src="https://img.shields.io/github/license/OSOceanAcoustics/echopype">
+  </a>
+</div>
+
+<div>
+  <a href="https://github.com/OSOceanAcoustics/echopype/actions/workflows/ci.yaml">
+    <img src="https://github.com/OSOceanAcoustics/echopype/actions/workflows/ci.yaml/badge.svg"/>
+  </a>
 
-<a href="https://codecov.io/gh/OSOceanAcoustics/echopype">
-  <img src="https://codecov.io/gh/OSOceanAcoustics/echopype/branch/master/graph/badge.svg?token=GT98F919XR"/>
-</a>
+  <a href="https://results.pre-commit.ci/latest/github/OSOceanAcoustics/echopype/master">
+    <img src="https://results.pre-commit.ci/badge/github/OSOceanAcoustics/echopype/master.svg"/>
+  </a>
 
-<a href="https://pypi.org/project/echopype/">
-  <img src="https://img.shields.io/pypi/v/echopype.svg"/>
-</a>
+  <a href="https://echopype.readthedocs.io/en/latest/?badge=latest">
+    <img src="https://readthedocs.org/projects/echopype/badge/?version=latest"/>
+  </a>
 
-<a href="https://anaconda.org/conda-forge/echopype">
-  <img src="https://img.shields.io/conda/vn/conda-forge/echopype.svg"/>
-</a>
+  <a href="https://codecov.io/gh/OSOceanAcoustics/echopype">
+    <img src="https://codecov.io/gh/OSOceanAcoustics/echopype/branch/master/graph/badge.svg?token=GT98F919XR"/>
+  </a>
+</div>
 
-<a href="https://doi.org/10.5281/zenodo.4066742">
-  <img src="https://zenodo.org/badge/DOI/10.5281/zenodo.4066742.svg" alt="DOI">
-</a>
+<div>
+  <a href="https://pypi.org/project/echopype/">
+    <img src="https://img.shields.io/pypi/v/echopype.svg"/>
+  </a>
 
-<a href="https://raw.githubusercontent.com/OSOceanAcoustics/echopype/master/LICENSE">
-  <img alt="GitHub License" src="https://img.shields.io/github/license/OSOceanAcoustics/echopype">
-</a>
+  <a href="https://anaconda.org/conda-forge/echopype">
+    <img src="https://img.shields.io/conda/vn/conda-forge/echopype.svg"/>
+  </a>
+</div>
 
 Echopype is a package built to enable interoperability and scalability in ocean sonar data processing. These data are widely used for obtaining information about the distribution and abundance of marine animals, such as fish and krill. Our ability to collect large volumes of sonar data from a variety of ocean platforms has grown significantly in the last decade. However, most of the new data remain under-utilized. echopype aims to address the root cause of this problem - the lack of interoperable data format and scalable analysis workflows that adapt well with increasing data volume - by providing open-source tools as entry points for scientists to make discovery using these new data.
 

diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,3 +1,4 @@
 sphinx_rtd_theme
 sphinx-automodapi
+sphinxcontrib-mermaid
 numpydoc