diff --git a/.github/workflows/pytests-upstream.yml b/.github/workflows/pytests-upstream.yml index ee52c59b2..cf5068eca 100644 --- a/.github/workflows/pytests-upstream.yml +++ b/.github/workflows/pytests-upstream.yml @@ -93,6 +93,7 @@ jobs: echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-core-free.yml" >> $GITHUB_ENV echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV echo "LOG_FILE=argopy-tests-Core-Free-Py${{matrix.python-version}}-${{matrix.os}}.log" >> $GITHUB_ENV + echo "PYTHONTRACEMALLOC=20" >> $GITHUB_ENV - name: Increase swapfile if: ${{matrix.os == 'ubuntu-latest'}} @@ -118,7 +119,7 @@ jobs: - name: Install argopy run: | - python -m pip install --no-deps -e . + python -m pip install --no-deps -e . - name: Version info run: | @@ -261,6 +262,7 @@ jobs: run: | echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-all-free.yml" >> $GITHUB_ENV echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV + echo "PYTHONTRACEMALLOC=20" >> $GITHUB_ENV - name: Increase swapfile if: ${{matrix.os == 'ubuntu-latest'}} @@ -287,7 +289,7 @@ jobs: - name: Install argopy run: | - python -m pip install --no-deps -e . + python -m pip install --no-deps -e . - name: Version info run: | @@ -313,6 +315,7 @@ jobs: - name: Test with pytest run: | + PYTHONTRACEMALLOC=1 python ci/resource_summary.py pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 \ --report-log output-${{ matrix.python-version }}-log.jsonl diff --git a/argopy/data_fetchers/gdac_data.py b/argopy/data_fetchers/gdac_data.py index 059cee2f7..ba154cb80 100644 --- a/argopy/data_fetchers/gdac_data.py +++ b/argopy/data_fetchers/gdac_data.py @@ -276,10 +276,10 @@ def to_xarray( and not self.parallelize and self.parallel_method == "sequential" ): - warnings.warn( - "Found more than 50 files to load, this may take a while to process sequentially ! " - "Consider using another data source (eg: 'erddap') or the 'parallel=True' option to improve processing time." - ) + msg = f"Found more than 50 files to load ({len(URI)} !), this may take a while to process sequentially ! Consider using another data source (eg: 'erddap') or the 'parallel=True' option to improve processing time." + warnings.warn(msg) + log.info(msg) + elif len(URI) == 0: raise DataNotFound("No data found for: %s" % self.indexfs.cname) diff --git a/argopy/extensions/optical_modeling.py b/argopy/extensions/optical_modeling.py index 4519b6579..f412b0bf6 100644 --- a/argopy/extensions/optical_modeling.py +++ b/argopy/extensions/optical_modeling.py @@ -104,7 +104,7 @@ def Zeu( But the euphotic depth can also be estimated using the exponential decay of light with depth, described by Beer's Law [1]_: .. math:: - I(z) = I_0 \exp(-K_{PAR}\\,z) + I(z) = I_0 \\exp(-K_{PAR}\\,z) If we solve for $I(Z_e)=0.01 I_0$ we get: @@ -288,11 +288,11 @@ def Z_iPAR_threshold( Notes ----- - This is the closest level $z$ in the vertical axis for which PAR is about a threshold value $t$, with some tolerance $\epsilon$: + This is the closest level $z$ in the vertical axis for which PAR is about a threshold value $t$, with some tolerance $\\epsilon$: .. math:: - z | abs(PAR(z) - t) < \epsilon + z | abs(PAR(z) - t) < \\epsilon A default value of 15 is used because it is the theoretical value below which the Fchla is no longer quenched (For correction of NPQ purposes). diff --git a/argopy/stores/float/spec.py b/argopy/stores/float/spec.py index 3d46bc99c..b94b1010a 100644 --- a/argopy/stores/float/spec.py +++ b/argopy/stores/float/spec.py @@ -426,7 +426,7 @@ def open_dataset( cast: bool, optional, default = True Determine if the dataset variables should be cast or not. This is similar to opening the dataset directly with :class:`xarray.open_dataset` using the ``engine=`argo``` option. This will be ignored if the ``netCDF4` kwarg is set to True. - \**kwargs + **kwargs All the other arguments are passed to the GDAC store `open_dataset` method. Returns @@ -461,7 +461,7 @@ def dataset(self, name: str = "prof", **kwargs) -> xr.Dataset: ---------- name: str, optional, default = "prof" Name of the dataset to open. It can be any key from the dictionary returned by :class:`ArgoFloat.ls_datasets`. - \**kwargs + **kwargs All the other arguments are passed to the :meth:`ArgoFloat.open_dataset` method. Returns @@ -804,7 +804,7 @@ def profile(self, name: str, **kwargs) -> xr.Dataset: ---------- name: str Name of the profile file to open. It can be any key from the dictionary returned by :class:`ArgoFloat.ls_profiles`. - \**kwargs + **kwargs All the other arguments are passed to the :meth:`ArgoFloat.open_profile` method. Returns diff --git a/argopy/stores/implementations/ftp.py b/argopy/stores/implementations/ftp.py index ccd6cf3bf..b85615c46 100644 --- a/argopy/stores/implementations/ftp.py +++ b/argopy/stores/implementations/ftp.py @@ -8,6 +8,7 @@ import warnings from typing import Literal from netCDF4 import Dataset +import threading from ...errors import InvalidMethod, DataNotFound from ...utils.transformers import drop_variables_not_in_all_datasets @@ -18,6 +19,9 @@ log = logging.getLogger("argopy.stores.implementation.ftp") +_cache_lock = threading.Lock() +# Used to lock threads to prevent race to the cached meda-data of fsspec + class ftpstore(httpstore): """Argo ftp file system @@ -110,7 +114,8 @@ def load_in_memory(url, errors="raise", xr_opts={}): try: this_url = self.fs._strip_protocol(url) - data = self.fs.cat_file(this_url) + with _cache_lock: + data = self.fs.cat_file(this_url) if data is None: if errors == "raise": raise DataNotFound(url) @@ -207,6 +212,9 @@ def load_lazily(url, errors="raise", xr_opts={}, akoverwrite: bool = False): if target is not None: if not netCDF4: ds = xr.open_dataset(target, **xr_opts) + if not lazy: + ds = ds.load() # materialize into plain numpy arrays, detach from the backend buffer + ds.close() # explicitly release the netCDF4/HDF5 handle right away if "source" not in ds.encoding: if isinstance(url, str): diff --git a/argopy/stores/implementations/http.py b/argopy/stores/implementations/http.py index 84f2f9202..05abc6a8a 100644 --- a/argopy/stores/implementations/http.py +++ b/argopy/stores/implementations/http.py @@ -17,6 +17,7 @@ from functools import lru_cache from netCDF4 import Dataset from urllib.parse import urlparse +import threading from ...errors import InvalidMethod, DataNotFound from ...utils import Registry, UriCName @@ -33,6 +34,9 @@ log = logging.getLogger("argopy.stores.implementation.http") +_cache_lock = threading.Lock() +# Used to lock threads to prevent race to the cached meda-data of fsspec + class httpstore(ArgoStoreProto): """Argo http file system @@ -118,7 +122,8 @@ def make_request( data = None if n_attempt <= max_attempt: try: - data = ffs.cat_file(url, **cat_opts) + with _cache_lock: + data = ffs.cat_file(url, **cat_opts) except FileNotFoundError as e: if errors == "raise": raise e @@ -369,7 +374,8 @@ def load_lazily( if not netCDF4: ds = xr.open_dataset(target, **xr_opts) if not lazy: - ds = ds.load() + ds = ds.load() # materialize into plain numpy arrays, detach from the backend buffer + ds.close() # explicitly release the netCDF4/HDF5 handle right away if "source" not in ds.encoding: if isinstance(url, str): @@ -905,9 +911,10 @@ def read_csv(self, url, **kwargs): """ url = self.curateurl(url) - # log.debug("Opening/reading csv from: %s" % url) - with self.open(url) as of: - df = pd.read_csv(of, **kwargs) + + with _cache_lock: + with self.open(url) as of: + df = pd.read_csv(of, **kwargs) self.register(url) return df diff --git a/argopy/stores/implementations/local.py b/argopy/stores/implementations/local.py index d1352c2b6..03a4d4908 100644 --- a/argopy/stores/implementations/local.py +++ b/argopy/stores/implementations/local.py @@ -11,6 +11,7 @@ from pathlib import Path import warnings from netCDF4 import Dataset +import threading from ...options import OPTIONS from ...errors import InvalidMethod, DataNotFound @@ -19,8 +20,12 @@ from ..filesystems import has_distributed, distributed from ..filesystems import tqdm + log = logging.getLogger("argopy.stores.implementation.local") +_cache_lock = threading.Lock() +# Used to lock threads to prevent race to the cached meda-data of fsspec + class filestore(ArgoStoreProto): """Argo local file system @@ -71,8 +76,9 @@ def open_json(self, url, errors: Literal['raise', 'silent', 'ignore'] = 'raise', if "js_opts" in kwargs: js_opts.update(kwargs["js_opts"]) - with self.open(url, **open_opts) as of: - js = json.load(of, **js_opts) + with _cache_lock: + with self.open(url, **open_opts) as of: + js = json.load(of, **js_opts) if len(js) == 0: if errors == "raise": @@ -121,7 +127,8 @@ def load_in_memory(path, errors="raise", xr_opts={}): tuple: (data, _) or (None, _) if errors == "ignore" """ try: - data = self.fs.cat_file(path) + with _cache_lock: + data = self.fs.cat_file(path) if data[0:3] != b"CDF" and data[0:3] != b"\x89HD": raise TypeError( @@ -198,6 +205,9 @@ def load_lazily(path, errors="raise", xr_opts={}, akoverwrite: bool = False): if target is not None: if not netCDF4: ds = xr.open_dataset(target, **xr_opts) + if not lazy: + ds = ds.load() # materialize into plain numpy arrays, detach from the backend buffer + ds.close() # explicitly release the netCDF4/HDF5 handle right away if "source" not in ds.encoding: if isinstance(path, str): @@ -426,6 +436,7 @@ def read_csv(self, path, **kwargs): :class:`pandas.DataFrame` """ log.debug("Reading csv: %s" % path) - with self.open(path) as of: - df = pd.read_csv(of, **kwargs) + with _cache_lock: + with self.open(path) as of: + df = pd.read_csv(of, **kwargs) return df diff --git a/argopy/tests/conftest.py b/argopy/tests/conftest.py index afa8ed682..00d04cdf8 100644 --- a/argopy/tests/conftest.py +++ b/argopy/tests/conftest.py @@ -2,10 +2,12 @@ import os import logging import shutil +import pytest +import threading sys.path.append(os.path.join(os.path.dirname(__file__), 'helpers')) -from mocked_ftp import mocked_ftpserver -from mocked_http import mocked_httpserver +from argopy.tests.helpers.mocked_ftp import mocked_ftpserver +from argopy.tests.helpers.mocked_http import mocked_httpserver log = logging.getLogger("argopy.tests.conftests") @@ -24,4 +26,34 @@ def pytest_sessionfinish(session, exitstatus): pass log.debug("Ending tests session") log.debug("Final session state: %s" % session) - pass \ No newline at end of file + pass + +@pytest.fixture(autouse=True) +def _resource_tracker(request): + """Track resource usage around every test to catch what leaks.""" + + proc = f"/proc/{os.getpid()}" + + def _fds(): + try: + return len(os.listdir(f"{proc}/fd")) + except Exception: + return -1 + + def _threads(): + return threading.active_count() + + before = dict(fds=_fds(), threads=_threads()) + yield + after = dict(fds=_fds(), threads=_threads()) + + delta_fds = after["fds"] - before["fds"] + delta_threads = after["threads"] - before["threads"] + + # Only log when something looks wrong + if delta_fds > 10 or delta_threads > 2: + print( + f"\n[RESOURCE LEAK] {request.node.nodeid}\n" + f" FDs: {before['fds']} : {after['fds']} (diff:{delta_fds:+d})\n" + f" Threads: {before['threads']} : {after['threads']} (diff:{delta_threads:+d})\n" + ) diff --git a/argopy/tests/helpers/mocked_http.py b/argopy/tests/helpers/mocked_http.py index b05249090..976fa840a 100644 --- a/argopy/tests/helpers/mocked_http.py +++ b/argopy/tests/helpers/mocked_http.py @@ -29,7 +29,7 @@ from pathlib import Path import threading from collections import ChainMap -from http.server import BaseHTTPRequestHandler, HTTPServer +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import pytest import logging from urllib.parse import unquote @@ -44,25 +44,6 @@ False # Should we list all files/uris available from the mocked server in the log ? ) -import socket - - -def _free_port() -> int: - """Return a free port number on localhost. - - bind("127.0.0.1", 0) asks the OS to assign a free port, which we read back - with getsockname""" - s = socket.socket() - s.bind(("127.0.0.1", 0)) - p = s.getsockname()[1] - s.close() - return p - - -port = _free_port() -mocked_server_address = "http://127.0.0.1:%i" % port - - """ Load test data and create a dictionary mapping of URL requests as keys, and expected responses as values @@ -174,22 +155,23 @@ def __init__(self, *args, **kwargs): def _respond(self, code=200, headers=None, data=b""): headers = headers or {} headers.update({"User-Agent": "Mocked http server for unit tests"}) - self.send_response(code) - for k, v in headers.items(): - self.send_header(k, str(v)) - self.end_headers() - if data: - if not isinstance(data, (bytes, bytearray)): - data = _read(data) - try: + try: + self.send_response(code) + for k, v in headers.items(): + self.send_header(k, str(v)) + self.end_headers() + if data: + if not isinstance(data, (bytes, bytearray)): + data = _read(data) self.wfile.write(data) - except socket.error as e: - # socket error [Errno 32] Broken pipe - # This might be happening when a client program doesn't wait till all the data from the server is - # received and simply closes a socket - if "32" not in str(e): - log.debug("socket error %s" % str(e)) - pass + except (BrokenPipeError, ConnectionResetError, socket.error) as e: + # socket error [Errno 32] Broken pipe + # This might be happening when a client program doesn't wait till all the data from the server is + # received and simply closes a socket + log.debug("socket error while responding: %s" % str(e)) + # if "32" not in str(e): + # log.debug("socket error %s" % str(e)) + # pass def log_message(self, format, *args): # Quiet logging ! @@ -301,8 +283,10 @@ def do_HEAD(self): @contextlib.contextmanager def serve_mocked_httpserver(): - server_address = ("", port) - httpd = HTTPServer(server_address, HTTPTestHandler) + httpd = ThreadingHTTPServer(("127.0.0.1", 0), HTTPTestHandler) + port = httpd.server_address[1] + mocked_server_address = "http://127.0.0.1:%i" % port + th = threading.Thread(target=httpd.serve_forever) th.daemon = True th.start() diff --git a/argopy/tests/helpers/utils.py b/argopy/tests/helpers/utils.py index 28263d7c4..2c2edf7c7 100644 --- a/argopy/tests/helpers/utils.py +++ b/argopy/tests/helpers/utils.py @@ -40,7 +40,7 @@ isAPIconnected, ) from argopy.options import OPTIONS -from mocked_http import mocked_server_address, serve_mocked_httpserver +from mocked_http import serve_mocked_httpserver log = logging.getLogger("argopy.tests.utils") @@ -92,7 +92,8 @@ def _xfail(name, msg): if CONNECTED: log.debug("Checking which Erddap dataset are available (eg: core, bgc, ref, index)") - with serve_mocked_httpserver() as s: # Use the mocked http server + with serve_mocked_httpserver() as mocked_server_address: # Use the mocked http server + # log.info(f"Use http server for the erddap: {mocked_server_address}") with set_options(erddap=mocked_server_address): res = erddap_ds_exists(["ArgoFloats", "ArgoFloats-synthetic-BGC", "ArgoFloats-reference", "ArgoFloats-index"]) DSEXISTS = res[0] diff --git a/argopy/tests/test_fetchers_dask_cluster.py b/argopy/tests/test_fetchers_dask_cluster.py index 73cdf2d38..7eb07b124 100644 --- a/argopy/tests/test_fetchers_dask_cluster.py +++ b/argopy/tests/test_fetchers_dask_cluster.py @@ -9,8 +9,8 @@ from collections import ChainMap import xarray as xr -from mocked_http import mocked_server_address, mocked_httpserver -from utils import ( +from argopy.tests.helpers.mocked_http import mocked_httpserver +from argopy.tests.helpers.utils import ( requires_argovis, requires_erddap, requires_gdac, @@ -95,10 +95,14 @@ class Test_Backend: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver + def setup_class(self): """setup any state specific to the execution of the given class""" self.client = Client(processes=False) - log.debug("Dask dashboard: %s" % self.client.dashboard_link) + # log.debug("Dask dashboard: %s" % self.client.dashboard_link) # Possibly prevent UTF-8 error on Windows def _test2fetcherargs(self, this_request): """Helper method to set up options for a fetcher creation""" @@ -108,8 +112,8 @@ def _test2fetcherargs(self, this_request): "chunks_maxsize": {"lon": 2.5, "lat": 2.5}, } if USE_MOCKED_SERVER: - defaults_args["server"] = mocked_server_address - defaults_args["gdac"] = mocked_server_address + defaults_args["server"] = self.mocked_server_address + defaults_args["gdac"] = self.mocked_server_address src = this_request.param["src"] dataset = this_request.param["ds"] diff --git a/argopy/tests/test_fetchers_data_argovis.py b/argopy/tests/test_fetchers_data_argovis.py index 0ae93b2a5..9dde0b6ed 100644 --- a/argopy/tests/test_fetchers_data_argovis.py +++ b/argopy/tests/test_fetchers_data_argovis.py @@ -10,7 +10,6 @@ create_temp_folder, ) -from mocked_http import mocked_server_address from mocked_http import mocked_httpserver as mocked_argovisserver import shutil @@ -122,6 +121,9 @@ class Test_Backend: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver def setup_class(self): """setup any state specific to the execution of the given class""" @@ -136,7 +138,7 @@ def _setup_fetcher(self, this_request, cached=False, parallel=False): "parallel": parallel, } if USE_MOCKED_SERVER: - defaults_args['server'] = mocked_server_address + defaults_args['server'] = self.mocked_server_address dataset = this_request.param['ds'] user_mode = this_request.param['mode'] diff --git a/argopy/tests/test_fetchers_data_erddap.py b/argopy/tests/test_fetchers_data_erddap.py index 3d7fec936..be7577f19 100644 --- a/argopy/tests/test_fetchers_data_erddap.py +++ b/argopy/tests/test_fetchers_data_erddap.py @@ -10,8 +10,7 @@ create_temp_folder, ) -from mocked_http import mocked_server_address -from mocked_http import mocked_httpserver as mocked_erddapserver +from mocked_http import mocked_httpserver import shutil from collections import ChainMap @@ -94,7 +93,7 @@ def core(fargs, apts): return fetcher -def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): +def assert_fetcher(mocked_httpserver, this_fetcher, cacheable=False): """Assert a data fetcher. This should be used by all tests asserting a fetcher @@ -142,6 +141,9 @@ class Test_Backend: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver def setup_class(self): """setup any state specific to the execution of the given class""" @@ -156,7 +158,7 @@ def _setup_fetcher(self, this_request, cached=False, parallel=False): "parallel": parallel, } if USE_MOCKED_SERVER: - defaults_args['server'] = mocked_server_address + defaults_args['server'] = self.mocked_server_address dataset = this_request.param['ds'] user_mode = this_request.param['mode'] @@ -209,17 +211,17 @@ def remove_test_dir(): @pytest.mark.parametrize("fetcher", VALID_ACCESS_POINTS, indirect=True, ids=VALID_ACCESS_POINTS_IDS) - def test_fetching(self, mocked_erddapserver, fetcher): - assert_fetcher(mocked_erddapserver, fetcher, cacheable=False) + def test_fetching(self, mocked_httpserver, fetcher): + assert_fetcher(mocked_httpserver, fetcher, cacheable=False) @pytest.mark.parametrize("cached_fetcher", VALID_ACCESS_POINTS, indirect=True, ids=VALID_ACCESS_POINTS_IDS) - def test_fetching_cached(self, mocked_erddapserver, cached_fetcher): - assert_fetcher(mocked_erddapserver, cached_fetcher, cacheable=True) + def test_fetching_cached(self, mocked_httpserver, cached_fetcher): + assert_fetcher(mocked_httpserver, cached_fetcher, cacheable=True) @pytest.mark.parametrize("parallel_fetcher", VALID_PARALLEL_ACCESS_POINTS, indirect=True, ids=VALID_PARALLEL_ACCESS_POINTS_IDS) - def test_fetching_parallel_thread(self, mocked_erddapserver, parallel_fetcher): - assert_fetcher(mocked_erddapserver, parallel_fetcher, cacheable=False) + def test_fetching_parallel_thread(self, mocked_httpserver, parallel_fetcher): + assert_fetcher(mocked_httpserver, parallel_fetcher, cacheable=False) diff --git a/argopy/tests/test_fetchers_data_erddap_bgc.py b/argopy/tests/test_fetchers_data_erddap_bgc.py index aeee91b11..902ed2f46 100644 --- a/argopy/tests/test_fetchers_data_erddap_bgc.py +++ b/argopy/tests/test_fetchers_data_erddap_bgc.py @@ -7,12 +7,11 @@ import pytest import xarray as xr -from utils import ( +from argopy.tests.helpers.utils import ( requires_erddap, create_temp_folder, ) -from mocked_http import mocked_server_address -from mocked_http import mocked_httpserver as mocked_erddapserver +from argopy.tests.helpers.mocked_http import mocked_httpserver import shutil from collections import ChainMap @@ -97,7 +96,7 @@ def core(fargs, apts): return fetcher -def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): +def assert_fetcher(mocked_httpserver, this_fetcher, cacheable=False): """Assert a data fetcher. This should be used by all tests asserting a fetcher @@ -145,6 +144,9 @@ class Test_Backend: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver def setup_class(self): """setup any state specific to the execution of the given class""" @@ -160,9 +162,9 @@ def _setup_fetcher(self, this_request, cached=False, parallel=False): } if USE_MOCKED_SERVER: - defaults_args['server'] = mocked_server_address + defaults_args['server'] = self.mocked_server_address defaults_args['indexfs'] = ArgoIndex( - host=mocked_server_address, + host=self.mocked_server_address, index_file='argo_synthetic-profile_index.txt', cache=cached, cachedir=self.cachedir, @@ -222,20 +224,20 @@ def remove_test_dir(): @pytest.mark.parametrize("fetcher", VALID_ACCESS_POINTS, indirect=True, ids=VALID_ACCESS_POINTS_IDS) - def test_fetching(self, mocked_erddapserver, fetcher): - assert_fetcher(mocked_erddapserver, fetcher, cacheable=False) + def test_fetching(self, mocked_httpserver, fetcher): + assert_fetcher(mocked_httpserver, fetcher, cacheable=False) @pytest.mark.parametrize("cached_fetcher", VALID_ACCESS_POINTS, indirect=True, ids=VALID_ACCESS_POINTS_IDS) - def test_fetching_cached(self, mocked_erddapserver, cached_fetcher): - assert_fetcher(mocked_erddapserver, cached_fetcher, cacheable=True) + def test_fetching_cached(self, mocked_httpserver, cached_fetcher): + assert_fetcher(mocked_httpserver, cached_fetcher, cacheable=True) @pytest.mark.parametrize("measured", [None, 'all', 'DOXY'], indirect=False, ids=["measured=%s" % m for m in [None, 'all', 'DOXY']] ) - def test_fetching_measured(self, mocked_erddapserver, measured): + def test_fetching_measured(self, mocked_httpserver, measured): class this_request: param = { 'ds': 'bgc', @@ -246,10 +248,10 @@ class this_request: } fetcher_args, access_point = self._setup_fetcher(this_request) fetcher = create_fetcher(fetcher_args, access_point) - assert_fetcher(mocked_erddapserver, fetcher) + assert_fetcher(mocked_httpserver, fetcher) @pytest.mark.parametrize("parallel_fetcher", VALID_PARALLEL_ACCESS_POINTS, indirect=True, ids=VALID_PARALLEL_ACCESS_POINTS_IDS) - def test_fetching_parallel_thread(self, mocked_erddapserver, parallel_fetcher): - assert_fetcher(mocked_erddapserver, parallel_fetcher, cacheable=False) + def test_fetching_parallel_thread(self, mocked_httpserver, parallel_fetcher): + assert_fetcher(mocked_httpserver, parallel_fetcher, cacheable=False) diff --git a/argopy/tests/test_fetchers_data_gdac.py b/argopy/tests/test_fetchers_data_gdac.py index 1697e805b..93a42dac5 100644 --- a/argopy/tests/test_fetchers_data_gdac.py +++ b/argopy/tests/test_fetchers_data_gdac.py @@ -20,9 +20,8 @@ CacheFileNotFound, ) from argopy.utils.checkers import is_list_of_strings, check_gdac_path -from utils import requires_gdac, create_temp_folder, patch_ftp, has_s3 -from mocked_http import mocked_httpserver -from mocked_http import mocked_server_address as MOCKHTTP +from argopy.tests.helpers.utils import requires_gdac, create_temp_folder, patch_ftp, has_s3 +from argopy.tests.helpers.mocked_http import mocked_httpserver log = logging.getLogger("argopy.tests.data.gdac") @@ -36,12 +35,12 @@ """ HOSTS = [ argopy.tutorial.open_dataset("gdac")[0], - MOCKHTTP, + "MOCKHTTP", "MOCKFTP", ] if has_s3: - # todo Create a mocked server for s3 tests + #todo Create a mocked server for s3 tests HOSTS.append("s3://argo-gdac-sandbox/pub") # todo: How do we mock a s3 server ? """ @@ -49,10 +48,10 @@ For each access points, we list 1-to-2 scenario to make sure all possibilities are tested """ ACCESS_POINTS = [ - {"float": [13857]}, - {"profile": [13857, 90]}, + # {"float": [13857]}, + # {"profile": [13857, 90]}, {"region": [-20, -16.0, 0, 1, 0, 100.0]}, - {"region": [-20, -16.0, 0, 1, 0, 100.0, "1997-07-01", "1997-09-01"]}, + # {"region": [-20, -16.0, 0, 1, 0, 100.0, "1997-07-01", "1997-09-01"]}, ] # PARALLEL_ACCESS_POINTS = [ # {"float": [1900468, 1900117, 1900386]}, @@ -78,6 +77,7 @@ List user modes to be tested """ USER_MODES = ["standard", "expert", "research"] +USER_MODES = ["standard"] @requires_gdac @@ -128,8 +128,10 @@ def gdac_shortname(gdac): """Get a short name for scenarios IDs, given a FTP host""" if gdac == "MOCKFTP": return "ftp_mocked" - elif "localhost" in gdac or "127.0.0.1" in gdac: + elif gdac == "MOCKHTTP": return "http_mocked" + # elif "localhost" in gdac or "127.0.0.1" in gdac: + # return "http_mocked" else: return (lambda x: "file" if x == "" else x)(urlparse(gdac).scheme) @@ -177,6 +179,9 @@ class TestBackend: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver def setup_class(self): """setup any state specific to the execution of the given class""" @@ -184,7 +189,12 @@ def setup_class(self): self.cachedir = create_temp_folder().folder def _patch_gdac(self, gdac): - return patch_ftp(gdac) + if gdac == 'MOCKFTP': + return patch_ftp(gdac) + elif gdac == 'MOCKHTTP': + return self.mocked_server_address + return gdac + def _setup_fetcher(self, this_request, cached=False, parallel=False): """Helper method to set up options for a fetcher creation""" @@ -202,6 +212,7 @@ def _setup_fetcher(self, this_request, cached=False, parallel=False): "cache": cached, "cachedir": self.cachedir, "parallel": False, + # "progress": True, "N_RECORDS": N_RECORDS, } diff --git a/argopy/tests/test_fetchers_facade_data.py b/argopy/tests/test_fetchers_facade_data.py index 1e7b13289..a69101bbd 100644 --- a/argopy/tests/test_fetchers_facade_data.py +++ b/argopy/tests/test_fetchers_facade_data.py @@ -28,8 +28,7 @@ ) -from mocked_http import mocked_httpserver -from mocked_http import mocked_server_address as MOCKHTTP +from argopy.tests.helpers.mocked_http import mocked_httpserver if has_matplotlib: @@ -112,7 +111,7 @@ def test_to_dataframe(self): ids=ids_params) def test_to_index(self, params, mocked_httpserver): full, coriolis_id = params - with argopy.set_options(server=MOCKHTTP): + with argopy.set_options(server=mocked_httpserver): assert isinstance(self.__get_fetcher()[1].to_index(full=full, coriolis_id=coriolis_id), pd.core.frame.DataFrame) def test_load(self): @@ -174,7 +173,7 @@ def test_domain(self): fetcher.domain def test_dashboard(self, mocked_httpserver): - with argopy.set_options(server=MOCKHTTP): + with argopy.set_options(server=mocked_httpserver): f, fetcher = self.__get_fetcher(pt='float') assert isinstance(fetcher.dashboard(url_only=True), str) diff --git a/argopy/tests/test_fetchers_index_erddap.py b/argopy/tests/test_fetchers_index_erddap.py index b42df83e2..dc04ade36 100644 --- a/argopy/tests/test_fetchers_index_erddap.py +++ b/argopy/tests/test_fetchers_index_erddap.py @@ -9,7 +9,7 @@ FileSystemHasNoCache, CacheFileNotFound ) -from utils import requires_connected_erddap_index, safe_to_server_errors, ci_erddap_index +from argopy.tests.helpers.utils import requires_connected_erddap_index, safe_to_server_errors, ci_erddap_index ERDDAP_TIMEOUT = 3 * 60 safe_to_no_cache = pytest.mark.skipif(True, reason="Cache disabled for erddap index fetcher") diff --git a/argopy/tests/test_fetchers_index_gdac.py b/argopy/tests/test_fetchers_index_gdac.py index 7b00e08ca..3437a7a6a 100644 --- a/argopy/tests/test_fetchers_index_gdac.py +++ b/argopy/tests/test_fetchers_index_gdac.py @@ -10,9 +10,8 @@ from argopy.errors import CacheFileNotFound, FileSystemHasNoCache, GdacPathError from argopy.utils.checkers import isconnected, is_list_of_strings -from utils import requires_gdac, create_temp_folder, patch_ftp -from mocked_http import mocked_httpserver -from mocked_http import mocked_server_address as MOCKHTTP +from argopy.tests.helpers.utils import requires_gdac, create_temp_folder, patch_ftp +from argopy.tests.helpers.mocked_http import mocked_httpserver log = logging.getLogger("argopy.tests.index.gdac") @@ -29,7 +28,7 @@ VALID_HOSTS = [ argopy.tutorial.open_dataset("gdac")[0], #'https://data-argo.ifremer.fr', - MOCKHTTP, + "MOCKHTTP", # 'ftp://ftp.ifremer.fr/ifremer/argo', "MOCKFTP", ] @@ -79,7 +78,7 @@ def core(fargs, apts): return fetcher -def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): +def assert_fetcher(mocked_httpserver, this_fetcher, cacheable=False): """Assert a data fetcher. This should be used by all tests @@ -116,6 +115,9 @@ class TestBackend: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver def setup_class(self): """setup any state specific to the execution of the given class""" @@ -123,7 +125,11 @@ def setup_class(self): self.cachedir = create_temp_folder().folder def _patch_gdac(self, gdac): - return patch_ftp(gdac) + if gdac == 'MOCKFTP': + return patch_ftp(gdac) + elif gdac == 'MOCKHTTP': + return self.mocked_server_address + return gdac def _setup_fetcher(self, this_request, cached=False): """Helper method to set up options for a fetcher creation""" @@ -149,6 +155,7 @@ def _setup_fetcher(self, this_request, cached=False): **fetcher_args, **{"cache": True, "cachedir": self.cachedir}, } + log.info(fetcher_args) if not isconnected(fetcher_args["gdac"]): pytest.xfail("Fails because %s not available" % fetcher_args["gdac"]) else: diff --git a/argopy/tests/test_fetchers_proto.py b/argopy/tests/test_fetchers_proto.py index f2eac41c9..50582fcba 100644 --- a/argopy/tests/test_fetchers_proto.py +++ b/argopy/tests/test_fetchers_proto.py @@ -5,7 +5,7 @@ import xarray from argopy.data_fetchers.proto import ArgoDataFetcherProto from argopy.utils import to_list -from mocked_http import mocked_httpserver, mocked_server_address +from argopy.tests.helpers.mocked_http import mocked_httpserver log = logging.getLogger("argopy.tests.fetchers.proto") @@ -52,7 +52,7 @@ def test_required_methods(): ids=["%s" % p for p in [[6901929, None], [6901929, 90]]]) def test_dashboard(profile, mocked_httpserver): - with argopy.set_options(server=mocked_server_address): + with argopy.set_options(server=mocked_httpserver): f = Fetcher() f.WMO, f.CYC = profile diff --git a/argopy/tests/test_options.py b/argopy/tests/test_options.py index df80564fb..3e8083786 100644 --- a/argopy/tests/test_options.py +++ b/argopy/tests/test_options.py @@ -4,8 +4,8 @@ import argopy from argopy.options import OPTIONS from argopy.errors import OptionValueError, GdacPathError, ErddapPathError -from utils import requires_gdac, create_read_only_folder -from mocked_http import mocked_httpserver, mocked_server_address +from argopy.tests.helpers.utils import requires_gdac, create_read_only_folder +from argopy.tests.helpers.mocked_http import mocked_httpserver import logging @@ -38,8 +38,8 @@ def test_opt_ifremer_erddap(mocked_httpserver): with pytest.raises(ErddapPathError): argopy.set_options(erddap="invalid_path") - with argopy.set_options(erddap=mocked_server_address): - assert OPTIONS["erddap"] == mocked_server_address + with argopy.set_options(erddap=mocked_httpserver): + assert OPTIONS["erddap"] == mocked_httpserver def test_opt_dataset(): diff --git a/argopy/tests/test_plot_argo_colors.py b/argopy/tests/test_plot_argo_colors.py index 5613ff2b8..d12b87476 100644 --- a/argopy/tests/test_plot_argo_colors.py +++ b/argopy/tests/test_plot_argo_colors.py @@ -5,7 +5,7 @@ import pytest import logging -from utils import ( +from argopy.tests.helpers.utils import ( requires_matplotlib, requires_seaborn, has_matplotlib, diff --git a/argopy/tests/test_plot_dashboards.py b/argopy/tests/test_plot_dashboards.py index 7717d54fb..3b536c4c3 100644 --- a/argopy/tests/test_plot_dashboards.py +++ b/argopy/tests/test_plot_dashboards.py @@ -6,13 +6,12 @@ import argopy from argopy.errors import InvalidDashboard -from utils import ( - requires_connection, +from argopy.tests.helpers.utils import ( requires_ipython, has_ipython, create_temp_folder, ) -from mocked_http import mocked_httpserver, mocked_server_address +from argopy.tests.helpers.mocked_http import mocked_httpserver if has_ipython: import IPython @@ -43,7 +42,7 @@ def test_valid_dashboard(board_type): @pytest.mark.parametrize("board_type", ["data", "meta", "ea", "argovis", "op", "ocean-ops", "bgc"], indirect=False) def test_valid_dashboard_float(board_type, mocked_httpserver): # Test types with 'wmo' (should be all) - with argopy.set_options(server=mocked_server_address): + with argopy.set_options(server=mocked_httpserver): assert isinstance(argopy.dashboard(6901929, type=board_type, url_only=True), str) @@ -51,7 +50,7 @@ def test_valid_dashboard_float(board_type, mocked_httpserver): def test_valid_dashboard_profile(board_type, mocked_httpserver): # Test types with 'cyc' with create_temp_folder() as cachedir: - with argopy.set_options(cachedir=cachedir, server=mocked_server_address): + with argopy.set_options(cachedir=cachedir, server=mocked_httpserver): assert isinstance(argopy.dashboard(5904797, 12, type=board_type, url_only=True), str) @@ -61,6 +60,6 @@ def test_valid_dashboard_profile(board_type, mocked_httpserver): indirect=False) def test_valid_dashboard_ipython_output(opts, mocked_httpserver): with create_temp_folder() as cachedir: - with argopy.set_options(cachedir=cachedir, server=mocked_server_address): + with argopy.set_options(cachedir=cachedir, server=mocked_httpserver): dsh = argopy.dashboard(**opts) assert isinstance(dsh, IPython.lib.display.IFrame) diff --git a/argopy/tests/test_plot_plot.py b/argopy/tests/test_plot_plot.py index dbb6b864d..49105d4b7 100644 --- a/argopy/tests/test_plot_plot.py +++ b/argopy/tests/test_plot_plot.py @@ -10,9 +10,8 @@ import pickle import argopy -from utils import ( +from argopy.tests.helpers.utils import ( requires_gdac, - requires_connection, requires_matplotlib, requires_ipython, requires_cartopy, @@ -33,7 +32,7 @@ from argopy.plot.argo_colors import ArgoColors from argopy.errors import InvalidDatasetStructure from argopy import DataFetcher -from mocked_http import mocked_server_address +from argopy.tests.helpers.mocked_http import mocked_httpserver if has_matplotlib: @@ -51,11 +50,15 @@ class Test_open_sat_altim_report: + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver + WMOs = [2901623, [2901623, 6901929]] - def test_load_mocked_server(self, mocked_httpserver): - """This will easily ensure that the module scope fixture is available to all methods !""" - assert True + # def test_load_mocked_server(self, mocked_httpserver): + # """This will easily ensure that the module scope fixture is available to all methods !""" + # assert True @pytest.mark.parametrize( "WMOs", WMOs, ids=["For unique WMO", "For a list of WMOs"], indirect=False @@ -68,7 +71,7 @@ def test_open_sat_altim_report(self, WMOs, embed): import IPython dsh = open_sat_altim_report( - WMO=WMOs, embed=embed, api_server=mocked_server_address + WMO=WMOs, embed=embed, api_server=self.mocked_server_address ) if has_ipython and embed is not None: @@ -92,7 +95,7 @@ def test_open_sat_altim_report(self, WMOs, embed): def test_invalid_method(self): with pytest.raises(ValueError): open_sat_altim_report( - WMO=self.WMOs[0], embed="dummy_method", api_server=mocked_server_address + WMO=self.WMOs[0], embed="dummy_method", api_server=self.mocked_server_address ) diff --git a/argopy/tests/test_reference.py b/argopy/tests/test_reference.py index 51069a046..1f16e5382 100644 --- a/argopy/tests/test_reference.py +++ b/argopy/tests/test_reference.py @@ -7,8 +7,8 @@ import logging from collections.abc import Iterable -from mocked_http import mocked_httpserver, mocked_server_address -from utils import ( +from argopy.tests.helpers.mocked_http import mocked_httpserver +from argopy.tests.helpers.utils import ( create_temp_folder, ) from argopy.errors import OptionValueError, NoDataLeft @@ -24,13 +24,16 @@ @pytest.mark.skip("Deprecated, out of the CI tests suite") class Test_ArgoNVSReferenceTables_Deprecated: + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver def setup_class(self): """setup any state specific to the execution of the given class""" # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests self.cachedir = create_temp_folder().folder self.nvs = ArgoNVSReferenceTables( - cache=True, cachedir=self.cachedir, nvs=mocked_server_address + cache=True, cachedir=self.cachedir, nvs=self.mocked_server_address ) def teardown_class(self): diff --git a/argopy/tests/test_related.py b/argopy/tests/test_related.py index c652273be..60048b2d6 100644 --- a/argopy/tests/test_related.py +++ b/argopy/tests/test_related.py @@ -4,9 +4,9 @@ from collections import ChainMap import shutil -from mocked_http import mocked_httpserver, mocked_server_address +from argopy.tests.helpers.mocked_http import mocked_httpserver -from utils import ( +from argopy.tests.helpers.utils import ( requires_matplotlib, requires_cartopy, requires_oops, @@ -40,6 +40,10 @@ class Test_TopoFetcher(): box = [81, 123, -67, -54] + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver + def setup_class(self): """setup any state specific to the execution of the given class""" # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests @@ -52,7 +56,7 @@ def remove_test_dir(): remove_test_dir() def make_a_fetcher(self, cached=False): - opts = {'ds': 'gebco', 'stride': [10, 10], 'server': mocked_server_address} + opts = {'ds': 'gebco', 'stride': [10, 10], 'server': self.mocked_server_address} if cached: opts = ChainMap(opts, {'cache': True, 'cachedir': self.cachedir}) return TopoFetcher(self.box, **opts) @@ -62,9 +66,9 @@ def assert_fetcher(self, f): assert isinstance(ds, xr.Dataset) assert 'elevation' in ds.data_vars - def test_load_mocked_server(self, mocked_httpserver): - """This will easily ensure that the module scope fixture is available to all methods !""" - assert True + # def test_load_mocked_server(self, mocked_httpserver): + # """This will easily ensure that the module scope fixture is available to all methods !""" + # assert True params = [True, False] ids_params = ["cached=%s" % p for p in params] @@ -101,6 +105,10 @@ class Test_OceanOPSDeployments: ([-90, 0, 0, 90, '2022-01-01', None], False)] scenarios_ids = ["%s, %s" % (opt[0], opt[1]) for opt in scenarios] + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver + @pytest.fixture def an_instance(self, request): """ Fixture to create a OceanOPS_Deployments instance for a given set of arguments """ @@ -115,14 +123,14 @@ def an_instance(self, request): oops = OceanOPSDeployments(**args) # Adjust server info to use the mocked HTTP server: - oops.api = mocked_server_address + oops.api = self.mocked_server_address oops.model = 'data/platform' return oops - def test_load_mocked_server(self, mocked_httpserver): - """This will easily ensure that the module scope fixture is available to all methods !""" - assert True + # def test_load_mocked_server(self, mocked_httpserver): + # """This will easily ensure that the module scope fixture is available to all methods !""" + # assert True @pytest.mark.parametrize("an_instance", scenarios, indirect=True, ids=scenarios_ids) def test_init(self, an_instance): @@ -152,6 +160,9 @@ def test_plot_status(self, an_instance): @pytest.mark.skipif(True, reason="Skipped temporarily, see http://github.com/euroargodev/argopy/issues/488") class Test_ArgoDocs: + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver @pytest.fixture def an_instance(self, request): @@ -161,14 +172,14 @@ def an_instance(self, request): Ad = ArgoDocs(docid=docid, cache=False) # Adjust server info to use the mocked HTTP server: - Ad._doiserver = mocked_server_address - Ad._archimer = mocked_server_address + Ad._doiserver = self.mocked_server_address + Ad._archimer = self.mocked_server_address return Ad - def test_load_mocked_server(self, mocked_httpserver): - """This will easily ensure that the module scope fixture is available to all methods !""" - assert True + # def test_load_mocked_server(self, mocked_httpserver): + # """This will easily ensure that the module scope fixture is available to all methods !""" + # assert True @pytest.mark.parametrize("an_instance", [None], indirect=True, ids=["docid=%s" % t for t in [None]]) def test_list(self, an_instance): @@ -251,11 +262,11 @@ def test_invalid_dictionnary_key(): @pytest.mark.parametrize("params", [[6901929, None], [6901929, 12]], indirect=False, ids=['float', 'profile']) def test_get_coriolis_profile_id(params, mocked_httpserver): with create_temp_folder() as temp_folder: - with argopy.set_options(cachedir=temp_folder, server=mocked_server_address): + with argopy.set_options(cachedir=temp_folder, server=mocked_httpserver): assert isinstance(get_coriolis_profile_id(params[0], params[1]), pd.core.frame.DataFrame) @pytest.mark.parametrize("params", [[6901929, None], [6901929, 12]], indirect=False, ids=['float', 'profile']) def test_get_ea_profile_page(params, mocked_httpserver): with create_temp_folder() as temp_folder: with argopy.set_options(cachedir=temp_folder): - assert is_list_of_strings(get_ea_profile_page(params[0], params[1], api_server=mocked_server_address)) + assert is_list_of_strings(get_ea_profile_page(params[0], params[1], api_server=mocked_httpserver)) diff --git a/argopy/tests/test_stores_float.py b/argopy/tests/test_stores_float.py index c0c7c3ebe..57f1751a4 100644 --- a/argopy/tests/test_stores_float.py +++ b/argopy/tests/test_stores_float.py @@ -20,7 +20,7 @@ FloatStore as ArgoFloatOnline, ) -from argopy.tests.helpers.mocked_http import mocked_httpserver, mocked_server_address +from argopy.tests.helpers.mocked_http import mocked_httpserver from argopy.tests.helpers.utils import patch_ftp, has_connection, has_s3 log = logging.getLogger("argopy.tests.floatstore") @@ -40,7 +40,7 @@ "local": argopy.tutorial.open_dataset("gdac")[0], # Use local files } VALID_REMOTE_HOSTS = { - "http1": mocked_server_address, # Use the mocked http server + "http1": "MOCKHTTP", # Use the mocked http server # 'http2': 'https://data-argo.ifremer.fr', "ftp": "MOCKFTP", # keyword to use a fake/mocked ftp server (running on localhost) } @@ -63,7 +63,7 @@ def id_for_host(host): """Get a short name for GDAC host to populate scenarios IDs""" if host == "MOCKFTP": return "ftp_mocked" - elif "localhost" in host or "127.0.0.1" in host: + elif host == "MOCKHTTP": return "http_mocked" else: return (lambda x: "local" if x == "" else x)(urlparse(host).scheme) @@ -98,6 +98,9 @@ class Test_FloatStore_Offline: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver def setup_class(self): """setup any state specific to the execution of the given class""" @@ -111,13 +114,20 @@ def remove_test_dir(): remove_test_dir() + def patch_host(self, gdac): + if gdac == 'MOCKFTP': + return patch_ftp(gdac) + elif gdac == 'MOCKHTTP': + return self.mocked_server_address + return gdac + @pytest.fixture def af(self, request) -> Generator[ArgoFloatOffline, Any, None]: """Fixture to create a Float store instance for a given wmo and host""" log.debug("-" * 50) # log.debug(request) wmo = request.param[0] - host = patch_ftp(VALID_LOCAL_HOSTS[request.param[1]]) + host = self.patch_host(VALID_LOCAL_HOSTS[request.param[1]]) cache = request.param[2] xfail, reason = False, "" @@ -144,9 +154,9 @@ def af(self, request) -> Generator[ArgoFloatOffline, Any, None]: # TESTS # ######### - def test_remotehost(self, mocked_httpserver): + def test_remotehost(self): with pytest.raises(OptionValueError): - self.floatstore(VALID_WMO[0], host=mocked_server_address) + self.floatstore(VALID_WMO[0], host=self.mocked_server_address) @pytest.mark.parametrize("af", scenarios, indirect=True, ids=scenarios_ids) def test_load_metadata(self, af): @@ -188,6 +198,10 @@ class Test_FloatStore_Online: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver + def setup_class(self): """setup any state specific to the execution of the given class""" @@ -201,13 +215,20 @@ def remove_test_dir(): remove_test_dir() - def _patch_host(self, host): + def patch_gdac(self, gdac): + if gdac == 'MOCKFTP': + return patch_ftp(gdac) + elif gdac == 'MOCKHTTP': + return self.mocked_server_address + return gdac + + def patch_host(self, host): if "s3" in host and not has_connection: log.info("Skip this test with 's3' because there is no internet connection") pytest.skip( "Skip this test with 's3' because there is no internet connection" ) - return patch_ftp(host) + return self.patch_gdac(host) @pytest.fixture def af(self, request) -> Generator[ArgoFloatOnline, Any, None]: @@ -215,7 +236,7 @@ def af(self, request) -> Generator[ArgoFloatOnline, Any, None]: log.debug("-" * 50) # log.debug(request) wmo = request.param[0] - host = self._patch_host(VALID_REMOTE_HOSTS[request.param[1]]) + host = self.patch_host(VALID_REMOTE_HOSTS[request.param[1]]) cache = request.param[2] xfail, reason = False, "" @@ -224,7 +245,7 @@ def af(self, request) -> Generator[ArgoFloatOnline, Any, None]: store_args = { "host": host, - "eafleetmonitoring_server": mocked_server_address, + "eafleetmonitoring_server": self.mocked_server_address, # also use mocked server for Euro-Argo meta data API calls } if cache: @@ -326,9 +347,13 @@ class Test_FloatStore_Spec: ) for opts in scenarios_core ] + ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver def setup_class(self): """setup any state specific to the execution of the given class""" @@ -342,19 +367,26 @@ def remove_test_dir(): remove_test_dir() - def _patch_host(self, host): + def patch_gdac(self, gdac): + if gdac == 'MOCKFTP': + return patch_ftp(gdac) + elif gdac == 'MOCKHTTP': + return self.mocked_server_address + return gdac + + def patch_host(self, host): if "s3" in host and not has_connection: log.info("Skip this test with 's3' because there is no internet connection") pytest.skip( "Skip this test with 's3' because there is no internet connection" ) - return patch_ftp(host) + return self.patch_gdac(host) @pytest.fixture def af(self, request) -> Generator[ArgoFloatOnline, Any, None]: """Fixture to create a Float store instance for a given wmo and host""" wmo = request.param[0] - host = self._patch_host(VALID_HOSTS[request.param[1]]) + host = self.patch_host(VALID_HOSTS[request.param[1]]) cache = request.param[2] xfail, reason = False, "" @@ -368,7 +400,7 @@ def af(self, request) -> Generator[ArgoFloatOnline, Any, None]: floatstore = ArgoFloatOnline store_args = { "host": host, - "eafleetmonitoring_server": mocked_server_address, + "eafleetmonitoring_server": self.mocked_server_address, # also use mocked server for Euro-Argo meta data API calls } @@ -393,7 +425,7 @@ def af(self, request) -> Generator[ArgoFloatOnline, Any, None]: # TESTS # ######### @pytest.mark.parametrize("af", scenarios, indirect=True, ids=scenarios_ids) - def test_attributes(self, mocked_httpserver, af): + def test_attributes(self, af): assert is_wmo(af.WMO) assert hasattr(af, "dac") @@ -410,7 +442,7 @@ def test_attributes(self, mocked_httpserver, af): assert isinstance(af.host_protocol, str) @pytest.mark.parametrize("af", scenarios, indirect=True, ids=scenarios_ids) - def test_list_directories(self, mocked_httpserver, af): + def test_list_directories(self, af): assert isinstance(af.ls_datasets(), dict) assert is_list_of_strings(af._ls()) @@ -421,7 +453,7 @@ def test_list_directories(self, mocked_httpserver, af): assert isinstance(af.profiles_to_dataframe(), pd.DataFrame) @pytest.mark.parametrize("af", scenarios, indirect=True, ids=scenarios_ids) - def test_open_dataset(self, mocked_httpserver, af): + def test_open_dataset(self, af): lds = af.ls_datasets() ds_key, _ = random.choice(list(lds.items())) assert isinstance(af[ds_key], xr.Dataset) @@ -432,7 +464,7 @@ def test_open_dataset(self, mocked_httpserver, af): af.open_dataset("dummy_ds_key") @pytest.mark.parametrize("af", scenarios, indirect=True, ids=scenarios_ids) - def test_open_profile(self, mocked_httpserver, af): + def test_open_profile(self, af): lds = af.ls_profiles() ds_key, _ = random.choice(list(lds.items())) assert isinstance(af[ds_key], xr.Dataset) @@ -443,12 +475,12 @@ def test_open_profile(self, mocked_httpserver, af): af.open_profile("dummy_ds_key") @pytest.mark.parametrize("af", scenarios, indirect=True, ids=scenarios_ids) - def test_open_profiles(self, mocked_httpserver, af): + def test_open_profiles(self, af): ds_list = af.open_profiles(af.CYCLE_NUMBERS[1:10]) - log.debug("^"*10) - log.debug(af._ds_profiles.keys()) - log.debug(af.fs.fs) - log.debug("^"*10) + # log.debug("^"*10) + # log.debug(af._ds_profiles.keys()) + # log.debug(af.fs.fs) + # log.debug("^"*10) assert all([isinstance(ds, xr.Dataset) for ds in ds_list]) # @pytest.mark.parametrize("af", scenarios, indirect=True, ids=scenarios_ids) diff --git a/argopy/tests/test_stores_float_metaconfig.py b/argopy/tests/test_stores_float_metaconfig.py index 19871a9b1..4bf54531b 100644 --- a/argopy/tests/test_stores_float_metaconfig.py +++ b/argopy/tests/test_stores_float_metaconfig.py @@ -18,10 +18,10 @@ ConfigParameters as config_off, ) -from utils import ( +from argopy.tests.helpers.utils import ( requires_gdac, ) -from mocked_http import mocked_httpserver, mocked_server_address +from argopy.tests.helpers.mocked_http import mocked_httpserver log = logging.getLogger("argopy.tests.floatstore.config") ar.clear_cache() @@ -131,16 +131,20 @@ def config(self, argo_float): class Test_FloatStore_Config_Online(FloatStore_Config_Proto): af: dict[int, OnlineArgoFloat] = {} - # Define a fixture for an online ArgoFloat instance (but using mocked http) + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver + + # Define a fixture for an online ArgoFloat instance @pytest.fixture - def argo_float(self, wmo, mocked_httpserver): + def argo_float(self, wmo): if wmo not in self.af: self.af[wmo] = OnlineArgoFloat( wmo, - host=mocked_server_address, + host=self.mocked_server_address, cache=True, cachedir=self.cachedir, - eafleetmonitoring_server=mocked_server_address, + eafleetmonitoring_server=self.mocked_server_address, ) return self.af[wmo] diff --git a/argopy/tests/test_stores_fs_gdac.py b/argopy/tests/test_stores_fs_gdac.py index 383098ff3..bfe2c9a49 100644 --- a/argopy/tests/test_stores_fs_gdac.py +++ b/argopy/tests/test_stores_fs_gdac.py @@ -7,8 +7,8 @@ import argopy from argopy.stores import gdacfs -from mocked_http import mocked_httpserver, mocked_server_address -from utils import patch_ftp, has_s3 +from argopy.tests.helpers.mocked_http import mocked_httpserver +from argopy.tests.helpers.utils import patch_ftp, has_s3 log = logging.getLogger("argopy.tests.gdacfs") @@ -22,7 +22,7 @@ """ VALID_HOSTS = { 'local': argopy.tutorial.open_dataset("gdac")[0], # Use local files - 'http': mocked_server_address, # Use the mocked http server + 'http': "MOCKHTTP", # Use the mocked http server 'ftp': "MOCKFTP", # keyword to use a fake/mocked ftp server (running on localhost) } @@ -33,7 +33,7 @@ def id_for_host(host): """Get a short name for scenarios IDs, given a FTP host""" if host == "MOCKFTP": return "ftp_mocked" - elif "localhost" in host or "127.0.0.1" in host: + elif host == "MOCKHTTP": return "http_mocked" else: return (lambda x: "local" if x == "" else x)(urlparse(host).scheme) @@ -49,6 +49,9 @@ class Test_Gdacfs: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver def setup_class(self): """setup any state specific to the execution of the given class""" @@ -62,9 +65,12 @@ def remove_test_dir(): remove_test_dir() - def _patch_ftp(self, ftp): - log.debug(ftp) - return patch_ftp(ftp) + def patch_gdac(self, gdac): + if gdac == 'MOCKFTP': + return patch_ftp(gdac) + elif gdac == 'MOCKHTTP': + return self.mocked_server_address + return gdac def call_gdacfs(self, host, xfail=False, reason="?"): def core(host): @@ -85,8 +91,8 @@ def get_a_gdacfs(self, host, **kwargs): @pytest.fixture def store_maker(self, request): """Fixture to create a GDAC store instance for a given host""" - host = self._patch_ftp(VALID_HOSTS[request.param[0]]) - log.debug(host) + host = self.patch_gdac(VALID_HOSTS[request.param[0]]) + # log.debug(host) # cache = request.param[1] xfail, reason = False, "" @@ -107,5 +113,5 @@ def assert_fs(self, fs): @pytest.mark.parametrize( "store_maker", scenarios, indirect=True, ids=scenarios_ids ) - def test_implementation(self, mocked_httpserver, store_maker): + def test_implementation(self, store_maker): self.assert_fs(store_maker) diff --git a/argopy/tests/test_stores_fsspec.py b/argopy/tests/test_stores_fsspec.py index bb537b322..5b7b5c2e6 100644 --- a/argopy/tests/test_stores_fsspec.py +++ b/argopy/tests/test_stores_fsspec.py @@ -35,8 +35,8 @@ is_list_of_datasets, is_list_of_dicts, ) -from utils import requires_connection, requires_connected_argovis, create_temp_folder -from mocked_http import mocked_httpserver, mocked_server_address +from argopy.tests.helpers.utils import requires_connection, create_temp_folder +from argopy.tests.helpers.mocked_http import mocked_httpserver log = logging.getLogger("argopy.tests.stores") @@ -220,6 +220,10 @@ class Test_HttpStore: ######### # UTILS # ######### + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver + def setup_class(self): """setup any state specific to the execution of the given class""" # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests @@ -241,12 +245,12 @@ def _mockeduri(self, uri): """ for server_to_mock in [self.repo, "https://api.ifremer.fr/"]: if server_to_mock in uri: - uri = uri.replace(server_to_mock, mocked_server_address + "/") + uri = uri.replace(server_to_mock, self.mocked_server_address + "/") return uri - def test_load_mocked_server(self, mocked_httpserver): - """This will easily ensure that the module scope fixture is available to all methods !""" - assert True + # def test_load_mocked_server(self, mocked_httpserver): + # """This will easily ensure that the module scope fixture is available to all methods !""" + # assert True ######### # TESTS # @@ -471,14 +475,14 @@ class Test_FtpStore: def host(self): # h = 'ftp.ifremer.fr' h = urlparse(pytest.MOCKFTP).hostname - log.debug("Using FTP host: %s" % h) + # log.debug("Using FTP host: %s" % h) return h @property def port(self): # p = 0 p = int(urlparse(pytest.MOCKFTP).port) - log.debug("Using FTP port: %i" % p) + # log.debug("Using FTP port: %i" % p) return p @pytest.fixture diff --git a/argopy/tests/test_stores_index.py b/argopy/tests/test_stores_index.py index 56cd48b4e..804fc306b 100644 --- a/argopy/tests/test_stores_index.py +++ b/argopy/tests/test_stores_index.py @@ -19,9 +19,8 @@ from argopy.utils.checkers import is_list_of_strings, is_wmo from argopy.stores.index import indexstore_pd from argopy.stores import ArgoFloat -from utils import create_temp_folder, has_s3 -from mocked_http import mocked_httpserver, mocked_server_address -from utils import patch_ftp +from argopy.tests.helpers.utils import create_temp_folder, has_s3, patch_ftp +from argopy.tests.helpers.mocked_http import mocked_httpserver log = logging.getLogger("argopy.tests.indexstores") @@ -43,7 +42,7 @@ """ VALID_HOSTS = [ argopy.tutorial.open_dataset("gdac")[0], # Use local files - mocked_server_address, # Use the mocked http server + "MOCKHTTP", # Use the mocked http server "MOCKFTP", # keyword to use a fake/mocked ftp server (running on localhost) ] @@ -102,6 +101,11 @@ def core(fargs, apts): nrows = None try: idx = idx_maker(**fargs) + + idx._nrows_index = nrows + # Internal trick to ensure the full index will load only nrows, + # because each search point triggers: idx.load(nrows=idx._nrows_index) + if "wmo" in apts: idx.query.wmo(apts["wmo"], nrows=nrows) if "cyc" in apts: @@ -160,11 +164,11 @@ def core(fargs, apts): return core(fetcher_args, search_point) -def ftp_shortname(ftp): +def host_shortname(ftp): """Get a short name for scenarios IDs, given a FTP host""" if ftp == "MOCKFTP": return "ftp_mocked" - elif "localhost" in ftp or "127.0.0.1" in ftp: + elif ftp == "MOCKHTTP": return "http_mocked" else: return (lambda x: "file" if x == "" else x)(urlparse(ftp).scheme) @@ -175,26 +179,26 @@ class IndexStore_test_proto: search_scenarios = [(h, ap) for h in VALID_HOSTS for ap in VALID_SEARCHES] search_scenarios = [ - (h, ap, n) for h in VALID_HOSTS for ap in VALID_SEARCHES for n in [2] + (h, ap, nrows) for h in VALID_HOSTS for ap in VALID_SEARCHES for nrows in [None, 2] ] search_scenarios_ids = [ - "%s, %s, nrows=%s" % (ftp_shortname(fix[0]), - "%s[n=%i]" % (list(fix[1].keys())[0], len(fix[1][list(fix[1].keys())[0]])), + "%s, %s, nrows=%s" % (host_shortname(fix[0]), + "%s[narg=%i]" % (list(fix[1].keys())[0], len(fix[1][list(fix[1].keys())[0]])), str(fix[2])) for fix in search_scenarios ] search_scenarios_bool = [ - (h, ap, n, b) + (h, ap, nrows, b) for h in VALID_HOSTS for ap in VALID_SEARCHES_LOGICAL - for n in [None, 2] + for nrows in [None, 2] for b in ["and", "or"] ] search_scenarios_bool_ids = [ "%s, %s, nrows=%s, logical='%s'" - % (ftp_shortname(fix[0]), - "%s[n=%i]" % (list(fix[1].keys())[0], len(fix[1][list(fix[1].keys())[0]])), + % (host_shortname(fix[0]), + "%s[narg=%i]" % (list(fix[1].keys())[0], len(fix[1][list(fix[1].keys())[0]])), str(fix[2]), str(fix[3])) for fix in search_scenarios_bool @@ -203,14 +207,17 @@ class IndexStore_test_proto: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver def setup_class(self): """setup any state specific to the execution of the given class""" # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests self.cachedir = create_temp_folder().folder if has_pyarrow: - log.warning(pa.cpu_count()) - log.warning(pa.io_thread_count()) + log.warning(f"cpu_count: {pa.cpu_count()}") + log.warning(f"io_thread_count: {pa.io_thread_count()}") def teardown_class(self): """Cleanup once we are finished.""" @@ -220,8 +227,12 @@ def remove_test_dir(): remove_test_dir() - def _patch_ftp(self, ftp): - return patch_ftp(ftp) + def _patch_gdac(self, gdac): + if gdac == 'MOCKFTP': + return patch_ftp(gdac) + elif gdac == 'MOCKHTTP': + return self.mocked_server_address + return gdac def create_store(self, store_args, xfail=False, reason="?"): def core(fargs): @@ -250,10 +261,10 @@ def _setup_store(self, this_request, cached=False): index_file = this_request["param"]["index_file"] convention = this_request["param"]["convention"] N_RECORDS = ( - None if "tutorial" in host or "MOCK" in host else 100 + None if "tutorial" in host or "MOCKFTP" in host else 1000 ) # Make sure we're not going to load the full index fetcher_args = { - "host": self._patch_ftp(host), + "host": self._patch_gdac(host), "index_file": index_file, "cache": False, "convention": convention, @@ -269,7 +280,7 @@ def new_idx(self, cache=False, cachedir=None, **kwargs): host = kwargs["host"] if "host" in kwargs else self.host index_file = kwargs["index_file"] if "index_file" in kwargs else self.index_file convention = kwargs["convention"] if "convention" in kwargs else None - fetcher_args, N_RECORDS = self._setup_store( + fetcher_args, _ = self._setup_store( { "param": { "host": host, @@ -296,14 +307,18 @@ def a_store(self, request): @pytest.fixture def a_search(self, request): """Fixture to create an Index fetcher for a given host and access point""" - host = request.param[0] + host = self._patch_gdac(request.param[0]) + srch = request.param[1] nrows = request.param[2] - srch["nrows"] = nrows + if len(request.param) == 4: logical = request.param[3] srch["logical"] = logical - # log.debug("a_search: %s, %s, %s" % (self.index_file, srch, xfail)) + + if nrows is None and ("tutorial" not in host and "MOCKFTP" not in host): + nrows = 1000 + srch["nrows"] = nrows xfail, reason = False, "" if not has_s3 and 's3' in host: @@ -311,7 +326,7 @@ def a_search(self, request): elif 's3' in host: xfail, reason = 0, 's3 is experimental (search)' - yield run_a_search(self.new_idx, {"host": host, "cache": True}, srch, xfail=xfail, reason=reason) + yield run_a_search(self.indexstore, {"host": host, "cache": True}, srch, xfail=xfail, reason=reason) def assert_index(self, this_idx, cacheable=False): assert hasattr(this_idx, "index") @@ -342,7 +357,7 @@ def assert_search(self, this_idx, cacheable=False): "a_store", VALID_HOSTS, indirect=True, - ids=["%s" % ftp_shortname(ftp) for ftp in VALID_HOSTS], + ids=["%s" % host_shortname(ftp) for ftp in VALID_HOSTS], ) def test_hosts(self, mocked_httpserver, a_store): self.assert_index( @@ -604,11 +619,32 @@ class Test_IndexStore_pandas_BGC_bio(IndexStore_test_proto): ############################# # TESTS FOR PYARROW BACKEND # ############################# +import gc + +@skip_nopyarrow +@skip_pyarrow +class IndexStore_test_proto_Monitored(IndexStore_test_proto): + @pytest.fixture(autouse=True) + def _pyarrow_snapshot(self): + gc.collect() + pool = pa.default_memory_pool() + before_bytes = pool.bytes_allocated() + before_cpu = pa.cpu_count() + + yield + + gc.collect() + after_bytes = pool.bytes_allocated() + leaked = after_bytes - before_bytes + + if leaked > 1 * 1024 * 1024: + print(f"[PYARROW LEAK] Mbytes still allocated after test: {leaked/1024/1024:,}") + @skip_nopyarrow @skip_pyarrow @skip_CORE -class Test_IndexStore_pyarrow_CORE(IndexStore_test_proto): +class Test_IndexStore_pyarrow_CORE(IndexStore_test_proto_Monitored): network = "core" from argopy.stores.index import indexstore_pa @@ -616,10 +652,11 @@ class Test_IndexStore_pyarrow_CORE(IndexStore_test_proto): index_file = "ar_index_global_prof.txt" + @skip_nopyarrow @skip_pyarrow @skip_BGCs -class Test_IndexStore_pyarrow_BGC_bio(IndexStore_test_proto): +class Test_IndexStore_pyarrow_BGC_bio(IndexStore_test_proto_Monitored): network = "bgc" from argopy.stores.index import indexstore_pa @@ -630,7 +667,7 @@ class Test_IndexStore_pyarrow_BGC_bio(IndexStore_test_proto): @skip_nopyarrow @skip_pyarrow @skip_BGCb -class Test_IndexStore_pyarrow_BGC_synthetic(IndexStore_test_proto): +class Test_IndexStore_pyarrow_BGC_synthetic(IndexStore_test_proto_Monitored): network = "bgc" from argopy.stores.index import indexstore_pa diff --git a/argopy/tests/test_stores_index_plot.py b/argopy/tests/test_stores_index_plot.py index c69d11e9b..17581674a 100644 --- a/argopy/tests/test_stores_index_plot.py +++ b/argopy/tests/test_stores_index_plot.py @@ -1,11 +1,10 @@ import pytest import logging -import importlib import argopy from argopy.stores import ArgoIndex -from utils import ( +from argopy.tests.helpers.utils import ( requires_gdac, requires_matplotlib, requires_cartopy, @@ -27,9 +26,6 @@ Select GDAC host to be use for plot accessor test """ VALID_HOST = argopy.tutorial.open_dataset("gdac")[0] # Use local files -# 'http1': mocked_server_address, # Use the mocked http server -# 'http2': 'https://data-argo.ifremer.fr', -# 'ftp': "MOCKFTP", # keyword to use a fake/mocked ftp server (running on localhost) """ List WMO to be tested, one for each mission diff --git a/argopy/tests/test_stores_nvs.py b/argopy/tests/test_stores_nvs.py index a33385821..48a55980f 100644 --- a/argopy/tests/test_stores_nvs.py +++ b/argopy/tests/test_stores_nvs.py @@ -1,6 +1,5 @@ import pytest import logging -import numpy as np from argopy.stores.nvs.implementations.offline.nvs import NVS as NVS_offline from argopy.stores.nvs.implementations.online.nvs import NVS as NVS_online @@ -20,7 +19,7 @@ sparql_mapping_request, ) -from mocked_http import mocked_httpserver, mocked_server_address +from argopy.tests.helpers.mocked_http import mocked_httpserver log = logging.getLogger("argopy.tests.nvsstores") @@ -168,11 +167,15 @@ class Test_NVS_Online: ############# # UTILITIES # ############# + @pytest.fixture(autouse=True) + def _setup(self, mocked_httpserver): + self.mocked_server_address = mocked_httpserver + @pytest.fixture def nvs(self): defaults_args = {} if USE_MOCKED_SERVER: - defaults_args["nvs"] = mocked_server_address + defaults_args["nvs"] = self.mocked_server_address return NVS_online(**defaults_args) @@ -182,9 +185,9 @@ def nvs(self): def test_init(self, nvs): assert nvs.online == True - def test_uniqueinstance(self, mocked_httpserver): - nvs1: NVS = NVS_online(nvs=mocked_server_address) - nvs2: NVS = NVS_online(nvs=mocked_server_address) + def test_uniqueinstance(self): + nvs1: NVS = NVS_online(nvs=self.mocked_server_address) + nvs2: NVS = NVS_online(nvs=self.mocked_server_address) assert nvs1.uid == nvs2.uid def test_readonlyinstance(self, nvs): diff --git a/argopy/tests/test_utils_checkers.py b/argopy/tests/test_utils_checkers.py index 73fc05b19..82fe7a03d 100644 --- a/argopy/tests/test_utils_checkers.py +++ b/argopy/tests/test_utils_checkers.py @@ -2,8 +2,8 @@ import logging import numpy as np -from mocked_http import mocked_httpserver, mocked_server_address -from utils import ( +from argopy.tests.helpers.mocked_http import mocked_httpserver +from argopy.tests.helpers.utils import ( requires_erddap, ) @@ -199,33 +199,38 @@ def test_check_gdac_path(): def test_isconnected(mocked_httpserver): - assert isinstance(isconnected(host=mocked_server_address), bool) + assert isinstance(isconnected(host=mocked_httpserver), bool) assert isconnected(host="http://dummyhost") is False def test_urlhaskeyword(mocked_httpserver): url = "https://api.ifremer.fr/argopy/data/ARGO-FULL.json" - url.replace("https://api.ifremer.fr", mocked_server_address) + url.replace("https://api.ifremer.fr", mocked_httpserver) assert isinstance(urlhaskeyword(url, "label"), bool) -params = [mocked_server_address, - {"url": mocked_server_address + "/argopy/data/ARGO-FULL.json", "keyword": "label"} +params = ["MOCKHTTP", + {"url": "MOCKHTTP" + "/argopy/data/ARGO-FULL.json", "keyword": "label"} ] params_ids = ["url is a %s" % str(type(p)) for p in params] @pytest.mark.parametrize("params", params, indirect=False, ids=params_ids) def test_isalive(params, mocked_httpserver): - assert isinstance(isalive(params), bool) + if type(params) == str and params == "MOCKHTTP": + api_server_check = mocked_httpserver + elif type(params) == dict and "MOCKHTTP" in params['url']: + api_server_check = params['url'].replace("MOCKHTTP",mocked_httpserver) + + assert isinstance(isalive(api_server_check), bool) @requires_erddap @pytest.mark.parametrize("data", [True, False], indirect=False, ids=["data=%s" % t for t in [True, False]]) def test_isAPIconnected(data, mocked_httpserver): - with argopy.set_options(erddap=mocked_server_address): + with argopy.set_options(erddap=mocked_httpserver): assert isinstance(isAPIconnected(src="erddap", data=data), bool) def test_erddap_ds_exists(mocked_httpserver): - with argopy.set_options(erddap=mocked_server_address): + with argopy.set_options(erddap=mocked_httpserver): assert isinstance(erddap_ds_exists(ds="ArgoFloats"), bool) assert erddap_ds_exists(ds="DummyDS") is False diff --git a/argopy/tests/test_xarray_accessor.py b/argopy/tests/test_xarray_accessor.py index fdca2cc1f..c0513570d 100644 --- a/argopy/tests/test_xarray_accessor.py +++ b/argopy/tests/test_xarray_accessor.py @@ -8,8 +8,8 @@ import argopy from argopy import DataFetcher as ArgoDataFetcher from argopy.errors import InvalidDatasetStructure, OptionValueError -from utils import requires_gdac, _importorskip, _connectskip -from mocked_http import mocked_server_address +from argopy.tests.helpers.utils import requires_gdac, _importorskip, _connectskip +from argopy.tests.helpers.mocked_http import mocked_httpserver has_gsw, requires_gsw = _importorskip("gsw") @@ -26,7 +26,7 @@ def ds_pts(mocked_httpserver): try: for user_mode in ['standard', 'expert']: data[user_mode] = ( - ArgoDataFetcher(src="erddap", mode=user_mode, server=mocked_server_address) + ArgoDataFetcher(src="erddap", mode=user_mode, server=mocked_httpserver) .region([-20, -16., 0, 1, 0, 100., "2004-01-01", "2004-01-31"]) .load() .data diff --git a/docs/whats-new.rst b/docs/whats-new.rst index ca47feaf5..6ae9f4d47 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -29,7 +29,19 @@ Features and front-end API Internals ^^^^^^^^^ -- **Annual upgrade to support last versions of dependencies**. (:pr:`681`) by |gmaze|. +- **Annual upgrade to support last versions of dependencies**. (:pr:`681`, :pr:``) by |gmaze|. This implied a few changes in the code: + + - Now kick-off a mocked HTTP server instance on each test module to improve performance and future proof use of parallel execution of the tests. + + - Hence we removed the freshly added ``_free_port()`` because a port could have been taken between the call to ``_free_port()`` and the time to reach ``serve_mocked_httpserver()``. So port number is now selected when the mocked server is quick started by a test module. + + - As another consequence, we needed to remove the explicit use of ``mocked_server_address`` in imports in test modules, because the server is now different for each (and all together useless in many locations). + + - Make the mocked server "threaded" with ``ThreadingHTTPServer``. This allows the mocked HTTP server used in CI tests to handle multiple concurrent connections instead of handling one request at a time (it was previously, based on the plain HTTPServer class). + + - As a consequence the mocked HTTP server was hit much often & faster, which caused a "race" problem for the cached fsspec store. We found that this is a known issue documented `here `_ but still has no fix. So we had to serialize the first download of each resource, to prevent that race to the meda-data registry managed internally by fsspec. Race to write the registry is now stop by the first cat to a file, all subsequent calls are ok. This is the ``_cache_lock`` addition in stores. + + - Following on "race" issues, we tried to remove a bunch of warnings and possible erratic errors due to inconsistency in how we create xarray dataset in out internal stores. We make this consistent by having data fetched from local, HTTP, and FTP stores systematically opening a dataset from an in-memory byte buffers (io.BytesIO). We also added an explicit ``.load()`` immediately after opening to copy all variables into plain in-memory numpy arrays, hence detaching the dataset from the lazy backend array wrapper and the buffer it was opened from. The subsequent ``.close()`` then aims to release the underlying netCDF4/HDF5 handle deterministically, rather than deferring cleanup to garbage collection. - **Improve import time** with lazy and/or deferred import of large dependencies, optimization and finally removing of the auto-discovery of data/index fetchers :issue:`585` (:pr:`676`) by |charles| and (:pr:`624`) by |gmaze|.