From 8283d873a595c3a69c27b8ec43ec1f4b794de4fa Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 26 Sep 2023 15:22:38 +0200 Subject: [PATCH 01/14] split upstream tests win vs (lin,mac) --- .../workflows/pytests-upstream-windows.yml | 212 ++++++++++++++++++ .github/workflows/pytests-upstream.yml | 4 +- 2 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pytests-upstream-windows.yml diff --git a/.github/workflows/pytests-upstream-windows.yml b/.github/workflows/pytests-upstream-windows.yml new file mode 100644 index 00000000..082c99c7 --- /dev/null +++ b/.github/workflows/pytests-upstream-windows.yml @@ -0,0 +1,212 @@ +name: CI tests Upstream for Windows +# Daily tests with all un-pinned dependencies +# Allows to be warned when argopy fail tests due to updates in dependencies + +on: + schedule: + - cron: "0 0 * * *" # Daily “At 00:00” UTC + workflow_dispatch: # allows you to trigger the workflow run manually + pull_request: + types: [synchronize] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + detect-ci-trigger: + name: detect upstream ci trigger + runs-on: ubuntu-latest + if: | + github.repository == 'euroargodev/argopy' + && (github.event_name == 'push' || github.event_name == 'pull_request') + outputs: + triggered: ${{ steps.detect-trigger.outputs.trigger-found }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: xarray-contrib/ci-trigger@v1 + id: detect-trigger + with: + keyword: "[test-upstream]" + + detect-ci-skip: + name: detect upstream ci skip + runs-on: ubuntu-latest + if: | + github.repository == 'euroargodev/argopy' + && (github.event_name == 'push' || github.event_name == 'pull_request') + outputs: + skipped: ${{ steps.detect-skip.outputs.trigger-found }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: xarray-contrib/ci-trigger@v1.2 + id: detect-skip + with: + keyword: "[skip-ci]" + + core-free: + # CI tests for environments with core requirements in free versions + + name: Core - Free - Py${{matrix.python-version}} - ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: [detect-ci-trigger, detect-ci-skip] + if: | + always() + && ( + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + || needs.detect-ci-trigger.outputs.triggered == 'true' + || needs.detect-ci-skip.outputs.skipped == 'false' + || contains(github.event.pull_request.labels.*.name, 'release') + ) + defaults: + run: + shell: bash -l {0} + timeout-minutes: 45 + strategy: + fail-fast: true + matrix: + python-version: ["3.8", "3.9"] + os: ["windows-latest"] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags. + + - name: Set environment variables + run: | + echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-core-free.yml" >> $GITHUB_ENV + echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV + + - name: Setup Micromamba ${{ matrix.python-version }} + uses: mamba-org/setup-micromamba@v1 + with: + environment-name: argopy-tests + environment-file: ${{ env.CONDA_ENV_FILE }} + init-shell: bash + cache-environment: true + cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" + create-args: >- + python=${{matrix.python-version}} + + - name: Install argopy + run: | + python -m pip install --no-deps -e . + + - name: Version info + run: | + micromamba info + micromamba list + + - name: Lint with flake8 + run: | + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Test with pytest + run: | + pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 \ + --report-log output-${{ matrix.python-version }}-log.jsonl + + - name: Generate and publish the report + if: | + failure() + && steps.status.outcome == 'failure' + && github.repository_owner == 'euroargodev' + uses: xarray-contrib/issue-from-pytest-log@v1 + with: + log-path: output-${{ matrix.python-version }}-log.jsonl + + - name: 'Save tests log as artifact' + uses: actions/upload-artifact@v3 + with: + name: Argopy-Tests + path: output-${{ matrix.python-version }}-log.jsonl + retention-days: 90 + + all-free: + # CI tests for environments with all possible requirements in free versions + + name: All - Free - Py${{matrix.python-version}} - ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: [detect-ci-trigger, detect-ci-skip] + if: | + always() + && ( + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + || needs.detect-ci-trigger.outputs.triggered == 'true' + || needs.detect-ci-skip.outputs.skipped == 'false' + || contains(github.event.pull_request.labels.*.name, 'release') + ) + defaults: + run: + shell: bash -l {0} + timeout-minutes: 45 + strategy: + fail-fast: true + matrix: + python-version: ["3.8", "3.9"] + os: ["windows-latest"] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags. + + - name: Set environment variables + run: | + echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-all-free.yml" >> $GITHUB_ENV + echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV + echo "LOG_FILE=argopy-tests-All-Min-Py${{matrix.python-version}}-${{matrix.os}}.log" >> $GITHUB_ENV + + - name: Setup Micromamba ${{ matrix.python-version }} + uses: mamba-org/setup-micromamba@v1 + with: + environment-name: argopy-tests + environment-file: ${{ env.CONDA_ENV_FILE }} + init-shell: bash + cache-environment: true + cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" + create-args: >- + python=${{matrix.python-version}} + pytest-reportlog + + - name: Install argopy + run: | + python -m pip install --no-deps -e . + + - name: Version info + run: | + micromamba info + micromamba list + + - name: Lint with flake8 + run: | + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Test with pytest + run: | + pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 \ + --report-log output-${{ matrix.python-version }}-log.jsonl + + - name: Generate and publish the report + if: | + failure() + && steps.status.outcome == 'failure' + && github.repository_owner == 'euroargodev' + uses: xarray-contrib/issue-from-pytest-log@v1 + with: + log-path: output-${{ matrix.python-version }}-log.jsonl + + - name: 'Save tests log as artifact' + uses: actions/upload-artifact@v3 + with: + name: Argopy-Tests + path: output-${{ matrix.python-version }}-log.jsonl + retention-days: 90 diff --git a/.github/workflows/pytests-upstream.yml b/.github/workflows/pytests-upstream.yml index 20971cdd..eaf8e875 100644 --- a/.github/workflows/pytests-upstream.yml +++ b/.github/workflows/pytests-upstream.yml @@ -71,7 +71,7 @@ jobs: fail-fast: true matrix: python-version: ["3.8", "3.9"] - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest", "macos-latest"] steps: - uses: actions/checkout@v4 @@ -151,7 +151,7 @@ jobs: fail-fast: true matrix: python-version: ["3.8", "3.9"] - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest", "macos-latest"] steps: - uses: actions/checkout@v4 From 559addf6888b6fae240d243fe1c5fdb3f2552c43 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 27 Sep 2023 16:11:21 +0200 Subject: [PATCH 02/14] Try to fix #294 --- argopy/stores/filesystems.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/argopy/stores/filesystems.py b/argopy/stores/filesystems.py index a74d321b..7c766e46 100644 --- a/argopy/stores/filesystems.py +++ b/argopy/stores/filesystems.py @@ -216,6 +216,15 @@ def register(self, uri): if path not in self.cache_registry: self.cache_registry.commit(path) + @property + def cached_files(self): + if version.parse(fsspec.__version__) <= version.parse("2023.6.0"): + return self.fs.cached_files + else: + # See https://github.com/euroargodev/argopy/issues/294 + return self._metadata.cached_files + + def cachepath(self, uri: str, errors: str = "raise"): """Return path to cached file for a given URI""" if not self.cache: @@ -223,10 +232,10 @@ def cachepath(self, uri: str, errors: str = "raise"): raise FileSystemHasNoCache("%s has no cache system" % type(self.fs)) elif uri is not None: store_path = self.store_path(uri) - self.fs.load_cache() # Read set of stored blocks from file and populate self.fs.cached_files - if store_path in self.fs.cached_files[-1]: + self.fs.load_cache() # Read set of stored blocks from file and populate self.cached_files + if store_path in self.cached_files[-1]: return os.path.sep.join( - [self.cachedir, self.fs.cached_files[-1][store_path]["fn"]] + [self.cachedir, self.cached_files[-1][store_path]["fn"]] ) elif errors == "raise": raise CacheFileNotFound( @@ -242,8 +251,8 @@ def _clear_cache_item(self, uri): # See the "save_cache()" method in: # https://filesystem-spec.readthedocs.io/en/latest/_modules/fsspec/implementations/cached.html#WholeFileCacheFileSystem fn = os.path.join(self.fs.storage[-1], "cache") - self.fs.load_cache() # Read set of stored blocks from file and populate self.fs.cached_files - cache = self.fs.cached_files[-1] + self.fs.load_cache() # Read set of stored blocks from file and populate self.cached_files + cache = self.cached_files[-1] if os.path.exists(fn): with open(fn, "rb") as f: cached_files = pickle.load( From 41e78989ef7fa947192e35a7f2998270665d1a2d Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 27 Sep 2023 16:23:39 +0200 Subject: [PATCH 03/14] Update filesystems.py --- argopy/stores/filesystems.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/argopy/stores/filesystems.py b/argopy/stores/filesystems.py index 7c766e46..f7a6db45 100644 --- a/argopy/stores/filesystems.py +++ b/argopy/stores/filesystems.py @@ -222,9 +222,8 @@ def cached_files(self): return self.fs.cached_files else: # See https://github.com/euroargodev/argopy/issues/294 - return self._metadata.cached_files + return self.fs._metadata.cached_files - def cachepath(self, uri: str, errors: str = "raise"): """Return path to cached file for a given URI""" if not self.cache: @@ -247,9 +246,7 @@ def cachepath(self, uri: str, errors: str = "raise"): ) def _clear_cache_item(self, uri): - """Open fsspec cache registry (pickle file) and remove entry for uri""" - # See the "save_cache()" method in: - # https://filesystem-spec.readthedocs.io/en/latest/_modules/fsspec/implementations/cached.html#WholeFileCacheFileSystem + """Open fsspec cache registry and remove entry for uri""" fn = os.path.join(self.fs.storage[-1], "cache") self.fs.load_cache() # Read set of stored blocks from file and populate self.cached_files cache = self.cached_files[-1] From 284cddebb0932006375e59ed6facc51611897421 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 09:32:45 +0200 Subject: [PATCH 04/14] more fsspec cache fix [skip-ci] --- argopy/stores/filesystems.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/argopy/stores/filesystems.py b/argopy/stores/filesystems.py index f7a6db45..8619436d 100644 --- a/argopy/stores/filesystems.py +++ b/argopy/stores/filesystems.py @@ -246,27 +246,40 @@ def cachepath(self, uri: str, errors: str = "raise"): ) def _clear_cache_item(self, uri): - """Open fsspec cache registry and remove entry for uri""" + """Remove medadata and file for fsspec cache uri""" fn = os.path.join(self.fs.storage[-1], "cache") self.fs.load_cache() # Read set of stored blocks from file and populate self.cached_files cache = self.cached_files[-1] + + # Read cache metadata: if os.path.exists(fn): - with open(fn, "rb") as f: - cached_files = pickle.load( - f - ) # nosec B301 because files controlled internally + if version.parse(fsspec.__version__) <= version.parse("2023.6.0"): + with open(fn, "rb") as f: + cached_files = pickle.load(f) # nosec B301 because files controlled internally + else: + with open(fn, "r") as f: + cached_files = json.load(f) else: cached_files = cache + + # Build new metadata without uri to delete, and delete corresponding cached file: cache = {} for k, v in cached_files.items(): if k != uri: cache[k] = v.copy() else: + # Delete file: os.remove(os.path.join(self.fs.storage[-1], v["fn"])) # log.debug("Removed %s -> %s" % (uri, v['fn'])) - with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: - pickle.dump(cache, f) - shutil.move(f.name, fn) + + # Update cache metadata file: + if version.parse(fsspec.__version__) <= version.parse("2023.6.0"): + with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: + pickle.dump(cache, f) + shutil.move(f.name, fn) + else: + with fsspec.atomic_write(fn, mode="w") as f: + json.dump(cache, f) def clear_cache(self): """Remove cache files and entry from uri open with this store instance""" From 172f1f2285ad16933f02f7f65241e2e179cd85d7 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 09:53:09 +0200 Subject: [PATCH 05/14] fix stupid typo ! [skip-ci] --- argopy/stores/filesystems.py | 2 +- ci/requirements/py3.8-all-free.yml | 1 + ci/requirements/py3.9-all-free.yml | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/argopy/stores/filesystems.py b/argopy/stores/filesystems.py index 8619436d..ca7d391c 100644 --- a/argopy/stores/filesystems.py +++ b/argopy/stores/filesystems.py @@ -278,7 +278,7 @@ def _clear_cache_item(self, uri): pickle.dump(cache, f) shutil.move(f.name, fn) else: - with fsspec.atomic_write(fn, mode="w") as f: + with fsspec.utils.atomic_write(fn, mode="w") as f: json.dump(cache, f) def clear_cache(self): diff --git a/ci/requirements/py3.8-all-free.yml b/ci/requirements/py3.8-all-free.yml index 1362ceb2..1cb5ac21 100644 --- a/ci/requirements/py3.8-all-free.yml +++ b/ci/requirements/py3.8-all-free.yml @@ -51,6 +51,7 @@ dependencies: - aiofiles - setuptools # - sphinx + - requests - pip: - pytest-reportlog \ No newline at end of file diff --git a/ci/requirements/py3.9-all-free.yml b/ci/requirements/py3.9-all-free.yml index e4efca35..8db3554f 100644 --- a/ci/requirements/py3.9-all-free.yml +++ b/ci/requirements/py3.9-all-free.yml @@ -12,7 +12,7 @@ dependencies: - packaging - scipy - toolz - - xarray + - xarray # EXT.UTIL: - gsw @@ -51,6 +51,7 @@ dependencies: - aiofiles - setuptools # - sphinx + - requests - pip: - pytest-reportlog From 3ae927fdb72696b9facba31850106592e37e44cb Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 10:32:44 +0200 Subject: [PATCH 06/14] Update caching.py --- argopy/utils/caching.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/argopy/utils/caching.py b/argopy/utils/caching.py index 5389aabd..dd95f54a 100644 --- a/argopy/utils/caching.py +++ b/argopy/utils/caching.py @@ -2,6 +2,7 @@ import shutil import logging import pickle +import json import fsspec import pandas as pd from packaging import version @@ -62,14 +63,18 @@ def convert_size(size_bytes): cached_files = [] fn = os.path.join(apath, "cache") if os.path.exists(fn): - with open(fn, "rb") as f: - loaded_cached_files = pickle.load( - f - ) # nosec B301 because files controlled internally - for c in loaded_cached_files.values(): - if isinstance(c["blocks"], list): - c["blocks"] = set(c["blocks"]) - cached_files.append(loaded_cached_files) + if version.parse(fsspec.__version__) <= version.parse("2023.6.0"): + with open(fn, "rb") as f: + loaded_cached_files = pickle.load( + f + ) # nosec B301 because files controlled internally + else: + with open(fn, "r") as f: + loaded_cached_files = json.load(f) + for c in loaded_cached_files.values(): + if isinstance(c["blocks"], list): + c["blocks"] = set(c["blocks"]) + cached_files.append(loaded_cached_files) else: raise FileSystemHasNoCache("No fsspec cache system at: %s" % apath) From 0894162df33589273e7b48de166db99c80e367b7 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 10:32:51 +0200 Subject: [PATCH 07/14] misc [skip-ci] --- argopy/tests/helpers/mocked_http.py | 14 +++++++++++--- argopy/utils/locals.py | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/argopy/tests/helpers/mocked_http.py b/argopy/tests/helpers/mocked_http.py index 92a54ed3..de36b6f1 100644 --- a/argopy/tests/helpers/mocked_http.py +++ b/argopy/tests/helpers/mocked_http.py @@ -8,9 +8,17 @@ in "argopy/cli" Servers covered by this fixture: -https://erddap.ifremer.fr/erddap -https://github.com/euroargodev/argopy-data/raw/master -https://api.ifremer.fr/argopy/data + "https://github.com/euroargodev/argopy-data/raw/master", + "https://erddap.ifremer.fr/erddap", + "https://data-argo.ifremer.fr", + "https://api.ifremer.fr", + "https://coastwatch.pfeg.noaa.gov/erddap", + "https://www.ocean-ops.org/api/1", + "https://dataselection.euro-argo.eu/api", + "https://vocab.nerc.ac.uk/collection", + "https://argovisbeta02.colorado.edu", + "https://dx.doi.org", + "https://archimer.ifremer.fr", The HTTPTestHandler class below is taken from the fsspec tests suite at: https://github.com/fsspec/filesystem_spec/blob/55c5d71e657445cbfbdba15049d660a5c9639ff0/fsspec/tests/conftest.py diff --git a/argopy/utils/locals.py b/argopy/utils/locals.py index af46b6e7..5d3b0039 100644 --- a/argopy/utils/locals.py +++ b/argopy/utils/locals.py @@ -168,7 +168,7 @@ def show_versions(file=sys.stdout, conda=False): # noqa: C901 ( "pytest_reportlog", lambda mod: mod.__version__, - ), # will come with pandas + ), ("setuptools", lambda mod: mod.__version__), ("aiofiles", lambda mod: mod.__version__), ("sphinx", lambda mod: mod.__version__), From 01728512593a12e0f47186297286d0fb20d9f79d Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 11:17:24 +0200 Subject: [PATCH 08/14] Update test_xarray_accessor.py --- argopy/tests/test_xarray_accessor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/argopy/tests/test_xarray_accessor.py b/argopy/tests/test_xarray_accessor.py index b804ae12..af8afdf7 100644 --- a/argopy/tests/test_xarray_accessor.py +++ b/argopy/tests/test_xarray_accessor.py @@ -191,17 +191,17 @@ def is_valid_mdata(self, this_mdata): """Validate structure of the output dataset """ check = [] # Check for dimensions: - check.append(argopy.utilities.is_list_equal(['m', 'n'], list(this_mdata.dims))) + check.append(argopy.utils.is_list_equal(['m', 'n'], list(this_mdata.dims))) # Check for coordinates: - check.append(argopy.utilities.is_list_equal(['m', 'n'], list(this_mdata.coords))) + check.append(argopy.utils.is_list_equal(['m', 'n'], list(this_mdata.coords))) # Check for data variables: check.append(np.all( [v in this_mdata.data_vars for v in ['PRES', 'TEMP', 'PTMP', 'SAL', 'DATES', 'LAT', 'LONG', 'PROFILE_NO']])) check.append(np.all( - [argopy.utilities.is_list_equal(['n'], this_mdata[v].dims) for v in ['LONG', 'LAT', 'DATES', 'PROFILE_NO'] + [argopy.utils.is_list_equal(['n'], this_mdata[v].dims) for v in ['LONG', 'LAT', 'DATES', 'PROFILE_NO'] if v in this_mdata.data_vars])) check.append(np.all( - [argopy.utilities.is_list_equal(['m', 'n'], this_mdata[v].dims) for v in ['PRES', 'TEMP', 'SAL', 'PTMP'] if + [argopy.utils.is_list_equal(['m', 'n'], this_mdata[v].dims) for v in ['PRES', 'TEMP', 'SAL', 'PTMP'] if v in this_mdata.data_vars])) return np.all(check) From ce2fa3384ab1fa02a6bf0cafda35f84bbc59c877 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 11:18:03 +0200 Subject: [PATCH 09/14] Fix another windows related bug in ci tests --- argopy/tests/helpers/utils.py | 72 ++++++++++++++++++++++++++++++++++- argopy/tests/test_options.py | 24 +----------- 2 files changed, 73 insertions(+), 23 deletions(-) diff --git a/argopy/tests/helpers/utils.py b/argopy/tests/helpers/utils.py index e688c589..06040750 100644 --- a/argopy/tests/helpers/utils.py +++ b/argopy/tests/helpers/utils.py @@ -20,6 +20,14 @@ from packaging import version import warnings import logging +import stat +import platform +import os +from contextlib import contextmanager +from enum import Enum +from subprocess import check_output +from pathlib import Path +from typing import Generator, List from argopy.options import set_options from argopy.errors import ErddapServerError, ArgovisServerError, DataNotFound, FtpPathError @@ -305,4 +313,66 @@ def test_wrapper(*args, **kwargs): fct_safe_to_server_errors(test_func)(*args, **kwargs) return test_wrapper -log.debug("%s TESTS UTILS %s" % ("="*50, "="*50)) \ No newline at end of file + +def create_read_only_folder_linux(folder_path): + try: + # Create the folder + os.makedirs(folder_path, exist_ok=True) + + # Get the current permissions of the folder + current_permissions = os.stat(folder_path).st_mode + + # Remove the write access for the owner and group + new_permissions = current_permissions & ~(stat.S_IWUSR | stat.S_IWGRP) + + # Set the new permissions + os.chmod(folder_path, new_permissions) + + except FileExistsError: + log.debug(f"Folder '{folder_path}' already exists.") + except PermissionError: + log.debug("Error: You do not have sufficient permissions to create the folder.") + + +def create_read_only_folder_windows(folder_path): + class AccessRight(Enum): + """Access Rights for files/folders""" + + DELETE = "D" + FULL = "F" # Edit Permissions + Create + Delete + Read + Write + NO_ACCESS = "N" + MODIFY = "M" # Create + Delete + Read + Write + READ_EXECUTE = "RX" + READ_ONLY = "R" + WRITE_ONLY = "W" + + def cmd(access_right: AccessRight, mode="grant:r") -> List[str]: + return [ + "icacls", + str(folder_path), + "/inheritance:r", + f"/{mode}", + f"Everyone:{access_right.value}", + ] + + try: + # Create the folder + os.makedirs(folder_path, exist_ok=True) + + # Change permissions + check_output(cmd(AccessRight.READ_EXECUTE)) + + except FileExistsError: + log.debug(f"Folder '{folder_path}' already exists.") + except PermissionError: + log.debug("Error: You do not have sufficient permissions to create the folder.") + + +def create_read_only_folder(folder_path): + if platform.system() == 'Windows': + create_read_only_folder_windows(folder_path) + else: + create_read_only_folder_linux(folder_path) + + +log.debug("%s TESTS UTILS %s" % ("="*50, "="*50)) diff --git a/argopy/tests/test_options.py b/argopy/tests/test_options.py index 37c44b34..436db7bd 100644 --- a/argopy/tests/test_options.py +++ b/argopy/tests/test_options.py @@ -3,7 +3,7 @@ import argopy from argopy.options import OPTIONS from argopy.errors import OptionValueError, FtpPathError, ErddapPathError -from utils import requires_gdac +from utils import requires_gdac, create_read_only_folder from mocked_http import mocked_httpserver, mocked_server_address import logging @@ -52,30 +52,10 @@ def test_opt_dataset(): assert OPTIONS["dataset"] == "ref" -@pytest.mark.skipif(True, reason="Need to be debugged for Windows support") +# @pytest.mark.skipif(True, reason="Need to be debugged for Windows support") def test_opt_invalid_cachedir(): # Cachedir is created if not exist. # OptionValueError is raised when it's not writable - import stat - def create_read_only_folder(folder_path): - try: - # Create the folder - os.makedirs(folder_path, exist_ok=True) - - # Get the current permissions of the folder - current_permissions = os.stat(folder_path).st_mode - - # Remove the write access for the owner and group - new_permissions = current_permissions & ~(stat.S_IWUSR | stat.S_IWGRP) - - # Set the new permissions - os.chmod(folder_path, new_permissions) - - except FileExistsError: - log.debug(f"Folder '{folder_path}' already exists.") - except PermissionError: - log.debug("Error: You do not have sufficient permissions to create the folder.") - folder_name = "read_only_folder" create_read_only_folder(folder_name) with pytest.raises(OptionValueError): From ce67f4dbcb5edb20d24a8b59f2e3c0f6f041a87e Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 12:22:58 +0200 Subject: [PATCH 10/14] Update test_options.py --- argopy/tests/test_options.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/argopy/tests/test_options.py b/argopy/tests/test_options.py index 436db7bd..042bfb30 100644 --- a/argopy/tests/test_options.py +++ b/argopy/tests/test_options.py @@ -1,5 +1,6 @@ import os import pytest +import warnings import argopy from argopy.options import OPTIONS from argopy.errors import OptionValueError, FtpPathError, ErddapPathError @@ -58,6 +59,8 @@ def test_opt_invalid_cachedir(): # OptionValueError is raised when it's not writable folder_name = "read_only_folder" create_read_only_folder(folder_name) + warnings.warn(os.stat(folder_name).st_mode) + warnings.warn(os.access(folder_name, os.W_OK)) with pytest.raises(OptionValueError): argopy.set_options(cachedir=folder_name) os.rmdir(folder_name) From 6ac34d8ed355fc6b105c0e12af73e27dfa6cf329 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 12:37:05 +0200 Subject: [PATCH 11/14] quicker ci tests for debug --- argopy/tests/helpers/utils.py | 2 +- argopy/tests/test_errors.py | 46 -- argopy/tests/test_fetchers_data_argovis.py | 235 -------- argopy/tests/test_fetchers_data_erddap.py | 216 ------- argopy/tests/test_fetchers_data_erddap_bgc.py | 267 --------- argopy/tests/test_fetchers_data_gdac.py | 230 ------- argopy/tests/test_fetchers_facade_data.py | 196 ------ argopy/tests/test_fetchers_facade_index.py | 131 ---- argopy/tests/test_fetchers_index_erddap.py | 137 ----- argopy/tests/test_fetchers_index_gdac.py | 201 ------- argopy/tests/test_fetchers_proto.py | 53 -- argopy/tests/test_options.py | 4 +- argopy/tests/test_plot_argo_colors.py | 101 ---- argopy/tests/test_plot_dashboards.py | 66 -- argopy/tests/test_plot_plot.py | 212 ------- argopy/tests/test_related.py | 327 ---------- argopy/tests/test_stores_fsspec.py | 563 ------------------ argopy/tests/test_stores_index.py | 494 --------------- argopy/tests/test_tutorial.py | 27 - argopy/tests/test_utils_accessories.py | 86 --- argopy/tests/test_utils_caching.py | 36 -- argopy/tests/test_utils_checkers.py | 226 ------- argopy/tests/test_utils_chunking.py | 196 ------ argopy/tests/test_utils_compute.py | 75 --- argopy/tests/test_utils_format.py | 60 -- argopy/tests/test_utils_geo.py | 42 -- argopy/tests/test_utils_lists.py | 7 - argopy/tests/test_utils_locals.py | 22 - argopy/tests/test_xarray_accessor.py | 238 -------- argopy/tests/test_xarray_engine.py | 91 --- 30 files changed, 3 insertions(+), 4584 deletions(-) delete mode 100644 argopy/tests/test_errors.py delete mode 100644 argopy/tests/test_fetchers_data_argovis.py delete mode 100644 argopy/tests/test_fetchers_data_erddap.py delete mode 100644 argopy/tests/test_fetchers_data_erddap_bgc.py delete mode 100644 argopy/tests/test_fetchers_data_gdac.py delete mode 100644 argopy/tests/test_fetchers_facade_data.py delete mode 100644 argopy/tests/test_fetchers_facade_index.py delete mode 100644 argopy/tests/test_fetchers_index_erddap.py delete mode 100644 argopy/tests/test_fetchers_index_gdac.py delete mode 100644 argopy/tests/test_fetchers_proto.py delete mode 100644 argopy/tests/test_plot_argo_colors.py delete mode 100644 argopy/tests/test_plot_dashboards.py delete mode 100644 argopy/tests/test_plot_plot.py delete mode 100644 argopy/tests/test_related.py delete mode 100644 argopy/tests/test_stores_fsspec.py delete mode 100644 argopy/tests/test_stores_index.py delete mode 100644 argopy/tests/test_tutorial.py delete mode 100644 argopy/tests/test_utils_accessories.py delete mode 100644 argopy/tests/test_utils_caching.py delete mode 100644 argopy/tests/test_utils_checkers.py delete mode 100644 argopy/tests/test_utils_chunking.py delete mode 100644 argopy/tests/test_utils_compute.py delete mode 100644 argopy/tests/test_utils_format.py delete mode 100644 argopy/tests/test_utils_geo.py delete mode 100644 argopy/tests/test_utils_lists.py delete mode 100644 argopy/tests/test_utils_locals.py delete mode 100644 argopy/tests/test_xarray_accessor.py delete mode 100644 argopy/tests/test_xarray_engine.py diff --git a/argopy/tests/helpers/utils.py b/argopy/tests/helpers/utils.py index 06040750..89d62833 100644 --- a/argopy/tests/helpers/utils.py +++ b/argopy/tests/helpers/utils.py @@ -360,7 +360,7 @@ def cmd(access_right: AccessRight, mode="grant:r") -> List[str]: os.makedirs(folder_path, exist_ok=True) # Change permissions - check_output(cmd(AccessRight.READ_EXECUTE)) + check_output(cmd(AccessRight.READ_ONLY)) except FileExistsError: log.debug(f"Folder '{folder_path}' already exists.") diff --git a/argopy/tests/test_errors.py b/argopy/tests/test_errors.py deleted file mode 100644 index 54e57371..00000000 --- a/argopy/tests/test_errors.py +++ /dev/null @@ -1,46 +0,0 @@ -import pytest -from argopy.errors import ( - DataNotFound, - FtpPathError, - ErddapPathError, - NetCDF4FileNotFoundError, - CacheFileNotFound, - FileSystemHasNoCache, - UnrecognisedProfileDirection, - InvalidDataset, - InvalidDatasetStructure, - InvalidFetcherAccessPoint, - InvalidFetcher, - InvalidOption, - OptionValueError, - InvalidMethod, - InvalidDashboard, - APIServerError, - ErddapServerError, - ArgovisServerError -) - - -@pytest.mark.parametrize("error", [ - DataNotFound, - FtpPathError, - ErddapPathError, - NetCDF4FileNotFoundError, - CacheFileNotFound, - FileSystemHasNoCache, - UnrecognisedProfileDirection, - InvalidDataset, - InvalidDatasetStructure, - InvalidFetcherAccessPoint, - InvalidFetcher, - InvalidOption, - OptionValueError, - InvalidMethod, - InvalidDashboard, - APIServerError, - ErddapServerError, - ArgovisServerError, - ], indirect=False) -def test_raise_all_errors(error): - with pytest.raises(error): - raise error() diff --git a/argopy/tests/test_fetchers_data_argovis.py b/argopy/tests/test_fetchers_data_argovis.py deleted file mode 100644 index d6f6f119..00000000 --- a/argopy/tests/test_fetchers_data_argovis.py +++ /dev/null @@ -1,235 +0,0 @@ -import warnings - -import numpy as np -import xarray as xr -import pandas as pd - -import pytest -import tempfile - -import argopy -from argopy import DataFetcher as ArgoDataFetcher -from argopy.errors import ( - CacheFileNotFound, - FileSystemHasNoCache, -) -from argopy.utils.checkers import is_list_of_strings -from utils import requires_connected_argovis, safe_to_server_errors - - -skip_this_for_debug = pytest.mark.skipif(False, reason="Skipped temporarily for debug") - - -@requires_connected_argovis -class Test_Backend: - """ Test main API facade for all available dataset and access points of the ARGOVIS data fetching backend """ - - src = "argovis" - requests = { - "float": [[1901393], [1901393, 6902746]], - # "profile": [[6902746, 12], [6902746, np.arange(12, 13)], [6902746, [1, 12]]], - "profile": [[6902746, 12]], - "region": [ - [-70, -65, 35.0, 40.0, 0, 10.0, "2012-01", "2012-03"], - [-70, -65, 35.0, 40.0, 0, 10.0, "2012-01", "2012-06"], - ], - } - - @skip_this_for_debug - def test_cachepath_notfound(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir): - fetcher = ArgoDataFetcher(src=self.src, cache=True).profile(*self.requests['profile'][0]).fetcher - with pytest.raises(CacheFileNotFound): - fetcher.cachepath - - @skip_this_for_debug - @safe_to_server_errors - def test_nocache(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir): - fetcher = ArgoDataFetcher(src=self.src, cache=False).profile(*self.requests['profile'][0]).fetcher - with pytest.raises(FileSystemHasNoCache): - fetcher.cachepath - - @skip_this_for_debug - @safe_to_server_errors - def test_clearcache(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir): - fetcher = ArgoDataFetcher(src=self.src, cache=True).float(self.requests['float'][0]).fetcher - fetcher.to_xarray() - fetcher.clear_cache() - with pytest.raises(CacheFileNotFound): - fetcher.cachepath - - @skip_this_for_debug - @safe_to_server_errors - def test_caching_float(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir): - fetcher = ( - ArgoDataFetcher(src=self.src, cache=True).float(self.requests['float'][0]).fetcher - ) - ds = fetcher.to_xarray() - assert isinstance(ds, xr.Dataset) - assert is_list_of_strings(fetcher.uri) - assert is_list_of_strings(fetcher.cachepath) - - @skip_this_for_debug - @safe_to_server_errors - def test_caching_profile(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir): - fetcher = ArgoDataFetcher(src=self.src, cache=True).profile(*self.requests['profile'][0]).fetcher - ds = fetcher.to_xarray() - assert isinstance(ds, xr.Dataset) - assert is_list_of_strings(fetcher.uri) - assert is_list_of_strings(fetcher.cachepath) - - @skip_this_for_debug - @safe_to_server_errors - def test_caching_region(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir): - fetcher = ( - ArgoDataFetcher(src=self.src, cache=True) - .region(self.requests['region'][1]) - .fetcher - ) - ds = fetcher.to_xarray() - assert isinstance(ds, xr.Dataset) - assert is_list_of_strings(fetcher.uri) - assert is_list_of_strings(fetcher.cachepath) - - def __testthis_profile(self, dataset): - fetcher_args = {"src": self.src, "ds": dataset} - for arg in self.args["profile"]: - f = ArgoDataFetcher(**fetcher_args).profile(*arg).fetcher - assert isinstance(f.to_xarray(), xr.Dataset) - # assert isinstance(f.to_dataframe(), pd.core.frame.DataFrame) - # ds = xr.Dataset.from_dataframe(f.to_dataframe()) - # ds = ds.sortby( - # ["TIME", "PRES"] - # ) # should already be sorted by date in descending order - # ds["N_POINTS"] = np.arange( - # 0, len(ds["N_POINTS"]) - # ) # Re-index to avoid duplicate values - # - # # Set coordinates: - # # ds = ds.set_coords('N_POINTS') - # coords = ("LATITUDE", "LONGITUDE", "TIME", "N_POINTS") - # ds = ds.reset_coords() - # ds["N_POINTS"] = ds["N_POINTS"] - # # Convert all coordinate variable names to upper case - # for v in ds.data_vars: - # ds = ds.rename({v: v.upper()}) - # ds = ds.set_coords(coords) - # - # # Cast data types and add variable attributes (not available in the csv download): - # warnings.warn(type(ds['TIME'].data)) - # warnings.warn(ds['TIME'].data[0]) - # ds['TIME'] = ds['TIME'].astype(np.datetime64) - # assert isinstance(ds, xr.Dataset) - assert is_list_of_strings(f.uri) - - def __testthis_float(self, dataset): - fetcher_args = {"src": self.src, "ds": dataset} - for arg in self.args["float"]: - f = ArgoDataFetcher(**fetcher_args).float(arg).fetcher - assert isinstance(f.to_xarray(), xr.Dataset) - assert is_list_of_strings(f.uri) - - def __testthis_region(self, dataset): - fetcher_args = {"src": self.src, "ds": dataset} - for arg in self.args["region"]: - f = ArgoDataFetcher(**fetcher_args).region(arg).fetcher - assert isinstance(f.to_xarray(), xr.Dataset) - assert is_list_of_strings(f.uri) - - def __testthis(self, dataset): - for access_point in self.args: - if access_point == "profile": - self.__testthis_profile(dataset) - elif access_point == "float": - self.__testthis_float(dataset) - elif access_point == "region": - self.__testthis_region(dataset) - - @skip_this_for_debug - @safe_to_server_errors - def test_phy_float(self): - self.args = {"float": self.requests['float']} - self.__testthis("phy") - - @safe_to_server_errors - def test_phy_profile(self): - self.args = {"profile": self.requests['profile']} - self.__testthis("phy") - - @skip_this_for_debug - @safe_to_server_errors - def test_phy_region(self): - self.args = {"region": self.requests['region']} - self.__testthis("phy") - - -@skip_this_for_debug -@requires_connected_argovis -class Test_BackendParallel: - """ This test backend for parallel requests """ - - src = "argovis" - requests = { - "region": [ - [-60, -55, 40.0, 45.0, 0.0, 10.0], - [-60, -55, 40.0, 45.0, 0.0, 10.0, "2007-08-01", "2007-09-01"], - ], - "wmo": [[6902766, 6902772, 6902914]], - } - - def test_methods(self): - args_list = [ - {"src": self.src, "parallel": "thread"}, - {"src": self.src, "parallel": True, "parallel_method": "thread"}, - ] - for fetcher_args in args_list: - loader = ArgoDataFetcher(**fetcher_args).float(self.requests["wmo"][0]) - assert isinstance(loader, argopy.fetchers.ArgoDataFetcher) - - args_list = [ - {"src": self.src, "parallel": True, "parallel_method": "toto"}, - {"src": self.src, "parallel": "process"}, - {"src": self.src, "parallel": True, "parallel_method": "process"}, - ] - for fetcher_args in args_list: - with pytest.raises(ValueError): - ArgoDataFetcher(**fetcher_args).float(self.requests["wmo"][0]) - - @safe_to_server_errors - def test_chunks_region(self): - for access_arg in self.requests["region"]: - fetcher_args = { - "src": self.src, - "parallel": True, - "chunks": {"lon": 1, "lat": 2, "dpt": 1, "time": 2}, - } - f = ArgoDataFetcher(**fetcher_args).region(access_arg).fetcher - assert isinstance(f.to_xarray(), xr.Dataset) - assert is_list_of_strings(f.uri) - assert len(f.uri) == np.prod( - [v for k, v in fetcher_args["chunks"].items()] - ) - - @safe_to_server_errors - def test_chunks_wmo(self): - for access_arg in self.requests["wmo"]: - fetcher_args = { - "src": self.src, - "parallel": True, - "chunks_maxsize": {"wmo": 1}, - } - f = ArgoDataFetcher(**fetcher_args).profile(access_arg, 12).fetcher - assert isinstance(f.to_xarray(), xr.Dataset) - assert is_list_of_strings(f.uri) - assert len(f.uri) == len(access_arg) diff --git a/argopy/tests/test_fetchers_data_erddap.py b/argopy/tests/test_fetchers_data_erddap.py deleted file mode 100644 index d57375ca..00000000 --- a/argopy/tests/test_fetchers_data_erddap.py +++ /dev/null @@ -1,216 +0,0 @@ -import logging - -from argopy import DataFetcher as ArgoDataFetcher -from argopy.utils.checkers import is_list_of_strings - -import pytest -import xarray as xr -from utils import ( - requires_erddap, -) - -from mocked_http import mocked_server_address -from mocked_http import mocked_httpserver as mocked_erddapserver - -import tempfile -import shutil -from collections import ChainMap - - -log = logging.getLogger("argopy.tests.data.erddap") - -USE_MOCKED_SERVER = True - -""" -List access points to be tested for each datasets: phy and ref. -For each access points, we list 1-to-2 scenario to make sure all possibilities are tested -""" -ACCESS_POINTS = [ - {"phy": [ - {"float": 1901393}, - {"float": [1901393, 6902746]}, - {"profile": [6902746, 34]}, - {"profile": [6902746, [1, 12]]}, - {"region": [-20, -16., 0, 1, 0, 100.]}, - {"region": [-20, -16., 0, 1, 0, 100., "2004-01-01", "2004-01-31"]}, - ]}, - {"ref": [ - {"region": [-70, -65, 35.0, 40.0, 0, 10.0]}, - {"region": [-70, -65, 35.0, 40.0, 0, 10.0, "2012-01-1", "2012-12-31"]}, - ]}, -] -PARALLEL_ACCESS_POINTS = [ - {"phy": [ - {"float": [1900468, 1900117, 1900386]}, - {"region": [-60, -55, 40.0, 45.0, 0.0, 20.0]}, - {"region": [-60, -55, 40.0, 45.0, 0.0, 20.0, "2007-08-01", "2007-09-01"]}, - ]}, -] - -""" -List user modes to be tested -""" -USER_MODES = ['standard', 'expert', 'research'] - - -""" -Make a list of VALID dataset/access_points to be tested -""" -VALID_ACCESS_POINTS, VALID_ACCESS_POINTS_IDS = [], [] -for entry in ACCESS_POINTS: - for ds in entry: - for mode in USER_MODES: - for ap in entry[ds]: - VALID_ACCESS_POINTS.append({'ds': ds, 'mode': mode, 'access_point': ap}) - VALID_ACCESS_POINTS_IDS.append("ds='%s', mode='%s', %s" % (ds, mode, ap)) - - -VALID_PARALLEL_ACCESS_POINTS, VALID_PARALLEL_ACCESS_POINTS_IDS = [], [] -for entry in PARALLEL_ACCESS_POINTS: - for ds in entry: - for mode in USER_MODES: - for ap in entry[ds]: - VALID_PARALLEL_ACCESS_POINTS.append({'ds': ds, 'mode': mode, 'access_point': ap}) - VALID_PARALLEL_ACCESS_POINTS_IDS.append("ds='%s', mode='%s', %s" % (ds, mode, ap)) - - -def create_fetcher(fetcher_args, access_point): - """ Create a fetcher for a given set of facade options and access point """ - def core(fargs, apts): - try: - f = ArgoDataFetcher(**fargs) - if "float" in apts: - f = f.float(apts['float']) - elif "profile" in apts: - f = f.profile(*apts['profile']) - elif "region" in apts: - f = f.region(apts['region']) - except Exception: - raise - return f - fetcher = core(fetcher_args, access_point) - return fetcher - - -def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): - """Assert a data fetcher. - - This should be used by all tests asserting a fetcher - """ - def assert_all(this_fetcher, cacheable): - # We use the facade to test 'to_xarray' in order to make sure to test all filters required by user mode - ds = this_fetcher.to_xarray(errors='raise') - assert isinstance(ds, xr.Dataset) - # - core = this_fetcher.fetcher - assert is_list_of_strings(core.uri) - assert (core.N_POINTS >= 1) # Make sure we found results - if cacheable: - assert is_list_of_strings(core.cachepath) - - log.debug("In assert, this fetcher is in '%s' user mode" % this_fetcher._mode) - if this_fetcher._dataset_id not in ['ref']: - if this_fetcher._mode == 'expert': - assert 'PRES_ADJUSTED' in ds - - elif this_fetcher._mode == 'standard': - assert 'PRES_ADJUSTED' not in ds - - elif this_fetcher._mode == 'research': - assert 'PRES_ADJUSTED' not in ds - assert 'PRES_QC' not in ds - else: - assert 'PTMP' in ds - - try: - assert_all(this_fetcher, cacheable) - except: - raise - assert False - - -@requires_erddap -class Test_Backend: - """ Test ERDDAP data fetching backend """ - src = 'erddap' - - ############# - # UTILITIES # - ############# - - def setup_class(self): - """setup any state specific to the execution of the given class""" - # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests - self.cachedir = tempfile.mkdtemp() - - def _setup_fetcher(self, this_request, cached=False, parallel=False): - """Helper method to set up options for a fetcher creation""" - defaults_args = {"src": self.src, - "cache": cached, - "cachedir": self.cachedir, - "parallel": parallel, - } - if USE_MOCKED_SERVER: - defaults_args['server'] = mocked_server_address - - dataset = this_request.param['ds'] - user_mode = this_request.param['mode'] - access_point = this_request.param['access_point'] - - fetcher_args = ChainMap(defaults_args, {"ds": dataset, 'mode': user_mode}) - if not cached: - # cache is False by default, so we don't need to clutter the arguments list - del fetcher_args["cache"] - del fetcher_args["cachedir"] - if not parallel: - # parallel is False by default, so we don't need to clutter the arguments list - del fetcher_args["parallel"] - - # log.debug("Setting up a new fetcher with the following arguments:") - # log.debug(fetcher_args) - return fetcher_args, access_point - - @pytest.fixture - def fetcher(self, request): - """ Fixture to create a ERDDAP data fetcher for a given dataset and access point """ - fetcher_args, access_point = self._setup_fetcher(request, cached=False) - yield create_fetcher(fetcher_args, access_point) - - @pytest.fixture - def cached_fetcher(self, request): - """ Fixture to create a cached ERDDAP data fetcher for a given dataset and access point """ - fetcher_args, access_point = self._setup_fetcher(request, cached=True) - yield create_fetcher(fetcher_args, access_point) - - @pytest.fixture - def parallel_fetcher(self, request): - """ Fixture to create a parallel ERDDAP data fetcher for a given dataset and access point """ - fetcher_args, access_point = self._setup_fetcher(request, parallel="thread") - yield create_fetcher(fetcher_args, access_point) - - def teardown_class(self): - """Cleanup once we are finished.""" - def remove_test_dir(): - shutil.rmtree(self.cachedir) - remove_test_dir() - - ######### - # TESTS # - ######### - @pytest.mark.parametrize("fetcher", VALID_ACCESS_POINTS, - indirect=True, - ids=VALID_ACCESS_POINTS_IDS) - def test_fetching(self, mocked_erddapserver, fetcher): - assert_fetcher(mocked_erddapserver, fetcher, cacheable=False) - - @pytest.mark.parametrize("cached_fetcher", VALID_ACCESS_POINTS, - indirect=True, - ids=VALID_ACCESS_POINTS_IDS) - def test_fetching_cached(self, mocked_erddapserver, cached_fetcher): - assert_fetcher(mocked_erddapserver, cached_fetcher, cacheable=True) - - @pytest.mark.parametrize("parallel_fetcher", VALID_PARALLEL_ACCESS_POINTS, - indirect=True, - ids=VALID_PARALLEL_ACCESS_POINTS_IDS) - def test_fetching_parallel(self, mocked_erddapserver, parallel_fetcher): - assert_fetcher(mocked_erddapserver, parallel_fetcher, cacheable=False) diff --git a/argopy/tests/test_fetchers_data_erddap_bgc.py b/argopy/tests/test_fetchers_data_erddap_bgc.py deleted file mode 100644 index ad0769b6..00000000 --- a/argopy/tests/test_fetchers_data_erddap_bgc.py +++ /dev/null @@ -1,267 +0,0 @@ -import logging -import numpy as np - -from argopy import DataFetcher as ArgoDataFetcher -from argopy.utils.checkers import is_list_of_strings -from argopy.stores import indexstore_pd as ArgoIndex # make sure to work with the Pandas index store with erddap-bgc - -import pytest -import xarray as xr -from utils import ( - requires_erddap, -) -from mocked_http import mocked_server_address -from mocked_http import mocked_httpserver as mocked_erddapserver - -import tempfile -import shutil -from collections import ChainMap - - -log = logging.getLogger("argopy.tests.data.erddap") - -USE_MOCKED_SERVER = True - -""" -List access points to be tested for each datasets: bgc. -For each access points, we list 1-to-2 scenario to make sure all possibilities are tested -""" -ACCESS_POINTS = [ - {"bgc": [ - {"float": 5903248}, - {"float": [5903248, 6904241]}, - {"profile": [5903248, 34]}, - {"profile": [5903248, np.arange(12, 14)]}, - {"region": [-55, -47, 55, 57, 0, 10]}, - {"region": [-55, -47, 55, 57, 0, 10, "2022-05-1", "2023-07-01"]}, - ]}, -] -PARALLEL_ACCESS_POINTS = [ - {"bgc": [ - {"float": [5903248, 6904241]}, - {"region": [-55, -47, 55, 57, 0, 10, "2022-05-1", "2023-07-01"]}, - ]}, -] - -""" -List user modes to be tested -""" -# USER_MODES = ['standard', 'expert', 'research'] -USER_MODES = ['expert'] - -""" -List of 'params' fetcher arguments to be tested -""" -PARAMS = ['all', 'DOXY'] - -""" -Make a list of VALID dataset/access_points to be tested -""" -VALID_ACCESS_POINTS, VALID_ACCESS_POINTS_IDS = [], [] -for entry in ACCESS_POINTS: - for ds in entry: - for mode in USER_MODES: - for params in PARAMS: - for ap in entry[ds]: - VALID_ACCESS_POINTS.append({'ds': ds, 'mode': mode, 'params': params, 'access_point': ap}) - VALID_ACCESS_POINTS_IDS.append("ds='%s', mode='%s', params='%s', %s" % (ds, mode, params, ap)) - - -VALID_PARALLEL_ACCESS_POINTS, VALID_PARALLEL_ACCESS_POINTS_IDS = [], [] -for entry in PARALLEL_ACCESS_POINTS: - for ds in entry: - for mode in USER_MODES: - for params in PARAMS: - for ap in entry[ds]: - VALID_PARALLEL_ACCESS_POINTS.append({'ds': ds, 'mode': mode, 'params': params, 'access_point': ap}) - VALID_PARALLEL_ACCESS_POINTS_IDS.append("ds='%s', mode='%s', params='%s', %s" % (ds, mode, params, ap)) - - -def create_fetcher(fetcher_args, access_point): - """ Create a fetcher for a given set of facade options and access point """ - def core(fargs, apts): - try: - f = ArgoDataFetcher(**fargs) - if "float" in apts: - f = f.float(apts['float']) - elif "profile" in apts: - f = f.profile(*apts['profile']) - elif "region" in apts: - f = f.region(apts['region']) - except Exception: - raise - return f - fetcher = core(fetcher_args, access_point) - return fetcher - - -def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): - """Assert a data fetcher. - - This should be used by all tests asserting a fetcher - """ - def assert_all(this_fetcher, cacheable): - - # We use the facade to test 'to_xarray' in order to make sure to test all filters required by user mode - ds = this_fetcher.to_xarray(errors='raise') - assert isinstance(ds, xr.Dataset) - - # Then apply checks on erddap fetcher: - core = this_fetcher.fetcher - assert is_list_of_strings(core.uri) - assert (core.N_POINTS >= 1) # Make sure we found results - if cacheable: - assert is_list_of_strings(core.cachepath) - - # log.debug("In assert, this fetcher is in '%s' user mode" % this_fetcher._mode) - if this_fetcher._mode == 'expert': - assert 'PRES_ADJUSTED' in ds - - elif this_fetcher._mode == 'standard': - assert 'PRES_ADJUSTED' not in ds - - elif this_fetcher._mode == 'research': - assert 'PRES_ADJUSTED' not in ds - assert 'PRES_QC' not in ds - - try: - assert_all(this_fetcher, cacheable) - except: - if this_fetcher._mode not in ['expert']: - pytest.xfail("BGC is not yet supported in '%s' user mode" % this_fetcher._mode) - else: - assert False - - -@requires_erddap -class Test_Backend: - """ Test ERDDAP data fetching backend """ - src = 'erddap' - - ############# - # UTILITIES # - ############# - - def setup_class(self): - """setup any state specific to the execution of the given class""" - # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests - self.cachedir = tempfile.mkdtemp() - - def _setup_fetcher(self, this_request, cached=False, parallel=False): - """Helper method to set up options for a fetcher creation""" - defaults_args = {"src": self.src, - "cache": cached, - "cachedir": self.cachedir, - "parallel": parallel, - } - - if USE_MOCKED_SERVER: - defaults_args['server'] = mocked_server_address - defaults_args['indexfs'] = ArgoIndex( - host=mocked_server_address, - index_file='argo_synthetic-profile_index.txt', - cache=True, - cachedir=self.cachedir, - timeout=5, - ) - - - dataset = this_request.param['ds'] - user_mode = this_request.param['mode'] - params = this_request.param['params'] - measured = this_request.param['measured'] if 'measured' in this_request.param else None - - access_point = this_request.param['access_point'] - - fetcher_args = ChainMap(defaults_args, {"ds": dataset, 'mode': user_mode, 'params': params, 'measured': measured}) - if not cached: - # cache is False by default, so we don't need to clutter the arguments list - del fetcher_args["cache"] - del fetcher_args["cachedir"] - if not parallel: - # parallel is False by default, so we don't need to clutter the arguments list - del fetcher_args["parallel"] - - # log.debug("Setting up a new fetcher with the following arguments:") - # log.debug(fetcher_args) - return fetcher_args, access_point - - @pytest.fixture - def fetcher(self, request): - """ Fixture to create a ERDDAP data fetcher for a given dataset and access point """ - fetcher_args, access_point = self._setup_fetcher(request, cached=False) - yield create_fetcher(fetcher_args, access_point) - - @pytest.fixture - def cached_fetcher(self, request): - """ Fixture to create a cached ERDDAP data fetcher for a given dataset and access point """ - fetcher_args, access_point = self._setup_fetcher(request, cached=True) - yield create_fetcher(fetcher_args, access_point) - - @pytest.fixture - def parallel_fetcher(self, request): - """ Fixture to create a parallel ERDDAP data fetcher for a given dataset and access point """ - fetcher_args, access_point = self._setup_fetcher(request, parallel="erddap") - yield create_fetcher(fetcher_args, access_point) - - def teardown_class(self): - """Cleanup once we are finished.""" - def remove_test_dir(): - shutil.rmtree(self.cachedir) - remove_test_dir() - - ######### - # TESTS # - ######### - @pytest.mark.parametrize("fetcher", VALID_ACCESS_POINTS, - indirect=True, - ids=VALID_ACCESS_POINTS_IDS) - def test_fetching(self, mocked_erddapserver, fetcher): - assert_fetcher(mocked_erddapserver, fetcher, cacheable=False) - - @pytest.mark.parametrize("cached_fetcher", VALID_ACCESS_POINTS, - indirect=True, - ids=VALID_ACCESS_POINTS_IDS) - def test_fetching_cached(self, mocked_erddapserver, cached_fetcher): - assert_fetcher(mocked_erddapserver, cached_fetcher, cacheable=True) - - @pytest.mark.parametrize("parallel_fetcher", VALID_PARALLEL_ACCESS_POINTS, - indirect=True, - ids=VALID_PARALLEL_ACCESS_POINTS_IDS) - def test_fetching_parallel(self, mocked_erddapserver, parallel_fetcher): - assert_fetcher(mocked_erddapserver, parallel_fetcher, cacheable=False) - - @pytest.mark.parametrize("measured", [None, 'all', 'DOXY'], - indirect=False, - ids=["measured=%s" % m for m in [None, 'all', 'DOXY']] - ) - def test_fetching_measured(self, mocked_erddapserver, measured): - class this_request: - param = { - 'ds': 'bgc', - 'mode': 'expert', - 'params': 'all', - 'measured': measured, - 'access_point': {"float": [5903248]}, - } - fetcher_args, access_point = self._setup_fetcher(this_request) - fetcher = create_fetcher(fetcher_args, access_point) - assert_fetcher(mocked_erddapserver, fetcher) - - @pytest.mark.parametrize("measured", ['all'], - indirect=False, - ids=["measured=%s" % m for m in ['all']], - ) - def test_fetching_failed_measured(self, mocked_erddapserver, measured): - class this_request: - param = { - 'ds': 'bgc', - 'mode': 'expert', - 'params': 'all', - 'measured': measured, - 'access_point': {"float": [6904240]}, - } - fetcher_args, access_point = self._setup_fetcher(this_request) - fetcher = create_fetcher(fetcher_args, access_point) - with pytest.raises(ValueError): - fetcher.to_xarray() \ No newline at end of file diff --git a/argopy/tests/test_fetchers_data_gdac.py b/argopy/tests/test_fetchers_data_gdac.py deleted file mode 100644 index 9bae3ee1..00000000 --- a/argopy/tests/test_fetchers_data_gdac.py +++ /dev/null @@ -1,230 +0,0 @@ -""" -Test the "GDAC ftp" data fetcher backend - -Here we try an approach based on fixtures and pytest parametrization -to make more explicit the full list of scenario tested. -""" -import xarray as xr - -import pytest -import tempfile -import shutil -from urllib.parse import urlparse -import logging -from collections import ChainMap - -import argopy -from argopy import DataFetcher as ArgoDataFetcher -from argopy.errors import ( - CacheFileNotFound, - FileSystemHasNoCache, - FtpPathError, -) -from argopy.utils.checkers import isconnected, is_list_of_strings -from utils import requires_gdac -from mocked_http import mocked_httpserver, mocked_server_address - - -log = logging.getLogger("argopy.tests.data.gdac") -skip_for_debug = pytest.mark.skipif(False, reason="Taking too long !") - - -""" -List ftp hosts to be tested. -Since the fetcher is compatible with host from local, http or ftp protocols, we -try to test them all: -""" -HOSTS = [argopy.tutorial.open_dataset("gdac")[0], - #'https://data-argo.ifremer.fr', # ok, but replaced by the mocked http server - mocked_server_address, - # 'ftp://ftp.ifremer.fr/ifremer/argo', - # 'ftp://usgodae.org/pub/outgoing/argo', # ok, but slow down CI and no need for 2 ftp tests - 'MOCKFTP', # keyword to use the fake/mocked ftp server (running on localhost) - ] - -""" -List access points to be tested. -For each access points, we list 1-to-2 scenario to make sure all possibilities are tested -""" -ACCESS_POINTS = [ - {"float": [13857]}, - {"profile": [13857, 90]}, - {"region": [-20, -16., 0, 1, 0, 100.]}, - {"region": [-20, -16., 0, 1, 0, 100., "1997-07-01", "1997-09-01"]}, - ] - -""" -List parallel methods to be tested. -""" -valid_parallel_opts = [ - {"parallel": "thread"}, - # {"parallel": True, "parallel_method": "thread"}, # opts0 - # {"parallel": True, "parallel_method": "process"} # opts1 -] - -""" -List user modes to be tested -""" -USER_MODES = ['standard', 'expert', 'research'] - - -@requires_gdac -def create_fetcher(fetcher_args, access_point): - """ Create a fetcher for a given set of facade options and access point - - """ - def core(fargs, apts): - try: - f = ArgoDataFetcher(**fargs) - if "float" in apts: - f = f.float(apts['float']) - elif "profile" in apts: - f = f.profile(*apts['profile']) - elif "region" in apts: - f = f.region(apts['region']) - except Exception: - raise - return f - fetcher = core(fetcher_args, access_point) - return fetcher - - -def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): - """Assert a data fetcher. - - This should be used by all tests - """ - assert isinstance(this_fetcher.to_xarray(errors='raise'), xr.Dataset) - core = this_fetcher.fetcher - assert is_list_of_strings(core.uri) - assert (core.N_RECORDS >= 1) # Make sure we loaded the index file content - assert (core.N_FILES >= 1) # Make sure we found results - if cacheable: - assert is_list_of_strings(core.cachepath) - - -def ftp_shortname(ftp): - """Get a short name for scenarios IDs, given a FTP host""" - if ftp == 'MOCKFTP': - return 'ftp_mocked' - elif 'localhost' in ftp or '127.0.0.1' in ftp: - return 'http_mocked' - else: - return (lambda x: 'file' if x == "" else x)(urlparse(ftp).scheme) - -""" -Make a list of VALID host/dataset/access_points to be tested -""" -VALID_ACCESS_POINTS, VALID_ACCESS_POINTS_IDS = [], [] -for host in HOSTS: - for mode in USER_MODES: - for ap in ACCESS_POINTS: - VALID_ACCESS_POINTS.append({'host': host, 'ds': 'phy', 'mode': mode, 'access_point': ap}) - VALID_ACCESS_POINTS_IDS.append("host='%s', ds='%s', mode='%s', %s" % (ftp_shortname(host), 'phy', mode, ap)) - - - -@requires_gdac -class TestBackend: - src = 'gdac' - - ############# - # UTILITIES # - ############# - - def setup_class(self): - """setup any state specific to the execution of the given class""" - # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests - self.cachedir = tempfile.mkdtemp() - - def _patch_ftp(self, ftp): - """Patch Mocked FTP server keyword""" - if ftp == 'MOCKFTP': - return pytest.MOCKFTP # this was set in conftest.py - else: - return ftp - - def _setup_fetcher(self, this_request, cached=False, parallel=False): - """Helper method to set up options for a fetcher creation""" - ftp = this_request.param['host'] - access_point = this_request.param['access_point'] - N_RECORDS = None if 'tutorial' in ftp or 'MOCK' in ftp else 100 # Make sure we're not going to load the full index - - fetcher_args = {"src": self.src, - "ftp": self._patch_ftp(ftp), - "ds": this_request.param['ds'], - "mode": this_request.param['mode'], - "cache": cached, - "cachedir": self.cachedir, - "parallel": False, - "N_RECORDS": N_RECORDS, - } - - if not cached: - # cache is False by default, so we don't need to clutter the arguments list - del fetcher_args["cache"] - del fetcher_args["cachedir"] - if not parallel: - # parallel is False by default, so we don't need to clutter the arguments list - del fetcher_args["parallel"] - - if not isconnected(fetcher_args['ftp']): - pytest.xfail("Fails because %s not available" % fetcher_args['ftp']) - else: - return fetcher_args, access_point - - @pytest.fixture - def fetcher(self, request, mocked_httpserver): - """ Fixture to create a GDAC fetcher for a given host and access point """ - fetcher_args, access_point = self._setup_fetcher(request, cached=False) - yield create_fetcher(fetcher_args, access_point) - - @pytest.fixture - def cached_fetcher(self, request): - """ Fixture to create a cached FTP fetcher for a given host and access point """ - fetcher_args, access_point = self._setup_fetcher(request, cached=True) - yield create_fetcher(fetcher_args, access_point) - - def teardown_class(self): - """Cleanup once we are finished.""" - def remove_test_dir(): - shutil.rmtree(self.cachedir) - remove_test_dir() - - ######### - # TESTS # - ######### - # def test_nocache(self, mocked_httpserver): - # this_fetcher = create_fetcher({"src": self.src, "ftp": self._patch_ftp(VALID_HOSTS[0]), "N_RECORDS": 10}, VALID_ACCESS_POINTS[0]) - # with pytest.raises(FileSystemHasNoCache): - # this_fetcher.cachepath - - # @pytest.mark.parametrize("fetcher", VALID_HOSTS, - # indirect=True, - # ids=["%s" % ftp_shortname(ftp) for ftp in VALID_HOSTS]) - # def test_hosts(self, mocked_httpserver, fetcher): - # assert (fetcher.N_RECORDS >= 1) - - # @pytest.mark.parametrize("ftp_host", ['invalid', 'https://invalid_ftp', 'ftp://invalid_ftp'], indirect=False) - # def test_hosts_invalid(self, ftp_host): - # # Invalid servers: - # with pytest.raises(FtpPathError): - # create_fetcher({"src": self.src, "ftp": ftp_host}, VALID_ACCESS_POINTS[0]) - - @pytest.mark.parametrize("fetcher", VALID_ACCESS_POINTS, indirect=True, ids=VALID_ACCESS_POINTS_IDS) - def test_fetching(self, mocked_httpserver, fetcher): - assert_fetcher(mocked_httpserver, fetcher, cacheable=False) - - @pytest.mark.parametrize("cached_fetcher", VALID_ACCESS_POINTS, indirect=True, ids=VALID_ACCESS_POINTS_IDS) - def test_fetching_cached(self, mocked_httpserver, cached_fetcher): - # Assert the fetcher (this trigger data fetching, hence caching as well): - assert_fetcher(mocked_httpserver, cached_fetcher, cacheable=True) - # and we also make sure we can clear the cache: - cached_fetcher.clear_cache() - with pytest.raises(CacheFileNotFound): - cached_fetcher.fetcher.cachepath - - def test_uri_mono2multi(self, mocked_httpserver): - ap = [v for v in ACCESS_POINTS if 'region' in v.keys()][0] - f = create_fetcher({"src": self.src, "ftp": HOSTS[0], "N_RECORDS": 100}, ap).fetcher - assert is_list_of_strings(f.uri_mono2multi(f.uri)) diff --git a/argopy/tests/test_fetchers_facade_data.py b/argopy/tests/test_fetchers_facade_data.py deleted file mode 100644 index 17e0646c..00000000 --- a/argopy/tests/test_fetchers_facade_data.py +++ /dev/null @@ -1,196 +0,0 @@ -import pandas as pd -import xarray as xr - -import pytest - -import argopy -from argopy import DataFetcher as ArgoDataFetcher -from argopy.errors import ( - InvalidFetcherAccessPoint, - InvalidFetcher, - OptionValueError, -) -from argopy.utils import is_list_of_strings -from utils import ( - requires_fetcher, - requires_connection, - requires_connected_erddap_phy, - requires_gdac, - requires_connected_gdac, - requires_connected_argovis, - requires_ipython, - safe_to_server_errors, - requires_matplotlib, - has_matplotlib, - has_seaborn, - has_cartopy, - has_ipython, -) - - -if has_matplotlib: - import matplotlib as mpl - -if has_cartopy: - import cartopy - -if has_ipython: - import IPython - -skip_for_debug = pytest.mark.skipif(True, reason="Taking too long !") - - -@requires_gdac -class Test_Facade: - - # Use the first valid data source: - src = 'gdac' - src_opts = {'ftp': argopy.tutorial.open_dataset("gdac")[0]} - - def __get_fetcher(self, empty: bool = False, pt: str = 'profile'): - f = ArgoDataFetcher(src=self.src, **self.src_opts) - # f.valid_access_points[0] - - if pt == 'float': - if not empty: - return f, ArgoDataFetcher(src=self.src, **self.src_opts).float(2901623) - else: - return f, ArgoDataFetcher(src=self.src, **self.src_opts).float(12) - - if pt == 'profile': - if not empty: - return f, ArgoDataFetcher(src=self.src, **self.src_opts).profile(2901623, 12) - else: - return f, ArgoDataFetcher(src=self.src, **self.src_opts).profile(12, 1200) - - if pt == 'region': - if not empty: - return f, ArgoDataFetcher(src=self.src, **self.src_opts).region([-60, -55, 40.0, 45.0, 0.0, 10.0, - "2007-08-01", "2007-09-01"]) - else: - return f, ArgoDataFetcher(src=self.src, **self.src_opts).region([-60, -55, 40.0, 45.0, 99.92, 99.99, - "2007-08-01", "2007-08-01"]) - - def test_invalid_mode(self): - with pytest.raises(OptionValueError): - ArgoDataFetcher(src=self.src, mode='invalid').to_xarray() - - def test_invalid_source(self): - with pytest.raises(OptionValueError): - ArgoDataFetcher(src="invalid").to_xarray() - - def test_invalid_dataset(self): - with pytest.raises(OptionValueError): - ArgoDataFetcher(src=self.src, ds='invalid') - - def test_invalid_accesspoint(self): - with pytest.raises(InvalidFetcherAccessPoint): - self.__get_fetcher()[0].invalid_accesspoint.to_xarray() - - def test_warnings(self): - with pytest.warns(UserWarning): - ArgoDataFetcher(src='erddap', ds='bgc', mode='standard') - with pytest.warns(UserWarning): - ArgoDataFetcher(src='erddap', ds='bgc', mode='research') - - def test_no_uri(self): - with pytest.raises(InvalidFetcherAccessPoint): - self.__get_fetcher()[0].uri - - def test_to_xarray(self): - assert isinstance(self.__get_fetcher()[1].to_xarray(), xr.Dataset) - with pytest.raises(InvalidFetcher): - assert self.__get_fetcher()[0].to_xarray() - - def test_to_dataframe(self): - assert isinstance(self.__get_fetcher()[1].to_dataframe(), pd.core.frame.DataFrame) - with pytest.raises(InvalidFetcher): - assert self.__get_fetcher()[0].to_dataframe() - - params = [(p, c) for p in [True, False] for c in [False]] - ids_params = ["full=%s, coriolis_id=%s" % (p[0], p[1]) for p in params] - @pytest.mark.parametrize("params", params, - indirect=False, - ids=ids_params) - def test_to_index(self, params): - full, coriolis_id = params - assert isinstance(self.__get_fetcher()[1].to_index(full=full, coriolis_id=coriolis_id), pd.core.frame.DataFrame) - - params = [(p, c) for p in [True, False] for c in [True]] - ids_params = ["full=%s, coriolis_id=%s" % (p[0], p[1]) for p in params] - @pytest.mark.parametrize("params", params, - indirect=False, - ids=ids_params) - @requires_connection - def test_to_index_coriolis(self, params): - full, coriolis_id = params - assert isinstance(self.__get_fetcher()[1].to_index(full=full, coriolis_id=coriolis_id), pd.core.frame.DataFrame) - - def test_load(self): - f, fetcher = self.__get_fetcher(pt='float') - - fetcher.load() - assert is_list_of_strings(fetcher.uri) - assert isinstance(fetcher.data, xr.Dataset) - assert isinstance(fetcher.index, pd.core.frame.DataFrame) - - # Change the access point: - new_fetcher = f.profile(fetcher._AccessPoint_data['wmo'], 1) - new_fetcher.load() - assert is_list_of_strings(new_fetcher.uri) - assert isinstance(new_fetcher.data, xr.Dataset) - assert isinstance(new_fetcher.index, pd.core.frame.DataFrame) - - @requires_matplotlib - def test_plot_trajectory(self): - f, fetcher = self.__get_fetcher(pt='float') - fig, ax = fetcher.plot(ptype='trajectory', - with_seaborn=has_seaborn, - with_cartopy=has_cartopy) - assert isinstance(fig, mpl.figure.Figure) - expected_ax_type = ( - cartopy.mpl.geoaxes.GeoAxesSubplot - if has_cartopy - else mpl.axes.Axes - ) - assert isinstance(ax, expected_ax_type) - assert isinstance(ax.get_legend(), mpl.legend.Legend) - mpl.pyplot.close(fig) - - @requires_matplotlib - @pytest.mark.parametrize("by", ["dac", "profiler"], indirect=False) - def test_plot_bar(self, by): - f, fetcher = self.__get_fetcher(pt='float') - fig, ax = fetcher.plot(ptype=by, with_seaborn=has_seaborn) - assert isinstance(fig, mpl.figure.Figure) - mpl.pyplot.close(fig) - - @requires_matplotlib - def test_plot_invalid(self): - f, fetcher = self.__get_fetcher(pt='float') - with pytest.raises(ValueError): - fetcher.plot(ptype='invalid_cat') - - @requires_matplotlib - def test_plot_qc_altimetry(self): - f, fetcher = self.__get_fetcher(pt='float') - dsh = fetcher.plot(ptype='qc_altimetry', embed='slide') - if has_ipython: - assert isinstance(dsh(0), IPython.display.Image) - else: - assert isinstance(dsh, dict) - - def test_domain(self): - f, fetcher = self.__get_fetcher(pt='float') - fetcher.domain - - def test_dashboard(self): - f, fetcher = self.__get_fetcher(pt='float') - assert isinstance(fetcher.dashboard(url_only=True), str) - - f, fetcher = self.__get_fetcher(pt='profile') - assert isinstance(fetcher.dashboard(url_only=True), str) - - with pytest.warns(UserWarning): - f, fetcher = self.__get_fetcher(pt='region') - fetcher.dashboard(url_only=True) diff --git a/argopy/tests/test_fetchers_facade_index.py b/argopy/tests/test_fetchers_facade_index.py deleted file mode 100644 index b10d5269..00000000 --- a/argopy/tests/test_fetchers_facade_index.py +++ /dev/null @@ -1,131 +0,0 @@ -import pytest -import importlib - -import argopy -from argopy import IndexFetcher as ArgoIndexFetcher -from argopy.errors import InvalidFetcherAccessPoint, InvalidFetcher -from utils import ( - # AVAILABLE_INDEX_SOURCES, - requires_fetcher_index, - requires_connected_erddap_index, - requires_connected_gdac, - requires_connection, - requires_ipython, - ci_erddap_index, - requires_matplotlib, - has_matplotlib, - has_seaborn, - has_cartopy -) - - -if has_matplotlib: - import matplotlib as mpl - -if has_cartopy: - import cartopy - - -skip_for_debug = pytest.mark.skipif(True, reason="Taking too long !") - - -@requires_connection -class Test_Facade: - local_ftp = argopy.tutorial.open_dataset("gdac")[0] - src = 'gdac' - src_opts = {'ftp': local_ftp} - - def __get_fetcher(self, empty: bool = False, pt: str = 'profile'): - f = ArgoIndexFetcher(src=self.src, **self.src_opts) - - if pt == 'float': - if not empty: - return f, ArgoIndexFetcher(src=self.src, **self.src_opts).float(2901623) - else: - return f, ArgoIndexFetcher(src=self.src, **self.src_opts).float(12) - - if pt == 'profile': - if not empty: - return f, ArgoIndexFetcher(src=self.src, **self.src_opts).profile(2901623, 12) - else: - return f, ArgoIndexFetcher(src=self.src, **self.src_opts).profile(12, 1200) - - if pt == 'region': - if not empty: - return f, ArgoIndexFetcher(src=self.src, **self.src_opts).region([-60, -55, 40.0, 45.0, 0.0, 10.0, - "2007-08-01", "2007-09-01"]) - else: - return f, ArgoIndexFetcher(src=self.src, **self.src_opts).region([-60, -55, 40.0, 45.0, 99.92, 99.99, - "2007-08-01", "2007-08-01"]) - - def test_invalid_fetcher(self): - with pytest.raises(InvalidFetcher): - ArgoIndexFetcher(src="invalid_fetcher").to_xarray() - - @requires_fetcher_index - def test_invalid_accesspoint(self): - # Use the first valid data source - with pytest.raises(InvalidFetcherAccessPoint): - ArgoIndexFetcher( - src=self.src, **self.src_opts - ).invalid_accesspoint.to_xarray() # Can't get data if access point not defined first - with pytest.raises(InvalidFetcherAccessPoint): - ArgoIndexFetcher( - src=self.src, **self.src_opts - ).to_xarray() # Can't get data if access point not defined first - - @requires_fetcher_index - def test_invalid_dataset(self): - with pytest.raises(ValueError): - ArgoIndexFetcher(src=self.src, ds='dummy_ds', **self.src_opts) - - @requires_matplotlib - def test_plot(self): - f, fetcher = self.__get_fetcher(pt='float') - - # Test 'trajectory' - for ws in [False, has_seaborn]: - for wc in [False, has_cartopy]: - for legend in [True, False]: - fig, ax = fetcher.plot(ptype='trajectory', with_seaborn=ws, with_cartopy=wc, add_legend=legend) - assert isinstance(fig, mpl.figure.Figure) - - expected_ax_type = ( - cartopy.mpl.geoaxes.GeoAxesSubplot - if has_cartopy and wc - else mpl.axes.Axes - ) - assert isinstance(ax, expected_ax_type) - - expected_lg_type = mpl.legend.Legend if legend else type(None) - assert isinstance(ax.get_legend(), expected_lg_type) - - mpl.pyplot.close(fig) - - # Test 'dac', 'profiler' - for ws in [False, has_seaborn]: - for by in [ - "dac", - "profiler" - ]: - fig, ax = fetcher.plot(ptype=by, with_seaborn=ws) - assert isinstance(fig, mpl.figure.Figure) - mpl.pyplot.close(fig) - - # Test 'qc_altimetry' - if importlib.util.find_spec('IPython') is not None: - import IPython - dsh = fetcher.plot(ptype='qc_altimetry', embed='slide') - assert isinstance(dsh(0), IPython.display.Image) - - # Test invalid plot - with pytest.raises(ValueError): - fetcher.plot(ptype='invalid_cat', with_seaborn=ws) - - @requires_ipython - @requires_matplotlib - def test_plot_qc_altimetry(self): - import IPython - f, fetcher = self.__get_fetcher(pt='float') - dsh = fetcher.plot(ptype='qc_altimetry', embed='slide') - assert isinstance(dsh(0), IPython.display.Image) diff --git a/argopy/tests/test_fetchers_index_erddap.py b/argopy/tests/test_fetchers_index_erddap.py deleted file mode 100644 index b42df83e..00000000 --- a/argopy/tests/test_fetchers_index_erddap.py +++ /dev/null @@ -1,137 +0,0 @@ -import pandas as pd - -import pytest -import tempfile - -import argopy -from argopy import IndexFetcher as ArgoIndexFetcher -from argopy.errors import ( - FileSystemHasNoCache, - CacheFileNotFound -) -from utils import requires_connected_erddap_index, safe_to_server_errors, ci_erddap_index - -ERDDAP_TIMEOUT = 3 * 60 -safe_to_no_cache = pytest.mark.skipif(True, reason="Cache disabled for erddap index fetcher") - - -@ci_erddap_index -@requires_connected_erddap_index -class Test_Backend_WMO: - """ Test ERDDAP index fetching backend for WMO access point""" - src = "erddap" - requests = { - "float": [[2901623], [2901623, 6901929]] - } - - def test_nocache(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir): - fetcher = ArgoIndexFetcher(src=self.src, cache=False).float(self.requests['float'][0]).fetcher - with pytest.raises(FileSystemHasNoCache): - fetcher.cachepath - - @safe_to_no_cache - def test_cachepath_notfound(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir): - fetcher = ArgoIndexFetcher(src=self.src, cache=True).float(self.requests['float'][0]).fetcher - with pytest.raises(CacheFileNotFound): - fetcher.cachepath - - @safe_to_no_cache - @safe_to_server_errors - def test_cached(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir, api_timeout=ERDDAP_TIMEOUT): - fetcher = ArgoIndexFetcher(src=self.src, cache=True).float(self.requests['float'][0]).fetcher - df = fetcher.to_dataframe() - assert isinstance(df, pd.core.frame.DataFrame) - assert isinstance(fetcher.cachepath, str) - - @safe_to_no_cache - @safe_to_server_errors - def test_clearcache(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir, api_timeout=ERDDAP_TIMEOUT): - fetcher = ArgoIndexFetcher(src=self.src, cache=True).float(self.requests['float'][0]).fetcher - fetcher.to_dataframe() - fetcher.clear_cache() - with pytest.raises(CacheFileNotFound): - fetcher.cachepath - - def test_url(self): - for arg in self.requests["float"]: - fetcher = ArgoIndexFetcher(src=self.src).float(arg).fetcher - assert isinstance(fetcher.url, str) - - @safe_to_server_errors - def test_phy_float(self): - for arg in self.requests["float"]: - with argopy.set_options(api_timeout=ERDDAP_TIMEOUT): - fetcher = ArgoIndexFetcher(src=self.src).float(arg).fetcher - df = fetcher.to_dataframe() - assert isinstance(df, pd.core.frame.DataFrame) - - -@ci_erddap_index -@requires_connected_erddap_index -class Test_Backend_BOX: - """ Test ERDDAP index fetching backend for the BOX access point """ - - src = "erddap" - requests = { - "region": [ - [-60, -50, 40.0, 50.0], - [-60, -55, 40.0, 45.0, "2007-08-01", "2007-09-01"], - ], - } - - def test_nocache(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir): - fetcher = ArgoIndexFetcher(src=self.src, cache=False).region(self.requests['region'][-1]).fetcher - with pytest.raises(FileSystemHasNoCache): - fetcher.cachepath - - @safe_to_no_cache - def test_cachepath_notfound(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir): - fetcher = ArgoIndexFetcher(src=self.src, cache=True).region(self.requests['region'][-1]).fetcher - with pytest.raises(CacheFileNotFound): - fetcher.cachepath - - @safe_to_no_cache - @safe_to_server_errors - def test_cached(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir, api_timeout=ERDDAP_TIMEOUT): - fetcher = ArgoIndexFetcher(src=self.src, cache=True).region(self.requests['region'][-1]).fetcher - df = fetcher.to_dataframe() - assert isinstance(df, pd.core.frame.DataFrame) - assert isinstance(fetcher.cachepath, str) - - @safe_to_no_cache - @safe_to_server_errors - def test_clearcache(self): - with tempfile.TemporaryDirectory() as testcachedir: - with argopy.set_options(cachedir=testcachedir, api_timeout=ERDDAP_TIMEOUT): - fetcher = ArgoIndexFetcher(src=self.src, cache=True).region(self.requests['region'][-1]).fetcher - fetcher.to_dataframe() - fetcher.clear_cache() - with pytest.raises(CacheFileNotFound): - fetcher.cachepath - - def test_url(self): - for arg in self.requests["region"]: - fetcher = ArgoIndexFetcher(src=self.src).region(arg).fetcher - assert isinstance(fetcher.url, str) - - @safe_to_server_errors - def test_phy_region(self): - for arg in self.requests["region"]: - with argopy.set_options(api_timeout=ERDDAP_TIMEOUT): - fetcher = ArgoIndexFetcher(src=self.src).region(arg).fetcher - df = fetcher.to_dataframe() - assert isinstance(df, pd.core.frame.DataFrame) diff --git a/argopy/tests/test_fetchers_index_gdac.py b/argopy/tests/test_fetchers_index_gdac.py deleted file mode 100644 index 02e71e92..00000000 --- a/argopy/tests/test_fetchers_index_gdac.py +++ /dev/null @@ -1,201 +0,0 @@ -import pandas as pd - -import pytest -import tempfile -import shutil -from urllib.parse import urlparse -import logging - -import argopy -from argopy import IndexFetcher as ArgoIndexFetcher -from argopy.errors import ( - CacheFileNotFound, - FileSystemHasNoCache, - FtpPathError -) -from argopy.utils.checkers import isconnected, is_list_of_strings -from utils import requires_gdac -from mocked_http import mocked_httpserver, mocked_server_address - - -log = logging.getLogger("argopy.tests.index.gdac") -skip_for_debug = pytest.mark.skipif(True, reason="Taking too long !") - - -""" -List ftp hosts to be tested. -Since the fetcher is compatible with host from local, http or ftp protocols, we -try to test them all: -""" -VALID_HOSTS = [argopy.tutorial.open_dataset("gdac")[0], - #'https://data-argo.ifremer.fr', - mocked_server_address, - # 'ftp://ftp.ifremer.fr/ifremer/argo', - # 'ftp://usgodae.org/pub/outgoing/argo', # ok, but slow down CI and no need for 2 ftp tests - 'MOCKFTP', # keyword to use the fake/mocked ftp server (running on localhost) - ] - -""" -List access points to be tested. -For each access points, we list 1-to-2 scenario to make sure all possibilities are tested -""" -VALID_ACCESS_POINTS = [ - {"float": [13857]}, - {"profile": [13857, 90]}, - {"region": [-20, -16., 0, 1]}, - {"region": [-20, -16., 0, 1, "1997-07-01", "1997-09-01"]}, - ] - -""" -List parallel methods to be tested. -""" -valid_parallel_opts = [ - {"parallel": "thread"}, - # {"parallel": True, "parallel_method": "thread"}, # opts0 - # {"parallel": True, "parallel_method": "process"} # opts1 -] - - -@requires_gdac -def create_fetcher(fetcher_args, access_point, xfail=False): - """ Create a fetcher for a given set of facade options and access point - - Use xfail=True when a test with this fetcher is expected to fail - """ - def core(fargs, apts): - try: - f = ArgoIndexFetcher(**fargs) - if "float" in apts: - f = f.float(apts['float']) - elif "profile" in apts: - f = f.profile(*apts['profile']) - elif "region" in apts: - f = f.region(apts['region']) - except Exception: - raise - return f - fetcher = core(fetcher_args, access_point).fetcher - return fetcher - - -def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): - """Assert a data fetcher. - - This should be used by all tests - """ - assert isinstance(this_fetcher.to_dataframe(), pd.core.frame.DataFrame) - assert (this_fetcher.N_RECORDS >= 1) # Make sure we loaded the index file content - assert (this_fetcher.N_FILES >= 1) # Make sure we found results - if cacheable: - assert is_list_of_strings(this_fetcher.cachepath) - - -def ftp_shortname(ftp): - """Get a short name for scenarios IDs, given a FTP host""" - if ftp == 'MOCKFTP': - return 'ftp_mocked' - elif 'localhost' in ftp or '127.0.0.1' in ftp: - return 'http_mocked' - else: - return (lambda x: 'file' if x == "" else x)(urlparse(ftp).scheme) - - -@requires_gdac -class TestBackend: - src = 'gdac' - - # Create list of tests scenarios - # combine all hosts with all access points: - scenarios = [(h, ap) for h in VALID_HOSTS for ap in VALID_ACCESS_POINTS] - - scenarios_ids = [ - "%s, %s" % (ftp_shortname(fix[0]), list(fix[1].keys())[0]) for - fix - in - scenarios] - - ############# - # UTILITIES # - ############# - - def setup_class(self): - """setup any state specific to the execution of the given class""" - # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests - self.cachedir = tempfile.mkdtemp() - - def _patch_ftp(self, ftp): - """Patch Mocked FTP server keyword""" - if ftp == 'MOCKFTP': - return pytest.MOCKFTP # this was set in conftest.py - else: - return ftp - - def _setup_fetcher(self, this_request, cached=False): - """Helper method to set up options for a fetcher creation""" - if isinstance(this_request.param, tuple): - ftp = this_request.param[0] - access_point = this_request.param[1] - else: - ftp = this_request.param - access_point = VALID_ACCESS_POINTS[0] # Use 1st valid access point - - N_RECORDS = None if 'tutorial' in ftp or 'MOCK' in ftp else 100 # Make sure we're not going to load the full index - fetcher_args = {"src": self.src, "ftp": self._patch_ftp(ftp), "cache": False, "N_RECORDS": N_RECORDS} - - if cached: - fetcher_args = {**fetcher_args, **{"cache": True, "cachedir": self.cachedir}} - if not isconnected(fetcher_args['ftp']): - pytest.xfail("Fails because %s not available" % fetcher_args['ftp']) - else: - return fetcher_args, access_point - - @pytest.fixture - def _make_a_fetcher(self, request, mocked_httpserver): - """ Fixture to create a GDAC fetcher for a given host and access point """ - fetcher_args, access_point = self._setup_fetcher(request, cached=False) - yield create_fetcher(fetcher_args, access_point) - - @pytest.fixture - def _make_a_cached_fetcher(self, request): - """ Fixture to create a cached FTP fetcher for a given host and access point """ - fetcher_args, access_point = self._setup_fetcher(request, cached=True) - yield create_fetcher(fetcher_args, access_point) - - def teardown_class(self): - """Cleanup once we are finished.""" - def remove_test_dir(): - shutil.rmtree(self.cachedir) - remove_test_dir() - - ######### - # TESTS # - ######### - def test_nocache(self, mocked_httpserver): - this_fetcher = create_fetcher({"src": self.src, "ftp": self._patch_ftp(VALID_HOSTS[0]), "N_RECORDS": 10}, VALID_ACCESS_POINTS[0]) - with pytest.raises(FileSystemHasNoCache): - this_fetcher.cachepath - - @pytest.mark.parametrize("_make_a_fetcher", VALID_HOSTS, - indirect=True, - ids=["%s" % ftp_shortname(ftp) for ftp in VALID_HOSTS]) - def test_hosts(self, mocked_httpserver, _make_a_fetcher): - assert (_make_a_fetcher.N_RECORDS >= 1) - - @pytest.mark.parametrize("ftp_host", ['invalid', 'https://invalid_ftp', 'ftp://invalid_ftp'], indirect=False) - def test_hosts_invalid(self, ftp_host): - # Invalid servers: - with pytest.raises(FtpPathError): - create_fetcher({"src": self.src, "ftp": ftp_host}, VALID_ACCESS_POINTS[0]) - - @pytest.mark.parametrize("_make_a_fetcher", scenarios, indirect=True, ids=scenarios_ids) - def test_fetching(self, mocked_httpserver, _make_a_fetcher): - assert_fetcher(mocked_httpserver, _make_a_fetcher, cacheable=False) - - @pytest.mark.parametrize("_make_a_cached_fetcher", scenarios, indirect=True, ids=scenarios_ids) - def test_fetching_cached(self, mocked_httpserver, _make_a_cached_fetcher): - # Assert the fetcher (this trigger data fetching, hence caching as well): - assert_fetcher(mocked_httpserver, _make_a_cached_fetcher, cacheable=True) - # and we also make sure we can clear the cache: - _make_a_cached_fetcher.clear_cache() - with pytest.raises(CacheFileNotFound): - _make_a_cached_fetcher.cachepath diff --git a/argopy/tests/test_fetchers_proto.py b/argopy/tests/test_fetchers_proto.py deleted file mode 100644 index 24f669ff..00000000 --- a/argopy/tests/test_fetchers_proto.py +++ /dev/null @@ -1,53 +0,0 @@ -import pytest -import xarray -from argopy.data_fetchers.proto import ArgoDataFetcherProto -from argopy.utils import to_list - - -class Fetcher(ArgoDataFetcherProto): - dataset_id = 'phy' - - def to_xarray(self, *args, **kwargs): - super(Fetcher, self).to_xarray(*args, **kwargs) - - def filter_data_mode(self, *args, **kwargs): - super(Fetcher, self).filter_data_mode(*args, **kwargs) - - def filter_qc(self, *args, **kwargs): - super(Fetcher, self).filter_qc(*args, **kwargs) - - def filter_researchmode(self, *args, **kwargs): - super(Fetcher, self).filter_researchmode(*args, **kwargs) - - -def test_required_methods(): - f = Fetcher() - with pytest.raises(NotImplementedError): - f.to_xarray() - - with pytest.raises(NotImplementedError): - f.filter_data_mode(xarray.Dataset, str) - - with pytest.raises(NotImplementedError): - f.filter_qc(xarray.Dataset) - - with pytest.raises(NotImplementedError): - f.filter_researchmode(xarray.Dataset) - - -@pytest.mark.parametrize("profile", [[13857, None], [13857, 90]], - indirect=False, - ids=["%s" % p for p in [[13857, None], [13857, 90]]]) -def test_dashboard(profile): - - f = Fetcher() - f.WMO, f.CYC = profile - f.WMO = to_list(f.WMO) - f.CYC = to_list(f.CYC) - assert isinstance(f.dashboard(url_only=True), str) - - with pytest.warns(UserWarning): - f = Fetcher() - f.WMO = [1901393, 6902746] - f.CYC = None - f.dashboard(url_only=True) diff --git a/argopy/tests/test_options.py b/argopy/tests/test_options.py index 042bfb30..c701e2f5 100644 --- a/argopy/tests/test_options.py +++ b/argopy/tests/test_options.py @@ -59,8 +59,8 @@ def test_opt_invalid_cachedir(): # OptionValueError is raised when it's not writable folder_name = "read_only_folder" create_read_only_folder(folder_name) - warnings.warn(os.stat(folder_name).st_mode) - warnings.warn(os.access(folder_name, os.W_OK)) + warnings.warn(str(os.stat(folder_name).st_mode)) + warnings.warn(str(os.access(folder_name, os.W_OK))) with pytest.raises(OptionValueError): argopy.set_options(cachedir=folder_name) os.rmdir(folder_name) diff --git a/argopy/tests/test_plot_argo_colors.py b/argopy/tests/test_plot_argo_colors.py deleted file mode 100644 index 5613ff2b..00000000 --- a/argopy/tests/test_plot_argo_colors.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -This file covers the plotters module -We test plotting functions from IndexFetcher and DataFetcher -""" -import pytest -import logging - -from utils import ( - requires_matplotlib, - requires_seaborn, - has_matplotlib, - has_seaborn, -) -from argopy.plot import ArgoColors - -if has_matplotlib: - import matplotlib as mpl - known_colormaps = ArgoColors().known_colormaps - list_valid_known_colormaps = ArgoColors().list_valid_known_colormaps - quantitative = ArgoColors().quantitative -else: - known_colormaps = [] - list_valid_known_colormaps = [] - quantitative = [] - -if has_seaborn: - import seaborn as sns - -log = logging.getLogger("argopy.tests.plot") - - -@requires_matplotlib -class Test_ArgoColors: - - @pytest.mark.parametrize("cname", known_colormaps, indirect=False) - def test_Argo_colormaps(self, cname): - ac = ArgoColors(name=cname) - assert ac.registered - assert isinstance(ac.cmap, mpl.colors.LinearSegmentedColormap) - - @pytest.mark.parametrize("cname", quantitative, indirect=False) - def test_quantitative_colormaps(self, cname): - ac = ArgoColors(name=cname) - assert ac.Ncolors == quantitative[cname] - assert isinstance(ac.cmap, mpl.colors.LinearSegmentedColormap) - - @pytest.mark.parametrize("opts", [('Spectral', None), - ('Blues', 13), - ('k', 1)], - ids = ["name='Spectral', N=None", - "name='Blues', N=13", - "name='k', N=1"], - indirect=False) - def test_other_colormaps(self, opts): - name, N = opts - ac = ArgoColors(name, N) - assert isinstance(ac.cmap, mpl.colors.LinearSegmentedColormap) - - @pytest.mark.parametrize("N", [12.35, '12'], - ids=['N is a float', 'N is a str'], - indirect=False) - def test_invalid_Ncolors(self, N): - with pytest.raises(ValueError): - ArgoColors(N=N) - - @pytest.mark.parametrize("cname", ['data_mode', 'dm'], ids=['key', 'aka'], indirect=False) - def test_definition(self, cname): - assert isinstance( ArgoColors(cname).definition, dict) - - def test_colors_lookup_dict(self): - ac = ArgoColors(list_valid_known_colormaps[0]) - assert isinstance(ac.lookup, dict) - - with pytest.raises(ValueError): - ArgoColors('Blues').lookup - - def test_ticklabels_dict(self): - ac = ArgoColors(list_valid_known_colormaps[0]) - assert isinstance(ac.ticklabels, dict) - - with pytest.raises(ValueError): - ArgoColors('Blues').ticklabels - - @requires_seaborn - def test_seaborn_palette(self): - assert isinstance(ArgoColors('Set1').palette, sns.palettes._ColorPalette) - assert isinstance(ArgoColors('k', N=1).palette, sns.palettes._ColorPalette) - - @pytest.mark.parametrize("cname", ['data_mode', 'Blues'], - ids=['known', 'other'], - indirect=False) - def test_repr_html_(self, cname): - ac = ArgoColors(cname) - assert isinstance(ac._repr_html_(), str) - - @pytest.mark.parametrize("cname", ['data_mode', 'Blues'], - ids=['known', 'other'], - indirect=False) - def test_show_COLORS(self, cname): - ac = ArgoColors(cname) - assert isinstance(ac.show_COLORS(), str) diff --git a/argopy/tests/test_plot_dashboards.py b/argopy/tests/test_plot_dashboards.py deleted file mode 100644 index 50ac891f..00000000 --- a/argopy/tests/test_plot_dashboards.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -This file covers the argopy.plot.dashboards submodule -""" -import pytest -import logging -import tempfile - -import argopy -from argopy.errors import InvalidDashboard -from utils import ( - requires_connection, - requires_ipython, - has_ipython, -) -from mocked_http import mocked_httpserver, mocked_server_address - -if has_ipython: - import IPython - -log = logging.getLogger("argopy.tests.plot.dashboards") - - -@pytest.mark.parametrize("board_type", ["invalid", "op", "ocean-ops", "coriolis"], indirect=False) -def test_invalid_dashboard(board_type): - # Test types without 'base' - with pytest.raises(InvalidDashboard): - argopy.dashboard(type=board_type, url_only=True) - - -@pytest.mark.parametrize("board_type", ["op", "ocean-ops", "coriolis"], indirect=False) -def test_invalid_dashboard_profile(board_type): - # Test types without 'cyc' - with pytest.raises(InvalidDashboard): - argopy.dashboard(6902755, 12, type=board_type, url_only=True) - - -@pytest.mark.parametrize("board_type", ["data", "meta", "ea", "argovis", "bgc"], indirect=False) -def test_valid_dashboard(board_type): - # Test types with 'base' - assert isinstance(argopy.dashboard(type=board_type, url_only=True), str) - - -@pytest.mark.parametrize("board_type", ["data", "meta", "ea", "argovis", "op", "ocean-ops", "bgc"], indirect=False) -def test_valid_dashboard_float(board_type, mocked_httpserver): - # Test types with 'wmo' (should be all) - with argopy.set_options(server=mocked_server_address): - assert isinstance(argopy.dashboard(6901929, type=board_type, url_only=True), str) - - -@pytest.mark.parametrize("board_type", ["data", "meta", "ea", "argovis", "bgc"], indirect=False) -def test_valid_dashboard_profile(board_type, mocked_httpserver): - # Test types with 'cyc' - with argopy.set_options(cachedir=tempfile.mkdtemp(), server=mocked_server_address): - assert isinstance(argopy.dashboard(5904797, 12, type=board_type, url_only=True), str) - - -@requires_ipython -@pytest.mark.parametrize("opts", [{}, {'wmo': 6901929}, {'wmo': 6901929, 'cyc': 3}], - ids=['', 'WMO', 'WMO, CYC'], - indirect=False) -def test_valid_dashboard_ipython_output(opts, mocked_httpserver): - with argopy.set_options(cachedir=tempfile.mkdtemp(), server=mocked_server_address): - dsh = argopy.dashboard(**opts) - assert isinstance(dsh, IPython.lib.display.IFrame) - - diff --git a/argopy/tests/test_plot_plot.py b/argopy/tests/test_plot_plot.py deleted file mode 100644 index f05bdfe1..00000000 --- a/argopy/tests/test_plot_plot.py +++ /dev/null @@ -1,212 +0,0 @@ -""" -This file covers the argopy.plot.plot submodule -""" -import pytest -import logging -from typing import Callable - -import argopy -from utils import ( - requires_gdac, - requires_connection, - requires_matplotlib, - requires_ipython, - requires_cartopy, - has_matplotlib, - has_seaborn, - has_cartopy, - has_ipython, - has_ipywidgets, -) - -from argopy.plot import bar_plot, plot_trajectory, open_sat_altim_report, scatter_map -from argopy.errors import InvalidDatasetStructure -from argopy import DataFetcher as ArgoDataFetcher -from mocked_http import mocked_server_address - -if has_matplotlib: - import matplotlib as mpl - -if has_cartopy: - import cartopy - -if has_ipython: - import IPython - - -log = logging.getLogger("argopy.tests.plot.plot") -argopy.clear_cache() - - -class Test_open_sat_altim_report: - WMOs = [2901623, [2901623, 6901929]] - - def test_load_mocked_server(self, mocked_httpserver): - """This will easily ensure that the module scope fixture is available to all methods !""" - assert True - - @pytest.mark.parametrize("WMOs", WMOs, - ids=['For unique WMO', 'For a list of WMOs'], - indirect=False) - @pytest.mark.parametrize("embed", ['dropdown', 'slide', 'list', None], - indirect=False) - def test_open_sat_altim_report(self, WMOs, embed): - if has_ipython: - import IPython - - dsh = open_sat_altim_report(WMO=WMOs, embed=embed, api_server=mocked_server_address) - - if has_ipython and embed is not None: - - if has_ipywidgets: - - if embed == "dropdown": - assert isinstance(dsh, Callable) - assert isinstance(dsh(2901623), IPython.display.Image) - - if embed == "slide": - assert isinstance(dsh, Callable) - - else: - assert dsh is None - - else: - assert isinstance(dsh, dict) - - @requires_ipython - def test_invalid_method(self): - with pytest.raises(ValueError): - open_sat_altim_report(WMO=self.WMOs[0], embed='dummy_method', api_server=mocked_server_address) - - -@requires_gdac -@requires_matplotlib -class Test_plot_trajectory: - src = "gdac" - local_ftp = argopy.tutorial.open_dataset("gdac")[0] - requests = { - # "float": [[2901623], [2901623, 6901929, 5906072]], - # "profile": [[2901623, 12], [6901929, [5, 45]]], - "region": [ - [-60, -40, 40.0, 60.0, 0., 10.], - [-60, -40, 40.0, 60.0, 0., 10., "2007-08-01", "2007-09-01"], - ], - } - - opts = [(ws, wc, lg) for ws in [False, has_seaborn] for wc in [False, has_cartopy] for lg in [True, False]] - opts_ids = ["with_seaborn=%s, with_cartopy=%s, add_legend=%s" % (opt[0], opt[1], opt[2]) for opt in opts] - - def __test_traj_plot(self, df, opts): - with_seaborn, with_cartopy, with_legend = opts - fig, ax = plot_trajectory( - df, with_seaborn=with_seaborn, with_cartopy=with_cartopy, add_legend=with_legend - ) - assert isinstance(fig, mpl.figure.Figure) - - expected_ax_type = ( - cartopy.mpl.geoaxes.GeoAxesSubplot - if has_cartopy and with_cartopy - else mpl.axes.Axes - ) - assert isinstance(ax, expected_ax_type) - - expected_lg_type = mpl.legend.Legend if with_legend else type(None) - assert isinstance(ax.get_legend(), expected_lg_type) - - mpl.pyplot.close(fig) - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_with_a_region(self, opts): - with argopy.set_options(src=self.src, ftp=self.local_ftp): - for arg in self.requests["region"]: - loader = ArgoDataFetcher(cache=True).region(arg).load() - self.__test_traj_plot(loader.index, opts) - - -@requires_gdac -@requires_matplotlib -class Test_bar_plot: - src = "gdac" - local_ftp = argopy.tutorial.open_dataset("gdac")[0] - requests = { - # "float": [[2901623], [2901623, 6901929, 5906072]], - # "profile": [[2901623, 12], [6901929, [5, 45]]], - "region": [ - [-60, -40, 40.0, 60.0, 0., 10.], - [-60, -40, 40.0, 60.0, 0., 10., "2007-08-01", "2007-09-01"], - ], - } - - opts = [(ws, by) for ws in [False, has_seaborn] for by in ['institution', 'profiler']] - opts_ids = ["with_seaborn=%s, by=%s" % (opt[0], opt[1]) for opt in opts] - - def __test_bar_plot(self, df, opts): - with_seaborn, by = opts - fig, ax = bar_plot(df, by=by, with_seaborn=with_seaborn) - assert isinstance(fig, mpl.figure.Figure) - mpl.pyplot.close(fig) - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_with_a_region(self, opts): - with argopy.set_options(src=self.src, ftp=self.local_ftp): - for arg in self.requests["region"]: - loader = ArgoDataFetcher().region(arg) - self.__test_bar_plot(loader.to_index(full=True), opts) - - -@requires_gdac -@requires_matplotlib -@requires_cartopy -class Test_scatter_map: - src = "gdac" - local_ftp = argopy.tutorial.open_dataset("gdac")[0] - requests = { - # "float": [[2901623], [2901623, 6901929, 5906072]], - # "profile": [[2901623, 12], [6901929, [5, 45]]], - "region": [ - [-60, -40, 40.0, 60.0, 0., 10.], - [-60, -40, 40.0, 60.0, 0., 10., "2007-08-01", "2007-09-01"], - ], - } - - opts = [(traj, lg) for traj in [True, False] for lg in [True, False]] - opts_ids = ["traj=%s, legend=%s" % (opt[0], opt[1]) for opt in opts] - - def __test(self, data, axis, opts): - traj, legend = opts - fig, ax = scatter_map( - data, x=axis[0], y=axis[1], hue=axis[2], traj=traj, traj_axis=axis[2], legend=legend - ) - assert isinstance(fig, mpl.figure.Figure) - - assert isinstance(ax, cartopy.mpl.geoaxes.GeoAxesSubplot) - - expected_lg_type = mpl.legend.Legend if legend else type(None) - assert isinstance(ax.get_legend(), expected_lg_type) - - mpl.pyplot.close(fig) - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_with_a_dataset_of_points(self, opts): - with argopy.set_options(src=self.src, ftp=self.local_ftp): - for arg in self.requests["region"]: - loader = ArgoDataFetcher(cache=True).region(arg).load() - with pytest.raises(InvalidDatasetStructure): - self.__test(loader.data, (None, None, None), opts) - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_with_a_dataset_of_profiles(self, opts): - with argopy.set_options(src=self.src, ftp=self.local_ftp): - for arg in self.requests["region"]: - loader = ArgoDataFetcher(cache=True).region(arg).load() - dsp = loader.data.argo.point2profile() - # with pytest.warns(UserWarning): - # self.__test(dsp, (None, None, None), opts) - self.__test(dsp.isel(N_LEVELS=0), (None, None, None), opts) - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_with_a_dataframe_of_index(self, opts): - with argopy.set_options(src=self.src, ftp=self.local_ftp): - for arg in self.requests["region"]: - loader = ArgoDataFetcher(cache=True).region(arg).load() - self.__test(loader.index, (None, None, None), opts) diff --git a/argopy/tests/test_related.py b/argopy/tests/test_related.py deleted file mode 100644 index 031d2710..00000000 --- a/argopy/tests/test_related.py +++ /dev/null @@ -1,327 +0,0 @@ -import pytest -import tempfile -import xarray as xr -import pandas as pd -from collections import ChainMap, OrderedDict -import shutil - -from mocked_http import mocked_httpserver, mocked_server_address -from utils import ( - requires_matplotlib, - requires_cartopy, - requires_oops, - has_matplotlib, - has_cartopy, - has_ipython, -) -import argopy -from argopy.related import ( - TopoFetcher, - ArgoNVSReferenceTables, - OceanOPSDeployments, - ArgoDocs, - load_dict, mapp_dict, - get_coriolis_profile_id, get_ea_profile_page -) -from argopy.utils.checkers import ( - is_list_of_strings, -) - -if has_matplotlib: - import matplotlib as mpl - -if has_cartopy: - import cartopy - -if has_ipython: - import IPython - - -class Test_TopoFetcher(): - box = [81, 123, -67, -54] - - def setup_class(self): - """setup any state specific to the execution of the given class""" - # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests - self.cachedir = tempfile.mkdtemp() - - def teardown_class(self): - """Cleanup once we are finished.""" - def remove_test_dir(): - shutil.rmtree(self.cachedir) - remove_test_dir() - - def make_a_fetcher(self, cached=False): - opts = {'ds': 'gebco', 'stride': [10, 10], 'server': mocked_server_address} - if cached: - opts = ChainMap(opts, {'cache': True, 'cachedir': self.cachedir}) - return TopoFetcher(self.box, **opts) - - def assert_fetcher(self, f): - ds = f.to_xarray() - assert isinstance(ds, xr.Dataset) - assert 'elevation' in ds.data_vars - - def test_load_mocked_server(self, mocked_httpserver): - """This will easily ensure that the module scope fixture is available to all methods !""" - assert True - - params = [True, False] - ids_params = ["cached=%s" % p for p in params] - @pytest.mark.parametrize("params", params, indirect=False, ids=ids_params) - def test_fetching(self, params): - fetcher = self.make_a_fetcher(cached=params) - self.assert_fetcher(fetcher) - - -class Test_ArgoNVSReferenceTables: - - def setup_class(self): - """setup any state specific to the execution of the given class""" - # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests - self.cachedir = tempfile.mkdtemp() - self.nvs = ArgoNVSReferenceTables(cache=True, cachedir=self.cachedir, nvs=mocked_server_address) - - def teardown_class(self): - """Cleanup once we are finished.""" - def remove_test_dir(): - shutil.rmtree(self.cachedir) - remove_test_dir() - - def test_load_mocked_server(self, mocked_httpserver): - """This will easily ensure that the module scope fixture is available to all methods !""" - assert True - - def test_valid_ref(self): - assert is_list_of_strings(self.nvs.valid_ref) - - opts = [3, 'R09'] - opts_ids = ["rtid is a %s" % type(o) for o in opts] - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_tbl(self, opts): - assert isinstance(self.nvs.tbl(opts), pd.DataFrame) - - opts = [3, 'R09'] - opts_ids = ["rtid is a %s" % type(o) for o in opts] - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_tbl_name(self, opts): - names = self.nvs.tbl_name(opts) - assert isinstance(names, tuple) - assert isinstance(names[0], str) - assert isinstance(names[1], str) - assert isinstance(names[2], str) - - def test_all_tbl(self): - all = self.nvs.all_tbl - assert isinstance(all, OrderedDict) - assert isinstance(all[list(all.keys())[0]], pd.DataFrame) - - def test_all_tbl_name(self): - all = self.nvs.all_tbl_name - assert isinstance(all, OrderedDict) - assert isinstance(all[list(all.keys())[0]], tuple) - - opts = ["ld+json", "rdf+xml", "text/turtle", "invalid"] - opts_ids = ["fmt=%s" % o for o in opts] - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_get_url(self, opts): - if opts != 'invalid': - url = self.nvs.get_url(3, fmt=opts) - assert isinstance(url, str) - if "json" in opts: - data = self.nvs.fs.open_json(url) - assert isinstance(data, dict) - elif "xml" in opts: - data = self.nvs.fs.fs.cat_file(url) - assert data[0:5] == b'= 1) # Make sure we loaded the index file content - - @pytest.mark.parametrize( - "ftp_host", - ["invalid", "https://invalid_ftp", "ftp://invalid_ftp"], - indirect=False, - ) - def test_hosts_invalid(self, ftp_host): - # Invalid servers: - with pytest.raises(FtpPathError): - self.indexstore(host=ftp_host) - - def test_index(self): - def new_idx(): - return self.indexstore( - host=self.host, index_file=self.index_file, cache=False - ) - - self.assert_index(new_idx().load()) - self.assert_index(new_idx().load(force=True)) - - N = np.random.randint(1, 100 + 1) - idx = new_idx().load(nrows=N) - self.assert_index(idx) - assert idx.index.shape[0] == N - # Since no search was triggered: - assert idx.N_FILES == idx.N_RECORDS - - with pytest.raises(OptionValueError): - idx = self.indexstore( - host=self.host, index_file="ar_greylist.txt", cache=False - ) - - # @pytest.mark.parametrize( - # "a_search", search_scenarios, indirect=True, ids=search_scenarios_ids - # ) - # def test_a_search(self, mocked_httpserver, a_search): - # self.assert_search(a_search, cacheable=False) - - @pytest.mark.parametrize( - "a_search", search_scenarios_bool, indirect=True, ids=search_scenarios_bool_ids - ) - def test_a_search_with_logical(self, mocked_httpserver, a_search): - self.assert_search(a_search, cacheable=False) - - def test_to_dataframe_index(self): - idx = self.new_idx() - assert isinstance(idx.to_dataframe(), pd.core.frame.DataFrame) - - df = idx.to_dataframe(index=True) - assert df.shape[0] == idx.N_RECORDS - - df = idx.to_dataframe() - assert df.shape[0] == idx.N_RECORDS - - N = np.random.randint(1, 20 + 1) - df = idx.to_dataframe(index=True, nrows=N) - assert df.shape[0] == N - - def test_to_dataframe_search(self): - idx = self.new_idx() - wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] - idx = idx.search_wmo(wmo) - - df = idx.to_dataframe() - assert isinstance(df, pd.core.frame.DataFrame) - assert df.shape[0] == idx.N_MATCH - - N = np.random.randint(1, 10 + 1) - df = idx.to_dataframe(nrows=N) - assert df.shape[0] == N - - def test_caching_index(self): - idx = self.new_idx(cache=True) - self.assert_index(idx, cacheable=True) - - def test_caching_search(self): - idx = self.new_idx(cache=True) - wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] - idx.search_wmo(wmo) - self.assert_search(idx, cacheable=True) - - @pytest.mark.parametrize( - "index", - [False, True], - indirect=False, - ids=["index=%s" % i for i in [False, True]], - ) - def test_read_wmo(self, index): - wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] - idx = self.new_idx().search_wmo(wmo) - WMOs = idx.read_wmo(index=index) - if index: - assert [str(w).isdigit() for w in WMOs] - else: - assert len(WMOs) == len(wmo) - - @pytest.mark.parametrize( - "index", - [False, True], - indirect=False, - ids=["index=%s" % i for i in [False, True]], - ) - def test_read_params(self, index): - wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] - idx = self.new_idx().search_wmo(wmo) - if self.network == "bgc": - params = idx.read_params(index=index) - assert is_list_of_strings(params) - else: - with pytest.raises(InvalidDatasetStructure): - idx.read_params(index=index) - - @pytest.mark.parametrize( - "index", - [False, True], - indirect=False, - ids=["index=%s" % i for i in [False, True]], - ) - def test_records_per_wmo(self, index): - wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] - idx = self.new_idx().search_wmo(wmo) - C = idx.records_per_wmo(index=index) - for w in C: - assert str(C[w]).isdigit() - - def test_to_indexfile(self): - # Create a store and make a simple float search: - idx0 = self.new_idx() - wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] - idx0 = idx0.search_wmo(wmo) - - # Then save this search as a new Argo index file: - tf = tempfile.NamedTemporaryFile(delete=False) - new_indexfile = idx0.to_indexfile(tf.name) - - # Finally try to load the new index file, like it was an official one: - idx = self.new_idx( - host=os.path.dirname(new_indexfile), - index_file=os.path.basename(new_indexfile), - convention=idx0.convention, - ) - self.assert_index(idx.load()) - - # Cleanup - tf.close() - -############################ -# TESTS FOR PANDAS BACKEND # -############################ - -# @skip_this -class Test_IndexStore_pandas_CORE(IndexStore_test_proto): - network = "core" - indexstore = indexstore_pandas - index_file = "ar_index_global_prof.txt" - -# @skip_this -class Test_IndexStore_pandas_BGC_synthetic(IndexStore_test_proto): - network = "bgc" - indexstore = indexstore_pandas - index_file = "argo_synthetic-profile_index.txt" - -# @skip_this -class Test_IndexStore_pandas_BGC_bio(IndexStore_test_proto): - network = "bgc" - indexstore = indexstore_pandas - index_file = "argo_bio-profile_index.txt" - -############################# -# TESTS FOR PYARROW BACKEND # -############################# - -# @skip_this -@skip_pyarrow -class Test_IndexStore_pyarrow_CORE(IndexStore_test_proto): - network = "core" - from argopy.stores.argo_index_pa import indexstore_pyarrow - - indexstore = indexstore_pyarrow - index_file = "ar_index_global_prof.txt" - -# @skip_this -@skip_pyarrow -class Test_IndexStore_pyarrow_BGC_bio(IndexStore_test_proto): - network = "bgc" - from argopy.stores.argo_index_pa import indexstore_pyarrow - - indexstore = indexstore_pyarrow - index_file = "argo_bio-profile_index.txt" - - -# @skip_this -@skip_pyarrow -class Test_IndexStore_pyarrow_BGC_synthetic(IndexStore_test_proto): - network = "bgc" - from argopy.stores.argo_index_pa import indexstore_pyarrow - - indexstore = indexstore_pyarrow - index_file = "argo_synthetic-profile_index.txt" diff --git a/argopy/tests/test_tutorial.py b/argopy/tests/test_tutorial.py deleted file mode 100644 index 17a5e8ca..00000000 --- a/argopy/tests/test_tutorial.py +++ /dev/null @@ -1,27 +0,0 @@ -import pytest -import argopy -from utils import requires_connection - - -def test_invalid_dataset(): - with pytest.raises(ValueError): - argopy.tutorial.open_dataset('invalid_dataset') - - -@requires_connection -def test_gdac_dataset(): - ftproot, flist = argopy.tutorial.open_dataset('gdac') - assert isinstance(ftproot, str) - assert isinstance(flist, list) - - -@requires_connection -def test_weekly_index_dataset(): - rpath, txtfile = argopy.tutorial.open_dataset('weekly_index_prof') - assert isinstance(txtfile, str) - - -@requires_connection -def test_global_index_dataset(): - rpath, txtfile = argopy.tutorial.open_dataset('global_index_prof') - assert isinstance(txtfile, str) diff --git a/argopy/tests/test_utils_accessories.py b/argopy/tests/test_utils_accessories.py deleted file mode 100644 index f786d8c1..00000000 --- a/argopy/tests/test_utils_accessories.py +++ /dev/null @@ -1,86 +0,0 @@ -import pytest -from argopy.utils.accessories import float_wmo, Registry - - -class Test_float_wmo(): - - def test_init(self): - assert isinstance(float_wmo(2901746), float_wmo) - assert isinstance(float_wmo(float_wmo(2901746)), float_wmo) - - def test_isvalid(self): - assert float_wmo(2901746).isvalid - assert not float_wmo(12, errors='ignore').isvalid - - def test_ppt(self): - assert isinstance(str(float_wmo(2901746)), str) - assert isinstance(repr(float_wmo(2901746)), str) - - def test_comparisons(self): - assert float_wmo(2901746) == float_wmo(2901746) - assert float_wmo(2901746) != float_wmo(2901745) - assert float_wmo(2901746) >= float_wmo(2901746) - assert float_wmo(2901746) > float_wmo(2901745) - assert float_wmo(2901746) <= float_wmo(2901746) - assert float_wmo(2901746) < float_wmo(2901747) - - def test_hashable(self): - assert isinstance(hash(float_wmo(2901746)), int) - - -class Test_Registry(): - - opts = [(None, 'str'), (['hello', 'world'], str), (None, float_wmo), ([2901746, 4902252], float_wmo)] - opts_ids = ["%s, %s" % ((lambda x: 'iterlist' if x is not None else x)(opt[0]), repr(opt[1])) for opt in opts] - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_init(self, opts): - assert isinstance(Registry(opts[0], dtype=opts[1]), Registry) - - opts = [(['hello', 'world'], str), ([2901746, 4902252], float_wmo)] - opts_ids = ["%s, %s" % ((lambda x: 'iterlist' if x is not None else x)(opt[0]), repr(opt[1])) for opt in opts] - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_commit(self, opts): - R = Registry(dtype=opts[1]) - R.commit(opts[0]) - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_append(self, opts): - R = Registry(dtype=opts[1]) - R.append(opts[0][0]) - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_extend(self, opts): - R = Registry(dtype=opts[1]) - R.append(opts[0]) - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_insert(self, opts): - R = Registry(opts[0][0], dtype=opts[1]) - R.insert(0, opts[0][-1]) - assert R[0] == opts[0][-1] - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_remove(self, opts): - R = Registry(opts[0], dtype=opts[1]) - R.remove(opts[0][0]) - assert opts[0][0] not in R - - @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) - def test_copy(self, opts): - R = Registry(opts[0], dtype=opts[1]) - assert R == R.copy() - - bad_opts = [(['hello', 12], str), ([2901746, 1], float_wmo)] - bad_opts_ids = ["%s, %s" % ((lambda x: 'iterlist' if x is not None else x)(opt[0]), repr(opt[1])) for opt in opts] - - @pytest.mark.parametrize("opts", bad_opts, indirect=False, ids=bad_opts_ids) - def test_invalid_dtype(self, opts): - with pytest.raises(ValueError): - Registry(opts[0][0], dtype=opts[1], invalid='raise').commit(opts[0][-1]) - with pytest.warns(UserWarning): - Registry(opts[0][0], dtype=opts[1], invalid='warn').commit(opts[0][-1]) - # Raise nothing: - Registry(opts[0][0], dtype=opts[1], invalid='ignore').commit(opts[0][-1]) - diff --git a/argopy/tests/test_utils_caching.py b/argopy/tests/test_utils_caching.py deleted file mode 100644 index e0841fdb..00000000 --- a/argopy/tests/test_utils_caching.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -import pandas as pd -import argopy -import tempfile -from argopy import DataFetcher as ArgoDataFetcher -from utils import ( - requires_gdac, -) -from argopy.utils.caching import lscache, clear_cache - - -@requires_gdac -def test_clear_cache(): - ftproot, flist = argopy.tutorial.open_dataset("gdac") - with tempfile.TemporaryDirectory() as cachedir: - with argopy.set_options(cachedir=cachedir): - loader = ArgoDataFetcher(src="gdac", ftp=ftproot, cache=True).profile(2902696, 12) - loader.to_xarray() - clear_cache() - assert os.path.exists(cachedir) is True - assert len(os.listdir(cachedir)) == 0 - - -@requires_gdac -def test_lscache(): - ftproot, flist = argopy.tutorial.open_dataset("gdac") - with tempfile.TemporaryDirectory() as cachedir: - with argopy.set_options(cachedir=cachedir): - loader = ArgoDataFetcher(src="gdac", ftp=ftproot, cache=True).profile(2902696, 12) - loader.to_xarray() - result = lscache(cache_path=cachedir, prt=True) - assert isinstance(result, str) - - result = lscache(cache_path=cachedir, prt=False) - assert isinstance(result, pd.DataFrame) - diff --git a/argopy/tests/test_utils_checkers.py b/argopy/tests/test_utils_checkers.py deleted file mode 100644 index b8c2d53d..00000000 --- a/argopy/tests/test_utils_checkers.py +++ /dev/null @@ -1,226 +0,0 @@ -import pytest -import numpy as np -from mocked_http import mocked_httpserver, mocked_server_address -from utils import ( - requires_erddap, -) -import argopy -from argopy.errors import FtpPathError -from argopy.utils.checkers import ( - is_box, is_indexbox, - check_wmo, is_wmo, - check_cyc, is_cyc, - check_gdac_path, - isconnected, urlhaskeyword, isAPIconnected, erddap_ds_exists, isalive -) - - -class Test_is_box: - @pytest.fixture(autouse=True) - def create_data(self): - self.BOX3d = [0, 20, 40, 60, 0, 1000] - self.BOX4d = [0, 20, 40, 60, 0, 1000, "2001-01", "2001-6"] - - def test_box_ok(self): - assert is_box(self.BOX3d) - assert is_box(self.BOX4d) - - def test_box_notok(self): - for box in [[], list(range(0, 12))]: - with pytest.raises(ValueError): - is_box(box) - with pytest.raises(ValueError): - is_box(box, errors="raise") - assert not is_box(box, errors="ignore") - - def test_box_invalid_num(self): - for i in [0, 1, 2, 3, 4, 5]: - box = self.BOX3d - box[i] = "str" - with pytest.raises(ValueError): - is_box(box) - with pytest.raises(ValueError): - is_box(box, errors="raise") - assert not is_box(box, errors="ignore") - - def test_box_invalid_range(self): - for i in [0, 1, 2, 3, 4, 5]: - box = self.BOX3d - box[i] = -1000 - with pytest.raises(ValueError): - is_box(box) - with pytest.raises(ValueError): - is_box(box, errors="raise") - assert not is_box(box, errors="ignore") - - def test_box_invalid_str(self): - for i in [6, 7]: - box = self.BOX4d - box[i] = "str" - with pytest.raises(ValueError): - is_box(box) - with pytest.raises(ValueError): - is_box(box, errors="raise") - assert not is_box(box, errors="ignore") - - -class Test_is_indexbox: - @pytest.fixture(autouse=True) - def create_data(self): - self.BOX2d = [0, 20, 40, 60] - self.BOX3d = [0, 20, 40, 60, "2001-01", "2001-6"] - - def test_box_ok(self): - assert is_indexbox(self.BOX2d) - assert is_indexbox(self.BOX3d) - - def test_box_notok(self): - for box in [[], list(range(0, 12))]: - with pytest.raises(ValueError): - is_indexbox(box) - with pytest.raises(ValueError): - is_indexbox(box, errors="raise") - assert not is_indexbox(box, errors="ignore") - - def test_box_invalid_num(self): - for i in [0, 1, 2, 3]: - box = self.BOX2d - box[i] = "str" - with pytest.raises(ValueError): - is_indexbox(box) - with pytest.raises(ValueError): - is_indexbox(box, errors="raise") - assert not is_indexbox(box, errors="ignore") - - def test_box_invalid_range(self): - for i in [0, 1, 2, 3]: - box = self.BOX2d - box[i] = -1000 - with pytest.raises(ValueError): - is_indexbox(box) - with pytest.raises(ValueError): - is_indexbox(box, errors="raise") - assert not is_indexbox(box, errors="ignore") - - def test_box_invalid_str(self): - for i in [4, 5]: - box = self.BOX3d - box[i] = "str" - with pytest.raises(ValueError): - is_indexbox(box) - with pytest.raises(ValueError): - is_indexbox(box, errors="raise") - assert not is_indexbox(box, errors="ignore") - - -def test_is_wmo(): - assert is_wmo(12345) - assert is_wmo([12345]) - assert is_wmo([12345, 1234567]) - - with pytest.raises(ValueError): - is_wmo(1234, errors="raise") - with pytest.raises(ValueError): - is_wmo(-1234, errors="raise") - with pytest.raises(ValueError): - is_wmo(1234.12, errors="raise") - with pytest.raises(ValueError): - is_wmo(12345.7, errors="raise") - - with pytest.warns(UserWarning): - is_wmo(1234, errors="warn") - with pytest.warns(UserWarning): - is_wmo(-1234, errors="warn") - with pytest.warns(UserWarning): - is_wmo(1234.12, errors="warn") - with pytest.warns(UserWarning): - is_wmo(12345.7, errors="warn") - - assert not is_wmo(12, errors="ignore") - assert not is_wmo(-12, errors="ignore") - assert not is_wmo(1234.12, errors="ignore") - assert not is_wmo(12345.7, errors="ignore") - - -def test_check_wmo(): - assert check_wmo(12345) == [12345] - assert check_wmo([1234567]) == [1234567] - assert check_wmo([12345, 1234567]) == [12345, 1234567] - assert check_wmo(np.array((12345, 1234567), dtype='int')) == [12345, 1234567] - - -def test_is_cyc(): - assert is_cyc(123) - assert is_cyc([123]) - assert is_cyc([12, 123, 1234]) - - with pytest.raises(ValueError): - is_cyc(12345, errors="raise") - with pytest.raises(ValueError): - is_cyc(-1234, errors="raise") - with pytest.raises(ValueError): - is_cyc(1234.12, errors="raise") - with pytest.raises(ValueError): - is_cyc(12345.7, errors="raise") - - with pytest.warns(UserWarning): - is_cyc(12345, errors="warn") - with pytest.warns(UserWarning): - is_cyc(-1234, errors="warn") - with pytest.warns(UserWarning): - is_cyc(1234.12, errors="warn") - with pytest.warns(UserWarning): - is_cyc(12345.7, errors="warn") - - assert not is_cyc(12345, errors="ignore") - assert not is_cyc(-12, errors="ignore") - assert not is_cyc(1234.12, errors="ignore") - assert not is_cyc(12345.7, errors="ignore") - - -def test_check_cyc(): - assert check_cyc(123) == [123] - assert check_cyc([12]) == [12] - assert check_cyc([12, 123]) == [12, 123] - assert check_cyc(np.array((123, 1234), dtype='int')) == [123, 1234] - - -def test_check_gdac_path(): - assert check_gdac_path("dummy_path", errors='ignore') is False - with pytest.raises(FtpPathError): - check_gdac_path("dummy_path", errors='raise') - with pytest.warns(UserWarning): - assert check_gdac_path("dummy_path", errors='warn') is False - - -def test_isconnected(mocked_httpserver): - assert isinstance(isconnected(host=mocked_server_address), bool) - assert isconnected(host="http://dummyhost") is False - - -def test_urlhaskeyword(mocked_httpserver): - url = "https://api.ifremer.fr/argopy/data/ARGO-FULL.json" - url.replace("https://api.ifremer.fr", mocked_server_address) - assert isinstance(urlhaskeyword(url, "label"), bool) - - -params = [mocked_server_address, - {"url": mocked_server_address + "/argopy/data/ARGO-FULL.json", "keyword": "label"} - ] -params_ids = ["url is a %s" % str(type(p)) for p in params] -@pytest.mark.parametrize("params", params, indirect=False, ids=params_ids) -def test_isalive(params, mocked_httpserver): - assert isinstance(isalive(params), bool) - - -@requires_erddap -@pytest.mark.parametrize("data", [True, False], indirect=False, ids=["data=%s" % t for t in [True, False]]) -def test_isAPIconnected(data, mocked_httpserver): - with argopy.set_options(erddap=mocked_server_address): - assert isinstance(isAPIconnected(src="erddap", data=data), bool) - - -def test_erddap_ds_exists(mocked_httpserver): - with argopy.set_options(erddap=mocked_server_address): - assert isinstance(erddap_ds_exists(ds="ArgoFloats"), bool) - assert erddap_ds_exists(ds="DummyDS") is False diff --git a/argopy/tests/test_utils_chunking.py b/argopy/tests/test_utils_chunking.py deleted file mode 100644 index 3aee8c86..00000000 --- a/argopy/tests/test_utils_chunking.py +++ /dev/null @@ -1,196 +0,0 @@ -import pytest -import types -import numpy as np -import pandas as pd - -from argopy.errors import InvalidFetcherAccessPoint -from argopy.utils.chunking import Chunker -from argopy.utils.checkers import is_box - - -class Test_Chunker: - @pytest.fixture(autouse=True) - def create_data(self): - self.WMO = [ - 6902766, - 6902772, - 6902914, - 6902746, - 6902916, - 6902915, - 6902757, - 6902771, - ] - self.BOX3d = [0, 20, 40, 60, 0, 1000] - self.BOX4d = [0, 20, 40, 60, 0, 1000, "2001-01", "2001-6"] - - def test_InvalidFetcherAccessPoint(self): - with pytest.raises(InvalidFetcherAccessPoint): - Chunker({"invalid": self.WMO}) - - def test_invalid_chunks(self): - with pytest.raises(ValueError): - Chunker({"box": self.BOX3d}, chunks='toto') - - def test_invalid_chunksize(self): - with pytest.raises(ValueError): - Chunker({"box": self.BOX3d}, chunksize='toto') - - def test_chunk_wmo(self): - C = Chunker({"wmo": self.WMO}) - assert all( - [all(isinstance(x, int) for x in chunk) for chunk in C.fit_transform()] - ) - - C = Chunker({"wmo": self.WMO}, chunks="auto") - assert all( - [all(isinstance(x, int) for x in chunk) for chunk in C.fit_transform()] - ) - - C = Chunker({"wmo": self.WMO}, chunks={"wmo": 1}) - assert all( - [all(isinstance(x, int) for x in chunk) for chunk in C.fit_transform()] - ) - assert len(C.fit_transform()) == 1 - - with pytest.raises(ValueError): - Chunker({"wmo": self.WMO}, chunks=["wmo", 1]) - - C = Chunker({"wmo": self.WMO}) - assert isinstance(C.this_chunker, types.FunctionType) or isinstance( - C.this_chunker, types.MethodType - ) - - def test_chunk_box3d(self): - C = Chunker({"box": self.BOX3d}) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - - C = Chunker({"box": self.BOX3d}, chunks="auto") - assert all([is_box(chunk) for chunk in C.fit_transform()]) - - C = Chunker({"box": self.BOX3d}, chunks={"lon": 12, "lat": 1, "dpt": 1}) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - assert len(C.fit_transform()) == 12 - - C = Chunker( - {"box": self.BOX3d}, chunks={"lat": 1, "dpt": 1}, chunksize={"lon": 10} - ) - chunks = C.fit_transform() - assert all([is_box(chunk) for chunk in chunks]) - assert chunks[0][1] - chunks[0][0] == 10 - - C = Chunker({"box": self.BOX3d}, chunks={"lon": 1, "lat": 12, "dpt": 1}) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - assert len(C.fit_transform()) == 12 - - C = Chunker( - {"box": self.BOX3d}, chunks={"lon": 1, "dpt": 1}, chunksize={"lat": 10} - ) - chunks = C.fit_transform() - assert all([is_box(chunk) for chunk in chunks]) - assert chunks[0][3] - chunks[0][2] == 10 - - C = Chunker({"box": self.BOX3d}, chunks={"lon": 1, "lat": 1, "dpt": 12}) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - assert len(C.fit_transform()) == 12 - - C = Chunker( - {"box": self.BOX3d}, chunks={"lon": 1, "lat": 1}, chunksize={"dpt": 10} - ) - chunks = C.fit_transform() - assert all([is_box(chunk) for chunk in chunks]) - assert chunks[0][5] - chunks[0][4] == 10 - - C = Chunker({"box": self.BOX3d}, chunks={"lon": 4, "lat": 2, "dpt": 1}) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - assert len(C.fit_transform()) == 2 * 4 - - C = Chunker({"box": self.BOX3d}, chunks={"lon": 2, "lat": 3, "dpt": 4}) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - assert len(C.fit_transform()) == 2 * 3 * 4 - - with pytest.raises(ValueError): - Chunker({"box": self.BOX3d}, chunks=["lon", 1]) - - C = Chunker({"box": self.BOX3d}) - assert isinstance(C.this_chunker, types.FunctionType) or isinstance( - C.this_chunker, types.MethodType - ) - - def test_chunk_box4d(self): - C = Chunker({"box": self.BOX4d}) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - - C = Chunker({"box": self.BOX4d}, chunks="auto") - assert all([is_box(chunk) for chunk in C.fit_transform()]) - - C = Chunker( - {"box": self.BOX4d}, chunks={"lon": 2, "lat": 1, "dpt": 1, "time": 1} - ) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - assert len(C.fit_transform()) == 2 - - C = Chunker( - {"box": self.BOX4d}, - chunks={"lat": 1, "dpt": 1, "time": 1}, - chunksize={"lon": 10}, - ) - chunks = C.fit_transform() - assert all([is_box(chunk) for chunk in chunks]) - assert chunks[0][1] - chunks[0][0] == 10 - - C = Chunker( - {"box": self.BOX4d}, chunks={"lon": 1, "lat": 2, "dpt": 1, "time": 1} - ) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - assert len(C.fit_transform()) == 2 - - C = Chunker( - {"box": self.BOX4d}, - chunks={"lon": 1, "dpt": 1, "time": 1}, - chunksize={"lat": 10}, - ) - chunks = C.fit_transform() - assert all([is_box(chunk) for chunk in chunks]) - assert chunks[0][3] - chunks[0][2] == 10 - - C = Chunker( - {"box": self.BOX4d}, chunks={"lon": 1, "lat": 1, "dpt": 2, "time": 1} - ) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - assert len(C.fit_transform()) == 2 - - C = Chunker( - {"box": self.BOX4d}, - chunks={"lon": 1, "lat": 1, "time": 1}, - chunksize={"dpt": 10}, - ) - chunks = C.fit_transform() - assert all([is_box(chunk) for chunk in chunks]) - assert chunks[0][5] - chunks[0][4] == 10 - - C = Chunker( - {"box": self.BOX4d}, chunks={"lon": 1, "lat": 1, "dpt": 1, "time": 2} - ) - assert all([is_box(chunk) for chunk in C.fit_transform()]) - assert len(C.fit_transform()) == 2 - - C = Chunker( - {"box": self.BOX4d}, - chunks={"lon": 1, "lat": 1, "dpt": 1}, - chunksize={"time": 5}, - ) - chunks = C.fit_transform() - assert all([is_box(chunk) for chunk in chunks]) - assert np.timedelta64( - pd.to_datetime(chunks[0][7]) - pd.to_datetime(chunks[0][6]), "D" - ) <= np.timedelta64(5, "D") - - with pytest.raises(ValueError): - Chunker({"box": self.BOX4d}, chunks=["lon", 1]) - - C = Chunker({"box": self.BOX4d}) - assert isinstance(C.this_chunker, types.FunctionType) or isinstance( - C.this_chunker, types.MethodType - ) - diff --git a/argopy/tests/test_utils_compute.py b/argopy/tests/test_utils_compute.py deleted file mode 100644 index 2806fd14..00000000 --- a/argopy/tests/test_utils_compute.py +++ /dev/null @@ -1,75 +0,0 @@ -import pytest -import numpy as np -import xarray as xr - -from argopy.utils.compute import linear_interpolation_remap - - -class Test_linear_interpolation_remap: - @pytest.fixture(autouse=True) - def create_data(self): - # create fake data to test interpolation: - temp = np.random.rand(200, 100) - pres = np.sort( - np.floor( - np.zeros([200, 100]) - + np.linspace(50, 950, 100) - + np.random.randint(-5, 5, [200, 100]) - ) - ) - self.dsfake = xr.Dataset( - { - "TEMP": (["N_PROF", "N_LEVELS"], temp), - "PRES": (["N_PROF", "N_LEVELS"], pres), - }, - coords={ - "N_PROF": ("N_PROF", range(200)), - "N_LEVELS": ("N_LEVELS", range(100)), - "Z_LEVELS": ("Z_LEVELS", np.arange(100, 900, 20)), - }, - ) - - def test_interpolation(self): - # Run it with success: - dsi = linear_interpolation_remap( - self.dsfake["PRES"], - self.dsfake["TEMP"], - self.dsfake["Z_LEVELS"], - z_dim="N_LEVELS", - z_regridded_dim="Z_LEVELS", - ) - assert "remapped" in dsi.dims - - def test_interpolation_1d(self): - # Run it with success: - dsi = linear_interpolation_remap( - self.dsfake["PRES"].isel(N_PROF=0), - self.dsfake["TEMP"].isel(N_PROF=0), - self.dsfake["Z_LEVELS"], - z_regridded_dim="Z_LEVELS", - ) - assert "remapped" in dsi.dims - - def test_error_zdim(self): - # Test error: - # catches error from _regular_interp linked to z_dim - with pytest.raises(RuntimeError): - linear_interpolation_remap( - self.dsfake["PRES"], - self.dsfake["TEMP"], - self.dsfake["Z_LEVELS"], - z_regridded_dim="Z_LEVELS", - ) - - def test_error_ds(self): - # Test error: - # catches error from linear_interpolation_remap linked to datatype - with pytest.raises(ValueError): - linear_interpolation_remap( - self.dsfake["PRES"], - self.dsfake, - self.dsfake["Z_LEVELS"], - z_dim="N_LEVELS", - z_regridded_dim="Z_LEVELS", - ) - diff --git a/argopy/tests/test_utils_format.py b/argopy/tests/test_utils_format.py deleted file mode 100644 index 6d3c161c..00000000 --- a/argopy/tests/test_utils_format.py +++ /dev/null @@ -1,60 +0,0 @@ -import os -import pytest -import argopy -from argopy.utils.format import format_oneline, argo_split_path - - -def test_format_oneline(): - s = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore" - assert isinstance(format_oneline(s), str) - assert isinstance(format_oneline(s[0:5]), str) - s = format_oneline(s, max_width=12) - assert isinstance(s, str) and len(s) == 12 - - -class Test_argo_split_path: - ############# - # UTILITIES # - ############# - # src = "https://data-argo.ifremer.fr/dac" - src = argopy.tutorial.open_dataset("gdac")[0] + "/dac" - list_of_files = [ - src + "/bodc/6901929/6901929_prof.nc", # core / multi-profile - src + "/coriolis/3902131/3902131_Sprof.nc", # bgc / synthetic multi-profile - - src + "/meds/4901079/profiles/D4901079_110.nc", # core / mono-profile / Delayed - src + "/aoml/13857/profiles/R13857_001.nc", # core / mono-profile / Real - - src + "/coriolis/3902131/profiles/SD3902131_001.nc", # bgc / synthetic mono-profile / Delayed - src + "/coriolis/3902131/profiles/SD3902131_001D.nc", # bgc / synthetic mono-profile / Delayed / Descent - src + "/coriolis/6903247/profiles/SR6903247_134.nc", # bgc / synthetic mono-profile / Real - src + "/coriolis/6903247/profiles/SR6903247_134D.nc", # bgc / synthetic mono-profile / Real / Descent - - src + "/coriolis/3902131/profiles/BR3902131_001.nc", # bgc / mono-profile / Real - src + "/coriolis/3902131/profiles/BR3902131_001D.nc", # bgc / mono-profile / Real / Descent - - src + "/aoml/5900446/5900446_Dtraj.nc", # traj / Delayed - src + "/csio/2902696/2902696_Rtraj.nc", # traj / Real - - src + "/coriolis/3902131/3902131_BRtraj.nc", # bgc / traj / Real - # src + "/coriolis/6903247/6903247_BRtraj.nc", # bgc / traj / Real - - src + "/incois/2902269/2902269_tech.nc", # technical - # src + "/nmdis/2901623/2901623_tech.nc", # technical - - src + "/jma/4902252/4902252_meta.nc", # meta-data - # src + "/coriolis/1900857/1900857_meta.nc", # meta-data - ] - list_of_files = [f.replace("/", os.path.sep) for f in list_of_files] - - ######### - # TESTS # - ######### - - @pytest.mark.parametrize("file", list_of_files, - indirect=False) - def test_argo_split_path(self, file): - desc = argo_split_path(file) - assert isinstance(desc, dict) - for key in ['origin', 'path', 'name', 'type', 'extension', 'wmo', 'dac']: - assert key in desc diff --git a/argopy/tests/test_utils_geo.py b/argopy/tests/test_utils_geo.py deleted file mode 100644 index 609242c9..00000000 --- a/argopy/tests/test_utils_geo.py +++ /dev/null @@ -1,42 +0,0 @@ -import pytest -import numpy as np -import pandas as pd -from argopy.utils.geo import wmo2box, wrap_longitude, toYearFraction, YearFraction_to_datetime -from argopy.utils.checkers import is_box - - -def test_wmo2box(): - with pytest.raises(ValueError): - wmo2box(12) - with pytest.raises(ValueError): - wmo2box(8000) - with pytest.raises(ValueError): - wmo2box(2000) - - def complete_box(b): - b2 = b.copy() - b2.insert(4, 0.) - b2.insert(5, 10000.) - return b2 - - assert is_box(complete_box(wmo2box(1212))) - assert is_box(complete_box(wmo2box(3324))) - assert is_box(complete_box(wmo2box(5402))) - assert is_box(complete_box(wmo2box(7501))) - - -def test_wrap_longitude(): - assert wrap_longitude(np.array([-20])) == 340 - assert wrap_longitude(np.array([40])) == 40 - assert np.all(np.equal(wrap_longitude(np.array([340, 20])), np.array([340, 380]))) - - -def test_toYearFraction(): - assert toYearFraction(pd.to_datetime('202001010000')) == 2020 - assert toYearFraction(pd.to_datetime('202001010000', utc=True)) == 2020 - assert toYearFraction(pd.to_datetime('202001010000')+pd.offsets.DateOffset(years=1)) == 2021 - - -def test_YearFraction_to_datetime(): - assert YearFraction_to_datetime(2020) == pd.to_datetime('202001010000') - assert YearFraction_to_datetime(2020+1) == pd.to_datetime('202101010000') diff --git a/argopy/tests/test_utils_lists.py b/argopy/tests/test_utils_lists.py deleted file mode 100644 index 06aaa893..00000000 --- a/argopy/tests/test_utils_lists.py +++ /dev/null @@ -1,7 +0,0 @@ -# import pytest -from argopy.utils.checkers import is_list_of_strings -from argopy.utils.lists import list_multiprofile_file_variables - - -def test_list_multiprofile_file_variables(): - assert is_list_of_strings(list_multiprofile_file_variables()) diff --git a/argopy/tests/test_utils_locals.py b/argopy/tests/test_utils_locals.py deleted file mode 100644 index 4ad32bfd..00000000 --- a/argopy/tests/test_utils_locals.py +++ /dev/null @@ -1,22 +0,0 @@ -import os -import pytest -import io -import argopy -from argopy.utils.locals import modified_environ - - -@pytest.mark.parametrize("conda", [False, True], - indirect=False, - ids=["conda=%s" % str(p) for p in [False, True]]) -def test_show_versions(conda): - f = io.StringIO() - argopy.show_versions(file=f, conda=conda) - assert "SYSTEM" in f.getvalue() - - -def test_modified_environ(): - os.environ["DUMMY_ENV_ARGOPY"] = 'initial' - with modified_environ(DUMMY_ENV_ARGOPY='toto'): - assert os.environ['DUMMY_ENV_ARGOPY'] == 'toto' - assert os.environ['DUMMY_ENV_ARGOPY'] == 'initial' - os.environ.pop('DUMMY_ENV_ARGOPY') diff --git a/argopy/tests/test_xarray_accessor.py b/argopy/tests/test_xarray_accessor.py deleted file mode 100644 index af8afdf7..00000000 --- a/argopy/tests/test_xarray_accessor.py +++ /dev/null @@ -1,238 +0,0 @@ -import os -import pytest -import warnings -import numpy as np -import tempfile -import xarray as xr - -import argopy -from argopy import DataFetcher as ArgoDataFetcher -from argopy.errors import InvalidDatasetStructure, OptionValueError -from utils import requires_gdac, _importorskip, _connectskip -from mocked_http import mocked_server_address - - -has_gsw, requires_gsw = _importorskip("gsw") -has_nogsw, requires_nogsw = _connectskip(not has_gsw, "that GSW module is NOT installed") - - -@pytest.fixture(scope="module") -def ds_pts(mocked_httpserver): - """ Create a dictionary of datasets to be used by tests - - Note that these datasets can be modified by tests, which can affect the behaviour of other tests ! - """ - data = {} - try: - for user_mode in ['standard', 'expert']: - data[user_mode] = ( - ArgoDataFetcher(src="erddap", mode=user_mode, server=mocked_server_address) - .region([-20, -16., 0, 1, 0, 100., "2004-01-01", "2004-01-31"]) - .load() - .data - ) - except Exception as e: - warnings.warn("Error when fetching tests data: %s" % str(e.args)) - pass - - if "expert" not in data or "standard" not in data: - # We don't have what we need for testing, skip this test module: - pytest.xfail("Could not retrieve erddap data in both standard and expert mode") - else: - return data - - -def test_point2profile(ds_pts): - assert "N_PROF" in ds_pts['standard'].argo.point2profile().dims - - -def test_profile2point(ds_pts): - with pytest.raises(InvalidDatasetStructure): - ds_pts['standard'].argo.profile2point() - - -def test_point2profile2point(ds_pts): - assert ds_pts['standard'].argo.point2profile().argo.profile2point().equals(ds_pts['standard']) - - -class Test_interp_std_levels: - def test_interpolation(self, ds_pts): - """Run with success""" - ds = ds_pts["standard"].argo.point2profile() - assert "PRES_INTERPOLATED" in ds.argo.interp_std_levels([20, 30, 40, 50]).dims - - def test_interpolation_expert(self, ds_pts): - """Run with success""" - ds = ds_pts["expert"].argo.point2profile() - assert "PRES_INTERPOLATED" in ds.argo.interp_std_levels([20, 30, 40, 50]).dims - - def test_std_error(self, ds_pts): - """Try to interpolate on a wrong axis""" - ds = ds_pts["standard"].argo.point2profile() - with pytest.raises(ValueError): - ds.argo.interp_std_levels([100, 20, 30, 40, 50]) - with pytest.raises(ValueError): - ds.argo.interp_std_levels([-20, 20, 30, 40, 50]) - with pytest.raises(ValueError): - ds.argo.interp_std_levels(12) - - -class Test_groupby_pressure_bins: - def test_groupby_ds_type(self, ds_pts): - """Run with success for standard/expert mode and point/profile""" - for user_mode, this in ds_pts.items(): - for format in ["point", "profile"]: - if format == 'profile': - that = this.argo.point2profile() - else: - that = this.copy() - bins = np.arange(0.0, np.max(that["PRES"]) + 10.0, 10.0) - assert "STD_PRES_BINS" in that.argo.groupby_pressure_bins(bins).coords - - def test_bins_error(self, ds_pts): - """Try to groupby over invalid bins """ - ds = ds_pts["standard"] - with pytest.raises(ValueError): - ds.argo.groupby_pressure_bins([100, 20, 30, 40, 50]) # un-sorted - with pytest.raises(ValueError): - ds.argo.groupby_pressure_bins([-20, 20, 30, 40, 50]) # Negative values - - def test_axis_error(self, ds_pts): - """Try to group by using invalid pressure axis """ - ds = ds_pts["standard"] - bins = np.arange(0.0, np.max(ds["PRES"]) + 10.0, 10.0) - with pytest.raises(ValueError): - ds.argo.groupby_pressure_bins(bins, axis='invalid') - - def test_empty_result(self, ds_pts): - """Try to groupby over bins without data""" - ds = ds_pts["standard"] - with pytest.warns(Warning): - out = ds.argo.groupby_pressure_bins([10000, 20000]) - assert out == None - - def test_all_select(self, ds_pts): - ds = ds_pts["standard"] - bins = np.arange(0.0, np.max(ds["PRES"]) + 10.0, 10.0) - for select in ["shallow", "deep", "middle", "random", "min", "max", "mean", "median"]: - assert "STD_PRES_BINS" in ds.argo.groupby_pressure_bins(bins).coords - - -class Test_teos10: - - @requires_nogsw - def test_gsw_not_available(self, ds_pts): - # Make sure we raise an error when GSW is not available - for key, this in ds_pts.items(): - that = this.copy() # To avoid modifying the original dataset - with pytest.raises(ModuleNotFoundError): - that.argo.teos10() - - @requires_gsw - def test_teos10_variables_default(self, ds_pts): - """Add default new set of TEOS10 variables""" - for key, this in ds_pts.items(): - for format in ["point", "profile"]: - that = this.copy() # To avoid modifying the original dataset - if format == "profile": - that = that.argo.point2profile() - that = that.argo.teos10() - assert "SA" in that.variables - assert "CT" in that.variables - - @requires_gsw - def test_teos10_variables_single(self, ds_pts): - """Add a single TEOS10 variables""" - for key, this in ds_pts.items(): - for format in ["point", "profile"]: - that = this.copy() # To avoid modifying the original dataset - if format == "profile": - that = that.argo.point2profile() - that = that.argo.teos10(["PV"]) - assert "PV" in that.variables - - @requires_gsw - def test_teos10_opt_variables_single(self, ds_pts): - """Add a single TEOS10 optional variables""" - for key, this in ds_pts.items(): - for format in ["point", "profile"]: - that = this.copy() # To avoid modifying the original dataset - if format == "profile": - that = that.argo.point2profile() - that = that.argo.teos10(["SOUND_SPEED"]) - assert "SOUND_SPEED" in that.variables - - @requires_gsw - def test_teos10_variables_inplace(self, ds_pts): - """Compute all default variables to a new dataset""" - for key, this in ds_pts.items(): - ds = this.argo.teos10(inplace=False) # So "SA" must be in 'ds' but not in 'this' - assert "SA" in ds.variables - assert "SA" not in this.variables - - @requires_gsw - def test_teos10_invalid_variable(self, ds_pts): - """Try to add an invalid variable""" - for key, this in ds_pts.items(): - for format in ["point", "profile"]: - that = this.copy() # To avoid modifying the original dataset - if format == "profile": - that = that.argo.point2profile() - with pytest.raises(ValueError): - that.argo.teos10(["InvalidVariable"]) - - -@requires_gsw -@requires_gdac -class Test_create_float_source: - local_ftp = argopy.tutorial.open_dataset("gdac")[0] - - def is_valid_mdata(self, this_mdata): - """Validate structure of the output dataset """ - check = [] - # Check for dimensions: - check.append(argopy.utils.is_list_equal(['m', 'n'], list(this_mdata.dims))) - # Check for coordinates: - check.append(argopy.utils.is_list_equal(['m', 'n'], list(this_mdata.coords))) - # Check for data variables: - check.append(np.all( - [v in this_mdata.data_vars for v in ['PRES', 'TEMP', 'PTMP', 'SAL', 'DATES', 'LAT', 'LONG', 'PROFILE_NO']])) - check.append(np.all( - [argopy.utils.is_list_equal(['n'], this_mdata[v].dims) for v in ['LONG', 'LAT', 'DATES', 'PROFILE_NO'] - if v in this_mdata.data_vars])) - check.append(np.all( - [argopy.utils.is_list_equal(['m', 'n'], this_mdata[v].dims) for v in ['PRES', 'TEMP', 'SAL', 'PTMP'] if - v in this_mdata.data_vars])) - return np.all(check) - - def test_error_user_mode(self): - with argopy.set_options(ftp=self.local_ftp): - with pytest.raises(InvalidDatasetStructure): - ds = ArgoDataFetcher(src="gdac", mode='standard').float([6901929, 2901623]).load().data - ds.argo.create_float_source() - - def test_opt_force(self): - with argopy.set_options(ftp=self.local_ftp): - expert_ds = ArgoDataFetcher(src="gdac", mode='expert').float([2901623]).load().data - - with pytest.raises(OptionValueError): - expert_ds.argo.create_float_source(force='dummy') - - ds_float_source = expert_ds.argo.create_float_source(path=None, force='default') - assert np.all([k in np.unique(expert_ds['PLATFORM_NUMBER']) for k in ds_float_source.keys()]) - assert np.all([isinstance(ds_float_source[k], xr.Dataset) for k in ds_float_source.keys()]) - assert np.all([self.is_valid_mdata(ds_float_source[k]) for k in ds_float_source.keys()]) - - ds_float_source = expert_ds.argo.create_float_source(path=None, force='raw') - assert np.all([k in np.unique(expert_ds['PLATFORM_NUMBER']) for k in ds_float_source.keys()]) - assert np.all([isinstance(ds_float_source[k], xr.Dataset) for k in ds_float_source.keys()]) - assert np.all([self.is_valid_mdata(ds_float_source[k]) for k in ds_float_source.keys()]) - - def test_filecreate(self): - with argopy.set_options(ftp=self.local_ftp): - expert_ds = ArgoDataFetcher(src="gdac", mode='expert').float([6901929, 2901623]).load().data - - N_file = len(np.unique(expert_ds['PLATFORM_NUMBER'])) - with tempfile.TemporaryDirectory() as folder_output: - expert_ds.argo.create_float_source(path=folder_output) - assert len(os.listdir(folder_output)) == N_file diff --git a/argopy/tests/test_xarray_engine.py b/argopy/tests/test_xarray_engine.py deleted file mode 100644 index b67701f0..00000000 --- a/argopy/tests/test_xarray_engine.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import pytest -import xarray as xr -import logging -import warnings -import argopy -from argopy.utils.format import argo_split_path - -log = logging.getLogger("argopy.tests.xarray.engine") - - -def print_desc(desc): - txt = [desc["type"]] - if "direction" in desc: - txt.append(desc["direction"]) - - if "data_mode" in desc: - txt.append(desc["data_mode"]) - - return ", ".join(txt) - - -class Test_Argo_Engine: - host = argopy.tutorial.open_dataset("gdac")[0] - src = host + "/dac" - list_of_files = [ - src + "/bodc/6901929/6901929_prof.nc", # core / multi-profile - src + "/coriolis/3902131/3902131_Sprof.nc", # bgc / synthetic multi-profile - src + "/meds/4901079/profiles/D4901079_110.nc", # core / mono-profile / Delayed - src + "/aoml/13857/profiles/R13857_001.nc", # core / mono-profile / Real - src - + "/coriolis/3902131/profiles/SD3902131_001.nc", # bgc / synthetic mono-profile / Delayed - src - + "/coriolis/3902131/profiles/SD3902131_001D.nc", # bgc / synthetic mono-profile / Delayed / Descent - src - + "/coriolis/6903247/profiles/SR6903247_134.nc", # bgc / synthetic mono-profile / Real - src - + "/coriolis/6903247/profiles/SR6903247_134D.nc", # bgc / synthetic mono-profile / Real / Descent - src - + "/coriolis/3902131/profiles/BR3902131_001.nc", # bgc / mono-profile / Real - src - + "/coriolis/3902131/profiles/BR3902131_001D.nc", # bgc / mono-profile / Real / Descent - src + "/aoml/5900446/5900446_Dtraj.nc", # traj / Delayed - src + "/csio/2902696/2902696_Rtraj.nc", # traj / Real - src + "/coriolis/3902131/3902131_BRtraj.nc", # bgc / traj / Real - # src + "/coriolis/6903247/6903247_BRtraj.nc", # bgc / traj / Real - src + "/incois/2902269/2902269_tech.nc", # technical - # src + "/nmdis/2901623/2901623_tech.nc", # technical - src + "/jma/4902252/4902252_meta.nc", # meta-data - # src + "/coriolis/1900857/1900857_meta.nc", # meta-data - ] - list_of_files = [f.replace("/", os.path.sep) for f in list_of_files] - - list_of_files_desc = [print_desc(argo_split_path(f)) for f in list_of_files] - # list_of_files_desc = [f for f in list_of_files] - - ############# - # UTILITIES # - ############# - - def how_many_casted(self, this_ds): - """Return the length of non casted variables in ds""" - # check which variables are not casted: - l = [] - for iv, v in enumerate(this_ds.variables): - if this_ds[v].dtype == "O": - l.append(v) - # log.debug("%i/%i variables not casted properly.\n%s" % (len(l), len(this_ds.variables), l)) - return len(l), len(this_ds.variables) - - ######### - # TESTS # - ######### - - @pytest.mark.parametrize( - "file", list_of_files, indirect=False, ids=list_of_files_desc - ) - def test_read(self, file, mocked_httpserver): - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", "invalid value encountered in cast", RuntimeWarning - ) - - # No Argo enfine: - # ds1 = xr.open_dataset(file) - # n1, N1, l1 = self.how_many_casted(ds1) - - # With Argo engine: - ds = xr.open_dataset(file, engine="argo") - n, N = self.how_many_casted(ds) - assert n == 0 From 824ef79467437761dfec14b970b6ee0658461d65 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 13:02:35 +0200 Subject: [PATCH 12/14] Update utils.py --- argopy/tests/helpers/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argopy/tests/helpers/utils.py b/argopy/tests/helpers/utils.py index 89d62833..930666ef 100644 --- a/argopy/tests/helpers/utils.py +++ b/argopy/tests/helpers/utils.py @@ -360,7 +360,7 @@ def cmd(access_right: AccessRight, mode="grant:r") -> List[str]: os.makedirs(folder_path, exist_ok=True) # Change permissions - check_output(cmd(AccessRight.READ_ONLY)) + warnings.warn(str(check_output(cmd(AccessRight.READ_ONLY)))) except FileExistsError: log.debug(f"Folder '{folder_path}' already exists.") From f523baa2021f397f05cee5989123c961be8e326d Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 13:09:43 +0200 Subject: [PATCH 13/14] Update test_options.py --- argopy/tests/test_options.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/argopy/tests/test_options.py b/argopy/tests/test_options.py index c701e2f5..c935189d 100644 --- a/argopy/tests/test_options.py +++ b/argopy/tests/test_options.py @@ -1,6 +1,6 @@ import os import pytest -import warnings +import platform import argopy from argopy.options import OPTIONS from argopy.errors import OptionValueError, FtpPathError, ErddapPathError @@ -53,14 +53,12 @@ def test_opt_dataset(): assert OPTIONS["dataset"] == "ref" -# @pytest.mark.skipif(True, reason="Need to be debugged for Windows support") +@pytest.mark.skipif(platform.system() == 'Windows', reason="Need to be debugged for Windows support") def test_opt_invalid_cachedir(): # Cachedir is created if not exist. # OptionValueError is raised when it's not writable folder_name = "read_only_folder" create_read_only_folder(folder_name) - warnings.warn(str(os.stat(folder_name).st_mode)) - warnings.warn(str(os.access(folder_name, os.W_OK))) with pytest.raises(OptionValueError): argopy.set_options(cachedir=folder_name) os.rmdir(folder_name) From 7a9ef654170bf4e7775e233cb3c191dd2c5d3351 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 28 Sep 2023 13:20:35 +0200 Subject: [PATCH 14/14] Resume all CI tests --- argopy/tests/test_errors.py | 46 ++ argopy/tests/test_fetchers_data_argovis.py | 235 ++++++++ argopy/tests/test_fetchers_data_erddap.py | 216 +++++++ argopy/tests/test_fetchers_data_erddap_bgc.py | 267 +++++++++ argopy/tests/test_fetchers_data_gdac.py | 230 +++++++ argopy/tests/test_fetchers_facade_data.py | 196 ++++++ argopy/tests/test_fetchers_facade_index.py | 131 ++++ argopy/tests/test_fetchers_index_erddap.py | 137 +++++ argopy/tests/test_fetchers_index_gdac.py | 201 +++++++ argopy/tests/test_fetchers_proto.py | 53 ++ argopy/tests/test_plot_argo_colors.py | 101 ++++ argopy/tests/test_plot_dashboards.py | 66 ++ argopy/tests/test_plot_plot.py | 212 +++++++ argopy/tests/test_related.py | 327 ++++++++++ argopy/tests/test_stores_fsspec.py | 563 ++++++++++++++++++ argopy/tests/test_stores_index.py | 494 +++++++++++++++ argopy/tests/test_tutorial.py | 27 + argopy/tests/test_utils_accessories.py | 86 +++ argopy/tests/test_utils_caching.py | 36 ++ argopy/tests/test_utils_checkers.py | 226 +++++++ argopy/tests/test_utils_chunking.py | 196 ++++++ argopy/tests/test_utils_compute.py | 75 +++ argopy/tests/test_utils_format.py | 60 ++ argopy/tests/test_utils_geo.py | 42 ++ argopy/tests/test_utils_lists.py | 7 + argopy/tests/test_utils_locals.py | 22 + argopy/tests/test_xarray_accessor.py | 238 ++++++++ argopy/tests/test_xarray_engine.py | 91 +++ docs/whats-new.rst | 1 + 29 files changed, 4582 insertions(+) create mode 100644 argopy/tests/test_errors.py create mode 100644 argopy/tests/test_fetchers_data_argovis.py create mode 100644 argopy/tests/test_fetchers_data_erddap.py create mode 100644 argopy/tests/test_fetchers_data_erddap_bgc.py create mode 100644 argopy/tests/test_fetchers_data_gdac.py create mode 100644 argopy/tests/test_fetchers_facade_data.py create mode 100644 argopy/tests/test_fetchers_facade_index.py create mode 100644 argopy/tests/test_fetchers_index_erddap.py create mode 100644 argopy/tests/test_fetchers_index_gdac.py create mode 100644 argopy/tests/test_fetchers_proto.py create mode 100644 argopy/tests/test_plot_argo_colors.py create mode 100644 argopy/tests/test_plot_dashboards.py create mode 100644 argopy/tests/test_plot_plot.py create mode 100644 argopy/tests/test_related.py create mode 100644 argopy/tests/test_stores_fsspec.py create mode 100644 argopy/tests/test_stores_index.py create mode 100644 argopy/tests/test_tutorial.py create mode 100644 argopy/tests/test_utils_accessories.py create mode 100644 argopy/tests/test_utils_caching.py create mode 100644 argopy/tests/test_utils_checkers.py create mode 100644 argopy/tests/test_utils_chunking.py create mode 100644 argopy/tests/test_utils_compute.py create mode 100644 argopy/tests/test_utils_format.py create mode 100644 argopy/tests/test_utils_geo.py create mode 100644 argopy/tests/test_utils_lists.py create mode 100644 argopy/tests/test_utils_locals.py create mode 100644 argopy/tests/test_xarray_accessor.py create mode 100644 argopy/tests/test_xarray_engine.py diff --git a/argopy/tests/test_errors.py b/argopy/tests/test_errors.py new file mode 100644 index 00000000..54e57371 --- /dev/null +++ b/argopy/tests/test_errors.py @@ -0,0 +1,46 @@ +import pytest +from argopy.errors import ( + DataNotFound, + FtpPathError, + ErddapPathError, + NetCDF4FileNotFoundError, + CacheFileNotFound, + FileSystemHasNoCache, + UnrecognisedProfileDirection, + InvalidDataset, + InvalidDatasetStructure, + InvalidFetcherAccessPoint, + InvalidFetcher, + InvalidOption, + OptionValueError, + InvalidMethod, + InvalidDashboard, + APIServerError, + ErddapServerError, + ArgovisServerError +) + + +@pytest.mark.parametrize("error", [ + DataNotFound, + FtpPathError, + ErddapPathError, + NetCDF4FileNotFoundError, + CacheFileNotFound, + FileSystemHasNoCache, + UnrecognisedProfileDirection, + InvalidDataset, + InvalidDatasetStructure, + InvalidFetcherAccessPoint, + InvalidFetcher, + InvalidOption, + OptionValueError, + InvalidMethod, + InvalidDashboard, + APIServerError, + ErddapServerError, + ArgovisServerError, + ], indirect=False) +def test_raise_all_errors(error): + with pytest.raises(error): + raise error() diff --git a/argopy/tests/test_fetchers_data_argovis.py b/argopy/tests/test_fetchers_data_argovis.py new file mode 100644 index 00000000..d6f6f119 --- /dev/null +++ b/argopy/tests/test_fetchers_data_argovis.py @@ -0,0 +1,235 @@ +import warnings + +import numpy as np +import xarray as xr +import pandas as pd + +import pytest +import tempfile + +import argopy +from argopy import DataFetcher as ArgoDataFetcher +from argopy.errors import ( + CacheFileNotFound, + FileSystemHasNoCache, +) +from argopy.utils.checkers import is_list_of_strings +from utils import requires_connected_argovis, safe_to_server_errors + + +skip_this_for_debug = pytest.mark.skipif(False, reason="Skipped temporarily for debug") + + +@requires_connected_argovis +class Test_Backend: + """ Test main API facade for all available dataset and access points of the ARGOVIS data fetching backend """ + + src = "argovis" + requests = { + "float": [[1901393], [1901393, 6902746]], + # "profile": [[6902746, 12], [6902746, np.arange(12, 13)], [6902746, [1, 12]]], + "profile": [[6902746, 12]], + "region": [ + [-70, -65, 35.0, 40.0, 0, 10.0, "2012-01", "2012-03"], + [-70, -65, 35.0, 40.0, 0, 10.0, "2012-01", "2012-06"], + ], + } + + @skip_this_for_debug + def test_cachepath_notfound(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir): + fetcher = ArgoDataFetcher(src=self.src, cache=True).profile(*self.requests['profile'][0]).fetcher + with pytest.raises(CacheFileNotFound): + fetcher.cachepath + + @skip_this_for_debug + @safe_to_server_errors + def test_nocache(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir): + fetcher = ArgoDataFetcher(src=self.src, cache=False).profile(*self.requests['profile'][0]).fetcher + with pytest.raises(FileSystemHasNoCache): + fetcher.cachepath + + @skip_this_for_debug + @safe_to_server_errors + def test_clearcache(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir): + fetcher = ArgoDataFetcher(src=self.src, cache=True).float(self.requests['float'][0]).fetcher + fetcher.to_xarray() + fetcher.clear_cache() + with pytest.raises(CacheFileNotFound): + fetcher.cachepath + + @skip_this_for_debug + @safe_to_server_errors + def test_caching_float(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir): + fetcher = ( + ArgoDataFetcher(src=self.src, cache=True).float(self.requests['float'][0]).fetcher + ) + ds = fetcher.to_xarray() + assert isinstance(ds, xr.Dataset) + assert is_list_of_strings(fetcher.uri) + assert is_list_of_strings(fetcher.cachepath) + + @skip_this_for_debug + @safe_to_server_errors + def test_caching_profile(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir): + fetcher = ArgoDataFetcher(src=self.src, cache=True).profile(*self.requests['profile'][0]).fetcher + ds = fetcher.to_xarray() + assert isinstance(ds, xr.Dataset) + assert is_list_of_strings(fetcher.uri) + assert is_list_of_strings(fetcher.cachepath) + + @skip_this_for_debug + @safe_to_server_errors + def test_caching_region(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir): + fetcher = ( + ArgoDataFetcher(src=self.src, cache=True) + .region(self.requests['region'][1]) + .fetcher + ) + ds = fetcher.to_xarray() + assert isinstance(ds, xr.Dataset) + assert is_list_of_strings(fetcher.uri) + assert is_list_of_strings(fetcher.cachepath) + + def __testthis_profile(self, dataset): + fetcher_args = {"src": self.src, "ds": dataset} + for arg in self.args["profile"]: + f = ArgoDataFetcher(**fetcher_args).profile(*arg).fetcher + assert isinstance(f.to_xarray(), xr.Dataset) + # assert isinstance(f.to_dataframe(), pd.core.frame.DataFrame) + # ds = xr.Dataset.from_dataframe(f.to_dataframe()) + # ds = ds.sortby( + # ["TIME", "PRES"] + # ) # should already be sorted by date in descending order + # ds["N_POINTS"] = np.arange( + # 0, len(ds["N_POINTS"]) + # ) # Re-index to avoid duplicate values + # + # # Set coordinates: + # # ds = ds.set_coords('N_POINTS') + # coords = ("LATITUDE", "LONGITUDE", "TIME", "N_POINTS") + # ds = ds.reset_coords() + # ds["N_POINTS"] = ds["N_POINTS"] + # # Convert all coordinate variable names to upper case + # for v in ds.data_vars: + # ds = ds.rename({v: v.upper()}) + # ds = ds.set_coords(coords) + # + # # Cast data types and add variable attributes (not available in the csv download): + # warnings.warn(type(ds['TIME'].data)) + # warnings.warn(ds['TIME'].data[0]) + # ds['TIME'] = ds['TIME'].astype(np.datetime64) + # assert isinstance(ds, xr.Dataset) + assert is_list_of_strings(f.uri) + + def __testthis_float(self, dataset): + fetcher_args = {"src": self.src, "ds": dataset} + for arg in self.args["float"]: + f = ArgoDataFetcher(**fetcher_args).float(arg).fetcher + assert isinstance(f.to_xarray(), xr.Dataset) + assert is_list_of_strings(f.uri) + + def __testthis_region(self, dataset): + fetcher_args = {"src": self.src, "ds": dataset} + for arg in self.args["region"]: + f = ArgoDataFetcher(**fetcher_args).region(arg).fetcher + assert isinstance(f.to_xarray(), xr.Dataset) + assert is_list_of_strings(f.uri) + + def __testthis(self, dataset): + for access_point in self.args: + if access_point == "profile": + self.__testthis_profile(dataset) + elif access_point == "float": + self.__testthis_float(dataset) + elif access_point == "region": + self.__testthis_region(dataset) + + @skip_this_for_debug + @safe_to_server_errors + def test_phy_float(self): + self.args = {"float": self.requests['float']} + self.__testthis("phy") + + @safe_to_server_errors + def test_phy_profile(self): + self.args = {"profile": self.requests['profile']} + self.__testthis("phy") + + @skip_this_for_debug + @safe_to_server_errors + def test_phy_region(self): + self.args = {"region": self.requests['region']} + self.__testthis("phy") + + +@skip_this_for_debug +@requires_connected_argovis +class Test_BackendParallel: + """ This test backend for parallel requests """ + + src = "argovis" + requests = { + "region": [ + [-60, -55, 40.0, 45.0, 0.0, 10.0], + [-60, -55, 40.0, 45.0, 0.0, 10.0, "2007-08-01", "2007-09-01"], + ], + "wmo": [[6902766, 6902772, 6902914]], + } + + def test_methods(self): + args_list = [ + {"src": self.src, "parallel": "thread"}, + {"src": self.src, "parallel": True, "parallel_method": "thread"}, + ] + for fetcher_args in args_list: + loader = ArgoDataFetcher(**fetcher_args).float(self.requests["wmo"][0]) + assert isinstance(loader, argopy.fetchers.ArgoDataFetcher) + + args_list = [ + {"src": self.src, "parallel": True, "parallel_method": "toto"}, + {"src": self.src, "parallel": "process"}, + {"src": self.src, "parallel": True, "parallel_method": "process"}, + ] + for fetcher_args in args_list: + with pytest.raises(ValueError): + ArgoDataFetcher(**fetcher_args).float(self.requests["wmo"][0]) + + @safe_to_server_errors + def test_chunks_region(self): + for access_arg in self.requests["region"]: + fetcher_args = { + "src": self.src, + "parallel": True, + "chunks": {"lon": 1, "lat": 2, "dpt": 1, "time": 2}, + } + f = ArgoDataFetcher(**fetcher_args).region(access_arg).fetcher + assert isinstance(f.to_xarray(), xr.Dataset) + assert is_list_of_strings(f.uri) + assert len(f.uri) == np.prod( + [v for k, v in fetcher_args["chunks"].items()] + ) + + @safe_to_server_errors + def test_chunks_wmo(self): + for access_arg in self.requests["wmo"]: + fetcher_args = { + "src": self.src, + "parallel": True, + "chunks_maxsize": {"wmo": 1}, + } + f = ArgoDataFetcher(**fetcher_args).profile(access_arg, 12).fetcher + assert isinstance(f.to_xarray(), xr.Dataset) + assert is_list_of_strings(f.uri) + assert len(f.uri) == len(access_arg) diff --git a/argopy/tests/test_fetchers_data_erddap.py b/argopy/tests/test_fetchers_data_erddap.py new file mode 100644 index 00000000..d57375ca --- /dev/null +++ b/argopy/tests/test_fetchers_data_erddap.py @@ -0,0 +1,216 @@ +import logging + +from argopy import DataFetcher as ArgoDataFetcher +from argopy.utils.checkers import is_list_of_strings + +import pytest +import xarray as xr +from utils import ( + requires_erddap, +) + +from mocked_http import mocked_server_address +from mocked_http import mocked_httpserver as mocked_erddapserver + +import tempfile +import shutil +from collections import ChainMap + + +log = logging.getLogger("argopy.tests.data.erddap") + +USE_MOCKED_SERVER = True + +""" +List access points to be tested for each datasets: phy and ref. +For each access points, we list 1-to-2 scenario to make sure all possibilities are tested +""" +ACCESS_POINTS = [ + {"phy": [ + {"float": 1901393}, + {"float": [1901393, 6902746]}, + {"profile": [6902746, 34]}, + {"profile": [6902746, [1, 12]]}, + {"region": [-20, -16., 0, 1, 0, 100.]}, + {"region": [-20, -16., 0, 1, 0, 100., "2004-01-01", "2004-01-31"]}, + ]}, + {"ref": [ + {"region": [-70, -65, 35.0, 40.0, 0, 10.0]}, + {"region": [-70, -65, 35.0, 40.0, 0, 10.0, "2012-01-1", "2012-12-31"]}, + ]}, +] +PARALLEL_ACCESS_POINTS = [ + {"phy": [ + {"float": [1900468, 1900117, 1900386]}, + {"region": [-60, -55, 40.0, 45.0, 0.0, 20.0]}, + {"region": [-60, -55, 40.0, 45.0, 0.0, 20.0, "2007-08-01", "2007-09-01"]}, + ]}, +] + +""" +List user modes to be tested +""" +USER_MODES = ['standard', 'expert', 'research'] + + +""" +Make a list of VALID dataset/access_points to be tested +""" +VALID_ACCESS_POINTS, VALID_ACCESS_POINTS_IDS = [], [] +for entry in ACCESS_POINTS: + for ds in entry: + for mode in USER_MODES: + for ap in entry[ds]: + VALID_ACCESS_POINTS.append({'ds': ds, 'mode': mode, 'access_point': ap}) + VALID_ACCESS_POINTS_IDS.append("ds='%s', mode='%s', %s" % (ds, mode, ap)) + + +VALID_PARALLEL_ACCESS_POINTS, VALID_PARALLEL_ACCESS_POINTS_IDS = [], [] +for entry in PARALLEL_ACCESS_POINTS: + for ds in entry: + for mode in USER_MODES: + for ap in entry[ds]: + VALID_PARALLEL_ACCESS_POINTS.append({'ds': ds, 'mode': mode, 'access_point': ap}) + VALID_PARALLEL_ACCESS_POINTS_IDS.append("ds='%s', mode='%s', %s" % (ds, mode, ap)) + + +def create_fetcher(fetcher_args, access_point): + """ Create a fetcher for a given set of facade options and access point """ + def core(fargs, apts): + try: + f = ArgoDataFetcher(**fargs) + if "float" in apts: + f = f.float(apts['float']) + elif "profile" in apts: + f = f.profile(*apts['profile']) + elif "region" in apts: + f = f.region(apts['region']) + except Exception: + raise + return f + fetcher = core(fetcher_args, access_point) + return fetcher + + +def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): + """Assert a data fetcher. + + This should be used by all tests asserting a fetcher + """ + def assert_all(this_fetcher, cacheable): + # We use the facade to test 'to_xarray' in order to make sure to test all filters required by user mode + ds = this_fetcher.to_xarray(errors='raise') + assert isinstance(ds, xr.Dataset) + # + core = this_fetcher.fetcher + assert is_list_of_strings(core.uri) + assert (core.N_POINTS >= 1) # Make sure we found results + if cacheable: + assert is_list_of_strings(core.cachepath) + + log.debug("In assert, this fetcher is in '%s' user mode" % this_fetcher._mode) + if this_fetcher._dataset_id not in ['ref']: + if this_fetcher._mode == 'expert': + assert 'PRES_ADJUSTED' in ds + + elif this_fetcher._mode == 'standard': + assert 'PRES_ADJUSTED' not in ds + + elif this_fetcher._mode == 'research': + assert 'PRES_ADJUSTED' not in ds + assert 'PRES_QC' not in ds + else: + assert 'PTMP' in ds + + try: + assert_all(this_fetcher, cacheable) + except: + raise + assert False + + +@requires_erddap +class Test_Backend: + """ Test ERDDAP data fetching backend """ + src = 'erddap' + + ############# + # UTILITIES # + ############# + + def setup_class(self): + """setup any state specific to the execution of the given class""" + # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests + self.cachedir = tempfile.mkdtemp() + + def _setup_fetcher(self, this_request, cached=False, parallel=False): + """Helper method to set up options for a fetcher creation""" + defaults_args = {"src": self.src, + "cache": cached, + "cachedir": self.cachedir, + "parallel": parallel, + } + if USE_MOCKED_SERVER: + defaults_args['server'] = mocked_server_address + + dataset = this_request.param['ds'] + user_mode = this_request.param['mode'] + access_point = this_request.param['access_point'] + + fetcher_args = ChainMap(defaults_args, {"ds": dataset, 'mode': user_mode}) + if not cached: + # cache is False by default, so we don't need to clutter the arguments list + del fetcher_args["cache"] + del fetcher_args["cachedir"] + if not parallel: + # parallel is False by default, so we don't need to clutter the arguments list + del fetcher_args["parallel"] + + # log.debug("Setting up a new fetcher with the following arguments:") + # log.debug(fetcher_args) + return fetcher_args, access_point + + @pytest.fixture + def fetcher(self, request): + """ Fixture to create a ERDDAP data fetcher for a given dataset and access point """ + fetcher_args, access_point = self._setup_fetcher(request, cached=False) + yield create_fetcher(fetcher_args, access_point) + + @pytest.fixture + def cached_fetcher(self, request): + """ Fixture to create a cached ERDDAP data fetcher for a given dataset and access point """ + fetcher_args, access_point = self._setup_fetcher(request, cached=True) + yield create_fetcher(fetcher_args, access_point) + + @pytest.fixture + def parallel_fetcher(self, request): + """ Fixture to create a parallel ERDDAP data fetcher for a given dataset and access point """ + fetcher_args, access_point = self._setup_fetcher(request, parallel="thread") + yield create_fetcher(fetcher_args, access_point) + + def teardown_class(self): + """Cleanup once we are finished.""" + def remove_test_dir(): + shutil.rmtree(self.cachedir) + remove_test_dir() + + ######### + # TESTS # + ######### + @pytest.mark.parametrize("fetcher", VALID_ACCESS_POINTS, + indirect=True, + ids=VALID_ACCESS_POINTS_IDS) + def test_fetching(self, mocked_erddapserver, fetcher): + assert_fetcher(mocked_erddapserver, fetcher, cacheable=False) + + @pytest.mark.parametrize("cached_fetcher", VALID_ACCESS_POINTS, + indirect=True, + ids=VALID_ACCESS_POINTS_IDS) + def test_fetching_cached(self, mocked_erddapserver, cached_fetcher): + assert_fetcher(mocked_erddapserver, cached_fetcher, cacheable=True) + + @pytest.mark.parametrize("parallel_fetcher", VALID_PARALLEL_ACCESS_POINTS, + indirect=True, + ids=VALID_PARALLEL_ACCESS_POINTS_IDS) + def test_fetching_parallel(self, mocked_erddapserver, parallel_fetcher): + assert_fetcher(mocked_erddapserver, parallel_fetcher, cacheable=False) diff --git a/argopy/tests/test_fetchers_data_erddap_bgc.py b/argopy/tests/test_fetchers_data_erddap_bgc.py new file mode 100644 index 00000000..ad0769b6 --- /dev/null +++ b/argopy/tests/test_fetchers_data_erddap_bgc.py @@ -0,0 +1,267 @@ +import logging +import numpy as np + +from argopy import DataFetcher as ArgoDataFetcher +from argopy.utils.checkers import is_list_of_strings +from argopy.stores import indexstore_pd as ArgoIndex # make sure to work with the Pandas index store with erddap-bgc + +import pytest +import xarray as xr +from utils import ( + requires_erddap, +) +from mocked_http import mocked_server_address +from mocked_http import mocked_httpserver as mocked_erddapserver + +import tempfile +import shutil +from collections import ChainMap + + +log = logging.getLogger("argopy.tests.data.erddap") + +USE_MOCKED_SERVER = True + +""" +List access points to be tested for each datasets: bgc. +For each access points, we list 1-to-2 scenario to make sure all possibilities are tested +""" +ACCESS_POINTS = [ + {"bgc": [ + {"float": 5903248}, + {"float": [5903248, 6904241]}, + {"profile": [5903248, 34]}, + {"profile": [5903248, np.arange(12, 14)]}, + {"region": [-55, -47, 55, 57, 0, 10]}, + {"region": [-55, -47, 55, 57, 0, 10, "2022-05-1", "2023-07-01"]}, + ]}, +] +PARALLEL_ACCESS_POINTS = [ + {"bgc": [ + {"float": [5903248, 6904241]}, + {"region": [-55, -47, 55, 57, 0, 10, "2022-05-1", "2023-07-01"]}, + ]}, +] + +""" +List user modes to be tested +""" +# USER_MODES = ['standard', 'expert', 'research'] +USER_MODES = ['expert'] + +""" +List of 'params' fetcher arguments to be tested +""" +PARAMS = ['all', 'DOXY'] + +""" +Make a list of VALID dataset/access_points to be tested +""" +VALID_ACCESS_POINTS, VALID_ACCESS_POINTS_IDS = [], [] +for entry in ACCESS_POINTS: + for ds in entry: + for mode in USER_MODES: + for params in PARAMS: + for ap in entry[ds]: + VALID_ACCESS_POINTS.append({'ds': ds, 'mode': mode, 'params': params, 'access_point': ap}) + VALID_ACCESS_POINTS_IDS.append("ds='%s', mode='%s', params='%s', %s" % (ds, mode, params, ap)) + + +VALID_PARALLEL_ACCESS_POINTS, VALID_PARALLEL_ACCESS_POINTS_IDS = [], [] +for entry in PARALLEL_ACCESS_POINTS: + for ds in entry: + for mode in USER_MODES: + for params in PARAMS: + for ap in entry[ds]: + VALID_PARALLEL_ACCESS_POINTS.append({'ds': ds, 'mode': mode, 'params': params, 'access_point': ap}) + VALID_PARALLEL_ACCESS_POINTS_IDS.append("ds='%s', mode='%s', params='%s', %s" % (ds, mode, params, ap)) + + +def create_fetcher(fetcher_args, access_point): + """ Create a fetcher for a given set of facade options and access point """ + def core(fargs, apts): + try: + f = ArgoDataFetcher(**fargs) + if "float" in apts: + f = f.float(apts['float']) + elif "profile" in apts: + f = f.profile(*apts['profile']) + elif "region" in apts: + f = f.region(apts['region']) + except Exception: + raise + return f + fetcher = core(fetcher_args, access_point) + return fetcher + + +def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): + """Assert a data fetcher. + + This should be used by all tests asserting a fetcher + """ + def assert_all(this_fetcher, cacheable): + + # We use the facade to test 'to_xarray' in order to make sure to test all filters required by user mode + ds = this_fetcher.to_xarray(errors='raise') + assert isinstance(ds, xr.Dataset) + + # Then apply checks on erddap fetcher: + core = this_fetcher.fetcher + assert is_list_of_strings(core.uri) + assert (core.N_POINTS >= 1) # Make sure we found results + if cacheable: + assert is_list_of_strings(core.cachepath) + + # log.debug("In assert, this fetcher is in '%s' user mode" % this_fetcher._mode) + if this_fetcher._mode == 'expert': + assert 'PRES_ADJUSTED' in ds + + elif this_fetcher._mode == 'standard': + assert 'PRES_ADJUSTED' not in ds + + elif this_fetcher._mode == 'research': + assert 'PRES_ADJUSTED' not in ds + assert 'PRES_QC' not in ds + + try: + assert_all(this_fetcher, cacheable) + except: + if this_fetcher._mode not in ['expert']: + pytest.xfail("BGC is not yet supported in '%s' user mode" % this_fetcher._mode) + else: + assert False + + +@requires_erddap +class Test_Backend: + """ Test ERDDAP data fetching backend """ + src = 'erddap' + + ############# + # UTILITIES # + ############# + + def setup_class(self): + """setup any state specific to the execution of the given class""" + # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests + self.cachedir = tempfile.mkdtemp() + + def _setup_fetcher(self, this_request, cached=False, parallel=False): + """Helper method to set up options for a fetcher creation""" + defaults_args = {"src": self.src, + "cache": cached, + "cachedir": self.cachedir, + "parallel": parallel, + } + + if USE_MOCKED_SERVER: + defaults_args['server'] = mocked_server_address + defaults_args['indexfs'] = ArgoIndex( + host=mocked_server_address, + index_file='argo_synthetic-profile_index.txt', + cache=True, + cachedir=self.cachedir, + timeout=5, + ) + + + dataset = this_request.param['ds'] + user_mode = this_request.param['mode'] + params = this_request.param['params'] + measured = this_request.param['measured'] if 'measured' in this_request.param else None + + access_point = this_request.param['access_point'] + + fetcher_args = ChainMap(defaults_args, {"ds": dataset, 'mode': user_mode, 'params': params, 'measured': measured}) + if not cached: + # cache is False by default, so we don't need to clutter the arguments list + del fetcher_args["cache"] + del fetcher_args["cachedir"] + if not parallel: + # parallel is False by default, so we don't need to clutter the arguments list + del fetcher_args["parallel"] + + # log.debug("Setting up a new fetcher with the following arguments:") + # log.debug(fetcher_args) + return fetcher_args, access_point + + @pytest.fixture + def fetcher(self, request): + """ Fixture to create a ERDDAP data fetcher for a given dataset and access point """ + fetcher_args, access_point = self._setup_fetcher(request, cached=False) + yield create_fetcher(fetcher_args, access_point) + + @pytest.fixture + def cached_fetcher(self, request): + """ Fixture to create a cached ERDDAP data fetcher for a given dataset and access point """ + fetcher_args, access_point = self._setup_fetcher(request, cached=True) + yield create_fetcher(fetcher_args, access_point) + + @pytest.fixture + def parallel_fetcher(self, request): + """ Fixture to create a parallel ERDDAP data fetcher for a given dataset and access point """ + fetcher_args, access_point = self._setup_fetcher(request, parallel="erddap") + yield create_fetcher(fetcher_args, access_point) + + def teardown_class(self): + """Cleanup once we are finished.""" + def remove_test_dir(): + shutil.rmtree(self.cachedir) + remove_test_dir() + + ######### + # TESTS # + ######### + @pytest.mark.parametrize("fetcher", VALID_ACCESS_POINTS, + indirect=True, + ids=VALID_ACCESS_POINTS_IDS) + def test_fetching(self, mocked_erddapserver, fetcher): + assert_fetcher(mocked_erddapserver, fetcher, cacheable=False) + + @pytest.mark.parametrize("cached_fetcher", VALID_ACCESS_POINTS, + indirect=True, + ids=VALID_ACCESS_POINTS_IDS) + def test_fetching_cached(self, mocked_erddapserver, cached_fetcher): + assert_fetcher(mocked_erddapserver, cached_fetcher, cacheable=True) + + @pytest.mark.parametrize("parallel_fetcher", VALID_PARALLEL_ACCESS_POINTS, + indirect=True, + ids=VALID_PARALLEL_ACCESS_POINTS_IDS) + def test_fetching_parallel(self, mocked_erddapserver, parallel_fetcher): + assert_fetcher(mocked_erddapserver, parallel_fetcher, cacheable=False) + + @pytest.mark.parametrize("measured", [None, 'all', 'DOXY'], + indirect=False, + ids=["measured=%s" % m for m in [None, 'all', 'DOXY']] + ) + def test_fetching_measured(self, mocked_erddapserver, measured): + class this_request: + param = { + 'ds': 'bgc', + 'mode': 'expert', + 'params': 'all', + 'measured': measured, + 'access_point': {"float": [5903248]}, + } + fetcher_args, access_point = self._setup_fetcher(this_request) + fetcher = create_fetcher(fetcher_args, access_point) + assert_fetcher(mocked_erddapserver, fetcher) + + @pytest.mark.parametrize("measured", ['all'], + indirect=False, + ids=["measured=%s" % m for m in ['all']], + ) + def test_fetching_failed_measured(self, mocked_erddapserver, measured): + class this_request: + param = { + 'ds': 'bgc', + 'mode': 'expert', + 'params': 'all', + 'measured': measured, + 'access_point': {"float": [6904240]}, + } + fetcher_args, access_point = self._setup_fetcher(this_request) + fetcher = create_fetcher(fetcher_args, access_point) + with pytest.raises(ValueError): + fetcher.to_xarray() \ No newline at end of file diff --git a/argopy/tests/test_fetchers_data_gdac.py b/argopy/tests/test_fetchers_data_gdac.py new file mode 100644 index 00000000..9bae3ee1 --- /dev/null +++ b/argopy/tests/test_fetchers_data_gdac.py @@ -0,0 +1,230 @@ +""" +Test the "GDAC ftp" data fetcher backend + +Here we try an approach based on fixtures and pytest parametrization +to make more explicit the full list of scenario tested. +""" +import xarray as xr + +import pytest +import tempfile +import shutil +from urllib.parse import urlparse +import logging +from collections import ChainMap + +import argopy +from argopy import DataFetcher as ArgoDataFetcher +from argopy.errors import ( + CacheFileNotFound, + FileSystemHasNoCache, + FtpPathError, +) +from argopy.utils.checkers import isconnected, is_list_of_strings +from utils import requires_gdac +from mocked_http import mocked_httpserver, mocked_server_address + + +log = logging.getLogger("argopy.tests.data.gdac") +skip_for_debug = pytest.mark.skipif(False, reason="Taking too long !") + + +""" +List ftp hosts to be tested. +Since the fetcher is compatible with host from local, http or ftp protocols, we +try to test them all: +""" +HOSTS = [argopy.tutorial.open_dataset("gdac")[0], + #'https://data-argo.ifremer.fr', # ok, but replaced by the mocked http server + mocked_server_address, + # 'ftp://ftp.ifremer.fr/ifremer/argo', + # 'ftp://usgodae.org/pub/outgoing/argo', # ok, but slow down CI and no need for 2 ftp tests + 'MOCKFTP', # keyword to use the fake/mocked ftp server (running on localhost) + ] + +""" +List access points to be tested. +For each access points, we list 1-to-2 scenario to make sure all possibilities are tested +""" +ACCESS_POINTS = [ + {"float": [13857]}, + {"profile": [13857, 90]}, + {"region": [-20, -16., 0, 1, 0, 100.]}, + {"region": [-20, -16., 0, 1, 0, 100., "1997-07-01", "1997-09-01"]}, + ] + +""" +List parallel methods to be tested. +""" +valid_parallel_opts = [ + {"parallel": "thread"}, + # {"parallel": True, "parallel_method": "thread"}, # opts0 + # {"parallel": True, "parallel_method": "process"} # opts1 +] + +""" +List user modes to be tested +""" +USER_MODES = ['standard', 'expert', 'research'] + + +@requires_gdac +def create_fetcher(fetcher_args, access_point): + """ Create a fetcher for a given set of facade options and access point + + """ + def core(fargs, apts): + try: + f = ArgoDataFetcher(**fargs) + if "float" in apts: + f = f.float(apts['float']) + elif "profile" in apts: + f = f.profile(*apts['profile']) + elif "region" in apts: + f = f.region(apts['region']) + except Exception: + raise + return f + fetcher = core(fetcher_args, access_point) + return fetcher + + +def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): + """Assert a data fetcher. + + This should be used by all tests + """ + assert isinstance(this_fetcher.to_xarray(errors='raise'), xr.Dataset) + core = this_fetcher.fetcher + assert is_list_of_strings(core.uri) + assert (core.N_RECORDS >= 1) # Make sure we loaded the index file content + assert (core.N_FILES >= 1) # Make sure we found results + if cacheable: + assert is_list_of_strings(core.cachepath) + + +def ftp_shortname(ftp): + """Get a short name for scenarios IDs, given a FTP host""" + if ftp == 'MOCKFTP': + return 'ftp_mocked' + elif 'localhost' in ftp or '127.0.0.1' in ftp: + return 'http_mocked' + else: + return (lambda x: 'file' if x == "" else x)(urlparse(ftp).scheme) + +""" +Make a list of VALID host/dataset/access_points to be tested +""" +VALID_ACCESS_POINTS, VALID_ACCESS_POINTS_IDS = [], [] +for host in HOSTS: + for mode in USER_MODES: + for ap in ACCESS_POINTS: + VALID_ACCESS_POINTS.append({'host': host, 'ds': 'phy', 'mode': mode, 'access_point': ap}) + VALID_ACCESS_POINTS_IDS.append("host='%s', ds='%s', mode='%s', %s" % (ftp_shortname(host), 'phy', mode, ap)) + + + +@requires_gdac +class TestBackend: + src = 'gdac' + + ############# + # UTILITIES # + ############# + + def setup_class(self): + """setup any state specific to the execution of the given class""" + # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests + self.cachedir = tempfile.mkdtemp() + + def _patch_ftp(self, ftp): + """Patch Mocked FTP server keyword""" + if ftp == 'MOCKFTP': + return pytest.MOCKFTP # this was set in conftest.py + else: + return ftp + + def _setup_fetcher(self, this_request, cached=False, parallel=False): + """Helper method to set up options for a fetcher creation""" + ftp = this_request.param['host'] + access_point = this_request.param['access_point'] + N_RECORDS = None if 'tutorial' in ftp or 'MOCK' in ftp else 100 # Make sure we're not going to load the full index + + fetcher_args = {"src": self.src, + "ftp": self._patch_ftp(ftp), + "ds": this_request.param['ds'], + "mode": this_request.param['mode'], + "cache": cached, + "cachedir": self.cachedir, + "parallel": False, + "N_RECORDS": N_RECORDS, + } + + if not cached: + # cache is False by default, so we don't need to clutter the arguments list + del fetcher_args["cache"] + del fetcher_args["cachedir"] + if not parallel: + # parallel is False by default, so we don't need to clutter the arguments list + del fetcher_args["parallel"] + + if not isconnected(fetcher_args['ftp']): + pytest.xfail("Fails because %s not available" % fetcher_args['ftp']) + else: + return fetcher_args, access_point + + @pytest.fixture + def fetcher(self, request, mocked_httpserver): + """ Fixture to create a GDAC fetcher for a given host and access point """ + fetcher_args, access_point = self._setup_fetcher(request, cached=False) + yield create_fetcher(fetcher_args, access_point) + + @pytest.fixture + def cached_fetcher(self, request): + """ Fixture to create a cached FTP fetcher for a given host and access point """ + fetcher_args, access_point = self._setup_fetcher(request, cached=True) + yield create_fetcher(fetcher_args, access_point) + + def teardown_class(self): + """Cleanup once we are finished.""" + def remove_test_dir(): + shutil.rmtree(self.cachedir) + remove_test_dir() + + ######### + # TESTS # + ######### + # def test_nocache(self, mocked_httpserver): + # this_fetcher = create_fetcher({"src": self.src, "ftp": self._patch_ftp(VALID_HOSTS[0]), "N_RECORDS": 10}, VALID_ACCESS_POINTS[0]) + # with pytest.raises(FileSystemHasNoCache): + # this_fetcher.cachepath + + # @pytest.mark.parametrize("fetcher", VALID_HOSTS, + # indirect=True, + # ids=["%s" % ftp_shortname(ftp) for ftp in VALID_HOSTS]) + # def test_hosts(self, mocked_httpserver, fetcher): + # assert (fetcher.N_RECORDS >= 1) + + # @pytest.mark.parametrize("ftp_host", ['invalid', 'https://invalid_ftp', 'ftp://invalid_ftp'], indirect=False) + # def test_hosts_invalid(self, ftp_host): + # # Invalid servers: + # with pytest.raises(FtpPathError): + # create_fetcher({"src": self.src, "ftp": ftp_host}, VALID_ACCESS_POINTS[0]) + + @pytest.mark.parametrize("fetcher", VALID_ACCESS_POINTS, indirect=True, ids=VALID_ACCESS_POINTS_IDS) + def test_fetching(self, mocked_httpserver, fetcher): + assert_fetcher(mocked_httpserver, fetcher, cacheable=False) + + @pytest.mark.parametrize("cached_fetcher", VALID_ACCESS_POINTS, indirect=True, ids=VALID_ACCESS_POINTS_IDS) + def test_fetching_cached(self, mocked_httpserver, cached_fetcher): + # Assert the fetcher (this trigger data fetching, hence caching as well): + assert_fetcher(mocked_httpserver, cached_fetcher, cacheable=True) + # and we also make sure we can clear the cache: + cached_fetcher.clear_cache() + with pytest.raises(CacheFileNotFound): + cached_fetcher.fetcher.cachepath + + def test_uri_mono2multi(self, mocked_httpserver): + ap = [v for v in ACCESS_POINTS if 'region' in v.keys()][0] + f = create_fetcher({"src": self.src, "ftp": HOSTS[0], "N_RECORDS": 100}, ap).fetcher + assert is_list_of_strings(f.uri_mono2multi(f.uri)) diff --git a/argopy/tests/test_fetchers_facade_data.py b/argopy/tests/test_fetchers_facade_data.py new file mode 100644 index 00000000..17e0646c --- /dev/null +++ b/argopy/tests/test_fetchers_facade_data.py @@ -0,0 +1,196 @@ +import pandas as pd +import xarray as xr + +import pytest + +import argopy +from argopy import DataFetcher as ArgoDataFetcher +from argopy.errors import ( + InvalidFetcherAccessPoint, + InvalidFetcher, + OptionValueError, +) +from argopy.utils import is_list_of_strings +from utils import ( + requires_fetcher, + requires_connection, + requires_connected_erddap_phy, + requires_gdac, + requires_connected_gdac, + requires_connected_argovis, + requires_ipython, + safe_to_server_errors, + requires_matplotlib, + has_matplotlib, + has_seaborn, + has_cartopy, + has_ipython, +) + + +if has_matplotlib: + import matplotlib as mpl + +if has_cartopy: + import cartopy + +if has_ipython: + import IPython + +skip_for_debug = pytest.mark.skipif(True, reason="Taking too long !") + + +@requires_gdac +class Test_Facade: + + # Use the first valid data source: + src = 'gdac' + src_opts = {'ftp': argopy.tutorial.open_dataset("gdac")[0]} + + def __get_fetcher(self, empty: bool = False, pt: str = 'profile'): + f = ArgoDataFetcher(src=self.src, **self.src_opts) + # f.valid_access_points[0] + + if pt == 'float': + if not empty: + return f, ArgoDataFetcher(src=self.src, **self.src_opts).float(2901623) + else: + return f, ArgoDataFetcher(src=self.src, **self.src_opts).float(12) + + if pt == 'profile': + if not empty: + return f, ArgoDataFetcher(src=self.src, **self.src_opts).profile(2901623, 12) + else: + return f, ArgoDataFetcher(src=self.src, **self.src_opts).profile(12, 1200) + + if pt == 'region': + if not empty: + return f, ArgoDataFetcher(src=self.src, **self.src_opts).region([-60, -55, 40.0, 45.0, 0.0, 10.0, + "2007-08-01", "2007-09-01"]) + else: + return f, ArgoDataFetcher(src=self.src, **self.src_opts).region([-60, -55, 40.0, 45.0, 99.92, 99.99, + "2007-08-01", "2007-08-01"]) + + def test_invalid_mode(self): + with pytest.raises(OptionValueError): + ArgoDataFetcher(src=self.src, mode='invalid').to_xarray() + + def test_invalid_source(self): + with pytest.raises(OptionValueError): + ArgoDataFetcher(src="invalid").to_xarray() + + def test_invalid_dataset(self): + with pytest.raises(OptionValueError): + ArgoDataFetcher(src=self.src, ds='invalid') + + def test_invalid_accesspoint(self): + with pytest.raises(InvalidFetcherAccessPoint): + self.__get_fetcher()[0].invalid_accesspoint.to_xarray() + + def test_warnings(self): + with pytest.warns(UserWarning): + ArgoDataFetcher(src='erddap', ds='bgc', mode='standard') + with pytest.warns(UserWarning): + ArgoDataFetcher(src='erddap', ds='bgc', mode='research') + + def test_no_uri(self): + with pytest.raises(InvalidFetcherAccessPoint): + self.__get_fetcher()[0].uri + + def test_to_xarray(self): + assert isinstance(self.__get_fetcher()[1].to_xarray(), xr.Dataset) + with pytest.raises(InvalidFetcher): + assert self.__get_fetcher()[0].to_xarray() + + def test_to_dataframe(self): + assert isinstance(self.__get_fetcher()[1].to_dataframe(), pd.core.frame.DataFrame) + with pytest.raises(InvalidFetcher): + assert self.__get_fetcher()[0].to_dataframe() + + params = [(p, c) for p in [True, False] for c in [False]] + ids_params = ["full=%s, coriolis_id=%s" % (p[0], p[1]) for p in params] + @pytest.mark.parametrize("params", params, + indirect=False, + ids=ids_params) + def test_to_index(self, params): + full, coriolis_id = params + assert isinstance(self.__get_fetcher()[1].to_index(full=full, coriolis_id=coriolis_id), pd.core.frame.DataFrame) + + params = [(p, c) for p in [True, False] for c in [True]] + ids_params = ["full=%s, coriolis_id=%s" % (p[0], p[1]) for p in params] + @pytest.mark.parametrize("params", params, + indirect=False, + ids=ids_params) + @requires_connection + def test_to_index_coriolis(self, params): + full, coriolis_id = params + assert isinstance(self.__get_fetcher()[1].to_index(full=full, coriolis_id=coriolis_id), pd.core.frame.DataFrame) + + def test_load(self): + f, fetcher = self.__get_fetcher(pt='float') + + fetcher.load() + assert is_list_of_strings(fetcher.uri) + assert isinstance(fetcher.data, xr.Dataset) + assert isinstance(fetcher.index, pd.core.frame.DataFrame) + + # Change the access point: + new_fetcher = f.profile(fetcher._AccessPoint_data['wmo'], 1) + new_fetcher.load() + assert is_list_of_strings(new_fetcher.uri) + assert isinstance(new_fetcher.data, xr.Dataset) + assert isinstance(new_fetcher.index, pd.core.frame.DataFrame) + + @requires_matplotlib + def test_plot_trajectory(self): + f, fetcher = self.__get_fetcher(pt='float') + fig, ax = fetcher.plot(ptype='trajectory', + with_seaborn=has_seaborn, + with_cartopy=has_cartopy) + assert isinstance(fig, mpl.figure.Figure) + expected_ax_type = ( + cartopy.mpl.geoaxes.GeoAxesSubplot + if has_cartopy + else mpl.axes.Axes + ) + assert isinstance(ax, expected_ax_type) + assert isinstance(ax.get_legend(), mpl.legend.Legend) + mpl.pyplot.close(fig) + + @requires_matplotlib + @pytest.mark.parametrize("by", ["dac", "profiler"], indirect=False) + def test_plot_bar(self, by): + f, fetcher = self.__get_fetcher(pt='float') + fig, ax = fetcher.plot(ptype=by, with_seaborn=has_seaborn) + assert isinstance(fig, mpl.figure.Figure) + mpl.pyplot.close(fig) + + @requires_matplotlib + def test_plot_invalid(self): + f, fetcher = self.__get_fetcher(pt='float') + with pytest.raises(ValueError): + fetcher.plot(ptype='invalid_cat') + + @requires_matplotlib + def test_plot_qc_altimetry(self): + f, fetcher = self.__get_fetcher(pt='float') + dsh = fetcher.plot(ptype='qc_altimetry', embed='slide') + if has_ipython: + assert isinstance(dsh(0), IPython.display.Image) + else: + assert isinstance(dsh, dict) + + def test_domain(self): + f, fetcher = self.__get_fetcher(pt='float') + fetcher.domain + + def test_dashboard(self): + f, fetcher = self.__get_fetcher(pt='float') + assert isinstance(fetcher.dashboard(url_only=True), str) + + f, fetcher = self.__get_fetcher(pt='profile') + assert isinstance(fetcher.dashboard(url_only=True), str) + + with pytest.warns(UserWarning): + f, fetcher = self.__get_fetcher(pt='region') + fetcher.dashboard(url_only=True) diff --git a/argopy/tests/test_fetchers_facade_index.py b/argopy/tests/test_fetchers_facade_index.py new file mode 100644 index 00000000..b10d5269 --- /dev/null +++ b/argopy/tests/test_fetchers_facade_index.py @@ -0,0 +1,131 @@ +import pytest +import importlib + +import argopy +from argopy import IndexFetcher as ArgoIndexFetcher +from argopy.errors import InvalidFetcherAccessPoint, InvalidFetcher +from utils import ( + # AVAILABLE_INDEX_SOURCES, + requires_fetcher_index, + requires_connected_erddap_index, + requires_connected_gdac, + requires_connection, + requires_ipython, + ci_erddap_index, + requires_matplotlib, + has_matplotlib, + has_seaborn, + has_cartopy +) + + +if has_matplotlib: + import matplotlib as mpl + +if has_cartopy: + import cartopy + + +skip_for_debug = pytest.mark.skipif(True, reason="Taking too long !") + + +@requires_connection +class Test_Facade: + local_ftp = argopy.tutorial.open_dataset("gdac")[0] + src = 'gdac' + src_opts = {'ftp': local_ftp} + + def __get_fetcher(self, empty: bool = False, pt: str = 'profile'): + f = ArgoIndexFetcher(src=self.src, **self.src_opts) + + if pt == 'float': + if not empty: + return f, ArgoIndexFetcher(src=self.src, **self.src_opts).float(2901623) + else: + return f, ArgoIndexFetcher(src=self.src, **self.src_opts).float(12) + + if pt == 'profile': + if not empty: + return f, ArgoIndexFetcher(src=self.src, **self.src_opts).profile(2901623, 12) + else: + return f, ArgoIndexFetcher(src=self.src, **self.src_opts).profile(12, 1200) + + if pt == 'region': + if not empty: + return f, ArgoIndexFetcher(src=self.src, **self.src_opts).region([-60, -55, 40.0, 45.0, 0.0, 10.0, + "2007-08-01", "2007-09-01"]) + else: + return f, ArgoIndexFetcher(src=self.src, **self.src_opts).region([-60, -55, 40.0, 45.0, 99.92, 99.99, + "2007-08-01", "2007-08-01"]) + + def test_invalid_fetcher(self): + with pytest.raises(InvalidFetcher): + ArgoIndexFetcher(src="invalid_fetcher").to_xarray() + + @requires_fetcher_index + def test_invalid_accesspoint(self): + # Use the first valid data source + with pytest.raises(InvalidFetcherAccessPoint): + ArgoIndexFetcher( + src=self.src, **self.src_opts + ).invalid_accesspoint.to_xarray() # Can't get data if access point not defined first + with pytest.raises(InvalidFetcherAccessPoint): + ArgoIndexFetcher( + src=self.src, **self.src_opts + ).to_xarray() # Can't get data if access point not defined first + + @requires_fetcher_index + def test_invalid_dataset(self): + with pytest.raises(ValueError): + ArgoIndexFetcher(src=self.src, ds='dummy_ds', **self.src_opts) + + @requires_matplotlib + def test_plot(self): + f, fetcher = self.__get_fetcher(pt='float') + + # Test 'trajectory' + for ws in [False, has_seaborn]: + for wc in [False, has_cartopy]: + for legend in [True, False]: + fig, ax = fetcher.plot(ptype='trajectory', with_seaborn=ws, with_cartopy=wc, add_legend=legend) + assert isinstance(fig, mpl.figure.Figure) + + expected_ax_type = ( + cartopy.mpl.geoaxes.GeoAxesSubplot + if has_cartopy and wc + else mpl.axes.Axes + ) + assert isinstance(ax, expected_ax_type) + + expected_lg_type = mpl.legend.Legend if legend else type(None) + assert isinstance(ax.get_legend(), expected_lg_type) + + mpl.pyplot.close(fig) + + # Test 'dac', 'profiler' + for ws in [False, has_seaborn]: + for by in [ + "dac", + "profiler" + ]: + fig, ax = fetcher.plot(ptype=by, with_seaborn=ws) + assert isinstance(fig, mpl.figure.Figure) + mpl.pyplot.close(fig) + + # Test 'qc_altimetry' + if importlib.util.find_spec('IPython') is not None: + import IPython + dsh = fetcher.plot(ptype='qc_altimetry', embed='slide') + assert isinstance(dsh(0), IPython.display.Image) + + # Test invalid plot + with pytest.raises(ValueError): + fetcher.plot(ptype='invalid_cat', with_seaborn=ws) + + @requires_ipython + @requires_matplotlib + def test_plot_qc_altimetry(self): + import IPython + f, fetcher = self.__get_fetcher(pt='float') + dsh = fetcher.plot(ptype='qc_altimetry', embed='slide') + assert isinstance(dsh(0), IPython.display.Image) diff --git a/argopy/tests/test_fetchers_index_erddap.py b/argopy/tests/test_fetchers_index_erddap.py new file mode 100644 index 00000000..b42df83e --- /dev/null +++ b/argopy/tests/test_fetchers_index_erddap.py @@ -0,0 +1,137 @@ +import pandas as pd + +import pytest +import tempfile + +import argopy +from argopy import IndexFetcher as ArgoIndexFetcher +from argopy.errors import ( + FileSystemHasNoCache, + CacheFileNotFound +) +from utils import requires_connected_erddap_index, safe_to_server_errors, ci_erddap_index + +ERDDAP_TIMEOUT = 3 * 60 +safe_to_no_cache = pytest.mark.skipif(True, reason="Cache disabled for erddap index fetcher") + + +@ci_erddap_index +@requires_connected_erddap_index +class Test_Backend_WMO: + """ Test ERDDAP index fetching backend for WMO access point""" + src = "erddap" + requests = { + "float": [[2901623], [2901623, 6901929]] + } + + def test_nocache(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir): + fetcher = ArgoIndexFetcher(src=self.src, cache=False).float(self.requests['float'][0]).fetcher + with pytest.raises(FileSystemHasNoCache): + fetcher.cachepath + + @safe_to_no_cache + def test_cachepath_notfound(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir): + fetcher = ArgoIndexFetcher(src=self.src, cache=True).float(self.requests['float'][0]).fetcher + with pytest.raises(CacheFileNotFound): + fetcher.cachepath + + @safe_to_no_cache + @safe_to_server_errors + def test_cached(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir, api_timeout=ERDDAP_TIMEOUT): + fetcher = ArgoIndexFetcher(src=self.src, cache=True).float(self.requests['float'][0]).fetcher + df = fetcher.to_dataframe() + assert isinstance(df, pd.core.frame.DataFrame) + assert isinstance(fetcher.cachepath, str) + + @safe_to_no_cache + @safe_to_server_errors + def test_clearcache(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir, api_timeout=ERDDAP_TIMEOUT): + fetcher = ArgoIndexFetcher(src=self.src, cache=True).float(self.requests['float'][0]).fetcher + fetcher.to_dataframe() + fetcher.clear_cache() + with pytest.raises(CacheFileNotFound): + fetcher.cachepath + + def test_url(self): + for arg in self.requests["float"]: + fetcher = ArgoIndexFetcher(src=self.src).float(arg).fetcher + assert isinstance(fetcher.url, str) + + @safe_to_server_errors + def test_phy_float(self): + for arg in self.requests["float"]: + with argopy.set_options(api_timeout=ERDDAP_TIMEOUT): + fetcher = ArgoIndexFetcher(src=self.src).float(arg).fetcher + df = fetcher.to_dataframe() + assert isinstance(df, pd.core.frame.DataFrame) + + +@ci_erddap_index +@requires_connected_erddap_index +class Test_Backend_BOX: + """ Test ERDDAP index fetching backend for the BOX access point """ + + src = "erddap" + requests = { + "region": [ + [-60, -50, 40.0, 50.0], + [-60, -55, 40.0, 45.0, "2007-08-01", "2007-09-01"], + ], + } + + def test_nocache(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir): + fetcher = ArgoIndexFetcher(src=self.src, cache=False).region(self.requests['region'][-1]).fetcher + with pytest.raises(FileSystemHasNoCache): + fetcher.cachepath + + @safe_to_no_cache + def test_cachepath_notfound(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir): + fetcher = ArgoIndexFetcher(src=self.src, cache=True).region(self.requests['region'][-1]).fetcher + with pytest.raises(CacheFileNotFound): + fetcher.cachepath + + @safe_to_no_cache + @safe_to_server_errors + def test_cached(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir, api_timeout=ERDDAP_TIMEOUT): + fetcher = ArgoIndexFetcher(src=self.src, cache=True).region(self.requests['region'][-1]).fetcher + df = fetcher.to_dataframe() + assert isinstance(df, pd.core.frame.DataFrame) + assert isinstance(fetcher.cachepath, str) + + @safe_to_no_cache + @safe_to_server_errors + def test_clearcache(self): + with tempfile.TemporaryDirectory() as testcachedir: + with argopy.set_options(cachedir=testcachedir, api_timeout=ERDDAP_TIMEOUT): + fetcher = ArgoIndexFetcher(src=self.src, cache=True).region(self.requests['region'][-1]).fetcher + fetcher.to_dataframe() + fetcher.clear_cache() + with pytest.raises(CacheFileNotFound): + fetcher.cachepath + + def test_url(self): + for arg in self.requests["region"]: + fetcher = ArgoIndexFetcher(src=self.src).region(arg).fetcher + assert isinstance(fetcher.url, str) + + @safe_to_server_errors + def test_phy_region(self): + for arg in self.requests["region"]: + with argopy.set_options(api_timeout=ERDDAP_TIMEOUT): + fetcher = ArgoIndexFetcher(src=self.src).region(arg).fetcher + df = fetcher.to_dataframe() + assert isinstance(df, pd.core.frame.DataFrame) diff --git a/argopy/tests/test_fetchers_index_gdac.py b/argopy/tests/test_fetchers_index_gdac.py new file mode 100644 index 00000000..02e71e92 --- /dev/null +++ b/argopy/tests/test_fetchers_index_gdac.py @@ -0,0 +1,201 @@ +import pandas as pd + +import pytest +import tempfile +import shutil +from urllib.parse import urlparse +import logging + +import argopy +from argopy import IndexFetcher as ArgoIndexFetcher +from argopy.errors import ( + CacheFileNotFound, + FileSystemHasNoCache, + FtpPathError +) +from argopy.utils.checkers import isconnected, is_list_of_strings +from utils import requires_gdac +from mocked_http import mocked_httpserver, mocked_server_address + + +log = logging.getLogger("argopy.tests.index.gdac") +skip_for_debug = pytest.mark.skipif(True, reason="Taking too long !") + + +""" +List ftp hosts to be tested. +Since the fetcher is compatible with host from local, http or ftp protocols, we +try to test them all: +""" +VALID_HOSTS = [argopy.tutorial.open_dataset("gdac")[0], + #'https://data-argo.ifremer.fr', + mocked_server_address, + # 'ftp://ftp.ifremer.fr/ifremer/argo', + # 'ftp://usgodae.org/pub/outgoing/argo', # ok, but slow down CI and no need for 2 ftp tests + 'MOCKFTP', # keyword to use the fake/mocked ftp server (running on localhost) + ] + +""" +List access points to be tested. +For each access points, we list 1-to-2 scenario to make sure all possibilities are tested +""" +VALID_ACCESS_POINTS = [ + {"float": [13857]}, + {"profile": [13857, 90]}, + {"region": [-20, -16., 0, 1]}, + {"region": [-20, -16., 0, 1, "1997-07-01", "1997-09-01"]}, + ] + +""" +List parallel methods to be tested. +""" +valid_parallel_opts = [ + {"parallel": "thread"}, + # {"parallel": True, "parallel_method": "thread"}, # opts0 + # {"parallel": True, "parallel_method": "process"} # opts1 +] + + +@requires_gdac +def create_fetcher(fetcher_args, access_point, xfail=False): + """ Create a fetcher for a given set of facade options and access point + + Use xfail=True when a test with this fetcher is expected to fail + """ + def core(fargs, apts): + try: + f = ArgoIndexFetcher(**fargs) + if "float" in apts: + f = f.float(apts['float']) + elif "profile" in apts: + f = f.profile(*apts['profile']) + elif "region" in apts: + f = f.region(apts['region']) + except Exception: + raise + return f + fetcher = core(fetcher_args, access_point).fetcher + return fetcher + + +def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): + """Assert a data fetcher. + + This should be used by all tests + """ + assert isinstance(this_fetcher.to_dataframe(), pd.core.frame.DataFrame) + assert (this_fetcher.N_RECORDS >= 1) # Make sure we loaded the index file content + assert (this_fetcher.N_FILES >= 1) # Make sure we found results + if cacheable: + assert is_list_of_strings(this_fetcher.cachepath) + + +def ftp_shortname(ftp): + """Get a short name for scenarios IDs, given a FTP host""" + if ftp == 'MOCKFTP': + return 'ftp_mocked' + elif 'localhost' in ftp or '127.0.0.1' in ftp: + return 'http_mocked' + else: + return (lambda x: 'file' if x == "" else x)(urlparse(ftp).scheme) + + +@requires_gdac +class TestBackend: + src = 'gdac' + + # Create list of tests scenarios + # combine all hosts with all access points: + scenarios = [(h, ap) for h in VALID_HOSTS for ap in VALID_ACCESS_POINTS] + + scenarios_ids = [ + "%s, %s" % (ftp_shortname(fix[0]), list(fix[1].keys())[0]) for + fix + in + scenarios] + + ############# + # UTILITIES # + ############# + + def setup_class(self): + """setup any state specific to the execution of the given class""" + # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests + self.cachedir = tempfile.mkdtemp() + + def _patch_ftp(self, ftp): + """Patch Mocked FTP server keyword""" + if ftp == 'MOCKFTP': + return pytest.MOCKFTP # this was set in conftest.py + else: + return ftp + + def _setup_fetcher(self, this_request, cached=False): + """Helper method to set up options for a fetcher creation""" + if isinstance(this_request.param, tuple): + ftp = this_request.param[0] + access_point = this_request.param[1] + else: + ftp = this_request.param + access_point = VALID_ACCESS_POINTS[0] # Use 1st valid access point + + N_RECORDS = None if 'tutorial' in ftp or 'MOCK' in ftp else 100 # Make sure we're not going to load the full index + fetcher_args = {"src": self.src, "ftp": self._patch_ftp(ftp), "cache": False, "N_RECORDS": N_RECORDS} + + if cached: + fetcher_args = {**fetcher_args, **{"cache": True, "cachedir": self.cachedir}} + if not isconnected(fetcher_args['ftp']): + pytest.xfail("Fails because %s not available" % fetcher_args['ftp']) + else: + return fetcher_args, access_point + + @pytest.fixture + def _make_a_fetcher(self, request, mocked_httpserver): + """ Fixture to create a GDAC fetcher for a given host and access point """ + fetcher_args, access_point = self._setup_fetcher(request, cached=False) + yield create_fetcher(fetcher_args, access_point) + + @pytest.fixture + def _make_a_cached_fetcher(self, request): + """ Fixture to create a cached FTP fetcher for a given host and access point """ + fetcher_args, access_point = self._setup_fetcher(request, cached=True) + yield create_fetcher(fetcher_args, access_point) + + def teardown_class(self): + """Cleanup once we are finished.""" + def remove_test_dir(): + shutil.rmtree(self.cachedir) + remove_test_dir() + + ######### + # TESTS # + ######### + def test_nocache(self, mocked_httpserver): + this_fetcher = create_fetcher({"src": self.src, "ftp": self._patch_ftp(VALID_HOSTS[0]), "N_RECORDS": 10}, VALID_ACCESS_POINTS[0]) + with pytest.raises(FileSystemHasNoCache): + this_fetcher.cachepath + + @pytest.mark.parametrize("_make_a_fetcher", VALID_HOSTS, + indirect=True, + ids=["%s" % ftp_shortname(ftp) for ftp in VALID_HOSTS]) + def test_hosts(self, mocked_httpserver, _make_a_fetcher): + assert (_make_a_fetcher.N_RECORDS >= 1) + + @pytest.mark.parametrize("ftp_host", ['invalid', 'https://invalid_ftp', 'ftp://invalid_ftp'], indirect=False) + def test_hosts_invalid(self, ftp_host): + # Invalid servers: + with pytest.raises(FtpPathError): + create_fetcher({"src": self.src, "ftp": ftp_host}, VALID_ACCESS_POINTS[0]) + + @pytest.mark.parametrize("_make_a_fetcher", scenarios, indirect=True, ids=scenarios_ids) + def test_fetching(self, mocked_httpserver, _make_a_fetcher): + assert_fetcher(mocked_httpserver, _make_a_fetcher, cacheable=False) + + @pytest.mark.parametrize("_make_a_cached_fetcher", scenarios, indirect=True, ids=scenarios_ids) + def test_fetching_cached(self, mocked_httpserver, _make_a_cached_fetcher): + # Assert the fetcher (this trigger data fetching, hence caching as well): + assert_fetcher(mocked_httpserver, _make_a_cached_fetcher, cacheable=True) + # and we also make sure we can clear the cache: + _make_a_cached_fetcher.clear_cache() + with pytest.raises(CacheFileNotFound): + _make_a_cached_fetcher.cachepath diff --git a/argopy/tests/test_fetchers_proto.py b/argopy/tests/test_fetchers_proto.py new file mode 100644 index 00000000..24f669ff --- /dev/null +++ b/argopy/tests/test_fetchers_proto.py @@ -0,0 +1,53 @@ +import pytest +import xarray +from argopy.data_fetchers.proto import ArgoDataFetcherProto +from argopy.utils import to_list + + +class Fetcher(ArgoDataFetcherProto): + dataset_id = 'phy' + + def to_xarray(self, *args, **kwargs): + super(Fetcher, self).to_xarray(*args, **kwargs) + + def filter_data_mode(self, *args, **kwargs): + super(Fetcher, self).filter_data_mode(*args, **kwargs) + + def filter_qc(self, *args, **kwargs): + super(Fetcher, self).filter_qc(*args, **kwargs) + + def filter_researchmode(self, *args, **kwargs): + super(Fetcher, self).filter_researchmode(*args, **kwargs) + + +def test_required_methods(): + f = Fetcher() + with pytest.raises(NotImplementedError): + f.to_xarray() + + with pytest.raises(NotImplementedError): + f.filter_data_mode(xarray.Dataset, str) + + with pytest.raises(NotImplementedError): + f.filter_qc(xarray.Dataset) + + with pytest.raises(NotImplementedError): + f.filter_researchmode(xarray.Dataset) + + +@pytest.mark.parametrize("profile", [[13857, None], [13857, 90]], + indirect=False, + ids=["%s" % p for p in [[13857, None], [13857, 90]]]) +def test_dashboard(profile): + + f = Fetcher() + f.WMO, f.CYC = profile + f.WMO = to_list(f.WMO) + f.CYC = to_list(f.CYC) + assert isinstance(f.dashboard(url_only=True), str) + + with pytest.warns(UserWarning): + f = Fetcher() + f.WMO = [1901393, 6902746] + f.CYC = None + f.dashboard(url_only=True) diff --git a/argopy/tests/test_plot_argo_colors.py b/argopy/tests/test_plot_argo_colors.py new file mode 100644 index 00000000..5613ff2b --- /dev/null +++ b/argopy/tests/test_plot_argo_colors.py @@ -0,0 +1,101 @@ +""" +This file covers the plotters module +We test plotting functions from IndexFetcher and DataFetcher +""" +import pytest +import logging + +from utils import ( + requires_matplotlib, + requires_seaborn, + has_matplotlib, + has_seaborn, +) +from argopy.plot import ArgoColors + +if has_matplotlib: + import matplotlib as mpl + known_colormaps = ArgoColors().known_colormaps + list_valid_known_colormaps = ArgoColors().list_valid_known_colormaps + quantitative = ArgoColors().quantitative +else: + known_colormaps = [] + list_valid_known_colormaps = [] + quantitative = [] + +if has_seaborn: + import seaborn as sns + +log = logging.getLogger("argopy.tests.plot") + + +@requires_matplotlib +class Test_ArgoColors: + + @pytest.mark.parametrize("cname", known_colormaps, indirect=False) + def test_Argo_colormaps(self, cname): + ac = ArgoColors(name=cname) + assert ac.registered + assert isinstance(ac.cmap, mpl.colors.LinearSegmentedColormap) + + @pytest.mark.parametrize("cname", quantitative, indirect=False) + def test_quantitative_colormaps(self, cname): + ac = ArgoColors(name=cname) + assert ac.Ncolors == quantitative[cname] + assert isinstance(ac.cmap, mpl.colors.LinearSegmentedColormap) + + @pytest.mark.parametrize("opts", [('Spectral', None), + ('Blues', 13), + ('k', 1)], + ids = ["name='Spectral', N=None", + "name='Blues', N=13", + "name='k', N=1"], + indirect=False) + def test_other_colormaps(self, opts): + name, N = opts + ac = ArgoColors(name, N) + assert isinstance(ac.cmap, mpl.colors.LinearSegmentedColormap) + + @pytest.mark.parametrize("N", [12.35, '12'], + ids=['N is a float', 'N is a str'], + indirect=False) + def test_invalid_Ncolors(self, N): + with pytest.raises(ValueError): + ArgoColors(N=N) + + @pytest.mark.parametrize("cname", ['data_mode', 'dm'], ids=['key', 'aka'], indirect=False) + def test_definition(self, cname): + assert isinstance( ArgoColors(cname).definition, dict) + + def test_colors_lookup_dict(self): + ac = ArgoColors(list_valid_known_colormaps[0]) + assert isinstance(ac.lookup, dict) + + with pytest.raises(ValueError): + ArgoColors('Blues').lookup + + def test_ticklabels_dict(self): + ac = ArgoColors(list_valid_known_colormaps[0]) + assert isinstance(ac.ticklabels, dict) + + with pytest.raises(ValueError): + ArgoColors('Blues').ticklabels + + @requires_seaborn + def test_seaborn_palette(self): + assert isinstance(ArgoColors('Set1').palette, sns.palettes._ColorPalette) + assert isinstance(ArgoColors('k', N=1).palette, sns.palettes._ColorPalette) + + @pytest.mark.parametrize("cname", ['data_mode', 'Blues'], + ids=['known', 'other'], + indirect=False) + def test_repr_html_(self, cname): + ac = ArgoColors(cname) + assert isinstance(ac._repr_html_(), str) + + @pytest.mark.parametrize("cname", ['data_mode', 'Blues'], + ids=['known', 'other'], + indirect=False) + def test_show_COLORS(self, cname): + ac = ArgoColors(cname) + assert isinstance(ac.show_COLORS(), str) diff --git a/argopy/tests/test_plot_dashboards.py b/argopy/tests/test_plot_dashboards.py new file mode 100644 index 00000000..50ac891f --- /dev/null +++ b/argopy/tests/test_plot_dashboards.py @@ -0,0 +1,66 @@ +""" +This file covers the argopy.plot.dashboards submodule +""" +import pytest +import logging +import tempfile + +import argopy +from argopy.errors import InvalidDashboard +from utils import ( + requires_connection, + requires_ipython, + has_ipython, +) +from mocked_http import mocked_httpserver, mocked_server_address + +if has_ipython: + import IPython + +log = logging.getLogger("argopy.tests.plot.dashboards") + + +@pytest.mark.parametrize("board_type", ["invalid", "op", "ocean-ops", "coriolis"], indirect=False) +def test_invalid_dashboard(board_type): + # Test types without 'base' + with pytest.raises(InvalidDashboard): + argopy.dashboard(type=board_type, url_only=True) + + +@pytest.mark.parametrize("board_type", ["op", "ocean-ops", "coriolis"], indirect=False) +def test_invalid_dashboard_profile(board_type): + # Test types without 'cyc' + with pytest.raises(InvalidDashboard): + argopy.dashboard(6902755, 12, type=board_type, url_only=True) + + +@pytest.mark.parametrize("board_type", ["data", "meta", "ea", "argovis", "bgc"], indirect=False) +def test_valid_dashboard(board_type): + # Test types with 'base' + assert isinstance(argopy.dashboard(type=board_type, url_only=True), str) + + +@pytest.mark.parametrize("board_type", ["data", "meta", "ea", "argovis", "op", "ocean-ops", "bgc"], indirect=False) +def test_valid_dashboard_float(board_type, mocked_httpserver): + # Test types with 'wmo' (should be all) + with argopy.set_options(server=mocked_server_address): + assert isinstance(argopy.dashboard(6901929, type=board_type, url_only=True), str) + + +@pytest.mark.parametrize("board_type", ["data", "meta", "ea", "argovis", "bgc"], indirect=False) +def test_valid_dashboard_profile(board_type, mocked_httpserver): + # Test types with 'cyc' + with argopy.set_options(cachedir=tempfile.mkdtemp(), server=mocked_server_address): + assert isinstance(argopy.dashboard(5904797, 12, type=board_type, url_only=True), str) + + +@requires_ipython +@pytest.mark.parametrize("opts", [{}, {'wmo': 6901929}, {'wmo': 6901929, 'cyc': 3}], + ids=['', 'WMO', 'WMO, CYC'], + indirect=False) +def test_valid_dashboard_ipython_output(opts, mocked_httpserver): + with argopy.set_options(cachedir=tempfile.mkdtemp(), server=mocked_server_address): + dsh = argopy.dashboard(**opts) + assert isinstance(dsh, IPython.lib.display.IFrame) + + diff --git a/argopy/tests/test_plot_plot.py b/argopy/tests/test_plot_plot.py new file mode 100644 index 00000000..f05bdfe1 --- /dev/null +++ b/argopy/tests/test_plot_plot.py @@ -0,0 +1,212 @@ +""" +This file covers the argopy.plot.plot submodule +""" +import pytest +import logging +from typing import Callable + +import argopy +from utils import ( + requires_gdac, + requires_connection, + requires_matplotlib, + requires_ipython, + requires_cartopy, + has_matplotlib, + has_seaborn, + has_cartopy, + has_ipython, + has_ipywidgets, +) + +from argopy.plot import bar_plot, plot_trajectory, open_sat_altim_report, scatter_map +from argopy.errors import InvalidDatasetStructure +from argopy import DataFetcher as ArgoDataFetcher +from mocked_http import mocked_server_address + +if has_matplotlib: + import matplotlib as mpl + +if has_cartopy: + import cartopy + +if has_ipython: + import IPython + + +log = logging.getLogger("argopy.tests.plot.plot") +argopy.clear_cache() + + +class Test_open_sat_altim_report: + WMOs = [2901623, [2901623, 6901929]] + + def test_load_mocked_server(self, mocked_httpserver): + """This will easily ensure that the module scope fixture is available to all methods !""" + assert True + + @pytest.mark.parametrize("WMOs", WMOs, + ids=['For unique WMO', 'For a list of WMOs'], + indirect=False) + @pytest.mark.parametrize("embed", ['dropdown', 'slide', 'list', None], + indirect=False) + def test_open_sat_altim_report(self, WMOs, embed): + if has_ipython: + import IPython + + dsh = open_sat_altim_report(WMO=WMOs, embed=embed, api_server=mocked_server_address) + + if has_ipython and embed is not None: + + if has_ipywidgets: + + if embed == "dropdown": + assert isinstance(dsh, Callable) + assert isinstance(dsh(2901623), IPython.display.Image) + + if embed == "slide": + assert isinstance(dsh, Callable) + + else: + assert dsh is None + + else: + assert isinstance(dsh, dict) + + @requires_ipython + def test_invalid_method(self): + with pytest.raises(ValueError): + open_sat_altim_report(WMO=self.WMOs[0], embed='dummy_method', api_server=mocked_server_address) + + +@requires_gdac +@requires_matplotlib +class Test_plot_trajectory: + src = "gdac" + local_ftp = argopy.tutorial.open_dataset("gdac")[0] + requests = { + # "float": [[2901623], [2901623, 6901929, 5906072]], + # "profile": [[2901623, 12], [6901929, [5, 45]]], + "region": [ + [-60, -40, 40.0, 60.0, 0., 10.], + [-60, -40, 40.0, 60.0, 0., 10., "2007-08-01", "2007-09-01"], + ], + } + + opts = [(ws, wc, lg) for ws in [False, has_seaborn] for wc in [False, has_cartopy] for lg in [True, False]] + opts_ids = ["with_seaborn=%s, with_cartopy=%s, add_legend=%s" % (opt[0], opt[1], opt[2]) for opt in opts] + + def __test_traj_plot(self, df, opts): + with_seaborn, with_cartopy, with_legend = opts + fig, ax = plot_trajectory( + df, with_seaborn=with_seaborn, with_cartopy=with_cartopy, add_legend=with_legend + ) + assert isinstance(fig, mpl.figure.Figure) + + expected_ax_type = ( + cartopy.mpl.geoaxes.GeoAxesSubplot + if has_cartopy and with_cartopy + else mpl.axes.Axes + ) + assert isinstance(ax, expected_ax_type) + + expected_lg_type = mpl.legend.Legend if with_legend else type(None) + assert isinstance(ax.get_legend(), expected_lg_type) + + mpl.pyplot.close(fig) + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_with_a_region(self, opts): + with argopy.set_options(src=self.src, ftp=self.local_ftp): + for arg in self.requests["region"]: + loader = ArgoDataFetcher(cache=True).region(arg).load() + self.__test_traj_plot(loader.index, opts) + + +@requires_gdac +@requires_matplotlib +class Test_bar_plot: + src = "gdac" + local_ftp = argopy.tutorial.open_dataset("gdac")[0] + requests = { + # "float": [[2901623], [2901623, 6901929, 5906072]], + # "profile": [[2901623, 12], [6901929, [5, 45]]], + "region": [ + [-60, -40, 40.0, 60.0, 0., 10.], + [-60, -40, 40.0, 60.0, 0., 10., "2007-08-01", "2007-09-01"], + ], + } + + opts = [(ws, by) for ws in [False, has_seaborn] for by in ['institution', 'profiler']] + opts_ids = ["with_seaborn=%s, by=%s" % (opt[0], opt[1]) for opt in opts] + + def __test_bar_plot(self, df, opts): + with_seaborn, by = opts + fig, ax = bar_plot(df, by=by, with_seaborn=with_seaborn) + assert isinstance(fig, mpl.figure.Figure) + mpl.pyplot.close(fig) + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_with_a_region(self, opts): + with argopy.set_options(src=self.src, ftp=self.local_ftp): + for arg in self.requests["region"]: + loader = ArgoDataFetcher().region(arg) + self.__test_bar_plot(loader.to_index(full=True), opts) + + +@requires_gdac +@requires_matplotlib +@requires_cartopy +class Test_scatter_map: + src = "gdac" + local_ftp = argopy.tutorial.open_dataset("gdac")[0] + requests = { + # "float": [[2901623], [2901623, 6901929, 5906072]], + # "profile": [[2901623, 12], [6901929, [5, 45]]], + "region": [ + [-60, -40, 40.0, 60.0, 0., 10.], + [-60, -40, 40.0, 60.0, 0., 10., "2007-08-01", "2007-09-01"], + ], + } + + opts = [(traj, lg) for traj in [True, False] for lg in [True, False]] + opts_ids = ["traj=%s, legend=%s" % (opt[0], opt[1]) for opt in opts] + + def __test(self, data, axis, opts): + traj, legend = opts + fig, ax = scatter_map( + data, x=axis[0], y=axis[1], hue=axis[2], traj=traj, traj_axis=axis[2], legend=legend + ) + assert isinstance(fig, mpl.figure.Figure) + + assert isinstance(ax, cartopy.mpl.geoaxes.GeoAxesSubplot) + + expected_lg_type = mpl.legend.Legend if legend else type(None) + assert isinstance(ax.get_legend(), expected_lg_type) + + mpl.pyplot.close(fig) + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_with_a_dataset_of_points(self, opts): + with argopy.set_options(src=self.src, ftp=self.local_ftp): + for arg in self.requests["region"]: + loader = ArgoDataFetcher(cache=True).region(arg).load() + with pytest.raises(InvalidDatasetStructure): + self.__test(loader.data, (None, None, None), opts) + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_with_a_dataset_of_profiles(self, opts): + with argopy.set_options(src=self.src, ftp=self.local_ftp): + for arg in self.requests["region"]: + loader = ArgoDataFetcher(cache=True).region(arg).load() + dsp = loader.data.argo.point2profile() + # with pytest.warns(UserWarning): + # self.__test(dsp, (None, None, None), opts) + self.__test(dsp.isel(N_LEVELS=0), (None, None, None), opts) + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_with_a_dataframe_of_index(self, opts): + with argopy.set_options(src=self.src, ftp=self.local_ftp): + for arg in self.requests["region"]: + loader = ArgoDataFetcher(cache=True).region(arg).load() + self.__test(loader.index, (None, None, None), opts) diff --git a/argopy/tests/test_related.py b/argopy/tests/test_related.py new file mode 100644 index 00000000..031d2710 --- /dev/null +++ b/argopy/tests/test_related.py @@ -0,0 +1,327 @@ +import pytest +import tempfile +import xarray as xr +import pandas as pd +from collections import ChainMap, OrderedDict +import shutil + +from mocked_http import mocked_httpserver, mocked_server_address +from utils import ( + requires_matplotlib, + requires_cartopy, + requires_oops, + has_matplotlib, + has_cartopy, + has_ipython, +) +import argopy +from argopy.related import ( + TopoFetcher, + ArgoNVSReferenceTables, + OceanOPSDeployments, + ArgoDocs, + load_dict, mapp_dict, + get_coriolis_profile_id, get_ea_profile_page +) +from argopy.utils.checkers import ( + is_list_of_strings, +) + +if has_matplotlib: + import matplotlib as mpl + +if has_cartopy: + import cartopy + +if has_ipython: + import IPython + + +class Test_TopoFetcher(): + box = [81, 123, -67, -54] + + def setup_class(self): + """setup any state specific to the execution of the given class""" + # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests + self.cachedir = tempfile.mkdtemp() + + def teardown_class(self): + """Cleanup once we are finished.""" + def remove_test_dir(): + shutil.rmtree(self.cachedir) + remove_test_dir() + + def make_a_fetcher(self, cached=False): + opts = {'ds': 'gebco', 'stride': [10, 10], 'server': mocked_server_address} + if cached: + opts = ChainMap(opts, {'cache': True, 'cachedir': self.cachedir}) + return TopoFetcher(self.box, **opts) + + def assert_fetcher(self, f): + ds = f.to_xarray() + assert isinstance(ds, xr.Dataset) + assert 'elevation' in ds.data_vars + + def test_load_mocked_server(self, mocked_httpserver): + """This will easily ensure that the module scope fixture is available to all methods !""" + assert True + + params = [True, False] + ids_params = ["cached=%s" % p for p in params] + @pytest.mark.parametrize("params", params, indirect=False, ids=ids_params) + def test_fetching(self, params): + fetcher = self.make_a_fetcher(cached=params) + self.assert_fetcher(fetcher) + + +class Test_ArgoNVSReferenceTables: + + def setup_class(self): + """setup any state specific to the execution of the given class""" + # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests + self.cachedir = tempfile.mkdtemp() + self.nvs = ArgoNVSReferenceTables(cache=True, cachedir=self.cachedir, nvs=mocked_server_address) + + def teardown_class(self): + """Cleanup once we are finished.""" + def remove_test_dir(): + shutil.rmtree(self.cachedir) + remove_test_dir() + + def test_load_mocked_server(self, mocked_httpserver): + """This will easily ensure that the module scope fixture is available to all methods !""" + assert True + + def test_valid_ref(self): + assert is_list_of_strings(self.nvs.valid_ref) + + opts = [3, 'R09'] + opts_ids = ["rtid is a %s" % type(o) for o in opts] + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_tbl(self, opts): + assert isinstance(self.nvs.tbl(opts), pd.DataFrame) + + opts = [3, 'R09'] + opts_ids = ["rtid is a %s" % type(o) for o in opts] + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_tbl_name(self, opts): + names = self.nvs.tbl_name(opts) + assert isinstance(names, tuple) + assert isinstance(names[0], str) + assert isinstance(names[1], str) + assert isinstance(names[2], str) + + def test_all_tbl(self): + all = self.nvs.all_tbl + assert isinstance(all, OrderedDict) + assert isinstance(all[list(all.keys())[0]], pd.DataFrame) + + def test_all_tbl_name(self): + all = self.nvs.all_tbl_name + assert isinstance(all, OrderedDict) + assert isinstance(all[list(all.keys())[0]], tuple) + + opts = ["ld+json", "rdf+xml", "text/turtle", "invalid"] + opts_ids = ["fmt=%s" % o for o in opts] + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_get_url(self, opts): + if opts != 'invalid': + url = self.nvs.get_url(3, fmt=opts) + assert isinstance(url, str) + if "json" in opts: + data = self.nvs.fs.open_json(url) + assert isinstance(data, dict) + elif "xml" in opts: + data = self.nvs.fs.fs.cat_file(url) + assert data[0:5] == b'= 1) # Make sure we loaded the index file content + + @pytest.mark.parametrize( + "ftp_host", + ["invalid", "https://invalid_ftp", "ftp://invalid_ftp"], + indirect=False, + ) + def test_hosts_invalid(self, ftp_host): + # Invalid servers: + with pytest.raises(FtpPathError): + self.indexstore(host=ftp_host) + + def test_index(self): + def new_idx(): + return self.indexstore( + host=self.host, index_file=self.index_file, cache=False + ) + + self.assert_index(new_idx().load()) + self.assert_index(new_idx().load(force=True)) + + N = np.random.randint(1, 100 + 1) + idx = new_idx().load(nrows=N) + self.assert_index(idx) + assert idx.index.shape[0] == N + # Since no search was triggered: + assert idx.N_FILES == idx.N_RECORDS + + with pytest.raises(OptionValueError): + idx = self.indexstore( + host=self.host, index_file="ar_greylist.txt", cache=False + ) + + # @pytest.mark.parametrize( + # "a_search", search_scenarios, indirect=True, ids=search_scenarios_ids + # ) + # def test_a_search(self, mocked_httpserver, a_search): + # self.assert_search(a_search, cacheable=False) + + @pytest.mark.parametrize( + "a_search", search_scenarios_bool, indirect=True, ids=search_scenarios_bool_ids + ) + def test_a_search_with_logical(self, mocked_httpserver, a_search): + self.assert_search(a_search, cacheable=False) + + def test_to_dataframe_index(self): + idx = self.new_idx() + assert isinstance(idx.to_dataframe(), pd.core.frame.DataFrame) + + df = idx.to_dataframe(index=True) + assert df.shape[0] == idx.N_RECORDS + + df = idx.to_dataframe() + assert df.shape[0] == idx.N_RECORDS + + N = np.random.randint(1, 20 + 1) + df = idx.to_dataframe(index=True, nrows=N) + assert df.shape[0] == N + + def test_to_dataframe_search(self): + idx = self.new_idx() + wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] + idx = idx.search_wmo(wmo) + + df = idx.to_dataframe() + assert isinstance(df, pd.core.frame.DataFrame) + assert df.shape[0] == idx.N_MATCH + + N = np.random.randint(1, 10 + 1) + df = idx.to_dataframe(nrows=N) + assert df.shape[0] == N + + def test_caching_index(self): + idx = self.new_idx(cache=True) + self.assert_index(idx, cacheable=True) + + def test_caching_search(self): + idx = self.new_idx(cache=True) + wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] + idx.search_wmo(wmo) + self.assert_search(idx, cacheable=True) + + @pytest.mark.parametrize( + "index", + [False, True], + indirect=False, + ids=["index=%s" % i for i in [False, True]], + ) + def test_read_wmo(self, index): + wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] + idx = self.new_idx().search_wmo(wmo) + WMOs = idx.read_wmo(index=index) + if index: + assert [str(w).isdigit() for w in WMOs] + else: + assert len(WMOs) == len(wmo) + + @pytest.mark.parametrize( + "index", + [False, True], + indirect=False, + ids=["index=%s" % i for i in [False, True]], + ) + def test_read_params(self, index): + wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] + idx = self.new_idx().search_wmo(wmo) + if self.network == "bgc": + params = idx.read_params(index=index) + assert is_list_of_strings(params) + else: + with pytest.raises(InvalidDatasetStructure): + idx.read_params(index=index) + + @pytest.mark.parametrize( + "index", + [False, True], + indirect=False, + ids=["index=%s" % i for i in [False, True]], + ) + def test_records_per_wmo(self, index): + wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] + idx = self.new_idx().search_wmo(wmo) + C = idx.records_per_wmo(index=index) + for w in C: + assert str(C[w]).isdigit() + + def test_to_indexfile(self): + # Create a store and make a simple float search: + idx0 = self.new_idx() + wmo = [s["wmo"] for s in VALID_SEARCHES if "wmo" in s.keys()][0] + idx0 = idx0.search_wmo(wmo) + + # Then save this search as a new Argo index file: + tf = tempfile.NamedTemporaryFile(delete=False) + new_indexfile = idx0.to_indexfile(tf.name) + + # Finally try to load the new index file, like it was an official one: + idx = self.new_idx( + host=os.path.dirname(new_indexfile), + index_file=os.path.basename(new_indexfile), + convention=idx0.convention, + ) + self.assert_index(idx.load()) + + # Cleanup + tf.close() + +############################ +# TESTS FOR PANDAS BACKEND # +############################ + +# @skip_this +class Test_IndexStore_pandas_CORE(IndexStore_test_proto): + network = "core" + indexstore = indexstore_pandas + index_file = "ar_index_global_prof.txt" + +# @skip_this +class Test_IndexStore_pandas_BGC_synthetic(IndexStore_test_proto): + network = "bgc" + indexstore = indexstore_pandas + index_file = "argo_synthetic-profile_index.txt" + +# @skip_this +class Test_IndexStore_pandas_BGC_bio(IndexStore_test_proto): + network = "bgc" + indexstore = indexstore_pandas + index_file = "argo_bio-profile_index.txt" + +############################# +# TESTS FOR PYARROW BACKEND # +############################# + +# @skip_this +@skip_pyarrow +class Test_IndexStore_pyarrow_CORE(IndexStore_test_proto): + network = "core" + from argopy.stores.argo_index_pa import indexstore_pyarrow + + indexstore = indexstore_pyarrow + index_file = "ar_index_global_prof.txt" + +# @skip_this +@skip_pyarrow +class Test_IndexStore_pyarrow_BGC_bio(IndexStore_test_proto): + network = "bgc" + from argopy.stores.argo_index_pa import indexstore_pyarrow + + indexstore = indexstore_pyarrow + index_file = "argo_bio-profile_index.txt" + + +# @skip_this +@skip_pyarrow +class Test_IndexStore_pyarrow_BGC_synthetic(IndexStore_test_proto): + network = "bgc" + from argopy.stores.argo_index_pa import indexstore_pyarrow + + indexstore = indexstore_pyarrow + index_file = "argo_synthetic-profile_index.txt" diff --git a/argopy/tests/test_tutorial.py b/argopy/tests/test_tutorial.py new file mode 100644 index 00000000..17a5e8ca --- /dev/null +++ b/argopy/tests/test_tutorial.py @@ -0,0 +1,27 @@ +import pytest +import argopy +from utils import requires_connection + + +def test_invalid_dataset(): + with pytest.raises(ValueError): + argopy.tutorial.open_dataset('invalid_dataset') + + +@requires_connection +def test_gdac_dataset(): + ftproot, flist = argopy.tutorial.open_dataset('gdac') + assert isinstance(ftproot, str) + assert isinstance(flist, list) + + +@requires_connection +def test_weekly_index_dataset(): + rpath, txtfile = argopy.tutorial.open_dataset('weekly_index_prof') + assert isinstance(txtfile, str) + + +@requires_connection +def test_global_index_dataset(): + rpath, txtfile = argopy.tutorial.open_dataset('global_index_prof') + assert isinstance(txtfile, str) diff --git a/argopy/tests/test_utils_accessories.py b/argopy/tests/test_utils_accessories.py new file mode 100644 index 00000000..f786d8c1 --- /dev/null +++ b/argopy/tests/test_utils_accessories.py @@ -0,0 +1,86 @@ +import pytest +from argopy.utils.accessories import float_wmo, Registry + + +class Test_float_wmo(): + + def test_init(self): + assert isinstance(float_wmo(2901746), float_wmo) + assert isinstance(float_wmo(float_wmo(2901746)), float_wmo) + + def test_isvalid(self): + assert float_wmo(2901746).isvalid + assert not float_wmo(12, errors='ignore').isvalid + + def test_ppt(self): + assert isinstance(str(float_wmo(2901746)), str) + assert isinstance(repr(float_wmo(2901746)), str) + + def test_comparisons(self): + assert float_wmo(2901746) == float_wmo(2901746) + assert float_wmo(2901746) != float_wmo(2901745) + assert float_wmo(2901746) >= float_wmo(2901746) + assert float_wmo(2901746) > float_wmo(2901745) + assert float_wmo(2901746) <= float_wmo(2901746) + assert float_wmo(2901746) < float_wmo(2901747) + + def test_hashable(self): + assert isinstance(hash(float_wmo(2901746)), int) + + +class Test_Registry(): + + opts = [(None, 'str'), (['hello', 'world'], str), (None, float_wmo), ([2901746, 4902252], float_wmo)] + opts_ids = ["%s, %s" % ((lambda x: 'iterlist' if x is not None else x)(opt[0]), repr(opt[1])) for opt in opts] + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_init(self, opts): + assert isinstance(Registry(opts[0], dtype=opts[1]), Registry) + + opts = [(['hello', 'world'], str), ([2901746, 4902252], float_wmo)] + opts_ids = ["%s, %s" % ((lambda x: 'iterlist' if x is not None else x)(opt[0]), repr(opt[1])) for opt in opts] + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_commit(self, opts): + R = Registry(dtype=opts[1]) + R.commit(opts[0]) + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_append(self, opts): + R = Registry(dtype=opts[1]) + R.append(opts[0][0]) + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_extend(self, opts): + R = Registry(dtype=opts[1]) + R.append(opts[0]) + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_insert(self, opts): + R = Registry(opts[0][0], dtype=opts[1]) + R.insert(0, opts[0][-1]) + assert R[0] == opts[0][-1] + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_remove(self, opts): + R = Registry(opts[0], dtype=opts[1]) + R.remove(opts[0][0]) + assert opts[0][0] not in R + + @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) + def test_copy(self, opts): + R = Registry(opts[0], dtype=opts[1]) + assert R == R.copy() + + bad_opts = [(['hello', 12], str), ([2901746, 1], float_wmo)] + bad_opts_ids = ["%s, %s" % ((lambda x: 'iterlist' if x is not None else x)(opt[0]), repr(opt[1])) for opt in opts] + + @pytest.mark.parametrize("opts", bad_opts, indirect=False, ids=bad_opts_ids) + def test_invalid_dtype(self, opts): + with pytest.raises(ValueError): + Registry(opts[0][0], dtype=opts[1], invalid='raise').commit(opts[0][-1]) + with pytest.warns(UserWarning): + Registry(opts[0][0], dtype=opts[1], invalid='warn').commit(opts[0][-1]) + # Raise nothing: + Registry(opts[0][0], dtype=opts[1], invalid='ignore').commit(opts[0][-1]) + diff --git a/argopy/tests/test_utils_caching.py b/argopy/tests/test_utils_caching.py new file mode 100644 index 00000000..e0841fdb --- /dev/null +++ b/argopy/tests/test_utils_caching.py @@ -0,0 +1,36 @@ +import os +import pandas as pd +import argopy +import tempfile +from argopy import DataFetcher as ArgoDataFetcher +from utils import ( + requires_gdac, +) +from argopy.utils.caching import lscache, clear_cache + + +@requires_gdac +def test_clear_cache(): + ftproot, flist = argopy.tutorial.open_dataset("gdac") + with tempfile.TemporaryDirectory() as cachedir: + with argopy.set_options(cachedir=cachedir): + loader = ArgoDataFetcher(src="gdac", ftp=ftproot, cache=True).profile(2902696, 12) + loader.to_xarray() + clear_cache() + assert os.path.exists(cachedir) is True + assert len(os.listdir(cachedir)) == 0 + + +@requires_gdac +def test_lscache(): + ftproot, flist = argopy.tutorial.open_dataset("gdac") + with tempfile.TemporaryDirectory() as cachedir: + with argopy.set_options(cachedir=cachedir): + loader = ArgoDataFetcher(src="gdac", ftp=ftproot, cache=True).profile(2902696, 12) + loader.to_xarray() + result = lscache(cache_path=cachedir, prt=True) + assert isinstance(result, str) + + result = lscache(cache_path=cachedir, prt=False) + assert isinstance(result, pd.DataFrame) + diff --git a/argopy/tests/test_utils_checkers.py b/argopy/tests/test_utils_checkers.py new file mode 100644 index 00000000..b8c2d53d --- /dev/null +++ b/argopy/tests/test_utils_checkers.py @@ -0,0 +1,226 @@ +import pytest +import numpy as np +from mocked_http import mocked_httpserver, mocked_server_address +from utils import ( + requires_erddap, +) +import argopy +from argopy.errors import FtpPathError +from argopy.utils.checkers import ( + is_box, is_indexbox, + check_wmo, is_wmo, + check_cyc, is_cyc, + check_gdac_path, + isconnected, urlhaskeyword, isAPIconnected, erddap_ds_exists, isalive +) + + +class Test_is_box: + @pytest.fixture(autouse=True) + def create_data(self): + self.BOX3d = [0, 20, 40, 60, 0, 1000] + self.BOX4d = [0, 20, 40, 60, 0, 1000, "2001-01", "2001-6"] + + def test_box_ok(self): + assert is_box(self.BOX3d) + assert is_box(self.BOX4d) + + def test_box_notok(self): + for box in [[], list(range(0, 12))]: + with pytest.raises(ValueError): + is_box(box) + with pytest.raises(ValueError): + is_box(box, errors="raise") + assert not is_box(box, errors="ignore") + + def test_box_invalid_num(self): + for i in [0, 1, 2, 3, 4, 5]: + box = self.BOX3d + box[i] = "str" + with pytest.raises(ValueError): + is_box(box) + with pytest.raises(ValueError): + is_box(box, errors="raise") + assert not is_box(box, errors="ignore") + + def test_box_invalid_range(self): + for i in [0, 1, 2, 3, 4, 5]: + box = self.BOX3d + box[i] = -1000 + with pytest.raises(ValueError): + is_box(box) + with pytest.raises(ValueError): + is_box(box, errors="raise") + assert not is_box(box, errors="ignore") + + def test_box_invalid_str(self): + for i in [6, 7]: + box = self.BOX4d + box[i] = "str" + with pytest.raises(ValueError): + is_box(box) + with pytest.raises(ValueError): + is_box(box, errors="raise") + assert not is_box(box, errors="ignore") + + +class Test_is_indexbox: + @pytest.fixture(autouse=True) + def create_data(self): + self.BOX2d = [0, 20, 40, 60] + self.BOX3d = [0, 20, 40, 60, "2001-01", "2001-6"] + + def test_box_ok(self): + assert is_indexbox(self.BOX2d) + assert is_indexbox(self.BOX3d) + + def test_box_notok(self): + for box in [[], list(range(0, 12))]: + with pytest.raises(ValueError): + is_indexbox(box) + with pytest.raises(ValueError): + is_indexbox(box, errors="raise") + assert not is_indexbox(box, errors="ignore") + + def test_box_invalid_num(self): + for i in [0, 1, 2, 3]: + box = self.BOX2d + box[i] = "str" + with pytest.raises(ValueError): + is_indexbox(box) + with pytest.raises(ValueError): + is_indexbox(box, errors="raise") + assert not is_indexbox(box, errors="ignore") + + def test_box_invalid_range(self): + for i in [0, 1, 2, 3]: + box = self.BOX2d + box[i] = -1000 + with pytest.raises(ValueError): + is_indexbox(box) + with pytest.raises(ValueError): + is_indexbox(box, errors="raise") + assert not is_indexbox(box, errors="ignore") + + def test_box_invalid_str(self): + for i in [4, 5]: + box = self.BOX3d + box[i] = "str" + with pytest.raises(ValueError): + is_indexbox(box) + with pytest.raises(ValueError): + is_indexbox(box, errors="raise") + assert not is_indexbox(box, errors="ignore") + + +def test_is_wmo(): + assert is_wmo(12345) + assert is_wmo([12345]) + assert is_wmo([12345, 1234567]) + + with pytest.raises(ValueError): + is_wmo(1234, errors="raise") + with pytest.raises(ValueError): + is_wmo(-1234, errors="raise") + with pytest.raises(ValueError): + is_wmo(1234.12, errors="raise") + with pytest.raises(ValueError): + is_wmo(12345.7, errors="raise") + + with pytest.warns(UserWarning): + is_wmo(1234, errors="warn") + with pytest.warns(UserWarning): + is_wmo(-1234, errors="warn") + with pytest.warns(UserWarning): + is_wmo(1234.12, errors="warn") + with pytest.warns(UserWarning): + is_wmo(12345.7, errors="warn") + + assert not is_wmo(12, errors="ignore") + assert not is_wmo(-12, errors="ignore") + assert not is_wmo(1234.12, errors="ignore") + assert not is_wmo(12345.7, errors="ignore") + + +def test_check_wmo(): + assert check_wmo(12345) == [12345] + assert check_wmo([1234567]) == [1234567] + assert check_wmo([12345, 1234567]) == [12345, 1234567] + assert check_wmo(np.array((12345, 1234567), dtype='int')) == [12345, 1234567] + + +def test_is_cyc(): + assert is_cyc(123) + assert is_cyc([123]) + assert is_cyc([12, 123, 1234]) + + with pytest.raises(ValueError): + is_cyc(12345, errors="raise") + with pytest.raises(ValueError): + is_cyc(-1234, errors="raise") + with pytest.raises(ValueError): + is_cyc(1234.12, errors="raise") + with pytest.raises(ValueError): + is_cyc(12345.7, errors="raise") + + with pytest.warns(UserWarning): + is_cyc(12345, errors="warn") + with pytest.warns(UserWarning): + is_cyc(-1234, errors="warn") + with pytest.warns(UserWarning): + is_cyc(1234.12, errors="warn") + with pytest.warns(UserWarning): + is_cyc(12345.7, errors="warn") + + assert not is_cyc(12345, errors="ignore") + assert not is_cyc(-12, errors="ignore") + assert not is_cyc(1234.12, errors="ignore") + assert not is_cyc(12345.7, errors="ignore") + + +def test_check_cyc(): + assert check_cyc(123) == [123] + assert check_cyc([12]) == [12] + assert check_cyc([12, 123]) == [12, 123] + assert check_cyc(np.array((123, 1234), dtype='int')) == [123, 1234] + + +def test_check_gdac_path(): + assert check_gdac_path("dummy_path", errors='ignore') is False + with pytest.raises(FtpPathError): + check_gdac_path("dummy_path", errors='raise') + with pytest.warns(UserWarning): + assert check_gdac_path("dummy_path", errors='warn') is False + + +def test_isconnected(mocked_httpserver): + assert isinstance(isconnected(host=mocked_server_address), bool) + assert isconnected(host="http://dummyhost") is False + + +def test_urlhaskeyword(mocked_httpserver): + url = "https://api.ifremer.fr/argopy/data/ARGO-FULL.json" + url.replace("https://api.ifremer.fr", mocked_server_address) + assert isinstance(urlhaskeyword(url, "label"), bool) + + +params = [mocked_server_address, + {"url": mocked_server_address + "/argopy/data/ARGO-FULL.json", "keyword": "label"} + ] +params_ids = ["url is a %s" % str(type(p)) for p in params] +@pytest.mark.parametrize("params", params, indirect=False, ids=params_ids) +def test_isalive(params, mocked_httpserver): + assert isinstance(isalive(params), bool) + + +@requires_erddap +@pytest.mark.parametrize("data", [True, False], indirect=False, ids=["data=%s" % t for t in [True, False]]) +def test_isAPIconnected(data, mocked_httpserver): + with argopy.set_options(erddap=mocked_server_address): + assert isinstance(isAPIconnected(src="erddap", data=data), bool) + + +def test_erddap_ds_exists(mocked_httpserver): + with argopy.set_options(erddap=mocked_server_address): + assert isinstance(erddap_ds_exists(ds="ArgoFloats"), bool) + assert erddap_ds_exists(ds="DummyDS") is False diff --git a/argopy/tests/test_utils_chunking.py b/argopy/tests/test_utils_chunking.py new file mode 100644 index 00000000..3aee8c86 --- /dev/null +++ b/argopy/tests/test_utils_chunking.py @@ -0,0 +1,196 @@ +import pytest +import types +import numpy as np +import pandas as pd + +from argopy.errors import InvalidFetcherAccessPoint +from argopy.utils.chunking import Chunker +from argopy.utils.checkers import is_box + + +class Test_Chunker: + @pytest.fixture(autouse=True) + def create_data(self): + self.WMO = [ + 6902766, + 6902772, + 6902914, + 6902746, + 6902916, + 6902915, + 6902757, + 6902771, + ] + self.BOX3d = [0, 20, 40, 60, 0, 1000] + self.BOX4d = [0, 20, 40, 60, 0, 1000, "2001-01", "2001-6"] + + def test_InvalidFetcherAccessPoint(self): + with pytest.raises(InvalidFetcherAccessPoint): + Chunker({"invalid": self.WMO}) + + def test_invalid_chunks(self): + with pytest.raises(ValueError): + Chunker({"box": self.BOX3d}, chunks='toto') + + def test_invalid_chunksize(self): + with pytest.raises(ValueError): + Chunker({"box": self.BOX3d}, chunksize='toto') + + def test_chunk_wmo(self): + C = Chunker({"wmo": self.WMO}) + assert all( + [all(isinstance(x, int) for x in chunk) for chunk in C.fit_transform()] + ) + + C = Chunker({"wmo": self.WMO}, chunks="auto") + assert all( + [all(isinstance(x, int) for x in chunk) for chunk in C.fit_transform()] + ) + + C = Chunker({"wmo": self.WMO}, chunks={"wmo": 1}) + assert all( + [all(isinstance(x, int) for x in chunk) for chunk in C.fit_transform()] + ) + assert len(C.fit_transform()) == 1 + + with pytest.raises(ValueError): + Chunker({"wmo": self.WMO}, chunks=["wmo", 1]) + + C = Chunker({"wmo": self.WMO}) + assert isinstance(C.this_chunker, types.FunctionType) or isinstance( + C.this_chunker, types.MethodType + ) + + def test_chunk_box3d(self): + C = Chunker({"box": self.BOX3d}) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + + C = Chunker({"box": self.BOX3d}, chunks="auto") + assert all([is_box(chunk) for chunk in C.fit_transform()]) + + C = Chunker({"box": self.BOX3d}, chunks={"lon": 12, "lat": 1, "dpt": 1}) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + assert len(C.fit_transform()) == 12 + + C = Chunker( + {"box": self.BOX3d}, chunks={"lat": 1, "dpt": 1}, chunksize={"lon": 10} + ) + chunks = C.fit_transform() + assert all([is_box(chunk) for chunk in chunks]) + assert chunks[0][1] - chunks[0][0] == 10 + + C = Chunker({"box": self.BOX3d}, chunks={"lon": 1, "lat": 12, "dpt": 1}) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + assert len(C.fit_transform()) == 12 + + C = Chunker( + {"box": self.BOX3d}, chunks={"lon": 1, "dpt": 1}, chunksize={"lat": 10} + ) + chunks = C.fit_transform() + assert all([is_box(chunk) for chunk in chunks]) + assert chunks[0][3] - chunks[0][2] == 10 + + C = Chunker({"box": self.BOX3d}, chunks={"lon": 1, "lat": 1, "dpt": 12}) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + assert len(C.fit_transform()) == 12 + + C = Chunker( + {"box": self.BOX3d}, chunks={"lon": 1, "lat": 1}, chunksize={"dpt": 10} + ) + chunks = C.fit_transform() + assert all([is_box(chunk) for chunk in chunks]) + assert chunks[0][5] - chunks[0][4] == 10 + + C = Chunker({"box": self.BOX3d}, chunks={"lon": 4, "lat": 2, "dpt": 1}) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + assert len(C.fit_transform()) == 2 * 4 + + C = Chunker({"box": self.BOX3d}, chunks={"lon": 2, "lat": 3, "dpt": 4}) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + assert len(C.fit_transform()) == 2 * 3 * 4 + + with pytest.raises(ValueError): + Chunker({"box": self.BOX3d}, chunks=["lon", 1]) + + C = Chunker({"box": self.BOX3d}) + assert isinstance(C.this_chunker, types.FunctionType) or isinstance( + C.this_chunker, types.MethodType + ) + + def test_chunk_box4d(self): + C = Chunker({"box": self.BOX4d}) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + + C = Chunker({"box": self.BOX4d}, chunks="auto") + assert all([is_box(chunk) for chunk in C.fit_transform()]) + + C = Chunker( + {"box": self.BOX4d}, chunks={"lon": 2, "lat": 1, "dpt": 1, "time": 1} + ) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + assert len(C.fit_transform()) == 2 + + C = Chunker( + {"box": self.BOX4d}, + chunks={"lat": 1, "dpt": 1, "time": 1}, + chunksize={"lon": 10}, + ) + chunks = C.fit_transform() + assert all([is_box(chunk) for chunk in chunks]) + assert chunks[0][1] - chunks[0][0] == 10 + + C = Chunker( + {"box": self.BOX4d}, chunks={"lon": 1, "lat": 2, "dpt": 1, "time": 1} + ) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + assert len(C.fit_transform()) == 2 + + C = Chunker( + {"box": self.BOX4d}, + chunks={"lon": 1, "dpt": 1, "time": 1}, + chunksize={"lat": 10}, + ) + chunks = C.fit_transform() + assert all([is_box(chunk) for chunk in chunks]) + assert chunks[0][3] - chunks[0][2] == 10 + + C = Chunker( + {"box": self.BOX4d}, chunks={"lon": 1, "lat": 1, "dpt": 2, "time": 1} + ) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + assert len(C.fit_transform()) == 2 + + C = Chunker( + {"box": self.BOX4d}, + chunks={"lon": 1, "lat": 1, "time": 1}, + chunksize={"dpt": 10}, + ) + chunks = C.fit_transform() + assert all([is_box(chunk) for chunk in chunks]) + assert chunks[0][5] - chunks[0][4] == 10 + + C = Chunker( + {"box": self.BOX4d}, chunks={"lon": 1, "lat": 1, "dpt": 1, "time": 2} + ) + assert all([is_box(chunk) for chunk in C.fit_transform()]) + assert len(C.fit_transform()) == 2 + + C = Chunker( + {"box": self.BOX4d}, + chunks={"lon": 1, "lat": 1, "dpt": 1}, + chunksize={"time": 5}, + ) + chunks = C.fit_transform() + assert all([is_box(chunk) for chunk in chunks]) + assert np.timedelta64( + pd.to_datetime(chunks[0][7]) - pd.to_datetime(chunks[0][6]), "D" + ) <= np.timedelta64(5, "D") + + with pytest.raises(ValueError): + Chunker({"box": self.BOX4d}, chunks=["lon", 1]) + + C = Chunker({"box": self.BOX4d}) + assert isinstance(C.this_chunker, types.FunctionType) or isinstance( + C.this_chunker, types.MethodType + ) + diff --git a/argopy/tests/test_utils_compute.py b/argopy/tests/test_utils_compute.py new file mode 100644 index 00000000..2806fd14 --- /dev/null +++ b/argopy/tests/test_utils_compute.py @@ -0,0 +1,75 @@ +import pytest +import numpy as np +import xarray as xr + +from argopy.utils.compute import linear_interpolation_remap + + +class Test_linear_interpolation_remap: + @pytest.fixture(autouse=True) + def create_data(self): + # create fake data to test interpolation: + temp = np.random.rand(200, 100) + pres = np.sort( + np.floor( + np.zeros([200, 100]) + + np.linspace(50, 950, 100) + + np.random.randint(-5, 5, [200, 100]) + ) + ) + self.dsfake = xr.Dataset( + { + "TEMP": (["N_PROF", "N_LEVELS"], temp), + "PRES": (["N_PROF", "N_LEVELS"], pres), + }, + coords={ + "N_PROF": ("N_PROF", range(200)), + "N_LEVELS": ("N_LEVELS", range(100)), + "Z_LEVELS": ("Z_LEVELS", np.arange(100, 900, 20)), + }, + ) + + def test_interpolation(self): + # Run it with success: + dsi = linear_interpolation_remap( + self.dsfake["PRES"], + self.dsfake["TEMP"], + self.dsfake["Z_LEVELS"], + z_dim="N_LEVELS", + z_regridded_dim="Z_LEVELS", + ) + assert "remapped" in dsi.dims + + def test_interpolation_1d(self): + # Run it with success: + dsi = linear_interpolation_remap( + self.dsfake["PRES"].isel(N_PROF=0), + self.dsfake["TEMP"].isel(N_PROF=0), + self.dsfake["Z_LEVELS"], + z_regridded_dim="Z_LEVELS", + ) + assert "remapped" in dsi.dims + + def test_error_zdim(self): + # Test error: + # catches error from _regular_interp linked to z_dim + with pytest.raises(RuntimeError): + linear_interpolation_remap( + self.dsfake["PRES"], + self.dsfake["TEMP"], + self.dsfake["Z_LEVELS"], + z_regridded_dim="Z_LEVELS", + ) + + def test_error_ds(self): + # Test error: + # catches error from linear_interpolation_remap linked to datatype + with pytest.raises(ValueError): + linear_interpolation_remap( + self.dsfake["PRES"], + self.dsfake, + self.dsfake["Z_LEVELS"], + z_dim="N_LEVELS", + z_regridded_dim="Z_LEVELS", + ) + diff --git a/argopy/tests/test_utils_format.py b/argopy/tests/test_utils_format.py new file mode 100644 index 00000000..6d3c161c --- /dev/null +++ b/argopy/tests/test_utils_format.py @@ -0,0 +1,60 @@ +import os +import pytest +import argopy +from argopy.utils.format import format_oneline, argo_split_path + + +def test_format_oneline(): + s = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore" + assert isinstance(format_oneline(s), str) + assert isinstance(format_oneline(s[0:5]), str) + s = format_oneline(s, max_width=12) + assert isinstance(s, str) and len(s) == 12 + + +class Test_argo_split_path: + ############# + # UTILITIES # + ############# + # src = "https://data-argo.ifremer.fr/dac" + src = argopy.tutorial.open_dataset("gdac")[0] + "/dac" + list_of_files = [ + src + "/bodc/6901929/6901929_prof.nc", # core / multi-profile + src + "/coriolis/3902131/3902131_Sprof.nc", # bgc / synthetic multi-profile + + src + "/meds/4901079/profiles/D4901079_110.nc", # core / mono-profile / Delayed + src + "/aoml/13857/profiles/R13857_001.nc", # core / mono-profile / Real + + src + "/coriolis/3902131/profiles/SD3902131_001.nc", # bgc / synthetic mono-profile / Delayed + src + "/coriolis/3902131/profiles/SD3902131_001D.nc", # bgc / synthetic mono-profile / Delayed / Descent + src + "/coriolis/6903247/profiles/SR6903247_134.nc", # bgc / synthetic mono-profile / Real + src + "/coriolis/6903247/profiles/SR6903247_134D.nc", # bgc / synthetic mono-profile / Real / Descent + + src + "/coriolis/3902131/profiles/BR3902131_001.nc", # bgc / mono-profile / Real + src + "/coriolis/3902131/profiles/BR3902131_001D.nc", # bgc / mono-profile / Real / Descent + + src + "/aoml/5900446/5900446_Dtraj.nc", # traj / Delayed + src + "/csio/2902696/2902696_Rtraj.nc", # traj / Real + + src + "/coriolis/3902131/3902131_BRtraj.nc", # bgc / traj / Real + # src + "/coriolis/6903247/6903247_BRtraj.nc", # bgc / traj / Real + + src + "/incois/2902269/2902269_tech.nc", # technical + # src + "/nmdis/2901623/2901623_tech.nc", # technical + + src + "/jma/4902252/4902252_meta.nc", # meta-data + # src + "/coriolis/1900857/1900857_meta.nc", # meta-data + ] + list_of_files = [f.replace("/", os.path.sep) for f in list_of_files] + + ######### + # TESTS # + ######### + + @pytest.mark.parametrize("file", list_of_files, + indirect=False) + def test_argo_split_path(self, file): + desc = argo_split_path(file) + assert isinstance(desc, dict) + for key in ['origin', 'path', 'name', 'type', 'extension', 'wmo', 'dac']: + assert key in desc diff --git a/argopy/tests/test_utils_geo.py b/argopy/tests/test_utils_geo.py new file mode 100644 index 00000000..609242c9 --- /dev/null +++ b/argopy/tests/test_utils_geo.py @@ -0,0 +1,42 @@ +import pytest +import numpy as np +import pandas as pd +from argopy.utils.geo import wmo2box, wrap_longitude, toYearFraction, YearFraction_to_datetime +from argopy.utils.checkers import is_box + + +def test_wmo2box(): + with pytest.raises(ValueError): + wmo2box(12) + with pytest.raises(ValueError): + wmo2box(8000) + with pytest.raises(ValueError): + wmo2box(2000) + + def complete_box(b): + b2 = b.copy() + b2.insert(4, 0.) + b2.insert(5, 10000.) + return b2 + + assert is_box(complete_box(wmo2box(1212))) + assert is_box(complete_box(wmo2box(3324))) + assert is_box(complete_box(wmo2box(5402))) + assert is_box(complete_box(wmo2box(7501))) + + +def test_wrap_longitude(): + assert wrap_longitude(np.array([-20])) == 340 + assert wrap_longitude(np.array([40])) == 40 + assert np.all(np.equal(wrap_longitude(np.array([340, 20])), np.array([340, 380]))) + + +def test_toYearFraction(): + assert toYearFraction(pd.to_datetime('202001010000')) == 2020 + assert toYearFraction(pd.to_datetime('202001010000', utc=True)) == 2020 + assert toYearFraction(pd.to_datetime('202001010000')+pd.offsets.DateOffset(years=1)) == 2021 + + +def test_YearFraction_to_datetime(): + assert YearFraction_to_datetime(2020) == pd.to_datetime('202001010000') + assert YearFraction_to_datetime(2020+1) == pd.to_datetime('202101010000') diff --git a/argopy/tests/test_utils_lists.py b/argopy/tests/test_utils_lists.py new file mode 100644 index 00000000..06aaa893 --- /dev/null +++ b/argopy/tests/test_utils_lists.py @@ -0,0 +1,7 @@ +# import pytest +from argopy.utils.checkers import is_list_of_strings +from argopy.utils.lists import list_multiprofile_file_variables + + +def test_list_multiprofile_file_variables(): + assert is_list_of_strings(list_multiprofile_file_variables()) diff --git a/argopy/tests/test_utils_locals.py b/argopy/tests/test_utils_locals.py new file mode 100644 index 00000000..4ad32bfd --- /dev/null +++ b/argopy/tests/test_utils_locals.py @@ -0,0 +1,22 @@ +import os +import pytest +import io +import argopy +from argopy.utils.locals import modified_environ + + +@pytest.mark.parametrize("conda", [False, True], + indirect=False, + ids=["conda=%s" % str(p) for p in [False, True]]) +def test_show_versions(conda): + f = io.StringIO() + argopy.show_versions(file=f, conda=conda) + assert "SYSTEM" in f.getvalue() + + +def test_modified_environ(): + os.environ["DUMMY_ENV_ARGOPY"] = 'initial' + with modified_environ(DUMMY_ENV_ARGOPY='toto'): + assert os.environ['DUMMY_ENV_ARGOPY'] == 'toto' + assert os.environ['DUMMY_ENV_ARGOPY'] == 'initial' + os.environ.pop('DUMMY_ENV_ARGOPY') diff --git a/argopy/tests/test_xarray_accessor.py b/argopy/tests/test_xarray_accessor.py new file mode 100644 index 00000000..af8afdf7 --- /dev/null +++ b/argopy/tests/test_xarray_accessor.py @@ -0,0 +1,238 @@ +import os +import pytest +import warnings +import numpy as np +import tempfile +import xarray as xr + +import argopy +from argopy import DataFetcher as ArgoDataFetcher +from argopy.errors import InvalidDatasetStructure, OptionValueError +from utils import requires_gdac, _importorskip, _connectskip +from mocked_http import mocked_server_address + + +has_gsw, requires_gsw = _importorskip("gsw") +has_nogsw, requires_nogsw = _connectskip(not has_gsw, "that GSW module is NOT installed") + + +@pytest.fixture(scope="module") +def ds_pts(mocked_httpserver): + """ Create a dictionary of datasets to be used by tests + + Note that these datasets can be modified by tests, which can affect the behaviour of other tests ! + """ + data = {} + try: + for user_mode in ['standard', 'expert']: + data[user_mode] = ( + ArgoDataFetcher(src="erddap", mode=user_mode, server=mocked_server_address) + .region([-20, -16., 0, 1, 0, 100., "2004-01-01", "2004-01-31"]) + .load() + .data + ) + except Exception as e: + warnings.warn("Error when fetching tests data: %s" % str(e.args)) + pass + + if "expert" not in data or "standard" not in data: + # We don't have what we need for testing, skip this test module: + pytest.xfail("Could not retrieve erddap data in both standard and expert mode") + else: + return data + + +def test_point2profile(ds_pts): + assert "N_PROF" in ds_pts['standard'].argo.point2profile().dims + + +def test_profile2point(ds_pts): + with pytest.raises(InvalidDatasetStructure): + ds_pts['standard'].argo.profile2point() + + +def test_point2profile2point(ds_pts): + assert ds_pts['standard'].argo.point2profile().argo.profile2point().equals(ds_pts['standard']) + + +class Test_interp_std_levels: + def test_interpolation(self, ds_pts): + """Run with success""" + ds = ds_pts["standard"].argo.point2profile() + assert "PRES_INTERPOLATED" in ds.argo.interp_std_levels([20, 30, 40, 50]).dims + + def test_interpolation_expert(self, ds_pts): + """Run with success""" + ds = ds_pts["expert"].argo.point2profile() + assert "PRES_INTERPOLATED" in ds.argo.interp_std_levels([20, 30, 40, 50]).dims + + def test_std_error(self, ds_pts): + """Try to interpolate on a wrong axis""" + ds = ds_pts["standard"].argo.point2profile() + with pytest.raises(ValueError): + ds.argo.interp_std_levels([100, 20, 30, 40, 50]) + with pytest.raises(ValueError): + ds.argo.interp_std_levels([-20, 20, 30, 40, 50]) + with pytest.raises(ValueError): + ds.argo.interp_std_levels(12) + + +class Test_groupby_pressure_bins: + def test_groupby_ds_type(self, ds_pts): + """Run with success for standard/expert mode and point/profile""" + for user_mode, this in ds_pts.items(): + for format in ["point", "profile"]: + if format == 'profile': + that = this.argo.point2profile() + else: + that = this.copy() + bins = np.arange(0.0, np.max(that["PRES"]) + 10.0, 10.0) + assert "STD_PRES_BINS" in that.argo.groupby_pressure_bins(bins).coords + + def test_bins_error(self, ds_pts): + """Try to groupby over invalid bins """ + ds = ds_pts["standard"] + with pytest.raises(ValueError): + ds.argo.groupby_pressure_bins([100, 20, 30, 40, 50]) # un-sorted + with pytest.raises(ValueError): + ds.argo.groupby_pressure_bins([-20, 20, 30, 40, 50]) # Negative values + + def test_axis_error(self, ds_pts): + """Try to group by using invalid pressure axis """ + ds = ds_pts["standard"] + bins = np.arange(0.0, np.max(ds["PRES"]) + 10.0, 10.0) + with pytest.raises(ValueError): + ds.argo.groupby_pressure_bins(bins, axis='invalid') + + def test_empty_result(self, ds_pts): + """Try to groupby over bins without data""" + ds = ds_pts["standard"] + with pytest.warns(Warning): + out = ds.argo.groupby_pressure_bins([10000, 20000]) + assert out == None + + def test_all_select(self, ds_pts): + ds = ds_pts["standard"] + bins = np.arange(0.0, np.max(ds["PRES"]) + 10.0, 10.0) + for select in ["shallow", "deep", "middle", "random", "min", "max", "mean", "median"]: + assert "STD_PRES_BINS" in ds.argo.groupby_pressure_bins(bins).coords + + +class Test_teos10: + + @requires_nogsw + def test_gsw_not_available(self, ds_pts): + # Make sure we raise an error when GSW is not available + for key, this in ds_pts.items(): + that = this.copy() # To avoid modifying the original dataset + with pytest.raises(ModuleNotFoundError): + that.argo.teos10() + + @requires_gsw + def test_teos10_variables_default(self, ds_pts): + """Add default new set of TEOS10 variables""" + for key, this in ds_pts.items(): + for format in ["point", "profile"]: + that = this.copy() # To avoid modifying the original dataset + if format == "profile": + that = that.argo.point2profile() + that = that.argo.teos10() + assert "SA" in that.variables + assert "CT" in that.variables + + @requires_gsw + def test_teos10_variables_single(self, ds_pts): + """Add a single TEOS10 variables""" + for key, this in ds_pts.items(): + for format in ["point", "profile"]: + that = this.copy() # To avoid modifying the original dataset + if format == "profile": + that = that.argo.point2profile() + that = that.argo.teos10(["PV"]) + assert "PV" in that.variables + + @requires_gsw + def test_teos10_opt_variables_single(self, ds_pts): + """Add a single TEOS10 optional variables""" + for key, this in ds_pts.items(): + for format in ["point", "profile"]: + that = this.copy() # To avoid modifying the original dataset + if format == "profile": + that = that.argo.point2profile() + that = that.argo.teos10(["SOUND_SPEED"]) + assert "SOUND_SPEED" in that.variables + + @requires_gsw + def test_teos10_variables_inplace(self, ds_pts): + """Compute all default variables to a new dataset""" + for key, this in ds_pts.items(): + ds = this.argo.teos10(inplace=False) # So "SA" must be in 'ds' but not in 'this' + assert "SA" in ds.variables + assert "SA" not in this.variables + + @requires_gsw + def test_teos10_invalid_variable(self, ds_pts): + """Try to add an invalid variable""" + for key, this in ds_pts.items(): + for format in ["point", "profile"]: + that = this.copy() # To avoid modifying the original dataset + if format == "profile": + that = that.argo.point2profile() + with pytest.raises(ValueError): + that.argo.teos10(["InvalidVariable"]) + + +@requires_gsw +@requires_gdac +class Test_create_float_source: + local_ftp = argopy.tutorial.open_dataset("gdac")[0] + + def is_valid_mdata(self, this_mdata): + """Validate structure of the output dataset """ + check = [] + # Check for dimensions: + check.append(argopy.utils.is_list_equal(['m', 'n'], list(this_mdata.dims))) + # Check for coordinates: + check.append(argopy.utils.is_list_equal(['m', 'n'], list(this_mdata.coords))) + # Check for data variables: + check.append(np.all( + [v in this_mdata.data_vars for v in ['PRES', 'TEMP', 'PTMP', 'SAL', 'DATES', 'LAT', 'LONG', 'PROFILE_NO']])) + check.append(np.all( + [argopy.utils.is_list_equal(['n'], this_mdata[v].dims) for v in ['LONG', 'LAT', 'DATES', 'PROFILE_NO'] + if v in this_mdata.data_vars])) + check.append(np.all( + [argopy.utils.is_list_equal(['m', 'n'], this_mdata[v].dims) for v in ['PRES', 'TEMP', 'SAL', 'PTMP'] if + v in this_mdata.data_vars])) + return np.all(check) + + def test_error_user_mode(self): + with argopy.set_options(ftp=self.local_ftp): + with pytest.raises(InvalidDatasetStructure): + ds = ArgoDataFetcher(src="gdac", mode='standard').float([6901929, 2901623]).load().data + ds.argo.create_float_source() + + def test_opt_force(self): + with argopy.set_options(ftp=self.local_ftp): + expert_ds = ArgoDataFetcher(src="gdac", mode='expert').float([2901623]).load().data + + with pytest.raises(OptionValueError): + expert_ds.argo.create_float_source(force='dummy') + + ds_float_source = expert_ds.argo.create_float_source(path=None, force='default') + assert np.all([k in np.unique(expert_ds['PLATFORM_NUMBER']) for k in ds_float_source.keys()]) + assert np.all([isinstance(ds_float_source[k], xr.Dataset) for k in ds_float_source.keys()]) + assert np.all([self.is_valid_mdata(ds_float_source[k]) for k in ds_float_source.keys()]) + + ds_float_source = expert_ds.argo.create_float_source(path=None, force='raw') + assert np.all([k in np.unique(expert_ds['PLATFORM_NUMBER']) for k in ds_float_source.keys()]) + assert np.all([isinstance(ds_float_source[k], xr.Dataset) for k in ds_float_source.keys()]) + assert np.all([self.is_valid_mdata(ds_float_source[k]) for k in ds_float_source.keys()]) + + def test_filecreate(self): + with argopy.set_options(ftp=self.local_ftp): + expert_ds = ArgoDataFetcher(src="gdac", mode='expert').float([6901929, 2901623]).load().data + + N_file = len(np.unique(expert_ds['PLATFORM_NUMBER'])) + with tempfile.TemporaryDirectory() as folder_output: + expert_ds.argo.create_float_source(path=folder_output) + assert len(os.listdir(folder_output)) == N_file diff --git a/argopy/tests/test_xarray_engine.py b/argopy/tests/test_xarray_engine.py new file mode 100644 index 00000000..b67701f0 --- /dev/null +++ b/argopy/tests/test_xarray_engine.py @@ -0,0 +1,91 @@ +import os +import pytest +import xarray as xr +import logging +import warnings +import argopy +from argopy.utils.format import argo_split_path + +log = logging.getLogger("argopy.tests.xarray.engine") + + +def print_desc(desc): + txt = [desc["type"]] + if "direction" in desc: + txt.append(desc["direction"]) + + if "data_mode" in desc: + txt.append(desc["data_mode"]) + + return ", ".join(txt) + + +class Test_Argo_Engine: + host = argopy.tutorial.open_dataset("gdac")[0] + src = host + "/dac" + list_of_files = [ + src + "/bodc/6901929/6901929_prof.nc", # core / multi-profile + src + "/coriolis/3902131/3902131_Sprof.nc", # bgc / synthetic multi-profile + src + "/meds/4901079/profiles/D4901079_110.nc", # core / mono-profile / Delayed + src + "/aoml/13857/profiles/R13857_001.nc", # core / mono-profile / Real + src + + "/coriolis/3902131/profiles/SD3902131_001.nc", # bgc / synthetic mono-profile / Delayed + src + + "/coriolis/3902131/profiles/SD3902131_001D.nc", # bgc / synthetic mono-profile / Delayed / Descent + src + + "/coriolis/6903247/profiles/SR6903247_134.nc", # bgc / synthetic mono-profile / Real + src + + "/coriolis/6903247/profiles/SR6903247_134D.nc", # bgc / synthetic mono-profile / Real / Descent + src + + "/coriolis/3902131/profiles/BR3902131_001.nc", # bgc / mono-profile / Real + src + + "/coriolis/3902131/profiles/BR3902131_001D.nc", # bgc / mono-profile / Real / Descent + src + "/aoml/5900446/5900446_Dtraj.nc", # traj / Delayed + src + "/csio/2902696/2902696_Rtraj.nc", # traj / Real + src + "/coriolis/3902131/3902131_BRtraj.nc", # bgc / traj / Real + # src + "/coriolis/6903247/6903247_BRtraj.nc", # bgc / traj / Real + src + "/incois/2902269/2902269_tech.nc", # technical + # src + "/nmdis/2901623/2901623_tech.nc", # technical + src + "/jma/4902252/4902252_meta.nc", # meta-data + # src + "/coriolis/1900857/1900857_meta.nc", # meta-data + ] + list_of_files = [f.replace("/", os.path.sep) for f in list_of_files] + + list_of_files_desc = [print_desc(argo_split_path(f)) for f in list_of_files] + # list_of_files_desc = [f for f in list_of_files] + + ############# + # UTILITIES # + ############# + + def how_many_casted(self, this_ds): + """Return the length of non casted variables in ds""" + # check which variables are not casted: + l = [] + for iv, v in enumerate(this_ds.variables): + if this_ds[v].dtype == "O": + l.append(v) + # log.debug("%i/%i variables not casted properly.\n%s" % (len(l), len(this_ds.variables), l)) + return len(l), len(this_ds.variables) + + ######### + # TESTS # + ######### + + @pytest.mark.parametrize( + "file", list_of_files, indirect=False, ids=list_of_files_desc + ) + def test_read(self, file, mocked_httpserver): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", "invalid value encountered in cast", RuntimeWarning + ) + + # No Argo enfine: + # ds1 = xr.open_dataset(file) + # n1, N1, l1 = self.how_many_casted(ds1) + + # With Argo engine: + ds = xr.open_dataset(file, engine="argo") + n, N = self.how_many_casted(ds) + assert n == 0 diff --git a/docs/whats-new.rst b/docs/whats-new.rst index 427bd7f1..157f458f 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -17,6 +17,7 @@ Coming up next **Internals** - Utilities refactoring. All classes and functions have been refactored to more appropriate locations like ``argopy.utils`` or ``argopy.related``. A deprecation warning message will be displayed every time utilities are being used from the former locations. (:pr:`290`) by `G. Maze `_ +- Fix bugs due to fsspec new internal cache handling and Windows specifics. (:pr:`293`) by `G. Maze `_ v0.1.14rc2 (27 Jul. 2023) -------------------------