From 58274591e227b3ec3c725b477bc8a8eb3dcf1e85 Mon Sep 17 00:00:00 2001 From: Renaud <38732257+renaudjester@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:16:27 +0200 Subject: [PATCH 1/5] chore: refactor to stop using the cache (#95) Not using any cache anymore Always get the data on the fly --- README.md | 10 ---------- poetry.lock | 1 - 2 files changed, 11 deletions(-) diff --git a/README.md b/README.md index 3fd561d2..b9b12a25 100644 --- a/README.md +++ b/README.md @@ -22,43 +22,36 @@ The `copernicusmarine` offers capabilities through both **Command Line Interface ## Installation - For installation, multiple options are available depending on your setup: ### Mamba | Conda - A `conda` package is available on [Anaconda](https://anaconda.org/conda-forge/copernicusmarine). You can install it using `mamba` (or conda) through the `conda-forge` channel with the following command: - ```bash mamba install conda-forge::copernicusmarine --yes ``` To upgrade the Toolbox with mamba (or conda): - ```bash mamba update --name copernicusmarine copernicusmarine --yes ``` ### Docker - A docker image is also available here: [https://hub.docker.com/r/copernicusmarine/copernicusmarine](https://hub.docker.com/r/copernicusmarine/copernicusmarine) First step is to pull the container image: - ```bash docker pull copernicusmarine/copernicusmarine:latest ``` Then run it: - ```bash docker run -it --rm copernicusmarine/copernicusmarine --version ``` @@ -75,14 +68,12 @@ python -m pip install copernicusmarine And to **upgrade the package** to the newest available version, run: - ```bash python -m pip install copernicusmarine --upgrade ``` ## User Guide - For more comprehensive details on how to use the `copernicusmarine` Toolbox, please refer to our [Help Center](https://help.marine.copernicus.eu/en/collections/9080063-copernicus-marine-toolbox). It ensures a smooth migration for existing users of legacy services such as MOTU, OPeNDAP or FTP. ### Network configuration @@ -174,7 +165,6 @@ You can use the `--skip-if-user-logged-in` option to skip the configuration file #### Access points migration and evolution - If you still have a configuration for legacy services (e.g. `~/motuclient/motuclient-python.ini`, `~/.netrc` or `~/_netrc`) in your home directory, it will automatically be taken into account with commands `get` and `subset` without the need for running the `login` command. If the configuration files are already available in another directory, when running commands `subset` or `get`, you can use the `--credentials-file` option to point to the files. diff --git a/poetry.lock b/poetry.lock index d83563ff..a7f759cc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1941,5 +1941,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" - content-hash = "f781539aa6604250d8715e3c6ef09165d031fbb95565f65e4368e41db7c50290" From ab5903c623969a18c3a7fed89e59d4e71db32b36 Mon Sep 17 00:00:00 2001 From: uriii3 Date: Tue, 30 Jul 2024 15:36:23 +0200 Subject: [PATCH 2/5] fix: adding again the positive attribute in the depth dimension --- copernicusmarine/download_functions/subset_xarray.py | 6 +++++- tests/test_command_line_interface.py | 9 ++++++--- tests/test_python_interface.py | 1 + 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/copernicusmarine/download_functions/subset_xarray.py b/copernicusmarine/download_functions/subset_xarray.py index 0a5e8660..211844c6 100644 --- a/copernicusmarine/download_functions/subset_xarray.py +++ b/copernicusmarine/download_functions/subset_xarray.py @@ -377,7 +377,11 @@ def _update_dataset_coordinate_attributes( coord.encoding["units"].replace("_", " ").title() ) coordinate_attributes.remove("units") - elif coordinate_label in ["latitude", "depth", "elevation"]: + elif coordinate_label in ["depth", "elevation"]: + attrs["valid_min"] = coord.values.min() + attrs["valid_max"] = coord.values.max() + coordinate_attributes.append("positive") + elif coordinate_label == "latitude": attrs["valid_min"] = coord.values.min() attrs["valid_max"] = coord.values.max() elif coordinate_label == "longitude": diff --git a/tests/test_command_line_interface.py b/tests/test_command_line_interface.py index 63d341a8..562a9c84 100644 --- a/tests/test_command_line_interface.py +++ b/tests/test_command_line_interface.py @@ -1011,24 +1011,27 @@ def then_I_have_correct_sign_for_depth_coordinates_values( assert dataset.elevation.min() >= -10 assert dataset.elevation.max() <= 0 - def then_I_have_correct_attribute_value(self, output_path, dimention_name): + def then_I_have_correct_attribute_value( + self, output_path, dimention_name, attribute_value + ): filepath = pathlib.Path(output_path, "data.zarr") dataset = xarray.open_dataset(filepath, engine="zarr") assert dataset[dimention_name].attrs["standard_name"] == dimention_name + assert dataset[dimention_name].attrs["positive"] == attribute_value def test_conversion_between_elevation_and_depth(self, tmp_path): self.when_I_request_subset_dataset_with_zarr_service(tmp_path, True) self.then_I_have_correct_sign_for_depth_coordinates_values( tmp_path, "positive" ) - self.then_I_have_correct_attribute_value(tmp_path, "depth") + self.then_I_have_correct_attribute_value(tmp_path, "depth", "down") def test_force_no_conversion_between_elevation_and_depth(self, tmp_path): self.when_I_request_subset_dataset_with_zarr_service(tmp_path, False) self.then_I_have_correct_sign_for_depth_coordinates_values( tmp_path, "negative" ) - self.then_I_have_correct_attribute_value(tmp_path, "elevation") + self.then_I_have_correct_attribute_value(tmp_path, "elevation", "up") def when_I_run_copernicus_marine_command_using_no_directories_option( self, tmp_path, force_service: GetServiceToTest, output_directory=None diff --git a/tests/test_python_interface.py b/tests/test_python_interface.py index 9b248060..0c0be704 100644 --- a/tests/test_python_interface.py +++ b/tests/test_python_interface.py @@ -227,6 +227,7 @@ def test_subset_modify_attr_for_depth(self): dataset = open_dataset( dataset_id="cmems_mod_arc_phy_anfc_6km_detided_P1D-m" ) + assert dataset.depth.attrs["positive"] == "down" assert dataset.depth.attrs["standard_name"] == "depth" assert dataset.depth.attrs["long_name"] == "Depth" From e915206f9b50c8ad2564a36880b645ab4cd063af Mon Sep 17 00:00:00 2001 From: uriii3 Date: Wed, 31 Jul 2024 13:07:51 +0200 Subject: [PATCH 3/5] fix: fixing the poetry lock file --- poetry.lock | 74 +++++++++++++++-------------------------------------- 1 file changed, 20 insertions(+), 54 deletions(-) diff --git a/poetry.lock b/poetry.lock index a7f759cc..6fcca9e5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -170,44 +170,41 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "boto3" -version = "1.34.143" +version = "1.25.0" description = "The AWS SDK for Python" optional = false -python-versions = ">=3.8" +python-versions = ">= 3.7" files = [ - {file = "boto3-1.34.143-py3-none-any.whl", hash = "sha256:0d16832f23e6bd3ae94e35ea8e625529850bfad9baccd426de96ad8f445d8e03"}, - {file = "boto3-1.34.143.tar.gz", hash = "sha256:b590ce80c65149194def43ebf0ea1cf0533945502507837389a8d22e3ecbcf05"}, + {file = "boto3-1.25.0-py3-none-any.whl", hash = "sha256:81139cc9da154a1672c7dd92da1678cae0ea1601a3e1f0394c6cd010eab1acb6"}, + {file = "boto3-1.25.0.tar.gz", hash = "sha256:170eab4a87592741933b6f8a02c3a6a8664162ef33bb12a2c2b4d431490d9ac2"}, ] [package.dependencies] -botocore = ">=1.34.143,<1.35.0" +botocore = ">=1.28.0,<1.29.0" jmespath = ">=0.7.1,<2.0.0" -s3transfer = ">=0.10.0,<0.11.0" +s3transfer = ">=0.6.0,<0.7.0" [package.extras] crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.143" +version = "1.28.5" description = "Low-level, data-driven core of boto 3." optional = false -python-versions = ">=3.8" +python-versions = ">= 3.7" files = [ - {file = "botocore-1.34.143-py3-none-any.whl", hash = "sha256:094aea179e8aaa1bc957ad49cc27d93b189dd3a1f3075d8b0ca7c445a2a88430"}, - {file = "botocore-1.34.143.tar.gz", hash = "sha256:059f032ec05733a836e04e869c5a15534420102f93116f3bc9a5b759b0651caf"}, + {file = "botocore-1.28.5-py3-none-any.whl", hash = "sha256:5eb98c85e80fab6640342377ce947c86a003bbaf8427c2e7a03c8da0bdbb796c"}, + {file = "botocore-1.28.5.tar.gz", hash = "sha256:f322d7b62163219ffeb787a116d318273dfb7243c3b49d95f5bfff8daa1df4e0"}, ] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" python-dateutil = ">=2.1,<3.0.0" -urllib3 = [ - {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, - {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}, -] +urllib3 = ">=1.25.4,<1.27" [package.extras] -crt = ["awscrt (==0.20.11)"] +crt = ["awscrt (==0.14.0)"] [[package]] name = "certifi" @@ -1533,20 +1530,20 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "s3transfer" -version = "0.10.2" +version = "0.6.2" description = "An Amazon S3 Transfer Manager" optional = false -python-versions = ">=3.8" +python-versions = ">= 3.7" files = [ - {file = "s3transfer-0.10.2-py3-none-any.whl", hash = "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69"}, - {file = "s3transfer-0.10.2.tar.gz", hash = "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6"}, + {file = "s3transfer-0.6.2-py3-none-any.whl", hash = "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084"}, + {file = "s3transfer-0.6.2.tar.gz", hash = "sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861"}, ] [package.dependencies] -botocore = ">=1.33.2,<2.0a.0" +botocore = ">=1.12.36,<2.0a.0" [package.extras] -crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "semver" @@ -1664,20 +1661,6 @@ files = [ [package.dependencies] types-urllib3 = "*" -[[package]] -name = "types-requests" -version = "2.32.0.20240622" -description = "Typing stubs for requests" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types-requests-2.32.0.20240622.tar.gz", hash = "sha256:ed5e8a412fcc39159d6319385c009d642845f250c63902718f605cd90faade31"}, - {file = "types_requests-2.32.0.20240622-py3-none-any.whl", hash = "sha256:97bac6b54b5bd4cf91d407e62f0932a74821bc2211f22116d9ee1dd643826caf"}, -] - -[package.dependencies] -urllib3 = ">=2" - [[package]] name = "types-urllib3" version = "1.26.25.14" @@ -1727,23 +1710,6 @@ brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotl secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -[[package]] -name = "urllib3" -version = "2.2.2" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.8" -files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] - [[package]] name = "virtualenv" version = "20.26.3" @@ -1941,4 +1907,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "f781539aa6604250d8715e3c6ef09165d031fbb95565f65e4368e41db7c50290" +content-hash = "c6b5c7b06e10318ef2ffd89841d0a52cb54bebed9ac5ddcddd8e1d408feb2c9d" From 66dd91ae19654cfa65150dca05f667d02cc1f6ca Mon Sep 17 00:00:00 2001 From: uriii3 Date: Wed, 31 Jul 2024 13:09:53 +0200 Subject: [PATCH 4/5] fix: reading the poetry lock --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 6fcca9e5..0cd3c121 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1907,4 +1907,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "c6b5c7b06e10318ef2ffd89841d0a52cb54bebed9ac5ddcddd8e1d408feb2c9d" +content-hash = "10f7e27b97411bfe5cf27642ad7b4ae7c5a65b0634daa48c1551eb67dbe941f4" From d54d94dca07ff6724f94e9bb3aea2a3e3cd09d4d Mon Sep 17 00:00:00 2001 From: uriii3 Date: Wed, 31 Jul 2024 13:15:24 +0200 Subject: [PATCH 5/5] fix: precommit run all files done --- copernicusmarine/core_functions/credentials_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/copernicusmarine/core_functions/credentials_utils.py b/copernicusmarine/core_functions/credentials_utils.py index 9373ac34..2d23304f 100644 --- a/copernicusmarine/core_functions/credentials_utils.py +++ b/copernicusmarine/core_functions/credentials_utils.py @@ -36,10 +36,12 @@ # TODO: handle cache of the credentials without cachier -class CredentialCannotBeNone(Exception): ... +class CredentialCannotBeNone(Exception): + ... -class InvalidUsernameOrPassword(Exception): ... +class InvalidUsernameOrPassword(Exception): + ... def _load_credential_from_copernicus_marine_configuration_file(