From 2a73311508ae522caa438083b53c83dc1cf0c1c7 Mon Sep 17 00:00:00 2001 From: grouazel Date: Wed, 27 Nov 2024 12:06:14 +0100 Subject: [PATCH 01/20] add 2 workflows and script/conf associated + define style linter --- .github/workflows/ci.yaml | 84 +++++++++++++++++++++++++ .github/workflows/upstream-dev.yaml | 98 +++++++++++++++++++++++++++++ ci/install-upstream-dev.sh | 14 +++++ ci/requirements/docs.yaml | 8 +++ ci/requirements/environment.yaml | 24 +++++++ pyproject.toml | 34 ++++++++++ 6 files changed, 262 insertions(+) create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/upstream-dev.yaml create mode 100644 ci/install-upstream-dev.sh create mode 100644 ci/requirements/docs.yaml create mode 100644 ci/requirements/environment.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..d09af46 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,84 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + detect-skip-ci-trigger: + name: "Detect CI Trigger: [skip-ci]" + if: | + github.repository == 'umr-lops/xradarsat2' + && ( + github.event_name == 'push' || github.event_name == 'pull_request' + ) + runs-on: ubuntu-latest + outputs: + triggered: ${{ steps.detect-trigger.outputs.trigger-found }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: xarray-contrib/ci-trigger@v1 + id: detect-trigger + with: + keyword: "[skip-ci]" + + ci: + name: ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + needs: detect-skip-ci-trigger + + if: needs.detect-skip-ci-trigger.outputs.triggered == 'false' + + defaults: + run: + shell: bash -l {0} + + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + with: + # need to fetch all tags to get a correct version + fetch-depth: 0 # fetch all branches and tags + + - name: Setup environment variables + run: | + echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + + echo "CONDA_ENV_FILE=ci/requirements/environment.yaml" >> $GITHUB_ENV + + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v2 + with: + environment-file: ${{ env.CONDA_ENV_FILE }} + environment-name: xradarsat2-tests + cache-environment: true + cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" + create-args: >- + python=${{matrix.python-version}} + + - name: Install xradarsat2 + run: | + python -m pip install --no-deps -e . + + - name: Import xradarsat2 + run: | + python -c "import xradarsat2" + + - name: Run tests + run: | + python -m pytest --cov=xradarsat2 diff --git a/.github/workflows/upstream-dev.yaml b/.github/workflows/upstream-dev.yaml new file mode 100644 index 0000000..023b57b --- /dev/null +++ b/.github/workflows/upstream-dev.yaml @@ -0,0 +1,98 @@ +name: upstream-dev CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "0 18 * * 0" # Weekly "On Sundays at 18:00" UTC + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + detect-test-upstream-trigger: + name: "Detect CI Trigger: [test-upstream]" + if: github.event_name == 'push' || github.event_name == 'pull_request' + runs-on: ubuntu-latest + outputs: + triggered: ${{ steps.detect-trigger.outputs.trigger-found }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: xarray-contrib/ci-trigger@v1.2 + id: detect-trigger + with: + keyword: "[test-upstream]" + + upstream-dev: + name: upstream-dev + runs-on: ubuntu-latest + needs: detect-test-upstream-trigger + + if: | + always() + && github.repository == 'umr-lops/xradarsat2' + && ( + github.event_name == 'schedule' + || github.event_name == 'workflow_dispatch' + || needs.detect-test-upstream-trigger.outputs.triggered == 'true' + || contains(github.event.pull_request.labels.*.name, 'run-upstream') + ) + + defaults: + run: + shell: bash -l {0} + + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - name: checkout the repository + uses: actions/checkout@v4 + with: + # need to fetch all tags to get a correct version + fetch-depth: 0 # fetch all branches and tags + + - name: set up conda environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ci/requirements/environment.yaml + environment-name: tests + create-args: >- + python=${{ matrix.python-version }} + pytest-reportlog + + - name: install upstream-dev dependencies + run: bash ci/install-upstream-dev.sh + + - name: install the package + run: python -m pip install --no-deps -e . + + - name: show versions + run: python -m pip list + + - name: import + run: | + python -c 'import xradarsat2' + + - name: run tests + if: success() + id: status + run: | + python -m pytest -rf --report-log=pytest-log.jsonl + + - name: report failures + if: | + failure() + && steps.tests.outcome == 'failure' + && github.event_name == 'schedule' + uses: xarray-contrib/issue-from-pytest-log@v1 + with: + log-path: pytest-log.jsonl diff --git a/ci/install-upstream-dev.sh b/ci/install-upstream-dev.sh new file mode 100644 index 0000000..8744bf9 --- /dev/null +++ b/ci/install-upstream-dev.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +conda remove -y --force cytoolz numpy xarray toolz python-dateutil rioxarray +python -m pip install \ + -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \ + --no-deps \ + --pre \ + --upgrade \ + numpy \ + rioxarray \ + xarray +python -m pip install --upgrade \ + git+https://github.com/pytoolz/toolz \ + git+https://github.com/dateutil/dateutil diff --git a/ci/requirements/docs.yaml b/ci/requirements/docs.yaml new file mode 100644 index 0000000..b287778 --- /dev/null +++ b/ci/requirements/docs.yaml @@ -0,0 +1,8 @@ +name: xradarsat2-docs +channels: + - conda-forge +dependencies: + - python=3.11 + - sphinx>=4 + - sphinx_book_theme + - ipython diff --git a/ci/requirements/environment.yaml b/ci/requirements/environment.yaml new file mode 100644 index 0000000..7cda729 --- /dev/null +++ b/ci/requirements/environment.yaml @@ -0,0 +1,24 @@ +name: xradarsat2-tests +channels: + - conda-forge +dependencies: + - ipython + - pre-commit + - pytest + - pytest-reportlog + - pytest-cov + - numpy + - toolz + - cytoolz + - python-dateutil + - construct + - xarray + - rioxarray + - aiohttp + - fsspec + - dask + - opencv + - pyyaml + - affine + - h5netcdf + - scipy diff --git a/pyproject.toml b/pyproject.toml index a0939e7..67425a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,3 +43,37 @@ dynamic = ["version"] [project.urls] "Homepage" = "https://github.com/umr-lops/xradarsat2" "Bug Tracker" = "https://github.com/umr-lops/xradarsat2/issues" + +[tool.coverage.report] +show_missing = true +exclude_lines = ["pragma: no cover", "if TYPE_CHECKING"] + +[tool.ruff.lint] +ignore = [ + "E402", # module level import not at top of file + "E501", # line too long - let black worry about that + "E731", # do not assign a lambda expression, use a def + "UP038", # type union instead of tuple for isinstance etc +] +select = [ + "F", # Pyflakes + "E", # Pycodestyle + "I", # isort + "UP", # Pyupgrade + "TID", # flake8-tidy-imports + "W", +] +extend-safe-fixes = [ + "TID252", # absolute imports + "UP031", # percent string interpolation +] +fixable = ["I", "TID252", "UP"] + +[tool.ruff.lint.isort] +known-first-party = ["xradarsat2"] +known-third-party = ["xarray", "toolz", "construct"] + +[tool.ruff.lint.flake8-tidy-imports] +# Disallow all relative imports. +ban-relative-imports = "all" + From c91b71c2d6f94ae827549cc6ca142e73d28b2472 Mon Sep 17 00:00:00 2001 From: grouazel Date: Wed, 27 Nov 2024 12:09:02 +0100 Subject: [PATCH 02/20] add xmltodict --- ci/requirements/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/requirements/environment.yaml b/ci/requirements/environment.yaml index 7cda729..9cec30d 100644 --- a/ci/requirements/environment.yaml +++ b/ci/requirements/environment.yaml @@ -22,3 +22,4 @@ dependencies: - affine - h5netcdf - scipy + - xmltodict From 87ca92e9e417b4de19d6773066f2f08bd047016e Mon Sep 17 00:00:00 2001 From: grouazel Date: Wed, 27 Nov 2024 12:22:53 +0100 Subject: [PATCH 03/20] make the unit test be portable --- .gitignore | 2 +- pyproject.toml | 34 ++++++------ src/xradarsat2/utils.py | 69 +++++++++++++++++++++++++ src/xradarsat2/xradarsat2-config.yaml | 2 +- test/test_opening_datatree_radarsat2.py | 12 +++-- 5 files changed, 95 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index f41feca..133e187 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,4 @@ tmp/ /.idea/ dask-worker-space/ -localxradarsat2-config.yaml \ No newline at end of file +localxradarsat2-config.yaml diff --git a/pyproject.toml b/pyproject.toml index 67425a8..1a2c6be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,28 +15,29 @@ fallback_version = "9999" [project] name = "xradarsat2" -authors = [ - { name="Yann Reynaud", email="Yann.Reynaud.2@ifremer.fr" }, -] -license = {text = "MIT"} +authors = [{ name = "Yann Reynaud", email = "Yann.Reynaud.2@ifremer.fr" }] +license = { text = "MIT" } description = "xarray/dask distributed L1 sar file reader for radarSat2" readme = "README.md" requires-python = ">=3.9" classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", ] dependencies = [ - 'xmltodict', - 'numpy', - "xarray>=2024.10.0", - 'rasterio', - 'rioxarray', - 'dask', - 'affine', - 'regex', - 'pyyaml' + 'xmltodict', + 'numpy', + "xarray>=2024.10.0", + 'rasterio', + 'rioxarray', + 'dask', + 'affine', + 'regex', + 'pyyaml', + 'fsspec', + 'aiohttp', + ] dynamic = ["version"] @@ -76,4 +77,3 @@ known-third-party = ["xarray", "toolz", "construct"] [tool.ruff.lint.flake8-tidy-imports] # Disallow all relative imports. ban-relative-imports = "all" - diff --git a/src/xradarsat2/utils.py b/src/xradarsat2/utils.py index bf91277..4defd02 100644 --- a/src/xradarsat2/utils.py +++ b/src/xradarsat2/utils.py @@ -3,6 +3,10 @@ import os import yaml import re +import aiohttp +import fsspec +import warnings +import zipfile def get_glob(strlist): @@ -45,3 +49,68 @@ def load_config(): stream = open(config_path, "r") conf = yaml.load(stream, Loader=yaml.CLoader) return conf + +def get_test_file(fname): + """ + get test file from https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata/ + file is unzipped and extracted to `config['data_dir']` + + Parameters + ---------- + fname: str + file name to get (without '.zip' extension) + + Returns + ------- + str + path to file, relative to `config['data_dir']` + + """ + config = {'data_dir': '/tmp'} + + def url_get(url, cache_dir=os.path.join(config['data_dir'], 'fsspec_cache')): + """ + Get fil from url, using caching. + + Parameters + ---------- + url: str + cache_dir: str + Cache dir to use. default to `os.path.join(config['data_dir'], 'fsspec_cache')` + + Raises + ------ + FileNotFoundError + + Returns + ------- + filename: str + The local file name + + Notes + ----- + Due to fsspec, the returned filename won't match the remote one. + """ + + if '://' in url: + with fsspec.open( + 'filecache::%s' % url, + https={'client_kwargs': {'timeout': aiohttp.ClientTimeout(total=3600)}}, + filecache={'cache_storage': os.path.join(os.path.join(config['data_dir'], 'fsspec_cache'))} + ) as f: + fname = f.name + else: + fname = url + + return fname + + res_path = config['data_dir'] + base_url = 'https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata' + file_url = '%s/%s.zip' % (base_url, fname) + if not os.path.exists(os.path.join(res_path, fname)): + warnings.warn("Downloading %s" % file_url) + local_file = url_get(file_url) + warnings.warn("Unzipping %s" % os.path.join(res_path, fname)) + with zipfile.ZipFile(local_file, 'r') as zip_ref: + zip_ref.extractall(res_path) + return os.path.join(res_path, fname) \ No newline at end of file diff --git a/src/xradarsat2/xradarsat2-config.yaml b/src/xradarsat2/xradarsat2-config.yaml index 4e19cc0..06bc2b1 100644 --- a/src/xradarsat2/xradarsat2-config.yaml +++ b/src/xradarsat2/xradarsat2-config.yaml @@ -1 +1 @@ -folder_path: ./2021/137/RS2_OK129673_PK1136693_DK1093025_SCWA_20210517_010235_VV_VH_SGF +folder_path: RS2_OK135107_PK1187782_DK1151894_SCWA_20220407_182127_VV_VH_SGF diff --git a/test/test_opening_datatree_radarsat2.py b/test/test_opening_datatree_radarsat2.py index 88111a8..a0df6cc 100644 --- a/test/test_opening_datatree_radarsat2.py +++ b/test/test_opening_datatree_radarsat2.py @@ -1,7 +1,8 @@ -from xradarsat2.utils import load_config -import xradarsat2 -import time import logging +import time + +import xradarsat2 +from xradarsat2.utils import load_config,get_test_file logging.basicConfig(level=logging.DEBUG) logging.debug("start opening RadarSAT-2 product") @@ -11,13 +12,14 @@ t0 = time.time() conf = load_config() folder_path = conf["folder_path"] -dt = xradarsat2.rs2_reader(folder_path) +rs2_product_path = get_test_file(conf['folder_path']) +dt = xradarsat2.rs2_reader(rs2_product_path) elapse_t = time.time() - t0 print(type(dt), dt) print("out of the reader") print(dt) -print("time to read the SAFE through nfs: %1.2f sec" % elapse_t) +print(f"time to read the SAFE through nfs: {elapse_t:1.2f} sec") dt = xradarsat2.load_digital_number( dt, chunks={"pol": "VV", "line": 6000, "sample": 8000} ) From 09c652182997f68cd4d1052d86293dfcbac08d21 Mon Sep 17 00:00:00 2001 From: grouazel Date: Wed, 27 Nov 2024 12:42:55 +0100 Subject: [PATCH 04/20] lint --- docs/doc_xradarsat2.ipynb | 20 ++++------- src/xradarsat2/__init__.py | 7 ++-- src/xradarsat2/radarSat2_tiff_reader.py | 10 ++---- src/xradarsat2/radarSat2_xarray_reader.py | 11 ++---- src/xradarsat2/utils.py | 43 +++++++++++++---------- test/test_opening_datatree_radarsat2.py | 4 +-- 6 files changed, 45 insertions(+), 50 deletions(-) diff --git a/docs/doc_xradarsat2.ipynb b/docs/doc_xradarsat2.ipynb index 0b131b8..293185e 100644 --- a/docs/doc_xradarsat2.ipynb +++ b/docs/doc_xradarsat2.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "03cfc89b-5ee1-49c2-8bc9-ca88c00eb63b", + "id": "0", "metadata": {}, "source": [ "# examples" @@ -11,7 +11,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9f901d61-adf5-4ad2-a721-b92fb8555642", + "id": "1", "metadata": {}, "outputs": [], "source": [ @@ -23,7 +23,7 @@ { "cell_type": "code", "execution_count": null, - "id": "84c1a31d-c27c-4614-8a39-f4dca2801bf6", + "id": "2", "metadata": {}, "outputs": [], "source": [ @@ -36,7 +36,7 @@ { "cell_type": "code", "execution_count": null, - "id": "90fb68ff-2c19-4432-8d95-98159ee546a8", + "id": "3", "metadata": {}, "outputs": [], "source": [ @@ -48,7 +48,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ff1730a0-caeb-4c53-8fa0-85c79adfb5a6", + "id": "4", "metadata": {}, "outputs": [], "source": [ @@ -59,7 +59,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6f74bb35-b8d0-4a17-82d2-4b151e3044fb", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -74,11 +74,6 @@ } ], "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -88,8 +83,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.0" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/src/xradarsat2/__init__.py b/src/xradarsat2/__init__.py index 1e614b3..423fde4 100644 --- a/src/xradarsat2/__init__.py +++ b/src/xradarsat2/__init__.py @@ -1,7 +1,10 @@ -from xradarsat2.radarSat2_xarray_reader import load_digital_number # noqa: F401 -from xradarsat2.radarSat2_xarray_reader import rs2_reader # noqa: F401 from importlib.metadata import version +from xradarsat2.radarSat2_xarray_reader import ( + load_digital_number, # noqa: F401 + rs2_reader, # noqa: F401 +) + try: __version__ = version("xradarsat2") except Exception: diff --git a/src/xradarsat2/radarSat2_tiff_reader.py b/src/xradarsat2/radarSat2_tiff_reader.py index 3cd6941..b508b44 100644 --- a/src/xradarsat2/radarSat2_tiff_reader.py +++ b/src/xradarsat2/radarSat2_tiff_reader.py @@ -8,6 +8,7 @@ import xarray as xr import yaml from affine import Affine + from xradarsat2.utils import get_glob, load_config # folder_path = "/home/datawork-cersat-public/cache/project/sarwing/data/RS2/L1/VV/2010/288/" \ @@ -43,12 +44,7 @@ def _load_digital_number( tiff_files = list_tiff_files(root_path) map_dims = {"pol": "band", "line": "y", "sample": "x"} if resolution is not None: - comment = 'resampled at "%s" with %s.%s.%s' % ( - resolution, - resampling.__module__, - resampling.__class__.__name__, - resampling.name, - ) + comment = f'resampled at "{resolution}" with {resampling.__module__}.{resampling.__class__.__name__}.{resampling.name}' else: comment = "read at full resolution" @@ -158,7 +154,7 @@ def _load_digital_number( var_name = "digital_number" dn.attrs = { - "comment": "%s digital number, %s" % (descr, comment), + "comment": f"{descr} digital number, {comment}", "history": yaml.safe_dump( {var_name: get_glob([p.replace(root_path + "/", "") for p in tiff_files])} ), diff --git a/src/xradarsat2/radarSat2_xarray_reader.py b/src/xradarsat2/radarSat2_xarray_reader.py index 721cdcd..1f2cd38 100644 --- a/src/xradarsat2/radarSat2_xarray_reader.py +++ b/src/xradarsat2/radarSat2_xarray_reader.py @@ -4,6 +4,7 @@ import os import re import traceback +from datetime import datetime import dask import numpy as np @@ -13,7 +14,6 @@ import xmltodict import yaml from affine import Affine -from datetime import datetime xpath_dict = { "geolocation_grid": { @@ -1896,12 +1896,7 @@ def load_digital_number( tiff_files, pols = sort_list_files_and_get_pols(tiff_files) map_dims = {"pol": "band", "line": "y", "sample": "x"} if resolution is not None: - comment = 'resampled at "%s" with %s.%s.%s' % ( - resolution, - resampling.__module__, - resampling.__class__.__name__, - resampling.name, - ) + comment = f'resampled at "{resolution}" with {resampling.__module__}.{resampling.__class__.__name__}.{resampling.name}' else: comment = "read at full resolution" @@ -2024,7 +2019,7 @@ def load_digital_number( var_name = "digital_number" dn.attrs = { - "comment": "%s digital number, %s" % (descr, comment), + "comment": f"{descr} digital number, {comment}", "history": yaml.safe_dump( { var_name: get_glob( diff --git a/src/xradarsat2/utils.py b/src/xradarsat2/utils.py index 4defd02..34c5626 100644 --- a/src/xradarsat2/utils.py +++ b/src/xradarsat2/utils.py @@ -1,13 +1,15 @@ -import xradarsat2 import logging import os -import yaml import re -import aiohttp -import fsspec import warnings import zipfile +import aiohttp +import fsspec +import yaml + +import xradarsat2 + def get_glob(strlist): # from list of str, replace diff by '?' @@ -46,10 +48,11 @@ def load_config(): ) logging.info("config path: %s", config_path) - stream = open(config_path, "r") + stream = open(config_path) conf = yaml.load(stream, Loader=yaml.CLoader) return conf + def get_test_file(fname): """ get test file from https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata/ @@ -66,9 +69,9 @@ def get_test_file(fname): path to file, relative to `config['data_dir']` """ - config = {'data_dir': '/tmp'} + config = {"data_dir": "/tmp"} - def url_get(url, cache_dir=os.path.join(config['data_dir'], 'fsspec_cache')): + def url_get(url, cache_dir=os.path.join(config["data_dir"], "fsspec_cache")): """ Get fil from url, using caching. @@ -92,11 +95,15 @@ def url_get(url, cache_dir=os.path.join(config['data_dir'], 'fsspec_cache')): Due to fsspec, the returned filename won't match the remote one. """ - if '://' in url: + if "://" in url: with fsspec.open( - 'filecache::%s' % url, - https={'client_kwargs': {'timeout': aiohttp.ClientTimeout(total=3600)}}, - filecache={'cache_storage': os.path.join(os.path.join(config['data_dir'], 'fsspec_cache'))} + f"filecache::{url}", + https={"client_kwargs": {"timeout": aiohttp.ClientTimeout(total=3600)}}, + filecache={ + "cache_storage": os.path.join( + os.path.join(config["data_dir"], "fsspec_cache") + ) + }, ) as f: fname = f.name else: @@ -104,13 +111,13 @@ def url_get(url, cache_dir=os.path.join(config['data_dir'], 'fsspec_cache')): return fname - res_path = config['data_dir'] - base_url = 'https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata' - file_url = '%s/%s.zip' % (base_url, fname) + res_path = config["data_dir"] + base_url = "https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata" + file_url = f"{base_url}/{fname}.zip" if not os.path.exists(os.path.join(res_path, fname)): - warnings.warn("Downloading %s" % file_url) + warnings.warn(f"Downloading {file_url}") local_file = url_get(file_url) - warnings.warn("Unzipping %s" % os.path.join(res_path, fname)) - with zipfile.ZipFile(local_file, 'r') as zip_ref: + warnings.warn(f"Unzipping {os.path.join(res_path, fname)}") + with zipfile.ZipFile(local_file, "r") as zip_ref: zip_ref.extractall(res_path) - return os.path.join(res_path, fname) \ No newline at end of file + return os.path.join(res_path, fname) diff --git a/test/test_opening_datatree_radarsat2.py b/test/test_opening_datatree_radarsat2.py index a0df6cc..431cd2e 100644 --- a/test/test_opening_datatree_radarsat2.py +++ b/test/test_opening_datatree_radarsat2.py @@ -2,7 +2,7 @@ import time import xradarsat2 -from xradarsat2.utils import load_config,get_test_file +from xradarsat2.utils import get_test_file, load_config logging.basicConfig(level=logging.DEBUG) logging.debug("start opening RadarSAT-2 product") @@ -12,7 +12,7 @@ t0 = time.time() conf = load_config() folder_path = conf["folder_path"] -rs2_product_path = get_test_file(conf['folder_path']) +rs2_product_path = get_test_file(conf["folder_path"]) dt = xradarsat2.rs2_reader(rs2_product_path) elapse_t = time.time() - t0 From 9c97d4cce9157704824a374b8328c6d391aae367 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Wed, 27 Nov 2024 18:17:37 +0100 Subject: [PATCH 05/20] try to get ride of rioxarray deprecation warning in unit tests --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1a2c6be..47462e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ 'numpy', "xarray>=2024.10.0", 'rasterio', - 'rioxarray', + 'rioxarray>=0.18.1', 'dask', 'affine', 'regex', From 471e548e0a81133e3a840ff29a4c49eb20ac4db4 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Wed, 27 Nov 2024 18:20:09 +0100 Subject: [PATCH 06/20] try to get ride of rioxarray deprecation warning in unit tests #2 --- ci/requirements/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/environment.yaml b/ci/requirements/environment.yaml index 9cec30d..9032887 100644 --- a/ci/requirements/environment.yaml +++ b/ci/requirements/environment.yaml @@ -13,7 +13,7 @@ dependencies: - python-dateutil - construct - xarray - - rioxarray + - rioxarray>=0.18.1 - aiohttp - fsspec - dask From 1f6de4ddb1ee6872baccd85c9a7fb22210528221 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 08:53:51 +0100 Subject: [PATCH 07/20] change description --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 47462e0..d5168aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ fallback_version = "9999" name = "xradarsat2" authors = [{ name = "Yann Reynaud", email = "Yann.Reynaud.2@ifremer.fr" }] license = { text = "MIT" } -description = "xarray/dask distributed L1 sar file reader for radarSat2" +description = "xarray Level-1 SAR file reader for radarSat2" readme = "README.md" requires-python = ">=3.9" classifiers = [ From 5744b6e3c92a7c21e35243d8538c601d13cca936 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 09:04:12 +0100 Subject: [PATCH 08/20] to not use the cache --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d09af46..04fb561 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -66,7 +66,7 @@ jobs: with: environment-file: ${{ env.CONDA_ENV_FILE }} environment-name: xradarsat2-tests - cache-environment: true + cache-environment: false cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" create-args: >- python=${{matrix.python-version}} From 052dc0339499c1033605063370f989b4a8dbad79 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 09:06:15 +0100 Subject: [PATCH 09/20] to not use the cache --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 04fb561..e3a372e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,7 +67,7 @@ jobs: environment-file: ${{ env.CONDA_ENV_FILE }} environment-name: xradarsat2-tests cache-environment: false - cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" + #cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" create-args: >- python=${{matrix.python-version}} From e0d249a29b55a42a6ad076934357b8efca6b1813 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 09:58:42 +0100 Subject: [PATCH 10/20] specify version of micromamba --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e3a372e..79a2701 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,6 +65,7 @@ jobs: uses: mamba-org/setup-micromamba@v2 with: environment-file: ${{ env.CONDA_ENV_FILE }} + micromamba-version: 2.0.2 environment-name: xradarsat2-tests cache-environment: false #cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" From b827ecc6959f9afdbb91f6c55c8eb7822bd243ab Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 09:59:59 +0100 Subject: [PATCH 11/20] specify version of micromamba --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 79a2701..7b2d985 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,7 +65,7 @@ jobs: uses: mamba-org/setup-micromamba@v2 with: environment-file: ${{ env.CONDA_ENV_FILE }} - micromamba-version: 2.0.2 + micromamba-version: latest environment-name: xradarsat2-tests cache-environment: false #cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" From 4d5fc082855d4b2e2f46952d62ad5706a890a2c6 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 10:01:31 +0100 Subject: [PATCH 12/20] specify version of micromamba --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7b2d985..1f4031b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,7 +65,8 @@ jobs: uses: mamba-org/setup-micromamba@v2 with: environment-file: ${{ env.CONDA_ENV_FILE }} - micromamba-version: latest + micromamba-version: 1.5.10-0 + micromamba-binary-path: /home/runner/micromamba-bin-versioned/micromamba environment-name: xradarsat2-tests cache-environment: false #cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" From 81817ad96d8e4275b1de8b5e9742920233147033 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 10:03:05 +0100 Subject: [PATCH 13/20] specify version of micromamba --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1f4031b..9cde7c7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -68,8 +68,8 @@ jobs: micromamba-version: 1.5.10-0 micromamba-binary-path: /home/runner/micromamba-bin-versioned/micromamba environment-name: xradarsat2-tests - cache-environment: false - #cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" + cache-environment: true + cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" create-args: >- python=${{matrix.python-version}} From 6fe6a3bab6af2853cdafdb58f24b8cf092cf73ef Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 10:08:52 +0100 Subject: [PATCH 14/20] remove version rioxarray --- ci/requirements/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/environment.yaml b/ci/requirements/environment.yaml index 9032887..9cec30d 100644 --- a/ci/requirements/environment.yaml +++ b/ci/requirements/environment.yaml @@ -13,7 +13,7 @@ dependencies: - python-dateutil - construct - xarray - - rioxarray>=0.18.1 + - rioxarray - aiohttp - fsspec - dask From 20f68bbd31ba96eb4703adaad553dc714d75aab0 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 10:10:35 +0100 Subject: [PATCH 15/20] default micromamba --- .github/workflows/ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9cde7c7..d09af46 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,8 +65,6 @@ jobs: uses: mamba-org/setup-micromamba@v2 with: environment-file: ${{ env.CONDA_ENV_FILE }} - micromamba-version: 1.5.10-0 - micromamba-binary-path: /home/runner/micromamba-bin-versioned/micromamba environment-name: xradarsat2-tests cache-environment: true cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" From 792a5f2c7c7679ca1e78340132e524f7f7cb96a8 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 10:16:30 +0100 Subject: [PATCH 16/20] put back the rioxarray version --- ci/requirements/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/environment.yaml b/ci/requirements/environment.yaml index 9cec30d..9032887 100644 --- a/ci/requirements/environment.yaml +++ b/ci/requirements/environment.yaml @@ -13,7 +13,7 @@ dependencies: - python-dateutil - construct - xarray - - rioxarray + - rioxarray>=0.18.1 - aiohttp - fsspec - dask From c12812a9959b4c74c10ca657be61f21073e73a08 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 10:21:56 +0100 Subject: [PATCH 17/20] try a fix --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d09af46..f907d8a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,6 +67,7 @@ jobs: environment-file: ${{ env.CONDA_ENV_FILE }} environment-name: xradarsat2-tests cache-environment: true + generate-run-shell: false cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" create-args: >- python=${{matrix.python-version}} From b3cd98180c01f7110a0016b8a62e3a408a5ce221 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Thu, 28 Nov 2024 10:26:30 +0100 Subject: [PATCH 18/20] try a fix --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f907d8a..0302867 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,7 +67,7 @@ jobs: environment-file: ${{ env.CONDA_ENV_FILE }} environment-name: xradarsat2-tests cache-environment: true - generate-run-shell: false + generate-run-shell: true cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" create-args: >- python=${{matrix.python-version}} From db4f733835375d28f4723400de575e1a45859a40 Mon Sep 17 00:00:00 2001 From: grouazel Date: Thu, 28 Nov 2024 15:07:10 +0100 Subject: [PATCH 19/20] add python as dependency --- ci/requirements/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/requirements/environment.yaml b/ci/requirements/environment.yaml index 9cec30d..0bed8bd 100644 --- a/ci/requirements/environment.yaml +++ b/ci/requirements/environment.yaml @@ -2,6 +2,7 @@ name: xradarsat2-tests channels: - conda-forge dependencies: + - python - ipython - pre-commit - pytest From 41e7254f50be37d66fe4818875e4a59d2175077b Mon Sep 17 00:00:00 2001 From: grouazel Date: Thu, 28 Nov 2024 15:10:44 +0100 Subject: [PATCH 20/20] fix rioxarray version since, only 0.17.0 is avail on conda-forge for now --- ci/requirements/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/environment.yaml b/ci/requirements/environment.yaml index cfd69bd..deb4a78 100644 --- a/ci/requirements/environment.yaml +++ b/ci/requirements/environment.yaml @@ -14,7 +14,7 @@ dependencies: - python-dateutil - construct - xarray - - rioxarray>=0.18.1 + - rioxarray>=0.17.0 - aiohttp - fsspec - dask