diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..0302867 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,85 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + detect-skip-ci-trigger: + name: "Detect CI Trigger: [skip-ci]" + if: | + github.repository == 'umr-lops/xradarsat2' + && ( + github.event_name == 'push' || github.event_name == 'pull_request' + ) + runs-on: ubuntu-latest + outputs: + triggered: ${{ steps.detect-trigger.outputs.trigger-found }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: xarray-contrib/ci-trigger@v1 + id: detect-trigger + with: + keyword: "[skip-ci]" + + ci: + name: ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + needs: detect-skip-ci-trigger + + if: needs.detect-skip-ci-trigger.outputs.triggered == 'false' + + defaults: + run: + shell: bash -l {0} + + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + with: + # need to fetch all tags to get a correct version + fetch-depth: 0 # fetch all branches and tags + + - name: Setup environment variables + run: | + echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + + echo "CONDA_ENV_FILE=ci/requirements/environment.yaml" >> $GITHUB_ENV + + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v2 + with: + environment-file: ${{ env.CONDA_ENV_FILE }} + environment-name: xradarsat2-tests + cache-environment: true + generate-run-shell: true + cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" + create-args: >- + python=${{matrix.python-version}} + + - name: Install xradarsat2 + run: | + python -m pip install --no-deps -e . + + - name: Import xradarsat2 + run: | + python -c "import xradarsat2" + + - name: Run tests + run: | + python -m pytest --cov=xradarsat2 diff --git a/.github/workflows/upstream-dev.yaml b/.github/workflows/upstream-dev.yaml new file mode 100644 index 0000000..023b57b --- /dev/null +++ b/.github/workflows/upstream-dev.yaml @@ -0,0 +1,98 @@ +name: upstream-dev CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "0 18 * * 0" # Weekly "On Sundays at 18:00" UTC + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + detect-test-upstream-trigger: + name: "Detect CI Trigger: [test-upstream]" + if: github.event_name == 'push' || github.event_name == 'pull_request' + runs-on: ubuntu-latest + outputs: + triggered: ${{ steps.detect-trigger.outputs.trigger-found }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: xarray-contrib/ci-trigger@v1.2 + id: detect-trigger + with: + keyword: "[test-upstream]" + + upstream-dev: + name: upstream-dev + runs-on: ubuntu-latest + needs: detect-test-upstream-trigger + + if: | + always() + && github.repository == 'umr-lops/xradarsat2' + && ( + github.event_name == 'schedule' + || github.event_name == 'workflow_dispatch' + || needs.detect-test-upstream-trigger.outputs.triggered == 'true' + || contains(github.event.pull_request.labels.*.name, 'run-upstream') + ) + + defaults: + run: + shell: bash -l {0} + + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - name: checkout the repository + uses: actions/checkout@v4 + with: + # need to fetch all tags to get a correct version + fetch-depth: 0 # fetch all branches and tags + + - name: set up conda environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ci/requirements/environment.yaml + environment-name: tests + create-args: >- + python=${{ matrix.python-version }} + pytest-reportlog + + - name: install upstream-dev dependencies + run: bash ci/install-upstream-dev.sh + + - name: install the package + run: python -m pip install --no-deps -e . + + - name: show versions + run: python -m pip list + + - name: import + run: | + python -c 'import xradarsat2' + + - name: run tests + if: success() + id: status + run: | + python -m pytest -rf --report-log=pytest-log.jsonl + + - name: report failures + if: | + failure() + && steps.tests.outcome == 'failure' + && github.event_name == 'schedule' + uses: xarray-contrib/issue-from-pytest-log@v1 + with: + log-path: pytest-log.jsonl diff --git a/.gitignore b/.gitignore index f41feca..133e187 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,4 @@ tmp/ /.idea/ dask-worker-space/ -localxradarsat2-config.yaml \ No newline at end of file +localxradarsat2-config.yaml diff --git a/ci/install-upstream-dev.sh b/ci/install-upstream-dev.sh new file mode 100644 index 0000000..8744bf9 --- /dev/null +++ b/ci/install-upstream-dev.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +conda remove -y --force cytoolz numpy xarray toolz python-dateutil rioxarray +python -m pip install \ + -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \ + --no-deps \ + --pre \ + --upgrade \ + numpy \ + rioxarray \ + xarray +python -m pip install --upgrade \ + git+https://github.com/pytoolz/toolz \ + git+https://github.com/dateutil/dateutil diff --git a/ci/requirements/docs.yaml b/ci/requirements/docs.yaml new file mode 100644 index 0000000..b287778 --- /dev/null +++ b/ci/requirements/docs.yaml @@ -0,0 +1,8 @@ +name: xradarsat2-docs +channels: + - conda-forge +dependencies: + - python=3.11 + - sphinx>=4 + - sphinx_book_theme + - ipython diff --git a/ci/requirements/environment.yaml b/ci/requirements/environment.yaml new file mode 100644 index 0000000..deb4a78 --- /dev/null +++ b/ci/requirements/environment.yaml @@ -0,0 +1,26 @@ +name: xradarsat2-tests +channels: + - conda-forge +dependencies: + - python + - ipython + - pre-commit + - pytest + - pytest-reportlog + - pytest-cov + - numpy + - toolz + - cytoolz + - python-dateutil + - construct + - xarray + - rioxarray>=0.17.0 + - aiohttp + - fsspec + - dask + - opencv + - pyyaml + - affine + - h5netcdf + - scipy + - xmltodict diff --git a/docs/doc_xradarsat2.ipynb b/docs/doc_xradarsat2.ipynb index 0b131b8..293185e 100644 --- a/docs/doc_xradarsat2.ipynb +++ b/docs/doc_xradarsat2.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "03cfc89b-5ee1-49c2-8bc9-ca88c00eb63b", + "id": "0", "metadata": {}, "source": [ "# examples" @@ -11,7 +11,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9f901d61-adf5-4ad2-a721-b92fb8555642", + "id": "1", "metadata": {}, "outputs": [], "source": [ @@ -23,7 +23,7 @@ { "cell_type": "code", "execution_count": null, - "id": "84c1a31d-c27c-4614-8a39-f4dca2801bf6", + "id": "2", "metadata": {}, "outputs": [], "source": [ @@ -36,7 +36,7 @@ { "cell_type": "code", "execution_count": null, - "id": "90fb68ff-2c19-4432-8d95-98159ee546a8", + "id": "3", "metadata": {}, "outputs": [], "source": [ @@ -48,7 +48,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ff1730a0-caeb-4c53-8fa0-85c79adfb5a6", + "id": "4", "metadata": {}, "outputs": [], "source": [ @@ -59,7 +59,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6f74bb35-b8d0-4a17-82d2-4b151e3044fb", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -74,11 +74,6 @@ } ], "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -88,8 +83,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.0" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/pyproject.toml b/pyproject.toml index a0939e7..d5168aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,31 +15,65 @@ fallback_version = "9999" [project] name = "xradarsat2" -authors = [ - { name="Yann Reynaud", email="Yann.Reynaud.2@ifremer.fr" }, -] -license = {text = "MIT"} -description = "xarray/dask distributed L1 sar file reader for radarSat2" +authors = [{ name = "Yann Reynaud", email = "Yann.Reynaud.2@ifremer.fr" }] +license = { text = "MIT" } +description = "xarray Level-1 SAR file reader for radarSat2" readme = "README.md" requires-python = ">=3.9" classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", ] dependencies = [ - 'xmltodict', - 'numpy', - "xarray>=2024.10.0", - 'rasterio', - 'rioxarray', - 'dask', - 'affine', - 'regex', - 'pyyaml' + 'xmltodict', + 'numpy', + "xarray>=2024.10.0", + 'rasterio', + 'rioxarray>=0.18.1', + 'dask', + 'affine', + 'regex', + 'pyyaml', + 'fsspec', + 'aiohttp', + ] dynamic = ["version"] [project.urls] "Homepage" = "https://github.com/umr-lops/xradarsat2" "Bug Tracker" = "https://github.com/umr-lops/xradarsat2/issues" + +[tool.coverage.report] +show_missing = true +exclude_lines = ["pragma: no cover", "if TYPE_CHECKING"] + +[tool.ruff.lint] +ignore = [ + "E402", # module level import not at top of file + "E501", # line too long - let black worry about that + "E731", # do not assign a lambda expression, use a def + "UP038", # type union instead of tuple for isinstance etc +] +select = [ + "F", # Pyflakes + "E", # Pycodestyle + "I", # isort + "UP", # Pyupgrade + "TID", # flake8-tidy-imports + "W", +] +extend-safe-fixes = [ + "TID252", # absolute imports + "UP031", # percent string interpolation +] +fixable = ["I", "TID252", "UP"] + +[tool.ruff.lint.isort] +known-first-party = ["xradarsat2"] +known-third-party = ["xarray", "toolz", "construct"] + +[tool.ruff.lint.flake8-tidy-imports] +# Disallow all relative imports. +ban-relative-imports = "all" diff --git a/src/xradarsat2/__init__.py b/src/xradarsat2/__init__.py index 1e614b3..423fde4 100644 --- a/src/xradarsat2/__init__.py +++ b/src/xradarsat2/__init__.py @@ -1,7 +1,10 @@ -from xradarsat2.radarSat2_xarray_reader import load_digital_number # noqa: F401 -from xradarsat2.radarSat2_xarray_reader import rs2_reader # noqa: F401 from importlib.metadata import version +from xradarsat2.radarSat2_xarray_reader import ( + load_digital_number, # noqa: F401 + rs2_reader, # noqa: F401 +) + try: __version__ = version("xradarsat2") except Exception: diff --git a/src/xradarsat2/radarSat2_tiff_reader.py b/src/xradarsat2/radarSat2_tiff_reader.py index 3cd6941..b508b44 100644 --- a/src/xradarsat2/radarSat2_tiff_reader.py +++ b/src/xradarsat2/radarSat2_tiff_reader.py @@ -8,6 +8,7 @@ import xarray as xr import yaml from affine import Affine + from xradarsat2.utils import get_glob, load_config # folder_path = "/home/datawork-cersat-public/cache/project/sarwing/data/RS2/L1/VV/2010/288/" \ @@ -43,12 +44,7 @@ def _load_digital_number( tiff_files = list_tiff_files(root_path) map_dims = {"pol": "band", "line": "y", "sample": "x"} if resolution is not None: - comment = 'resampled at "%s" with %s.%s.%s' % ( - resolution, - resampling.__module__, - resampling.__class__.__name__, - resampling.name, - ) + comment = f'resampled at "{resolution}" with {resampling.__module__}.{resampling.__class__.__name__}.{resampling.name}' else: comment = "read at full resolution" @@ -158,7 +154,7 @@ def _load_digital_number( var_name = "digital_number" dn.attrs = { - "comment": "%s digital number, %s" % (descr, comment), + "comment": f"{descr} digital number, {comment}", "history": yaml.safe_dump( {var_name: get_glob([p.replace(root_path + "/", "") for p in tiff_files])} ), diff --git a/src/xradarsat2/radarSat2_xarray_reader.py b/src/xradarsat2/radarSat2_xarray_reader.py index 721cdcd..1f2cd38 100644 --- a/src/xradarsat2/radarSat2_xarray_reader.py +++ b/src/xradarsat2/radarSat2_xarray_reader.py @@ -4,6 +4,7 @@ import os import re import traceback +from datetime import datetime import dask import numpy as np @@ -13,7 +14,6 @@ import xmltodict import yaml from affine import Affine -from datetime import datetime xpath_dict = { "geolocation_grid": { @@ -1896,12 +1896,7 @@ def load_digital_number( tiff_files, pols = sort_list_files_and_get_pols(tiff_files) map_dims = {"pol": "band", "line": "y", "sample": "x"} if resolution is not None: - comment = 'resampled at "%s" with %s.%s.%s' % ( - resolution, - resampling.__module__, - resampling.__class__.__name__, - resampling.name, - ) + comment = f'resampled at "{resolution}" with {resampling.__module__}.{resampling.__class__.__name__}.{resampling.name}' else: comment = "read at full resolution" @@ -2024,7 +2019,7 @@ def load_digital_number( var_name = "digital_number" dn.attrs = { - "comment": "%s digital number, %s" % (descr, comment), + "comment": f"{descr} digital number, {comment}", "history": yaml.safe_dump( { var_name: get_glob( diff --git a/src/xradarsat2/utils.py b/src/xradarsat2/utils.py index bf91277..34c5626 100644 --- a/src/xradarsat2/utils.py +++ b/src/xradarsat2/utils.py @@ -1,8 +1,14 @@ -import xradarsat2 import logging import os -import yaml import re +import warnings +import zipfile + +import aiohttp +import fsspec +import yaml + +import xradarsat2 def get_glob(strlist): @@ -42,6 +48,76 @@ def load_config(): ) logging.info("config path: %s", config_path) - stream = open(config_path, "r") + stream = open(config_path) conf = yaml.load(stream, Loader=yaml.CLoader) return conf + + +def get_test_file(fname): + """ + get test file from https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata/ + file is unzipped and extracted to `config['data_dir']` + + Parameters + ---------- + fname: str + file name to get (without '.zip' extension) + + Returns + ------- + str + path to file, relative to `config['data_dir']` + + """ + config = {"data_dir": "/tmp"} + + def url_get(url, cache_dir=os.path.join(config["data_dir"], "fsspec_cache")): + """ + Get fil from url, using caching. + + Parameters + ---------- + url: str + cache_dir: str + Cache dir to use. default to `os.path.join(config['data_dir'], 'fsspec_cache')` + + Raises + ------ + FileNotFoundError + + Returns + ------- + filename: str + The local file name + + Notes + ----- + Due to fsspec, the returned filename won't match the remote one. + """ + + if "://" in url: + with fsspec.open( + f"filecache::{url}", + https={"client_kwargs": {"timeout": aiohttp.ClientTimeout(total=3600)}}, + filecache={ + "cache_storage": os.path.join( + os.path.join(config["data_dir"], "fsspec_cache") + ) + }, + ) as f: + fname = f.name + else: + fname = url + + return fname + + res_path = config["data_dir"] + base_url = "https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata" + file_url = f"{base_url}/{fname}.zip" + if not os.path.exists(os.path.join(res_path, fname)): + warnings.warn(f"Downloading {file_url}") + local_file = url_get(file_url) + warnings.warn(f"Unzipping {os.path.join(res_path, fname)}") + with zipfile.ZipFile(local_file, "r") as zip_ref: + zip_ref.extractall(res_path) + return os.path.join(res_path, fname) diff --git a/src/xradarsat2/xradarsat2-config.yaml b/src/xradarsat2/xradarsat2-config.yaml index 4e19cc0..06bc2b1 100644 --- a/src/xradarsat2/xradarsat2-config.yaml +++ b/src/xradarsat2/xradarsat2-config.yaml @@ -1 +1 @@ -folder_path: ./2021/137/RS2_OK129673_PK1136693_DK1093025_SCWA_20210517_010235_VV_VH_SGF +folder_path: RS2_OK135107_PK1187782_DK1151894_SCWA_20220407_182127_VV_VH_SGF diff --git a/test/test_opening_datatree_radarsat2.py b/test/test_opening_datatree_radarsat2.py index 88111a8..431cd2e 100644 --- a/test/test_opening_datatree_radarsat2.py +++ b/test/test_opening_datatree_radarsat2.py @@ -1,7 +1,8 @@ -from xradarsat2.utils import load_config -import xradarsat2 -import time import logging +import time + +import xradarsat2 +from xradarsat2.utils import get_test_file, load_config logging.basicConfig(level=logging.DEBUG) logging.debug("start opening RadarSAT-2 product") @@ -11,13 +12,14 @@ t0 = time.time() conf = load_config() folder_path = conf["folder_path"] -dt = xradarsat2.rs2_reader(folder_path) +rs2_product_path = get_test_file(conf["folder_path"]) +dt = xradarsat2.rs2_reader(rs2_product_path) elapse_t = time.time() - t0 print(type(dt), dt) print("out of the reader") print(dt) -print("time to read the SAFE through nfs: %1.2f sec" % elapse_t) +print(f"time to read the SAFE through nfs: {elapse_t:1.2f} sec") dt = xradarsat2.load_digital_number( dt, chunks={"pol": "VV", "line": 6000, "sample": 8000} )