diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index ffef8950..90c92821 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -9,8 +9,11 @@ Release Notes -.. Upcoming Release -.. ================ +Upcoming Release +================ + +* The functions ``add_raster()`` and ``add_geometry()`` of the ``ExclusionContainer`` can now directly read from URL. + Version 0.2.11 ============== diff --git a/atlite/gis.py b/atlite/gis.py index d68755d7..902fe044 100644 --- a/atlite/gis.py +++ b/atlite/gis.py @@ -8,11 +8,14 @@ """ import logging +import os +import validators import multiprocessing as mp from collections import OrderedDict from functools import wraps from pathlib import Path from warnings import catch_warnings, simplefilter, warn +from urllib.request import urlretrieve import geopandas as gpd import numpy as np @@ -35,6 +38,26 @@ logger = logging.getLogger(__name__) +# for the writable data directory follow the XDG guidelines +# https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html +_writable_dir = os.path.join(os.path.expanduser("~"), ".local", "share") +_data_dir = os.path.join( + os.environ.get("XDG_DATA_HOME", os.environ.get("APPDATA", _writable_dir)), + "atlite-excluders", +) +_data_dir = Path(_data_dir) +try: + _data_dir.mkdir(exist_ok=True) +except FileNotFoundError: + os.makedirs(_data_dir) + + +def _retrieve_from_url(path): + local_path = _data_dir / os.path.basename(path) + logger.info(f"Retrieving network data from {path}") + urlretrieve(path, local_path) + return str(local_path) + def get_coords(x, y, time, dx=0.25, dy=0.25, dt="h", **kwargs): """ @@ -426,7 +449,7 @@ def add_raster( Parameters ---------- raster : str/rasterio.DatasetReader - Raster or path to raster which to exclude. + Raster, path or URL to raster which to exclude. codes : int/list/function, optional Codes in the raster which to exclude. Can be a callable function which takes the mask (np.array) as argument and performs a @@ -465,7 +488,7 @@ def add_geometry(self, geometry, buffer=0, invert=False): Parameters ---------- geometry : str/path/geopandas.GeoDataFrame - Path to geometries or geometries which to exclude. + Path or URL to geometries or geometries which to exclude. buffer : float, optional Buffer around the excluded areas in units of ExclusionContainer.crs. The default is 0. @@ -483,6 +506,8 @@ def open_files(self): for d in self.rasters: raster = d["raster"] if isinstance(raster, (str, Path)): + if validators.url(str(raster)): + raster = _retrieve_from_url(raster) raster = rio.open(raster) else: assert isinstance(raster, rio.DatasetReader) diff --git a/environment.yaml b/environment.yaml index ad1bbfbd..ae62f89f 100644 --- a/environment.yaml +++ b/environment.yaml @@ -13,7 +13,7 @@ dependencies: - numpy - scipy - pandas>=0.25 -- xarray>=0.16.2 +- xarray>=0.20 - netcdf4 - dask>=2021.10.0 - toolz @@ -26,6 +26,7 @@ dependencies: - shapely - progressbar2 - tqdm +- validators # dev tools - black diff --git a/setup.py b/setup.py index 70339ce4..26b3aa8e 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ "pyproj>=2", "geopandas", "cdsapi", + "validators", ], extras_require={ "docs": [