Skip to content

Commit

Permalink
mldataset now supported
Browse files Browse the repository at this point in the history
  • Loading branch information
konstntokas committed Jul 1, 2024
1 parent 4151bea commit 341f33f
Show file tree
Hide file tree
Showing 10 changed files with 2,110 additions and 257 deletions.
848 changes: 814 additions & 34 deletions examples/geotiff_nonsearchable_catalog.ipynb

Large diffs are not rendered by default.

756 changes: 712 additions & 44 deletions examples/geotiff_searchable_catalog.ipynb

Large diffs are not rendered by default.

106 changes: 67 additions & 39 deletions examples/netcdf_searchable_catalog.ipynb

Large diffs are not rendered by default.

405 changes: 309 additions & 96 deletions examples/xcube_server_stac_s3.ipynb

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,12 @@ interactions:
Level-2A","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a"},{"rel":"child","type":"application/json","title":"MODIS
Leaf Area Index/FPAR 8-Day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-15A2H-061"},{"rel":"child","type":"application/json","title":"MODIS
Land Surface Temperature/Emissivity Daily","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-11A1-061"},{"rel":"child","type":"application/json","title":"MODIS
Leaf Area Index/FPAR 4-Day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-15A3H-061"},{"rel":"child","type":"application/json","title":"ECMWF
Open Data (real-time)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/ecmwf-forecast"},{"rel":"child","type":"application/json","title":"MODIS
Leaf Area Index/FPAR 4-Day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-15A3H-061"},{"rel":"child","type":"application/json","title":"MODIS
Vegetation Indices 16-Day (500m)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-13A1-061"},{"rel":"child","type":"application/json","title":"Daymet
Daily North America","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-daily-na"},{"rel":"child","type":"application/json","title":"Land
Cover of Canada","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/nrcan-landcover"},{"rel":"child","type":"application/json","title":"MODIS
Snow Cover 8-day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-10A2-061"},{"rel":"child","type":"application/json","title":"NOAA
Snow Cover 8-day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-10A2-061"},{"rel":"child","type":"application/json","title":"ECMWF
Open Data (real-time)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/ecmwf-forecast"},{"rel":"child","type":"application/json","title":"NOAA
MRMS QPE 24-Hour Pass 2","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/noaa-mrms-qpe-24h-pass2"},{"rel":"child","type":"application/json","title":"Sentinel
1 Level-1 Ground Range Detected (GRD)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-1-grd"},{"rel":"child","type":"application/json","title":"NASADEM
HGT v001","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/nasadem"},{"rel":"child","type":"application/json","title":"Esri
Expand Down Expand Up @@ -159,11 +159,11 @@ interactions:
Connection:
- keep-alive
Content-Length:
- '3324'
- '3321'
Content-Type:
- application/json
Date:
- Fri, 28 Jun 2024 14:17:20 GMT
- Mon, 01 Jul 2024 06:55:56 GMT
Strict-Transport-Security:
- max-age=31536000; includeSubDomains
X-Cache:
Expand All @@ -173,7 +173,7 @@ interactions:
vary:
- Accept-Encoding
x-azure-ref:
- 20240628T141720Z-r16b4c9c64d9ndrgwr6tpuhs7c000000058g000000002gwg
- 20240701T065556Z-1586678d48fvb4sdmzpm9r2n1s00000009f000000000nk4u
status:
code: 200
message: OK
Expand Down Expand Up @@ -244,7 +244,7 @@ interactions:
Content-Type:
- application/geo+json
Date:
- Fri, 28 Jun 2024 14:17:21 GMT
- Mon, 01 Jul 2024 06:55:56 GMT
Strict-Transport-Security:
- max-age=31536000; includeSubDomains
X-Cache:
Expand All @@ -254,7 +254,7 @@ interactions:
vary:
- Accept-Encoding
x-azure-ref:
- 20240628T141720Z-r16b4c9c64dpf7xzq9mfyssp98000000052g00000000hwca
- 20240701T065556Z-16dbcf9f44cgmttlasnpk649ss0000000a9g00000000bb13
status:
code: 200
message: OK
Expand All @@ -273,26 +273,26 @@ interactions:
uri: https://planetarycomputer.microsoft.com/api/sas/v1/token/cpdataeuwest/era5
response:
body:
string: '{"msft:expiry":"2024-06-28T15:02:21Z","token":"st=2024-06-27T14%3A17%3A21Z&se=2024-06-28T15%3A02%3A21Z&sp=rl&sv=2024-05-04&sr=c&skoid=9c8ff44a-6a2c-4dfb-b298-1c9212f64d9a&sktid=72f988bf-86f1-41af-91ab-2d7cd011db47&skt=2024-06-27T16%3A29%3A39Z&ske=2024-07-04T16%3A29%3A39Z&sks=b&skv=2024-05-04&sig=mhOqQXY8zcfKPArTB4hGFRQQEbcqlx5ftzKtxA8fh6w%3D"}'
string: '{"msft:expiry":"2024-07-01T07:40:56Z","token":"st=2024-06-30T06%3A55%3A56Z&se=2024-07-01T07%3A40%3A56Z&sp=rl&sv=2024-05-04&sr=c&skoid=9c8ff44a-6a2c-4dfb-b298-1c9212f64d9a&sktid=72f988bf-86f1-41af-91ab-2d7cd011db47&skt=2024-06-30T03%3A12%3A52Z&ske=2024-07-07T03%3A12%3A52Z&sks=b&skv=2024-05-04&sig=UmlrxTFBYe39lA/B4%2BTzD5xpiTrVIv%2BUPv6dPBgd6tI%3D"}'
headers:
Accept-Ranges:
- bytes
Connection:
- keep-alive
Content-Length:
- '345'
- '349'
Content-Type:
- application/json
Date:
- Fri, 28 Jun 2024 14:17:21 GMT
- Mon, 01 Jul 2024 06:55:56 GMT
Request-Context:
- appId=cid-v1:75161b1b-6883-4b66-9410-715040c44427
Strict-Transport-Security:
- max-age=31536000; includeSubDomains
X-Cache:
- CONFIG_NOCACHE
x-azure-ref:
- 20240628T141721Z-r16b4c9c64dkjwxnu435st6c6n00000005rg00000000n2q4
- 20240701T065556Z-16dbcf9f44cgmttlasnpk649ss0000000a7g00000000r1zn
status:
code: 200
message: OK
Expand Down
112 changes: 96 additions & 16 deletions test/cassettes/test_store/StacDataStoreTest.test_open_data_tiff.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -416,17 +416,19 @@ interactions:
Content-Type:
- application/json
Date:
- Tue, 25 Jun 2024 12:52:55 GMT
- Mon, 01 Jul 2024 13:47:23 GMT
ETag:
- '"326fbfb3eeb064cf4c2092261381fb89"'
Last-Modified:
- Fri, 15 Jul 2022 14:06:49 GMT
Server:
- WasabiS3/7.20.2951-2024-06-14-7fcea49969 (head21)
- WasabiS3/7.20.2951-2024-06-14-7fcea49969 (F8-R407-U17)
x-amz-id-2:
- Kduqm/oaSMgyjSr2i+4xwylRA3fxiXnzUSLjFxafTHjPtPu7uQLjbvzAz2+3zr9sopzTe5um+kKM
- EW3jQonzeHLU+K4FSzgjFNwsa7Qm/WFuqV+/9U3lXuVeSrcGe6qAIKqF6BSEJhsyrSkF3MWV1GQk
x-amz-request-id:
- 583613E097821208:B
- 94406B614BF26B04:B
x-wasabi-cm-reference-id:
- 1719841643075 130.117.252.103 ConID:546413410/EngineConID:5182779/Core:75
status:
code: 200
message: OK
Expand Down Expand Up @@ -847,17 +849,19 @@ interactions:
Content-Type:
- application/json
Date:
- Tue, 25 Jun 2024 12:52:55 GMT
- Mon, 01 Jul 2024 13:47:23 GMT
ETag:
- '"326fbfb3eeb064cf4c2092261381fb89"'
Last-Modified:
- Fri, 15 Jul 2022 14:06:49 GMT
Server:
- WasabiS3/7.20.2951-2024-06-14-7fcea49969 (head18)
- WasabiS3/7.20.2951-2024-06-14-7fcea49969 (F8-R407-U16)
x-amz-id-2:
- Rwlm0OJbMVgJgSQVorPcTXXfg7dIM4HP/BMHRDJAgcXwIbOtAgMirV8UuqP9UhrrXVPeZw4zOjk7
- XGCnxfrp9ja05K1zNo68nu4UTqj4PF8Bko3/mEo98ssKoGQcbL25CYmRYvTdW/8gEH7DPv5sqt6B
x-amz-request-id:
- 802457FB6A641FF8:B
- 3F24AC19E6E90CEC:B
x-wasabi-cm-reference-id:
- 1719841643182 130.117.252.103 ConID:546413429/EngineConID:5245528/Core:5
status:
code: 200
message: OK
Expand Down Expand Up @@ -923,17 +927,17 @@ interactions:
Content-Type:
- application/json
Date:
- Tue, 25 Jun 2024 12:52:55 GMT
- Mon, 01 Jul 2024 13:47:23 GMT
ETag:
- '"751b3e3d2f8eaffcf4ecd41619ec1fae"'
Last-Modified:
- Thu, 19 May 2022 19:07:42 GMT
Server:
- WasabiS3/7.20.2951-2024-06-14-7fcea49969 (head10)
- WasabiS3/7.20.2951-2024-06-14-7fcea49969 (head12)
x-amz-id-2:
- 7SKnOQKl/dYtTpFv43d7IwnxQ0fiZkowPIrcFtbmACK+hwuPEKQno8VpN/G6q90tsAWoo6niDKAN
- nb/cdvupbC1rnWB+ftZuq+E2zr52jJojx7WqGRPh24XVohrKQaRGD5lIfz3pI3GGRX3YRWdHbQ1G
x-amz-request-id:
- 9567EA84343DAD20:A
- 1BA359A7128A8942:B
status:
code: 200
message: OK
Expand Down Expand Up @@ -999,17 +1003,93 @@ interactions:
Content-Type:
- application/json
Date:
- Tue, 25 Jun 2024 12:52:58 GMT
- Mon, 01 Jul 2024 13:47:25 GMT
ETag:
- '"751b3e3d2f8eaffcf4ecd41619ec1fae"'
Last-Modified:
- Thu, 19 May 2022 19:07:42 GMT
Server:
- WasabiS3/7.20.2951-2024-06-14-7fcea49969 (head13)
- WasabiS3/7.20.2951-2024-06-14-7fcea49969 (head22)
x-amz-id-2:
- EzhbB/KJ9hEfMedmPzXuSIChwICp0VRBOPAX3d2oA2g1vOJEyTJEShJ5SJWQyOAcTs5w0pXOsrdU
- 7rtwWPlWMUXgFGmioI+htFKUcokAYb4qbD3/PXkNsOYVCsCrQ5bzdfaLFomjREsiT6rGp2dCtAZM
x-amz-request-id:
- 9AF0D763FEA63CDA:A
- 3160D5CD17868DDA:B
status:
code: 200
message: OK
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate, br
Connection:
- keep-alive
User-Agent:
- python-requests/2.32.3
method: GET
uri: https://s3.eu-central-1.wasabisys.com/stac/odse/lcv_blue_landsat.glad.ard/lcv_blue_landsat.glad.ard_1999.12.02..2000.03.20/lcv_blue_landsat.glad.ard_1999.12.02..2000.03.20.json
response:
body:
string: "{\n \"type\": \"Feature\",\n \"stac_version\": \"1.0.0\",\n \"id\":
\"lcv_blue_landsat.glad.ard_1999.12.02..2000.03.20\",\n \"properties\": {\n
\ \"start_datetime\": \"1999-12-02\",\n \"end_datetime\": \"2000-03-20\",\n
\ \"gsd\": \"30\",\n \"instruments\": [\n \"TM\",\n \"ETM+\",\n
\ \"OLI\"\n ],\n \"platform\": \"landsat\",\n \"datetime\": \"1999-12-02T00:00:00Z\"\n
\ },\n \"geometry\": {\n \"type\": \"Polygon\",\n \"coordinates\":
[\n [\n [\n -23.550818268711048,\n 24.399543432891665\n
\ ],\n [\n -23.550818268711048,\n 77.69295185585888\n
\ ],\n [\n 63.352379098951936,\n 77.69295185585888\n
\ ],\n [\n 63.352379098951936,\n 24.399543432891665\n
\ ],\n [\n -23.550818268711048,\n 24.399543432891665\n
\ ]\n ]\n ]\n },\n \"links\": [\n {\n \"rel\": \"root\",\n
\ \"href\": \"../../catalog.json\",\n \"type\": \"application/json\",\n
\ \"title\": \"OpenDataScience Europe\"\n },\n {\n \"rel\":
\"collection\",\n \"href\": \"../collection.json\",\n \"type\":
\"application/json\",\n \"title\": \"Quarterly blue band of GLAD landsat
ARD (2000\\u20132020)\"\n },\n {\n \"rel\": \"parent\",\n \"href\":
\"../collection.json\",\n \"type\": \"application/json\",\n \"title\":
\"Quarterly blue band of GLAD landsat ARD (2000\\u20132020)\"\n }\n ],\n
\ \"assets\": {\n \"blue_p50\": {\n \"href\": \"https://s3.eu-central-1.wasabisys.com/eumap/lcv/lcv_blue_landsat.glad.ard_p50_30m_0..0cm_1999.12.02..2000.03.20_eumap_epsg3035_v1.1.tif\",\n
\ \"type\": \"image/tiff; application=geotiff\",\n \"main\": true,\n
\ \"roles\": [\n \"data\"\n ]\n },\n \"blue_p25\": {\n
\ \"href\": \"https://s3.eu-central-1.wasabisys.com/eumap/lcv/lcv_blue_landsat.glad.ard_p25_30m_0..0cm_1999.12.02..2000.03.20_eumap_epsg3035_v1.1.tif\",\n
\ \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n
\ \"data\"\n ]\n },\n \"blue_p75\": {\n \"href\": \"https://s3.eu-central-1.wasabisys.com/eumap/lcv/lcv_blue_landsat.glad.ard_p75_30m_0..0cm_1999.12.02..2000.03.20_eumap_epsg3035_v1.1.tif\",\n
\ \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n
\ \"data\"\n ]\n },\n \"qa_f\": {\n \"href\": \"https://s3.eu-central-1.wasabisys.com/eumap/lcv/lcv_qa_landsat.glad.ard_f_30m_0..0cm_1999.12.02..2000.03.20_eumap_epsg3035_v1.1.tif\",\n
\ \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n
\ \"data\"\n ]\n },\n \"thumbnail\": {\n \"href\": \"https://s3.eu-central-1.wasabisys.com/stac/odse/lcv_blue_landsat.glad.ard/lcv_blue_landsat.glad.ard_1999.12.02..2000.03.20/lcv_blue_landsat.glad.ard_p50_30m_0..0cm_1999.12.02..2000.03.20_eumap_epsg3035_v1.1.png\",\n
\ \"type\": \"image/png\",\n \"roles\": [\n \"thumbnail\"\n
\ ]\n },\n \"blue_p25_preview\": {\n \"href\": \"https://s3.eu-central-1.wasabisys.com/stac/odse/lcv_blue_landsat.glad.ard/lcv_blue_landsat.glad.ard_1999.12.02..2000.03.20/lcv_blue_landsat.glad.ard_p25_30m_0..0cm_1999.12.02..2000.03.20_eumap_epsg3035_v1.1.png\",\n
\ \"type\": \"image/png\",\n \"roles\": []\n },\n \"blue_p75_preview\":
{\n \"href\": \"https://s3.eu-central-1.wasabisys.com/stac/odse/lcv_blue_landsat.glad.ard/lcv_blue_landsat.glad.ard_1999.12.02..2000.03.20/lcv_blue_landsat.glad.ard_p75_30m_0..0cm_1999.12.02..2000.03.20_eumap_epsg3035_v1.1.png\",\n
\ \"type\": \"image/png\",\n \"roles\": []\n },\n \"qa_f_preview\":
{\n \"href\": \"https://s3.eu-central-1.wasabisys.com/stac/odse/lcv_blue_landsat.glad.ard/lcv_blue_landsat.glad.ard_1999.12.02..2000.03.20/lcv_qa_landsat.glad.ard_f_30m_0..0cm_1999.12.02..2000.03.20_eumap_epsg3035_v1.1.png\",\n
\ \"type\": \"image/png\",\n \"roles\": []\n }\n },\n \"bbox\":
[\n -23.550818268711048,\n 24.399543432891665,\n 63.352379098951936,\n
\ 77.69295185585888\n ],\n \"stac_extensions\": [\n \"https://stac-extensions.github.io/eo/v1.0.0/schema.json\"\n
\ ],\n \"collection\": \"lcv_blue_landsat.glad.ard\"\n}"
headers:
Accept-Ranges:
- bytes
Content-Length:
- '3945'
Content-Type:
- application/json
Date:
- Mon, 01 Jul 2024 13:47:26 GMT
ETag:
- '"751b3e3d2f8eaffcf4ecd41619ec1fae"'
Last-Modified:
- Thu, 19 May 2022 19:07:42 GMT
Server:
- WasabiS3/7.20.2951-2024-06-14-7fcea49969 (head16)
x-amz-id-2:
- KTZ5PdT++jwCMgUbG+X67zmlVe1+H6SojRGbxM79j6mLp3eZKFf0MzEh1y0RB+JhxhZ+JZhEDrbN
x-amz-request-id:
- 56892BC3F24A3FD4:A
status:
code: 200
message: OK
Expand Down
40 changes: 37 additions & 3 deletions test/test_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import pytest
import requests
import xarray as xr
from xcube.core.mldataset import MultiLevelDataset
from xcube.core.store import DatasetDescriptor, DataStoreError
from xcube.core.store.store import new_data_store
from xcube.util.jsonschema import JsonObjectSchema
Expand Down Expand Up @@ -238,13 +239,27 @@ def test_open_data_tiff(self):
)

# open data with open_params
ds = store.open_data(self.data_id_time_range, asset_names=["blue_p25"])
mlds = store.open_data(self.data_id_time_range, asset_names=["blue_p25"])
self.assertIsInstance(mlds, MultiLevelDataset)
ds = mlds.base_dataset
self.assertCountEqual(["blue_p25"], list(ds.data_vars))
self.assertCountEqual([151000, 188000], [ds.sizes["y"], ds.sizes["x"]])
self.assertCountEqual(
[512, 512], [ds.chunksizes["x"][0], ds.chunksizes["y"][0]]
)

# open data where multiple assets are stored in one mldataset
mlds = store.open_data(
self.data_id_time_range, asset_names=["blue_p25", "blue_p75"]
)
self.assertIsInstance(mlds, MultiLevelDataset)
ds = mlds.base_dataset
self.assertCountEqual(["blue_p25", "blue_p75"], list(ds.data_vars))
self.assertCountEqual([151000, 188000], [ds.sizes["y"], ds.sizes["x"]])
self.assertCountEqual(
[512, 512], [ds.chunksizes["x"][0], ds.chunksizes["y"][0]]
)

@pytest.mark.vcr()
def test_open_data_netcdf(self):
store = new_data_store(DATA_STORE_ID, url=self.url_netcdf)
Expand Down Expand Up @@ -272,7 +287,7 @@ def test_open_data_abfs(self):
# open data without open_params
data_id = "collections/era5-pds/items/era5-pds-2020-12-an"
with self.assertRaises(DataStoreError) as cm:
_ = store.open_data(data_id)
_ = store.open_data(data_id, asset_names=["surface_air_pressure"])
self.assertEqual(
(
"Only 's3' and 'https' protocols are supported, not 'abfs'. The asset "
Expand Down Expand Up @@ -339,7 +354,26 @@ def test_open_data_xcube_server_zarr(self):
)
self.assertCountEqual([1387, 1491], [ds.sizes["y"], ds.sizes["x"]])
# open data store in cloud-optimized tif format
ds = store.open_data("collections/datacubes/items/cog_geotiff_file")
ds = store.open_data(
"collections/datacubes/items/cog_geotiff_file",
opener_id="dataset:geotiff:s3",
)
self.assertIsInstance(ds, xr.Dataset)
self.assertCountEqual(
[
"analytic_band_1",
"analytic_band_2",
"analytic_band_3",
"analytic_spatial_ref",
],
list(ds.data_vars),
)
self.assertCountEqual([343, 343], [ds.sizes["y"], ds.sizes["x"]])
# open data store in cloud-optimized tif format in mldataaset
ds = store.open_data(
"collections/datacubes/items/cog_geotiff_file",
opener_id="mldataset:geotiff:s3",
)
self.assertIsInstance(ds, xr.Dataset)
self.assertCountEqual(
[
Expand Down
18 changes: 16 additions & 2 deletions xcube_stac/opener.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import Union

import xarray as xr
from xcube.util.jsonschema import JsonObjectSchema
from xcube.core.mldataset import MultiLevelDataset
from xcube.core.store import DataOpener, new_data_store


Expand All @@ -41,12 +43,18 @@ def __init__(self, root: str, opener_id: str):
def root(self):
return self._root

@property
def opener_id(self):
return self._opener_id

def get_open_data_params_schema(self, data_id: str = None) -> JsonObjectSchema:
return self._https_accessor.get_open_data_params_schema(
data_id=data_id, opener_id=self._opener_id
)

def open_data(self, data_id: str, **open_params) -> xr.Dataset:
def open_data(
self, data_id: str, **open_params
) -> Union[xr.Dataset, MultiLevelDataset]:
stac_schema = self.get_open_data_params_schema()
stac_schema.validate_instance(open_params)
return self._https_accessor.open_data(
Expand Down Expand Up @@ -109,14 +117,20 @@ def __init__(
def root(self):
return self._root

@property
def opener_id(self):
return self._opener_id

def get_open_data_params_schema(
self, data_id: str = None, opener_id: str = None
) -> JsonObjectSchema:
return self._s3_accessor.get_open_data_params_schema(
data_id=data_id, opener_id=opener_id
)

def open_data(self, data_id: str, **open_params) -> xr.Dataset:
def open_data(
self, data_id: str, **open_params
) -> Union[xr.Dataset, MultiLevelDataset]:
stac_schema = self.get_open_data_params_schema()
stac_schema.validate_instance(open_params)
return self._s3_accessor.open_data(
Expand Down
Loading

0 comments on commit 341f33f

Please sign in to comment.