Merge pull request #627 from opendatacube/integrate_1.9

Ready for 1.9 release

.gitignore

@@ -84,3 +84,6 @@ docs/notebooks/
 .run/
 
 docker/wheels/
+
+.jj
+.ropeproject

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       - id: yamllint
         args: ['-c', '.yamllint']
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: end-of-file-fixer
       - id: check-docstring-first
@@ -26,14 +26,14 @@ repos:
         #        name: isort (python)
         #        args: [ "--profile", "black", "--filter-files" ]
   - repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 24.10.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/pylint
-    rev: 'v3.2.3'  # Use the sha / tag you want to point at
+    rev: 'v3.3.2'  # Use the sha / tag you want to point at
     hooks:
       - id: pylint
   - repo: https://github.com/PyCQA/flake8
-    rev: '7.0.0'
+    rev: '7.1.1'
     hooks:
       - id: flake8

apps/dc_tools/odc/apps/dc_tools/_stac.py

@@ -9,8 +9,9 @@
 
 import numpy
 from datacube.model import Dataset
-from datacube.utils.geometry import Geometry, box
+from odc.geo.geom import Geometry, box
 from eodatasets3.stac import to_stac_item
+from eodatasets3.serialise import from_doc
 from toolz import get_in
 from urllib.parse import urlparse
 
@@ -452,7 +453,6 @@ def transform_geom_json_coordinates_to_list(geom_json):
 
 def ds_to_stac(ds: Dataset) -> dict:
     """Get STAC document from dataset with eo3 metadata"""
-    from eodatasets3.serialise import from_doc
     if ds.is_eo3:
         if not ds.uris:
             raise ValueError("Can't find dataset location")

apps/dc_tools/odc/apps/dc_tools/_version.py

@@ -1 +1 @@
-__version__ = "0.2.18"
+__version__ = "1.9.0"

apps/dc_tools/odc/apps/dc_tools/add_update_products.py

@@ -14,6 +14,8 @@
 
 import datacube
 from datacube import Datacube
+from datacube.cfg import ODCEnvironment
+from datacube.ui.click import environment_option, pass_config
 from odc.apps.dc_tools.utils import (
     update_if_exists_flag,
     statsd_gauge_reporting,
@@ -123,12 +125,16 @@ def add_update_products(
 
 
 @click.command("dc-sync-products")
+@environment_option
+@pass_config
 @click.argument("csv-path", nargs=1)
 @update_if_exists_flag
 @statsd_setting
-def cli(csv_path: str, update_if_exists: bool, statsd_setting: str):
+def cli(
+    cfg_env: ODCEnvironment, csv_path: str, update_if_exists: bool, statsd_setting: str
+):
     # Check we can connect to the Datacube
-    dc = datacube.Datacube(app="add_update_products")
+    dc = datacube.Datacube(app="add_update_products", env=cfg_env)
     logging.info(
         "Starting up: connected to Datacube, and update-if-exists is: %s",
         update_if_exists,

apps/dc_tools/odc/apps/dc_tools/azure_to_dc.py

@@ -10,7 +10,9 @@
 
 import click
 from datacube import Datacube
+from datacube.cfg import ODCEnvironment
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from odc.apps.dc_tools._stac import stac_transform
 from odc.apps.dc_tools.utils import (
     SkippedException,
@@ -128,6 +130,8 @@ def dump_list_to_odc(
 
 
 @click.command("azure-to-dc")
+@environment_option
+@pass_config
 @update_flag
 @update_if_exists_flag
 @allow_unsafe
@@ -151,6 +155,7 @@ def dump_list_to_odc(
 @click.argument("suffix", type=str, nargs=1)
 @rename_product
 def cli(
+    cfg_env: ODCEnvironment,
     update: bool,
     update_if_exists: bool,
     allow_unsafe: bool,
@@ -166,7 +171,7 @@ def cli(
     rename_product: str,
 ):
     # Set up the datacube first, to ensure we have a connection
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
     print(f"Opening AZ Container {container_name} on {account_url}")
     print(f"Searching on prefix '{prefix}' for files matching suffix '{suffix}'")
     yaml_urls = find_blobs(

apps/dc_tools/odc/apps/dc_tools/cop_dem_to_dc.py

@@ -14,6 +14,7 @@
 
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from datacube.utils import read_documents
 from odc.apps.dc_tools.utils import (
     SkippedException,
@@ -202,6 +203,8 @@ def cop_dem_to_dc(
 
 
 @click.command("cop-dem-to-dc")
+@environment_option
+@pass_config
 @limit
 @update_if_exists_flag
 @bbox
@@ -226,6 +229,7 @@ def cop_dem_to_dc(
     help="Number of threads to use to process, default 20",
 )
 def cli(
+    cfg_env,
     limit,
     update_if_exists,
     bbox,
@@ -244,7 +248,7 @@ def cli(
             f"Unknown product {product}, must be one of {' '.join(PRODUCTS)}"
         )
 
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
 
     if add_product:
         add_cop_dem_product(dc, product)

apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py

@@ -14,6 +14,7 @@
 
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from datacube.utils import read_documents
 from odc.apps.dc_tools.utils import (
     bbox,
@@ -112,6 +113,7 @@ def get_tile_uris(bounding_box: str) -> Tuple[str, str]:
             )
 
 
+# pylint: disable=too-many-positional-arguments
 def process_uri_tile(
     uri_tile: Tuple[str, str, str],
     dc: Datacube,
@@ -160,6 +162,7 @@ def select_map_version(version: str):
         map_version["algo"] = "v200"
 
 
+# pylint: disable=too-many-positional-arguments
 def esa_wc_to_dc(
     dc: Datacube,
     bounding_box,
@@ -204,6 +207,7 @@ def esa_wc_to_dc(
                     sys.stdout.write(f"\rAdded {success} datasets...")
             except rasterio.errors.RasterioIOError:
                 logging.info("Couldn't read file %s", uri, exc_info=True)
+                failure += 1
             except Exception:  # pylint:disable=broad-except
                 logging.exception("Failed to handle uri %s", uri)
                 failure += 1
@@ -213,6 +217,8 @@ def esa_wc_to_dc(
 
 
 @click.command("esa-wc-to-dc")
+@environment_option
+@pass_config
 @limit
 @update_if_exists_flag
 @bbox
@@ -237,7 +243,8 @@ def esa_wc_to_dc(
     type=str,
     help="Select version of world cover map, default 2020",
 )
-def cli(
+def cli(  # pylint: disable=too-many-positional-arguments
+    cfg_env,
     limit,
     update_if_exists,
     bbox,
@@ -255,7 +262,7 @@ def cli(
     # Select map version
     select_map_version(version)
 
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
 
     if add_product:
         add_odc_product(dc)

apps/dc_tools/odc/apps/dc_tools/fs_to_dc.py

@@ -6,6 +6,7 @@
 
 import datacube
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from odc.apps.dc_tools._stac import stac_transform
 from odc.apps.dc_tools.utils import (
     allow_unsafe,
@@ -26,6 +27,8 @@
 
 
 @click.command("fs-to-dc")
+@environment_option
+@pass_config
 @click.argument("input_directory", type=str, nargs=1)
 @update_if_exists_flag
 @allow_unsafe
@@ -39,6 +42,7 @@
     help="File system glob to use, defaults to **/*.yaml or **/*.json for STAC.",
 )
 def cli(
+    cfg_env,
     input_directory,
     update_if_exists,
     allow_unsafe,
@@ -48,7 +52,7 @@ def cli(
     archive_less_mature,
     publish_action,
 ):
-    dc = datacube.Datacube()
+    dc = datacube.Datacube(env=cfg_env)
     doc2ds = Doc2Dataset(dc.index)
 
     if glob is None:

apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py

@@ -10,6 +10,7 @@
 
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from odc.apps.dc_tools._docs import parse_doc_stream
 from odc.apps.dc_tools._stac import stac_transform
 from odc.apps.dc_tools.utils import (
@@ -62,6 +63,9 @@ def dump_to_odc(
 
     found_docs = False
     for uri, metadata in uris_docs:
+        if metadata is None:
+            ds_skipped += 1
+            continue
         found_docs = True
         stac_doc = None
         if transform:
@@ -93,6 +97,8 @@ def dump_to_odc(
 
 
 @click.command("s3-to-dc")
+@environment_option
+@pass_config
 @click.option(
     "--log",
     type=click.Choice(
@@ -118,6 +124,7 @@ def dump_to_odc(
 @click.argument("uris", nargs=-1)
 @click.argument("product", type=str, nargs=1, required=False)
 def cli(
+    cfg_env,
     log,
     skip_lineage,
     fail_on_missing_lineage,
@@ -156,7 +163,7 @@ def cli(
     if request_payer:
         opts["RequestPayer"] = "requester"
 
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
 
     # if it's a uri, a product wasn't provided, and 'product' is actually another uri
     if product.startswith("s3://"):

apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py

@@ -19,6 +19,7 @@
 
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
+from datacube.ui.click import environment_option, pass_config
 from datacube.utils import documents
 from odc.apps.dc_tools.utils import (
     IndexingException,
@@ -306,6 +307,8 @@ def queue_to_odc(
 
 
 @click.command("sqs-to-dc")
+@environment_option
+@pass_config
 @skip_lineage
 @fail_on_missing_lineage
 @verify_lineage
@@ -342,6 +345,7 @@ def queue_to_odc(
 @click.argument("queue_name", type=str, nargs=1)
 @click.argument("product", type=str, nargs=1)
 def cli(
+    cfg_env,
     skip_lineage,
     fail_on_missing_lineage,
     verify_lineage,
@@ -369,7 +373,7 @@ def cli(
     queue = sqs.get_queue_by_name(QueueName=queue_name)
 
     # Do the thing
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
     success, failed, skipped = queue_to_odc(
         queue,
         dc,

apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py

@@ -10,7 +10,7 @@
 import click
 from datacube import Datacube
 from datacube.model import Dataset
-
+from datacube.ui.click import environment_option, pass_config
 from odc.stac.eo3 import stac2ds
 
 from odc.apps.dc_tools.utils import (
@@ -90,7 +90,9 @@ def item_to_meta_uri(
             "Couldn't find matching product for product name: %s",
             product_name_sanitised,
         )
-        raise SkippedException(f"Couldn't find matching product for product name: {product_name_sanitised}")
+        raise SkippedException(
+            f"Couldn't find matching product for product name: {product_name_sanitised}"
+        )
 
     # Convert the STAC Item to a Dataset
     dataset = next(stac2ds([item]))
@@ -183,6 +185,8 @@ def stac_api_to_odc(
 
 
 @click.command("stac-to-dc")
+@environment_option
+@pass_config
 @limit
 @update_if_exists_flag
 @allow_unsafe
@@ -216,6 +220,7 @@ def stac_api_to_odc(
 @publish_action
 @statsd_setting
 def cli(
+    cfg_env,
     limit,
     update_if_exists,
     allow_unsafe,
@@ -249,7 +254,7 @@ def cli(
     config["max_items"] = limit
 
     # Do the thing
-    dc = Datacube()
+    dc = Datacube(env=cfg_env)
     added, failed, skipped = stac_api_to_odc(
         dc,
         update_if_exists,