Skip to content

Commit

Permalink
sqme: dask mode from env
Browse files Browse the repository at this point in the history
  • Loading branch information
Kirill888 committed Jun 7, 2024
1 parent f69891a commit 6e7569f
Showing 1 changed file with 13 additions and 2 deletions.
15 changes: 13 additions & 2 deletions odc/loader/_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

import dataclasses
import os
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from typing import (
Expand Down Expand Up @@ -51,6 +52,8 @@
T,
)

DaskBuilderMode = Literal["mem", "concurrency"]


class MkArray(Protocol):
"""Internal interface."""
Expand Down Expand Up @@ -143,6 +146,13 @@ def resolve_sources_dask(
]


def _default_dask_mode() -> DaskBuilderMode:
mode = os.environ.get("ODC_STAC_DASK_MODE", "mem")
if mode == "concurrency":
return "concurrency"
return "mem"


class DaskGraphBuilder:
"""
Build xarray from parsed metadata.
Expand All @@ -160,7 +170,7 @@ def __init__(
env: Dict[str, Any],
rdr: ReaderDriver,
chunks: Mapping[str, int],
mode: Literal["auto"] | Literal["mem"] | Literal["concurrency"] = "auto",
mode: DaskBuilderMode | Literal["auto"] = "auto",
) -> None:
gbox = gbt.base
assert isinstance(gbox, GeoBox)
Expand All @@ -169,7 +179,8 @@ def __init__(
chunks = {**chunks}
chunks.update(dict(zip(["time", "y", "x"], chunk_tyx)))
if mode == "auto":
mode = "mem" # current default
# "mem" unless overwriten by env var
mode = _default_dask_mode()

self.cfg = cfg
self.template = template
Expand Down

0 comments on commit 6e7569f

Please sign in to comment.