From 785ff118f3999e6badf736fbe8e1896cf01e1d4d Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Mon, 6 Nov 2023 13:13:18 +1100 Subject: [PATCH 01/11] Add DEA Tools and minimum pyTMD version (#273) * Add DEA Tools and minimum pyTMD version * Add dea-tools to requirements.txt * pin datacube as no-init-users failed in 1.8.16 --------- Co-authored-by: Emma Ai --- docker/env.yaml | 2 +- docker/requirements.txt | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/env.yaml b/docker/env.yaml index 88a2b19..22a924f 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -201,7 +201,7 @@ dependencies: - spyndex - urbanaccess - contextily - - pyTMD + - pyTMD>=2.0.8 # jupyter things - autopep8 - black diff --git a/docker/requirements.txt b/docker/requirements.txt index 698318d..dc0db46 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -18,7 +18,8 @@ s2cloudmask opencv-python-headless opencv-contrib-python-headless -datacube[performance,s3] +jsonschema < 4.18 +datacube[performance,s3] == 1.8.15 odc-algo odc-cloud[ASYNC] odc-dscache @@ -27,6 +28,7 @@ odc-stac odc-stats[ows] odc-ui odc-geo +dea-tools thredds-crawler hdstats==0.1.8.post1 From ffe1343a27dc705dc14fd45bdb3ea28dc8b60df6 Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Wed, 8 Nov 2023 04:15:50 +0000 Subject: [PATCH 02/11] update odc core version and remove install dea-tools from local folder --- docker/assets/sync_repo | 4 ---- docker/env.yaml | 2 +- docker/requirements.txt | 3 +-- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/docker/assets/sync_repo b/docker/assets/sync_repo index 1722443..1089c2c 100755 --- a/docker/assets/sync_repo +++ b/docker/assets/sync_repo @@ -15,9 +15,5 @@ BRANCH="${2:-master}" rm -rf "${WORKDIR}"/{.github,.gitignore,LICENSE,README.md,README.rst,CITATION.cff,USAGE.rst,docker-compose.yml,DEAfrica_notebooks_template.ipynb,DEA_notebooks_template.ipynb,Scientific_workflows,Tests} || true cp -Rv ${WORKDIR}/. ~/ rm -rf "${WORKDIR}" - # Install Tools folder if available - if [ -e $HOME/Tools/setup.py ]; then - python -m pip install -e "$HOME/Tools" || true - fi rm -rf "$HOME/.git" } diff --git a/docker/env.yaml b/docker/env.yaml index 22a924f..b54b7c8 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -75,7 +75,7 @@ dependencies: - itsdangerous - Jinja2 - jmespath - - jsonschema<4.18 + - jsonschema>4.18 - kiwisolver - lark - python-lmdb diff --git a/docker/requirements.txt b/docker/requirements.txt index dc0db46..483731b 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -18,8 +18,7 @@ s2cloudmask opencv-python-headless opencv-contrib-python-headless -jsonschema < 4.18 -datacube[performance,s3] == 1.8.15 +datacube[performance,s3] >= 1.8.17 odc-algo odc-cloud[ASYNC] odc-dscache From 5244811b8fae78a892bc63881cc937c329521e15 Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Mon, 13 Nov 2023 00:12:48 +0000 Subject: [PATCH 03/11] sync the change in stats production docker --- docker/env.yaml | 2 +- docker/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/env.yaml b/docker/env.yaml index b54b7c8..d56d4d7 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -4,7 +4,7 @@ channels: dependencies: - python=3.10 - libgdal - - gdal + - gdal>=3.7.3 - proj - rasterio>=1.3.2 - gcc_linux-64 diff --git a/docker/requirements.txt b/docker/requirements.txt index 483731b..7613f49 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -19,7 +19,7 @@ opencv-python-headless opencv-contrib-python-headless datacube[performance,s3] >= 1.8.17 -odc-algo +odc-algo @ git+https://github.com/opendatacube/odc-algo@b8dcfce odc-cloud[ASYNC] odc-dscache odc-io From d4d25e8022f64096f6037f288129d1534227c6fc Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Mon, 13 Nov 2023 00:43:09 +0000 Subject: [PATCH 04/11] conflicts between gdal and pygeos is unsolvable hence remove pin and pin tf higher stable version --- docker/env.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/env.yaml b/docker/env.yaml index d56d4d7..cb17221 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -4,7 +4,7 @@ channels: dependencies: - python=3.10 - libgdal - - gdal>=3.7.3 + - gdal - proj - rasterio>=1.3.2 - gcc_linux-64 @@ -173,7 +173,7 @@ dependencies: - dask-ml - pathos - scikit-learn - - tensorflow=2.7 + - tensorflow>=2.10 - xgboost - zarr - bokeh From e42fe3b60a80ed0e1d262a12b33f3cedd29c198a Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Tue, 14 Nov 2023 04:05:10 +0000 Subject: [PATCH 05/11] pin bokeh due to dask dashboard issue --- docker/env.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/env.yaml b/docker/env.yaml index cb17221..0b0c010 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -176,7 +176,7 @@ dependencies: - tensorflow>=2.10 - xgboost - zarr - - bokeh + - bokeh=3.2.2 - descartes - matplotlib - seaborn From 72833d759579eb7115ff81013c462f784f8e14ae Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Mon, 29 Jan 2024 11:03:36 +1100 Subject: [PATCH 06/11] Update odc geo (#277) --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index 7613f49..d32aaaa 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -26,7 +26,7 @@ odc-io odc-stac odc-stats[ows] odc-ui -odc-geo +odc-geo >= 0.4.2 dea-tools thredds-crawler From ef03485f1fbfe1fcaf0076d94e166168df41ab06 Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Tue, 5 Mar 2024 10:37:25 +1030 Subject: [PATCH 07/11] make everything compatible with jupyterlab=3.6.x (#280) * make everything compatible with jupyterlab=3.6.x * increase size limit to 8G * pin pytest as 8.1.0 not working --------- Co-authored-by: Emma Ai --- .github/workflows/dive.yml | 2 +- docker/Dockerfile | 6 +++--- docker/env.yaml | 7 ++++--- docker/requirements.txt | 3 +++ 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dive.yml b/.github/workflows/dive.yml index 4f37cc4..6d7863a 100644 --- a/.github/workflows/dive.yml +++ b/.github/workflows/dive.yml @@ -91,4 +91,4 @@ jobs: uses: wemake-services/docker-image-size-limit@master with: image: ${{ env.ORG }}/${{ env.IMAGE}}:_build - size: "7 GiB" + size: "8 GiB" diff --git a/docker/Dockerfile b/docker/Dockerfile index fd52837..d1d613c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM mambaorg/micromamba:git-e4f55ab-jammy as sandbox-conda +FROM mambaorg/micromamba:git-df79b72-jammy as sandbox-conda USER root COPY env.yaml /conf/ @@ -16,7 +16,7 @@ ENV CC=/env/bin/x86_64-conda_cos6-linux-gnu-gcc \ RUN micromamba run -p /env pip install --no-cache-dir \ --no-build-isolation -r /conf/requirements.txt -FROM ubuntu:jammy-20230816 +FROM ubuntu:jammy-20240212 ARG nb_user=jovyan ARG nb_uid=1000 @@ -38,7 +38,7 @@ ENV LC_ALL=C.UTF-8 \ USER $nb_user # NOTE - JupyterLab extensions can now be distributed as prebuilt extensions and available to PyPI RUN echo "Enable jupyter lab extensions" \ - && jupyter labextension install jupyterlab-theme-toggle \ + && jupyter labextension install jupyterlab-topbar-extension jupyterlab-theme-toggle \ && jupyter labextension list \ && echo "...done" diff --git a/docker/env.yaml b/docker/env.yaml index 0b0c010..1011679 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -52,7 +52,7 @@ dependencies: - datadog - deepdiff - defusedxml - - distributed + - distributed>=2023.11 - docutils - ephem - fiona @@ -109,7 +109,7 @@ dependencies: - pyrsistent - pystac - pystac-client - - pytest + - pytest!=8.1.0 - python-dateutil - python-rapidjson - python-slugify @@ -221,7 +221,8 @@ dependencies: - jupyterlab-topbar - jupyterlab-system-monitor - jupyterhub - - jupytext + - jupytext<1.14 + - jupyterlab_pygments<0.3.0 - jupyter-resource-usage - jupyter_server - jupyter-server-proxy diff --git a/docker/requirements.txt b/docker/requirements.txt index d32aaaa..379674b 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -6,6 +6,9 @@ jupyterlab-logout jupyter-contrib-core jupyter-contrib-nbextensions jupyter-nbextensions-configurator +# pin for jupyterlab=3.x +jupyterlab-topbar-text==0.6.2 +jupyterlab-logout==0.5.0 # ODC/DEA: these are installed in builder stage otps From c2d04747d8e70d22d54d174da2dc460ecd7dd0ea Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Tue, 5 Mar 2024 13:06:53 +1030 Subject: [PATCH 08/11] fix jupytext (#281) Co-authored-by: Emma Ai --- docker/env.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/env.yaml b/docker/env.yaml index 1011679..047f899 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -221,7 +221,7 @@ dependencies: - jupyterlab-topbar - jupyterlab-system-monitor - jupyterhub - - jupytext<1.14 + - jupytext<1.16 - jupyterlab_pygments<0.3.0 - jupyter-resource-usage - jupyter_server From e9804143e450ddc63bd74d73a2c7dbe418e2f89d Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Wed, 3 Apr 2024 18:42:09 +1100 Subject: [PATCH 09/11] Update DEA Sandbox image (#282) * Update requirements.txt * Retrigger build * Remove restriction on odc-geo --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index 379674b..964995e 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -29,7 +29,7 @@ odc-io odc-stac odc-stats[ows] odc-ui -odc-geo >= 0.4.2 +odc-geo dea-tools thredds-crawler From ffce84d526c3f31d4e8649222b766c73087b7009 Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Mon, 15 Apr 2024 16:36:52 +1000 Subject: [PATCH 10/11] Add s5cmd to Sandbox image --- docker/env.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/env.yaml b/docker/env.yaml index 047f899..9ebfcfb 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -21,6 +21,7 @@ dependencies: - access - aiobotocore - boto3 + - s5cmd - affine - aiohttp - aioitertools From f40b1e87e7603247a68ed1963adda35cf8374bdb Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Wed, 17 Apr 2024 10:29:31 +1000 Subject: [PATCH 11/11] Update pyTMD --- docker/env.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/env.yaml b/docker/env.yaml index 9ebfcfb..18db878 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -202,7 +202,7 @@ dependencies: - spyndex - urbanaccess - contextily - - pyTMD>=2.0.8 + - pyTMD<=2.1.0 # jupyter things - autopep8 - black