Add minimum package versions for ESRI/GDAL nodata fix (#288)

* Add minimum package versions for ESRI/GDAL nodata fix

* Use latest versions of odc-geo and eodatasets

* Download only subset of tide models

* install awscliv2 from conda

* fix indentation of workflow yaml

* output python package version

* Remove pyTMD restriction

---------

Co-authored-by: Emma Ai <[email protected]>

.github/workflows/integration-test.yml

@@ -46,7 +46,9 @@ jobs:
           aws-region: ap-southeast-2
 
       - name: Copy tide modelling files with the AWS CLI
-        run: aws s3 sync s3://dea-non-public-data/tide_models/tide_models tide_models
+        run: |
+          aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014
+          aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide
 
       - name: Start docker-compose
         run: |

docker/Dockerfile

@@ -48,18 +48,11 @@ RUN echo "Enable server extensions" \
    && jupyter server extension list \
    && echo "...done"
 
-RUN echo "Install AWS CLI v2" \
-   && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" \
-   && unzip /tmp/awscliv2.zip -d /tmp \
-   && /tmp/aws/install -i /env/bin/aws-cli -b /env/bin \
-   && aws --version \
-   && rm -f /tmp/awscliv2.zip \
-   && rm -fr /tmp/aws \
-   && echo "...done"
-
 COPY assets/sync_repo assets/jupyterhub-singleuser /usr/local/bin/
 COPY assets/overrides.json /env/share/jupyter/lab/settings/
 
+RUN pip list --format=freeze 
+
 WORKDIR "/home/$nb_user"
 ENTRYPOINT ["/env/bin/tini", "--"]
 CMD ["jupyter", "lab", \

docker/env.yaml

@@ -21,6 +21,7 @@ dependencies:
   - rio-cogeo
   - access
   - aiobotocore
+  - awscliv2
   - boto3
   - s5cmd
   - affine
@@ -204,7 +205,7 @@ dependencies:
   - spyndex
   - urbanaccess
   - contextily
-  - pyTMD<=2.1.0
+  - pyTMD
   - xarray-spatial
 # jupyter things
   - autopep8

docker/requirements.txt

@@ -12,7 +12,7 @@ jupyterlab-logout==0.5.0
 
 # ODC/DEA: these are installed in builder stage
 otps
-eodatasets3
+eodatasets3 >= 0.30.6  # min version required for ESRI/GDAL nodata fix
 
 # Dale's s2cloudmask
 #  https://github.com/daleroberts/s2cloudmask
@@ -21,15 +21,15 @@ s2cloudmask
 opencv-python-headless
 opencv-contrib-python-headless
 
-datacube[performance,s3] >= 1.8.19
+datacube[performance,s3] >= 1.8.19  # min version required for ESRI/GDAL nodata fix
 odc-algo @ git+https://github.com/opendatacube/odc-algo@b8dcfce
 odc-cloud[ASYNC]
 odc-dscache
 odc-io
 odc-stac
 odc-stats[ows]
 odc-ui
-odc-geo
+odc-geo >= 0.4.8  # min version required for ESRI/GDAL nodata fix
 dea-tools
 
 thredds-crawler