Merge pull request #152 from alan-turing-institute/fix-coordinate-pro…

…jections

Refactor and fix coordinate projection errors

.dockerignore

@@ -14,3 +14,5 @@ python/tests/data/test-run-results*
 **/*.aux.xml
 .**/*
 !.conda-linux-64.lock
+!python/tests/data/local-cache
+.git

.github/workflows/ci.yaml

@@ -5,6 +5,7 @@ env:
   DOCKER_BUILDKIT: 1
   COMPOSE_DOCKER_CLI_BUILD: 1
   CONDA_ENV_NAME: clim-recal
+  # CONDA_ENV_PATH: conda-lock.yml
   CONDA_ENV_PATH: .conda-linux-64.lock
   MIN_PYTHON_VERSION: 3.12
   PYTHON_MODULE_FOLDER: python/
@@ -16,11 +17,11 @@ env:
 
 on:
   pull_request:
-    branches: ['main', 'raw-data-resample', ]
+    branches: ['main', 'cache-cpm-for-hads', ]
     paths-ignore: ['docs/**']
 
   push:
-    branches: ['main', 'raw-data-resample', ]
+    branches: ['main', 'cache-cpm-for-hads', ]
     paths-ignore: ['docs/**']
 
 concurrency:
@@ -62,15 +63,20 @@ jobs:
       - name: Build Conda Environment
         uses: conda-incubator/setup-miniconda@v3
         with:
-          auto-update-conda: false
           activate-environment: ${{ env.CONDA_ENV_NAME }}
           environment-file: ${{ env.CONDA_ENV_PATH }}
+          auto-activate-base: false
           miniforge-version: latest
           use-mamba: true
 
       - name: Run Python Tests
+        # The first line below may be temporary,
+        # see this solution:
+        # https://github.com/pyenv/pyenv/issues/2625
         run: |
+          mamba install libsqlite --force-reinstall
           mamba run -n ${{ env.CONDA_ENV_NAME }} --cwd python pytest -m "not darwin and not mount"
+        shell: bash -el {0}
 
       - name: Copy test coverage results
         run: |

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 exclude: "data"
 repos:
   - repo: https://github.com/psf/black
-    rev: "23.12.1"
+    rev: "24.4.2"
     hooks:
       - id: black-jupyter
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -27,7 +27,7 @@ repos:
       - id: rst-inline-touching-normal
 
   - repo: https://github.com/pdm-project/pdm
-    rev: 2.12.4 # a PDM release exposing the hook
+    rev: 2.17.1 # a PDM release exposing the hook
     hooks:
       - id: pdm-lock-check
         args: [--lockfile, python/pdm.lock]

_quarto.yml

@@ -43,6 +43,8 @@ website:
         href: "setup-instructions.md"
       - text: "Pipeline"
         href: "docs/pipeline_guidance.qmd"
+      - text: "Generated Datasets"
+        href: "docs/datasets.qmd"
       # - section: "Pipeline"
       #   contents:
       #     - text: "Guidance"
@@ -109,7 +111,7 @@ quartodoc:
   # see: https://github.com/alan-turing-institute/clim-recal/issues/128
   sections:
     - title: Data Source Management
-      desc: How data is downloaded for use
+      desc: How data is downloaded and processed
       contents:
         # the functions being documented in the package.
         # you can refer to anything: class methods, modules, etc..

compose.yml

@@ -6,6 +6,13 @@ services:
       dockerfile: ./compose/jupyter/Dockerfile
     ports:
       - "8888:8888"
+    # user: root
+    environment:
+      - NB_UID=1000
+      - NB_GID=100
+      - CHOWN_HOME=yes
+      - CHOWN_HOME_OPTS=-R
+      - JUPYTER_ENABLE_LAB=yes
     volumes:
       - .:/home/jovyan:rw
 

compose/docs/Dockerfile

@@ -8,7 +8,7 @@ ARG EXTERNAL_PORT=$LOCAL_PORT
 # an ARG
 ENV HOST_IP=0.0.0.0
 ENV LOCAL_PORT=${LOCAL_PORT}
-ARG py_ver=3.12
+ARG py_ver=3.15
 ENV DEBIAN_FRONTEND=noninteractive
 
 COPY . /app
@@ -24,7 +24,7 @@ USER root
 
 # Install the local python environment set via
 # `pre-commit` in .conda-linux-64
-RUN micromamba create -f .conda-linux-64.lock -n clim-recal
+RUN micromamba create -n clim-recal gdal=3.8.5 pdm rsync quartodoc jupyter -c conda-forge
 
 
 CMD micromamba run -n clim-recal quartodoc build && \

compose/jupyter/Dockerfile

@@ -9,36 +9,29 @@ ARG HOST_DATA_PATH=/Volumes/vmfileshare
 # The local_data_path is an absolute path to mount ClimateData within `docker`
 ARG DOCKER_DATA_PATH=/mnt/vmfileshare
 
-USER ${NB_UID}
-COPY --chown=$NB_USER:$NB_USER conda-lock.yml /tmp/conda-lock.yml
-
-RUN conda install -y conda-lock
-
-RUN \
-  conda-lock install --copy -n ${env_name} /tmp/conda-lock.yml && \
-  conda clean --all -y && \
-  fix-permissions "${CONDA_DIR}" && \
-  fix-permissions "/home/${NB_USER}"
-
-# Note the below originally was run just prior to `conda clean`
-# but may no longer be needed
-# conda run -n ${env_name} python -m ipykernel install --user --name ${env_name} && \
 
 # Needed for write permission
 USER root
 
+# Build-essential for gdal and potentially packages like scipy
+RUN sudo apt-get update && sudo apt-get install -y build-essential
+
 # First line changes a startup hook, which will activate the custom environment
-RUN \
-    echo conda activate "${env_name}" >> /usr/local/bin/before-notebook.d/10activate-conda-env.sh && \
-    echo conda activate "${env_name}" >> /etc/skel/.bashrc
-# Above line makes the custom environment default in Jupyter Terminals for all users which might be created later
+# RUN echo conda activate "${env_name}" >> /usr/local/bin/before-notebook.d/10activate-conda-env.sh
 
-RUN chown -R ${NB_USER} /home/${NB_USER}
+COPY . .
 
-USER ${NB_UID}
+RUN chown -R ${NB_USER}:${NB_GID} /home/${NB_USER}
 
-COPY . .
+USER ${NB_USER}
+
+RUN conda create -n ${env_name}
+# Todo: either return to an update to environment.yml or document this
+RUN conda install -n ${env_name} -y gdal=3.8.5 pdm rsync
 
 # This makes the custom environment default in Jupyter Terminals for already existing NB_USER
 # Note this *doesn't* apply if simply using sh -c
 RUN echo conda activate "${env_name}" >> "/home/${NB_USER}/.bashrc"
+
+# Install dependencies
+RUN cd python && conda run -n ${env_name} pdm install

compose/linux-compose.yml → compose/ua-linux-compose.yml

@@ -1,6 +1,10 @@
 services:
 
   jupyter:
+    user: root
+    environment:
+      - NB_UID=1001
+      - NB_GID=1001
     volumes:
       - /mnt/vmfileshare:/mnt/vmfileshare