Update and correct docs

.github/workflows/ci.yaml

@@ -78,9 +78,9 @@ jobs:
           mamba run -n ${{ env.CONDA_ENV_NAME }} --cwd python pytest -m "not darwin and not mount"
         shell: bash -el {0}
 
-      - name: Copy test coverage results
-        run: |
-          cp ${{ env.PYTHON_MODULE_FOLDER }}${{ env.COVERAGE_SVG_PATH }} ${{ env.COVERAGE_SVG_FOLDER }}
+      # - name: Copy test coverage results
+      #   run: |
+      #     cp ${{ env.PYTHON_MODULE_FOLDER }}${{ env.COVERAGE_SVG_PATH }} ${{ env.COVERAGE_SVG_FOLDER }}
 
       - name: Archive coverage svg
         uses: actions/upload-artifact@v3
@@ -101,6 +101,9 @@ jobs:
           name: coverage-badge
           path: ${{ env.GH_PAGE_PATH }}${{ env.COVERAGE_SVG_PATH }}
 
+      - name: Set up Quarto
+        uses: quarto-dev/quarto-actions/setup@v2
+
       - name: Build Conda Environment
         uses: conda-incubator/setup-miniconda@v3
         with:
@@ -114,15 +117,18 @@ jobs:
         run: |
           mamba run -n ${{ env.CONDA_ENV_NAME }} quartodoc build
 
-      - name: Set up Quarto
-        uses: quarto-dev/quarto-actions/setup@v2
-
       - name: Install Python and Dependencies
         uses: actions/setup-python@v4
         with:
           python-version: '3.12'
           cache: 'pip'
-      - run: pip install jupyter
+
+      - name: Install gdal and local libraries
+        run: |
+          sudo apt-add-repository ppa:ubuntugis/ubuntugis-unstable
+          sudo apt-get update
+          sudo apt-get install gdal-bin libgdal-dev
+          pip install ./python
 
       - name: Render and Publish
         uses: quarto-dev/quarto-actions/publish@v2

.gitignore

@@ -286,6 +286,7 @@ _extensions
 _reference/
 _site/
 _sidebar.yml
+*.quarto_ipynb
 # See https://github.com/machow/quartodoc/issues/83
 objects.json
 

README.md

@@ -1,4 +1,8 @@
 # Welcome to the `clim-recal` repository!
+![mit-license](https://img.shields.io/github/license/alan-turing-institute/clim-recal)
+![coverage](https://alan-turing-institute.github.io/clim-recal/assets/coverage.svg)
+![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)
+![CI](https://github.com/alan-turing-institute/clim-recal/actions/workflows/ci.yaml/badge.svg)
 
 Welcome to `clim-recal`, a specialized resource designed to tackle systematic errors or biases in **Regional Climate Models (RCMs)**. As researchers, policy-makers, and various stakeholders explore publicly available RCMs, they need to consider the challenge of biases that can affect the accurate representation of climate change signals.
 
@@ -11,21 +15,7 @@ Welcome to `clim-recal`, a specialized resource designed to tackle systematic er
 - Was developed in partnership with the MetOffice to ensure the propriety, quality, and usability of our work
 - Provides a framework for open additions of new software libraries/bias correction methods (in planning)
 
-## Data results and documentation updates
-
-*WARNING*: The documentation below is out of date in some cases, and we are in the process of updating this. In the meantime, see our [Exported Datasets](/docs/datasets.qmd) page for updates on datasets released for three UK cities.
-
-## Table of Contents
-
-1. [Overview: Bias Correction Pipeline](#overview-bias-correction-pipeline)
-1. [Documentation](#documentation)
-1. [The Datasets](#the-datasets)
-1. [Why Bias Correction?](#why-bias-correction)
-1. [Contributing](#contributing)
-1. [Future Plans](#future-plans)
-1. [License](/LICENSE)
-
-## Overview: Bias Correction Pipeline
+# Overview: Bias Correction Pipeline
 
 `clim-recal` is a debiasing pipeline,  with the following steps:
 
@@ -38,111 +28,61 @@ Welcome to `clim-recal`, a specialized resource designed to tackle systematic er
 4. **Assess the debiased data**
     *We have developed a way to assess the quality of the debiasing step across multiple alternative methods*
 
-For a quick start on bias correction, refer to our [comprehensive analysis pipeline guide](https://github.com/alan-turing-institute/clim-recal/blob/documentation/docs/pipeline_guidance.md).
+For a quick start on bias correction, refer to our [pipeline guide](python/README.md).
 
-## Documentation
+# Documentation
 
 We are in the process of developing comprehensive documentation for our code base to supplement the guidance provided in this and other `README.md` files. In the interim, there is documentation available in the following forms:
 
-- Comments within `R` scripts
-- Command line `--help` documentation for some of our `python` scripts
-- `python` `function` and `class` [`docstrings`](https://docs.python.org/3/library/doctest.html)
-- Local render of documentation via [`quarto`](https://quarto.org/)
-
-### `R`
-
-For `R` scripts, please refer to contextual information and usage guidelines, and feel free to reach out with any specific queries.
-
-### `python`
-
-For many of our `python` command line scripts, you can use the `--help` flag to access a summary of the available options and usage information:
-
-```sh
-$ python resampling_hads.py --help
-
-usage: resampling_hads.py [-h] --input INPUT [--output OUTPUT] [--grid_data GRID_DATA]
+## User documentation
 
-options:
--h, --help            show this help message and exit
---input INPUT         Path where the .nc files to resample is located
---output OUTPUT       Path to save the resampled data data
---grid_data GRID_DATA Path where the .nc file with the grid to resample is located
-```
+- See [setup instructions](setup-instructions.md)
+- See `python` [`README`](python/README.md) for an overview of the pipeline
+- Once installed, using the `clim-recal --help` option for details
+- See the [reproducibility page](docs/reproducibility.qmd) for information on how we used `clim-recal`
 
-This will display all available options for the script, including their purposes.
+## To use `clim-recal` programmatically
 
-### Quarto
-
-We also hope to provide comprehensive documentation via [`quarto`](https://quarto.org/). This is a work in progress, but if you would like to render documentation locally you can do so via `quarto` and [`conda`](https://docs.conda.io):
-
-1. Ensure you have a [local installation](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) of `conda` or [`anaconda`](https://www.anaconda.com/download) .
-1. Checkout a copy of our `git` repository
-1. Create a local `conda` `environment` via our `environment.yml` file. This should install `quarto`.
-1. Activate that environment
-1. Run `quarto preview`.
-
-Below are example `bash` shell commands to render locally after installing `conda`:
+- There are extensive [`API Reference`](docs/reference) within the python code.
+- Comments within `R` scripts
 
-```sh
-$ git clone https://github.com/alan-turing-institute/clim-recal
-$ cd clim-recal
-$ conda create -n clim-recal -f environment.yml
-$ conda activate clim-recal
-$ quarto preview
-```
+## To contribute to `clim-recal`
 
-We appreciate your patience and encourage you to check back for updates on our ongoing documentation efforts.
+- See the [Contributing](docs/contributing.md) section below
 
-## The Datasets
+# The Datasets
 
-### UKCP18
-The UK Climate Projections 2018 (UKCP18) dataset offers insights into the potential climate changes in the UK. UKCP18 is an advancement of the UKCP09 projections and delivers the latest evaluations of the UK's possible climate alterations in land and marine regions throughout the 21st century. This crucial information aids in future Climate Change Risk Assessments and supports the UK’s adaptation to climate change challenges and opportunities as per the National Adaptation Programme.
+## UKCP18
+The [UK Climate Projections 2018 (UKCP18)](https://www.metoffice.gov.uk/research/approach/collaboration/ukcp) dataset offers insights into the potential climate changes in the UK. UKCP18 is an advancement of the UKCP09 projections and delivers the latest evaluations of the UK's possible climate alterations in land and marine regions throughout the 21st century. This crucial information aids in future Climate Change Risk Assessments and supports the UK’s adaptation to climate change challenges and opportunities as per the National Adaptation Programme.
 
-### HADS
+## HADS
 [HadUK-Grid](https://www.metoffice.gov.uk/research/climate/maps-and-data/data/haduk-grid/haduk-grid) is a comprehensive collection of climate data for the UK, compiled from various land surface observations across the country. This data is organized into a uniform grid to ensure consistent coverage throughout the UK at up to 1km x 1km resolution. The dataset, spanning from 1836 to the present, includes a variety of climate variables such as air temperature, precipitation, sunshine, and wind speed, available on daily, monthly, seasonal, and annual timescales.
 
-### Geographical Dataset
-The geographical dataset can be used for visualising climate data. It mainly includes administrative boundaries published by the Office for National Statistics (ONS). The dataset is sharable under the [Open Government Licence v.3.0](https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) and is available for download via this [link](https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/NUTS_Level_1_January_2018_FCB_in_the_United_Kingdom_2022/FeatureServer/replicafilescache/NUTS_Level_1_January_2018_FCB_in_the_United_Kingdom_2022_7279368953270783580.zip). We include a copy in the `data/Geofiles` folder for convenience. In addition, the clips for three cities' boundaries from the same dataset are copied to `three.cities` subfolder.
-
-## Why Bias Correction?
+# Why Bias Correction?
 
-Regional climate models contain systematic errors, or biases in their output [1]. Biases arise in RCMs for a number of reasons, such as the assumptions in the general circulation models (GCMs), and in the downscaling process from GCM to RCM [1,2].
+Regional climate models contain systematic errors, or biases in their output [^1]. Biases arise in RCMs for a number of reasons, such as the assumptions in the general circulation models (GCMs), and in the downscaling process from GCM to RCM.
 
-Researchers, policy-makers and other stakeholders wishing to use publicly available RCMs need to consider a range of "bias correction” methods (sometimes referred to as "bias adjustment" or "recalibration"). Bias correction methods offer a means of adjusting the outputs of RCM in a manner that might better reflect future climate change signals whilst preserving the natural and internal variability of climate [2].
+Researchers, policy-makers and other stakeholders wishing to use publicly available RCMs need to consider a range of "bias correction” methods (sometimes referred to as "bias adjustment" or "recalibration"). Bias correction methods offer a means of adjusting the outputs of RCM in a manner that might better reflect future climate change signals whilst preserving the natural and internal variability of climate [^2].
 
 Part of the `clim-recal` project is to review several bias correction methods. This work is ongoing and you can find our initial [taxonomy here](https://docs.google.com/spreadsheets/d/18LIc8omSMTzOWM60aFNv1EZUl1qQN_DG8HFy1_0NdWk/edit?usp=sharing). When we've completed our literature review, it will be submitted for publication in an open peer-reviewed journal.
 
 Our work is however, just like climate data, intended to be dynamic, and we are in the process of setting up a pipeline for researchers creating new methods of bias correction to be able to submit their methods for inclusion on in the `clim-recal` repository.
 
- 1. Senatore et al., 2022, https://doi.org/10.1016/j.ejrh.2022.101120
- 2. Ayar et al., 2021, https://doi.org/10.1038/s41598-021-82715-1
-
-
-## Contributing
-
-We hope to bring together the extensive work already undertaken by the climate science community and showcase a range of libraries and techniques. If you have suggestions on the repository, or would like to include a new method (see below) or library, please raise an issue or [get in touch](mailto:[email protected])!
-
-### Adding to the conda environment file
-
-To use `R` in `anaconda` you may need to specify the `conda-forge` channel:
-
-```sh
-$ conda config --env --add channels conda-forge
-```
+[^1]: Senatore et al., 2022, <https://doi.org/10.1016/j.ejrh.2022.101120>
+[^2]: Ayar et al., 2021, <https://doi.org/10.1038/s41598-021-82715-1>
 
-Some libraries may be only available through `pip`, for example, these may
-require the generation / update of a `requirements.txt`:
 
-```sh
-$ pip freeze > requirements.txt
-```
+# Contributing
 
-and installing with:
+If you have suggestions on the repository, or would like to include a new method (see below) or library, please
+- raise an [issue](https://github.com/alan-turing-institute/clim-recal/issues)
+- [get in touch](mailto:[email protected])
+- see our [contributing](docs/contributing.md) section, which includes details on contriubting to the documentation.
 
-```sh
-$ pip install -r requirements.txt
-```
+All are welcome and appreciated.
 
-## Future plans
+# Future plans
+- **Finish refactor for BC**: The infrastructure for testing bias correction methods needs some reworking and documentation.
+- **Release BC results**: Provide results from example BC runs.
 - **More BC Methods**: Further bias correction of UKCP18 products. *This is planned for a future release and is not available yet.*
 - **Pipeline for adding new methods**: *This is planned for a future release and is not available yet.*

_quarto.yml

@@ -12,19 +12,25 @@ project:
     - "R/README.md"
     - "R/misc/Identifying_Runs.md"
     - "R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.md"
+    - "docs/cpm_projection.qmd"
     - "docs/reference"
-    # - "docs/pipeline.qmd"
     - "docs/contributing.md"
+    - "docs/download.qmd"
     - "docs/datasets.qmd"
-    - "docs/pipeline_guidance.qmd"
     - "docs/docker-configurations.qmd"
+    - "docs/deprecated_pipeline.qmd"
     - "python/README.md"
+    - "notebooks/cpm_projection_diff_plots.ipynb"
+    - "notebooks/cpm_projection_diff_plots_linear_nearest.ipynb"
       # - "notebooks/Assessing_bc_data/MethodsAssessment_DecWorkshop.Rmd"
       # Requires dataset mounted to run notebook
 
 toc: True
 number-sections: True
 
+theme:
+  light: flatly
+  dark: darkly
 
 # tell quarto to read the generated sidebar
 metadata-files:
@@ -41,17 +47,12 @@ website:
         href: "README.md"
       - text: "Install"
         href: "setup-instructions.md"
-      - text: "Pipeline"
-        href: "docs/pipeline_guidance.qmd"
-      - text: "Generated Datasets"
-        href: "docs/datasets.qmd"
-      # - section: "Pipeline"
-      #   contents:
-      #     - text: "Guidance"
-      #       href: "docs/pipeline_guidance.qmd"
-      #     - text: "Diagram"
-      #       href: "docs/pipeline.qmd"
-      - section: "R"
+      - text: "Pipeline Description"
+        href: "python/README.md"
+      - text: "Download Datasets"
+        href: "docs/download.qmd"
+      # TODO: change name to deprecated?
+      - section: "Analysis in R"
         href: "R/README.md"
         contents:
           - href: "R/misc/Identifying_Runs.md"
@@ -61,41 +62,51 @@ website:
               - href: "R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.md"
                 text: "WIP Comparing HADs grids"
 
-      - section: "python"
-        href: "python/README.md"
+      - section: "API Reference"
+        href: "docs/reference/index.qmd"
         contents:
-          - section: "Reference"
-            href: "docs/reference/index.qmd"
+          - href: "docs/reference/clim_recal.pipeline.qmd"
+            text: "Pipeline"
+          - href: "docs/reference/clim_recal.config.qmd"
+            text: "Configure"
+          - href: "docs/reference/clim_recal.ceda_ftp_download.qmd"
+            text: "CEDA Data Access"
+          - href: "docs/reference/clim_recal.data_loader.qmd"
+            text: "Data Loading"
+          - href: "docs/reference/clim_recal.resample.qmd"
+            text: "Data Resampling"
+          - section: "Debiasing"
             contents:
-              - href: "docs/reference/clim_recal.pipeline.qmd"
-                text: "Pipeline"
-              - href: "docs/reference/clim_recal.config.qmd"
-                text: "Configure"
-              - href: "docs/reference/clim_recal.ceda_ftp_download.qmd"
-                text: "CEDA Data Access"
-              - href: "docs/reference/clim_recal.data_loader.qmd"
-                text: "Data Loading"
-              - href: "docs/reference/clim_recal.resample.qmd"
-                text: "Data Resampling"
-              - section: "Debiasing"
-                contents:
-                  - href: "docs/reference/clim_recal.debiasing.debias_wrapper.qmd"
-                    text: "Wrapper"
-              - section: "Utilities"
-                contents:
-                  - href: "docs/reference/clim_recal.utils.core.qmd"
-                    text: "core"
-                  - href: "docs/reference/clim_recal.utils.server.qmd"
-                    text: "server"
-                  - href: "docs/reference/clim_recal.utils.xarray.qmd"
-                    text: "xarray"
-                  - href: "docs/reference/clim_recal.utils.data.qmd"
-                    text: "data"
-      - text: "Docker"
-        href: "docs/docker-configurations.qmd"
+              - href: "docs/reference/clim_recal.debiasing.debias_wrapper.qmd"
+                text: "Wrapper"
+          - section: "Utilities"
+            contents:
+              - href: "docs/reference/clim_recal.utils.core.qmd"
+                text: "core"
+              - href: "docs/reference/clim_recal.utils.server.qmd"
+                text: "server"
+              - href: "docs/reference/clim_recal.utils.xarray.qmd"
+                text: "xarray"
+              - href: "docs/reference/clim_recal.utils.data.qmd"
+                text: "data"
       - text: "Contributing"
         href: "docs/contributing.md"
 
+      - section: "Appendix"
+        contents:
+          - section: "CPM Projection Analysis"
+            href: "docs/cpm_projection.qmd"
+            contents:
+              - text: "Temporal Interpolation Artefacts"
+                href: "notebooks/cpm_projection_diff_plots.ipynb"
+              - text: "Nearest vs Linear Temporal Interpolation"
+                href: "notebooks/cpm_projection_diff_plots_linear_nearest.ipynb"
+          - text: "Docker"
+            href: "docs/docker-configurations.qmd"
+          - text: "Deprecated"
+            href: "docs/deprecated_pipeline.qmd"
+
+
 
 quartodoc:
   # the name used to import the package you want to create reference docs for

compose/docs/Dockerfile

@@ -24,7 +24,7 @@ USER root
 
 # Install the local python environment set via
 # `pre-commit` in .conda-linux-64
-RUN micromamba create -n clim-recal gdal=3.8.5 pdm rsync quartodoc jupyter -c conda-forge
+RUN micromamba create -n clim-recal gdal=3.8.5 pdm rsync quartodoc jupyter jupyter-cache -c conda-forge
 
 
 CMD micromamba run -n clim-recal quartodoc build && \