Wrapper to build multi-version API docs via pdoc

Including the necessary pdoc-templates and settings files for the
wrapper script to build docs for all of our current packages.

pdoc-templates/index.html.jinja2

@@ -0,0 +1,32 @@
+{% extends "default/index.html.jinja2" %}
+
+{% block title %}IMCF Python API Documentation{% endblock %}
+
+
+{% block nav %}
+<a class="pdoc-button module-list-button" href="../">
+    {% include "resources/box-arrow-in-left.svg" %}
+    &nbsp;
+    Back
+</a>
+<style>
+    nav.pdoc {
+        background-image: url("https://imcf.one/hugo/images/bg.jpg");
+        background-size: cover;
+    }
+</style>
+{% endblock %}
+
+
+{% block content %}
+<main class="pdoc">
+
+    <h1>API docs for IMCF Python packages.</h1>
+
+    <ul>
+        {% for submodule in all_modules if "." not in submodule %}
+        <li><a href="{{ submodule }}.html">{{submodule}}</a></li>
+        {% endfor %}
+    </ul>
+</main>
+{% endblock %}

pdoc-templates/module.html.jinja2

@@ -0,0 +1,58 @@
+{% extends "default/module.html.jinja2" %}
+
+{% block nav_members %}
+    {% if module.members %}
+        {% set filtered_modules = {} %}
+        {% for key, value in module.members.items() %}
+            {% if not key.startswith('__') %}
+                {% set _ = filtered_modules.update({key: value}) %}
+            {% endif %}
+        {% endfor %}
+
+        {% if filtered_modules %}
+            <h2>API Documentation</h2>
+            {{ nav_members(module.members.values()) }}
+        {% endif %}
+    {% endif %}
+{% endblock nav_members %}
+
+{% set versions = env.get("VERSIONS", "").strip().split(" ") %}
+
+{% set package = env.get("PACKAGENAME", "") %}
+
+{% block nav_footer %}
+    <footer>
+      <h3><label for="page-select">API Version</label></h3>
+      <select id="page-select" onchange="redirectToPage()">
+        <option value="">Select an option</option>
+        {% for item in versions %}
+            <option value="{{ item }}">{{ item }}</option>
+        {% endfor %}
+        </select>
+
+      <script>
+        function redirectToPage() {
+          var select = document.getElementById("page-select");
+          var selectedOption = select.options[select.selectedIndex].value;
+
+          if (selectedOption !== "") {
+            var currentURL = window.location.pathname;
+            var match = currentURL.match(/^(.*\/{{ package }}\/)[0-9]+[0-9a-z.]+(\/.*)$/);
+            var redirectURL = match[1] + selectedOption + match[2];
+            window.location.href = redirectURL;
+          }
+        }
+
+        // Set the default value of the dropdown to the selected option
+        window.onload = function() {
+          var select = document.getElementById("page-select");
+          var currentURL = window.location.pathname;
+          var match = currentURL.match(/^.*\/{{ package }}\/([0-9]+[0-9a-z.]+)\/.+$/);
+
+          if (match && match[1]) {
+            select.value = match[1];
+          }
+        };
+      </script>
+    </footer>
+{% endblock nav_footer %}

pdoc-templates/syntax-highlighting.css

@@ -0,0 +1,80 @@
+/* monokai color scheme, see pdoc/template/README.md */
+pre { line-height: 125%; }
+span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 20px; }
+.pdoc-code .hll { background-color: #49483e }
+.pdoc-code { background: #272822; color: #f8f8f2 }
+.pdoc-code .c { color: #75715e } /* Comment */
+.pdoc-code .err { color: #960050; background-color: #1e0010 } /* Error */
+.pdoc-code .esc { color: #f8f8f2 } /* Escape */
+.pdoc-code .g { color: #f8f8f2 } /* Generic */
+.pdoc-code .k { color: #66d9ef } /* Keyword */
+.pdoc-code .l { color: #ae81ff } /* Literal */
+.pdoc-code .n { color: #f8f8f2 } /* Name */
+.pdoc-code .o { color: #f92672 } /* Operator */
+.pdoc-code .x { color: #f8f8f2 } /* Other */
+.pdoc-code .p { color: #f8f8f2 } /* Punctuation */
+.pdoc-code .ch { color: #75715e } /* Comment.Hashbang */
+.pdoc-code .cm { color: #75715e } /* Comment.Multiline */
+.pdoc-code .cp { color: #75715e } /* Comment.Preproc */
+.pdoc-code .cpf { color: #75715e } /* Comment.PreprocFile */
+.pdoc-code .c1 { color: #75715e } /* Comment.Single */
+.pdoc-code .cs { color: #75715e } /* Comment.Special */
+.pdoc-code .gd { color: #f92672 } /* Generic.Deleted */
+.pdoc-code .ge { color: #f8f8f2; font-style: italic } /* Generic.Emph */
+.pdoc-code .gr { color: #f8f8f2 } /* Generic.Error */
+.pdoc-code .gh { color: #f8f8f2 } /* Generic.Heading */
+.pdoc-code .gi { color: #a6e22e } /* Generic.Inserted */
+.pdoc-code .go { color: #66d9ef } /* Generic.Output */
+.pdoc-code .gp { color: #f92672; font-weight: bold } /* Generic.Prompt */
+.pdoc-code .gs { color: #f8f8f2; font-weight: bold } /* Generic.Strong */
+.pdoc-code .gu { color: #75715e } /* Generic.Subheading */
+.pdoc-code .gt { color: #f8f8f2 } /* Generic.Traceback */
+.pdoc-code .kc { color: #66d9ef } /* Keyword.Constant */
+.pdoc-code .kd { color: #66d9ef } /* Keyword.Declaration */
+.pdoc-code .kn { color: #f92672 } /* Keyword.Namespace */
+.pdoc-code .kp { color: #66d9ef } /* Keyword.Pseudo */
+.pdoc-code .kr { color: #66d9ef } /* Keyword.Reserved */
+.pdoc-code .kt { color: #66d9ef } /* Keyword.Type */
+.pdoc-code .ld { color: #e6db74 } /* Literal.Date */
+.pdoc-code .m { color: #ae81ff } /* Literal.Number */
+.pdoc-code .s { color: #e6db74 } /* Literal.String */
+.pdoc-code .na { color: #a6e22e } /* Name.Attribute */
+.pdoc-code .nb { color: #f8f8f2 } /* Name.Builtin */
+.pdoc-code .nc { color: #a6e22e } /* Name.Class */
+.pdoc-code .no { color: #66d9ef } /* Name.Constant */
+.pdoc-code .nd { color: #a6e22e } /* Name.Decorator */
+.pdoc-code .ni { color: #f8f8f2 } /* Name.Entity */
+.pdoc-code .ne { color: #a6e22e } /* Name.Exception */
+.pdoc-code .nf { color: #a6e22e } /* Name.Function */
+.pdoc-code .nl { color: #f8f8f2 } /* Name.Label */
+.pdoc-code .nn { color: #f8f8f2 } /* Name.Namespace */
+.pdoc-code .nx { color: #a6e22e } /* Name.Other */
+.pdoc-code .py { color: #f8f8f2 } /* Name.Property */
+.pdoc-code .nt { color: #f92672 } /* Name.Tag */
+.pdoc-code .nv { color: #f8f8f2 } /* Name.Variable */
+.pdoc-code .ow { color: #f92672 } /* Operator.Word */
+.pdoc-code .w { color: #f8f8f2 } /* Text.Whitespace */
+.pdoc-code .mb { color: #ae81ff } /* Literal.Number.Bin */
+.pdoc-code .mf { color: #ae81ff } /* Literal.Number.Float */
+.pdoc-code .mh { color: #ae81ff } /* Literal.Number.Hex */
+.pdoc-code .mi { color: #ae81ff } /* Literal.Number.Integer */
+.pdoc-code .mo { color: #ae81ff } /* Literal.Number.Oct */
+.pdoc-code .sa { color: #e6db74 } /* Literal.String.Affix */
+.pdoc-code .sb { color: #e6db74 } /* Literal.String.Backtick */
+.pdoc-code .sc { color: #e6db74 } /* Literal.String.Char */
+.pdoc-code .dl { color: #e6db74 } /* Literal.String.Delimiter */
+.pdoc-code .sd { color: #e6db74 } /* Literal.String.Doc */
+.pdoc-code .s2 { color: #e6db74 } /* Literal.String.Double */
+.pdoc-code .se { color: #ae81ff } /* Literal.String.Escape */
+.pdoc-code .sh { color: #e6db74 } /* Literal.String.Heredoc */
+.pdoc-code .si { color: #e6db74 } /* Literal.String.Interpol */
+.pdoc-code .sx { color: #e6db74 } /* Literal.String.Other */
+.pdoc-code .sr { color: #e6db74 } /* Literal.String.Regex */
+.pdoc-code .s1 { color: #e6db74 } /* Literal.String.Single */
+.pdoc-code .ss { color: #e6db74 } /* Literal.String.Symbol */
+.pdoc-code .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
+.pdoc-code .fm { color: #a6e22e } /* Name.Function.Magic */
+.pdoc-code .vc { color: #f8f8f2 } /* Name.Variable.Class */
+.pdoc-code .vg { color: #f8f8f2 } /* Name.Variable.Global */
+.pdoc-code .vi { color: #f8f8f2 } /* Name.Variable.Instance */
+.pdoc-code .vm { color: #f8f8f2 } /* Name.Variable.Magic */

pdoc-templates/theme.css

@@ -0,0 +1,20 @@
+:root {
+    --pdoc-background: #212529;
+}
+
+.pdoc {
+    --text: #f7f7f7;
+    --muted: #9d9d9d;
+    --link: #58a6ff;
+    --link-hover: #3989ff;
+    --code: #333;
+    --active: #555;
+
+    --accent: #343434;
+    --accent2: #555;
+
+    --nav-hover: rgba(0, 0, 0, 0.1);
+    --name: #77C1FF;
+    --def: #0cdd0c;
+    --annotation: #00c037;
+}

scripts/multiversion_pdoc_wrapper

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+
+if [ -n "$1" ]; then
+    SETTINGS_FILE="$1"
+fi
+
+if [ -z "$SETTINGS_FILE" ]; then
+    echo "Error: SETTINGS_FILE needs to be given as an env variable or directly"
+    echo "as a parameter to this script - stopping!"
+    exit 1
+fi
+
+# derive the package name from the name of the settings file (w/o the suffix):
+PACKAGENAME=$(basename "${SETTINGS_FILE%.inc.sh}")
+
+# settings from a sourced file (PRIORITY!) or the environment:
+echo "Reading settings from file [$SETTINGS_FILE]..."
+source "$SETTINGS_FILE"
+
+cd "$(dirname "$0")"
+
+# set default values for empty variables:
+PYTHON_CMD="${PYTHON_CMD:=python3}"
+USE_VENV="${USE_VENV:=$HOME/.virtualenvs/pdoc}"
+PDOC="${USE_VENV}/bin/pdoc"
+PDOC_TEMPLATES="${PDOC_TEMPLATES:=$PWD/../pdoc-templates}"  # templates for pdoc
+DOCS_BASEDIR="${DOCS_BASEDIR:=$PWD/../PUBLIC/apidocs-mv}"  # output base directory
+REPO_BASEDIR="${REPO_BASEDIR:=$PWD/../build-tmp}"
+INCLUDE_TAGS=${INCLUDE_TAGS:=^(${REPO_NAME}-|v)[0-9]+}
+MAIN_BRANCH="${MAIN_BRANCH:=main}"  # name of the repo's main branch
+SKIP_PULL="${SKIP_PULL:=0}"  # set to '1' to skip pulling the package repo
+EXCLUDE_VERSIONS="${EXCLUDE_VERSIONS:=^$}"  # tags to exclude from docs (egrep pattern)
+PKG_SRC="${PKG_SRC:=src}"  # location of the sources inside their repository
+PREPROC_VERSIONS="${PREPROC_VERSIONS:=^$}"
+CACHE_DIR="${CACHE_DIR:=$HOME/.cache/imcf}"
+WHEELS_DIR="$CACHE_DIR/wheels"
+
+# hard coded settings:
+LOGO="https://imcf.one/images/raeppli.png"
+LOGO_LINK="https://imcf.one/apidocs/"
+FAVICON="https://imcf.one/images/raeppli-code.png"
+PIP_PKGS_DEFAULT="click pdoc requests"
+
+# a sed command to process the stdout of the pdoc call, by default everything
+# will be passed on without changes, only in "dry-run" mode (see below) the
+# output will be modified to improve readability:
+PDOC_STDOUT_FILTER=""
+
+if [ -n "$DRY_RUN" ]; then
+    PDOC="echo $PDOC"
+    PDOC_STDOUT_FILTER='s, ,\n    ,g'
+fi
+
+set -o nounset
+test -n "$REPO_NAME"
+test -n "$REPO_URI"
+
+#
+#
+### functions
+
+function prepare_wheels_cache() {
+    mkdir -p "$WHEELS_DIR"
+    cd "$WHEELS_DIR"
+    set +o nounset
+    test -z "$CACHE_WHEELS" && return
+    echo "Caching extra wheels at: [$WHEELS_DIR]"
+    for URI in $CACHE_WHEELS; do
+        echo "Trying to cache [${URI}]..."
+        wget --no-verbose --no-clobber "$URI"
+    done
+    set +o nounset
+    cd -
+}
+
+function prepare_venv() {
+    if ! [ -d "$USE_VENV" ]; then
+        echo "Creating new venv at [$USE_VENV]..."
+        $PYTHON_CMD -m venv "$USE_VENV"
+    fi
+    PIP="${USE_VENV}/bin/pip"
+    $PIP install --upgrade pip
+    # shellcheck disable=SC2086
+    $PIP install --upgrade $PIP_PKGS_DEFAULT
+    set +o nounset
+    if [ -n "$PIP_PKGS_EXTRA" ]; then
+        # shellcheck disable=SC2086
+        $PIP install --upgrade $PIP_PKGS_EXTRA --find-links "$WHEELS_DIR"
+    fi
+    set -o nounset
+}
+
+### functions
+#
+#
+
+prepare_wheels_cache
+prepare_venv
+
+mkdir -p "$REPO_BASEDIR"
+cd "$REPO_BASEDIR"
+if ! [ -d "$REPO_NAME" ]; then
+    git clone "$REPO_URI"
+    cd "$REPO_NAME"
+else
+    cd "$REPO_NAME"
+    git checkout -- .
+    git checkout "$MAIN_BRANCH"
+    if [ "$SKIP_PULL" != "0" ]; then
+        echo "Not pulling repo, using current state!"
+    else
+        git pull --tags -v
+    fi
+fi
+
+# version-tags will be prefixed with the repo-name or start with 'v':
+ALL_VERSIONS=$(
+    git tag -l --sort=committerdate |
+        grep -E "$INCLUDE_TAGS" |
+        sed 's,.*-\([0-9]\+\.\),\1, ; s,^v\([0-9]\+\.\),\1,' |
+        sort -V -r |
+        grep -vE "$EXCLUDE_VERSIONS"
+)
+LATEST_STABLE="$(echo "$ALL_VERSIONS" |
+    grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' |  # exclude alpha/beta/pre/dev etc.
+    head -n 1)"
+
+echo -e "\n==== using versions ====\n${ALL_VERSIONS}"
+echo -e "==== latest stable =====\n${LATEST_STABLE}\n========================"
+
+# all versions which should be documented
+VERSIONS="$(echo "${ALL_VERSIONS}" | xargs)"
+
+# export all variables required by pdoc:
+export VERSIONS
+export PACKAGENAME
+
+run_pdoc() {
+    echo -e "\n*** Generating docs for version [$CUR_VERSION]..."
+    # git reset -- .
+    git checkout -- .
+
+    # find back the tag name to check out from the version (required e.g. for
+    # repos where the release-tag-naming-scheme has changed, note this will fail
+    # in situations where it returns more than one tag!):
+    TAG_NAME=$(git tag -l "*${CUR_VERSION}")
+    git checkout "${TAG_NAME}" --
+
+    ## this doesn't seem to be required for the moment:
+    # PKG_INIT="${PKG_SRC}/${PACKAGENAME}/__init__.py"
+    # sed -i 's/\${project.version}/'"${CUR_VERSION}"'/g' "${PKG_INIT}"
+
+    if echo "$CUR_VERSION" | grep -qsE "$PREPROC_VERSIONS"; then
+        echo "Preprocessing source tree for version [$CUR_VERSION]..."
+
+        PDOC_SKIP="pdoc: skip"               # pragma to look for in the code
+        PDOC_CMNT="disabled while running pdoc"  # prefix for disabled lines
+
+        find "${PKG_SRC}/${PACKAGENAME}/" -iname '*.py' -print0 |
+            xargs -0 sed -i "
+                s,\(.*\)# $PDOC_SKIP,## $PDOC_CMNT ## \1,
+                s,\(.*plugins\.\)in \(.*\),\1_in \2,
+            "
+    fi
+
+    OUTPUT="$DOCS_BASEDIR/$PACKAGENAME/$CUR_VERSION"
+    # echo "Output directory: [$OUTPUT]"
+    mkdir -p "$OUTPUT"
+    $PDOC \
+        --template-directory "$PDOC_TEMPLATES" \
+        --output-directory "$OUTPUT" \
+        --docformat numpy \
+        --logo "$LOGO" \
+        --logo-link "$LOGO_LINK" \
+        --favicon "$FAVICON" \
+        "${PKG_SRC}"/* |
+            sed "$PDOC_STDOUT_FILTER"
+}
+
+for CUR_VERSION in $VERSIONS; do
+    run_pdoc
+done
+
+# create an index.html pointing to the latest release
+INDEX_REDIRECT="$DOCS_BASEDIR/$PACKAGENAME/index.html"
+echo "<meta
+    http-equiv=\"Refresh\"
+    content=\"0;
+    url='./${LATEST_STABLE}/index.html'\"
+/>" > "$INDEX_REDIRECT"
+
+echo -e "\nDone. Output can be found at [$INDEX_REDIRECT].\n"