Merge pull request #8 from xcube-dev/thomas_xxx_development

continued development

.github/workflows/workflow.yaml

@@ -13,7 +13,6 @@ env:
   SKIP_UNITTESTS: "0"
   WAIT_FOR_STARTUP: "1"
 
-
 jobs:
   unittest:
     runs-on: ubuntu-latest
@@ -23,31 +22,90 @@ jobs:
     steps:
       - run: |
           echo "SKIP_UNITTESTS: ${{ env.SKIP_UNITTESTS }}"
-      - uses: actions/checkout@v2
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: actions/checkout@v3
+        name: Checkout xcube-geodb-openeo repo
+      - name: Checkout xcube repo
+        uses: actions/checkout@v3
+        with:
+          repository: dcs4cop/xcube
+          path: "xcube"
+      - uses: mamba-org/setup-micromamba@v1
         if: ${{ env.SKIP_UNITTESTS == '0' }}
         with:
-          mamba-version: "*"
-          channels: conda-forge
-          auto-update-conda: false
-          activate-environment: xcube-geodb-openeo
+          micromamba-version: '1.4.8-0'
           environment-file: environment.yml
+          init-shell: >-
+            bash
+          cache-environment: true
+          post-cleanup: 'all'
       - run: |
           conda info
           conda list
-          conda config --show-sources
-          conda config --show
-          printenv | sort
+      - name: setup-xcube
+        if: ${{ env.SKIP_UNITTESTS == '0' }}
+        run: |
+          cd xcube
+          micromamba env update -n xcube-geodb-openeo -f environment.yml
+          pip install -e .
+          cd ..
       - name: setup-xcube-geodb-openeo
         if: ${{ env.SKIP_UNITTESTS == '0' }}
         run: |
-          python setup.py develop
+          pip install -e .
       - name: unittest-xcube-geodb-openeo
         if: ${{ env.SKIP_UNITTESTS == '0' }}
         run: |
-          pytest --cov=./ --cov-report=xml --tb=native
-      - uses: codecov/codecov-action@v2
+          pytest --cov=./ --cov-report=xml --tb=native tests
+      - name: Upload coverage reports to Codecov
         if: ${{ env.SKIP_UNITTESTS == '0' }}
+        uses: codecov/[email protected]
         with:
           fail_ci_if_error: true
-          verbose: false
+          token: ${{ secrets.CODECOV_TOKEN }}
+          codecov_yml_path: ./codecov.yml
+
+  build-docker-image:
+    runs-on: ubuntu-latest
+    needs: [unittest]
+    name: build-docker-image
+    if: ${{ github.event_name == 'release'}}
+    steps:
+      # Checkout xcube-geodb-openeo (this project)
+      - name: git-checkout
+        uses: actions/checkout@v2
+      # Get the base release tag used in docker images
+      - name: get-release-tag
+        id: release
+        run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
+      # The docker image always needs the version as in version.py
+      - name: get-xcube-geodb-openeo-version
+        id: real-version
+        run: |
+          VERSION=$(echo "`cat xcube_geodb_openeo/version.py | grep version | cut -d "'" -f2`")
+          echo ::set-output name=version::${VERSION}
+      # Determine the deployment phase (dev/stage/prod) will be 'ignore' if a dev branch is processed
+      - name: deployment-phase
+        id: deployment-phase
+        uses: bc-org/[email protected]
+        with:
+          event_name: ${{ github.event_name }}
+          tag: ${{ steps.release.outputs.tag }}
+      - name: info
+        id: info
+        run: |
+          echo "TAG: ${{ steps.release.outputs.tag }}"
+          echo "DEPLOYMENT_PHASE: ${{ steps.deployment-phase.outputs.phase }}"
+          echo "REAL_VERSION: ${{ steps.real-version.outputs.version }}"
+          echo "EVENT: ${{ github.event_name }}"
+      # Build docker image
+      - uses: mr-smithers-excellent/[email protected]
+        if: ${{ steps.deployment-phase.outputs.phase != 'ignore' || env.FORCE_DOCKER_BUILD == '1'}}
+        name: build-and-push-docker-image
+        with:
+          image: ${{ env.ORG_NAME }}/${{ env.APP_NAME }}-server
+          directory: .
+          dockerfile: docker/Dockerfile
+          addLatest: true
+          registry: quay.io
+          username: ${{ secrets.QUAY_REG_USERNAME }}
+          password: ${{ secrets.QUAY_REG_PASSWORD }}

.gitignore

@@ -1,3 +1,6 @@
+# Files that may contain private data
+xcube_geodb_openeo/config.yml
+
 # IDEs
 .idea/
 
@@ -130,3 +133,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+xcube_geodb_openeo.iml
+/notebooks/credentials.py
+/xcube-geodb-openeo.iml

CHANGES.md

@@ -1,3 +1,12 @@
+## 0.0.2 (in development)
+
+This version:
+- extends the GitHub actions workflow to build and deploy a Docker image
+- adds the respective Dockerfile
+- uses xcube server NG, implemented in xcube v0.13.0
+- updates the implemented STAC version to 1.0.0
+- aims to implement the complete STAC Catalog Specification
+
 ## Initial version 0.0.1
 
 This version comprises:

docker/Dockerfile

@@ -0,0 +1,63 @@
+ARG MICROMAMBA_VERSION=1.3.1
+FROM mambaorg/micromamba:${MICROMAMBA_VERSION}
+
+LABEL maintainer="[email protected]"
+LABEL name=xcube_geodb_openeo
+
+ARG NEW_MAMBA_USER=xcube-geodb-openeo
+ARG NEW_MAMBA_USER_ID=1000
+ARG NEW_MAMBA_USER_GID=1000
+
+# <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+USER root
+
+# Update system and ensure that basic commands are available.
+RUN apt-get -y update && \
+    apt-get -y upgrade vim jq curl wget && \
+    apt-get -y install git && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Magic taken from https://hub.docker.com/r/mambaorg/micromamba,
+# section "Changing the user id or name"
+RUN usermod "--login=${NEW_MAMBA_USER}" "--home=/home/${NEW_MAMBA_USER}" \
+        --move-home "-u ${NEW_MAMBA_USER_ID}" "${MAMBA_USER}" && \
+    groupmod "--new-name=${NEW_MAMBA_USER}" \
+             "-g ${NEW_MAMBA_USER_GID}" "${MAMBA_USER}" && \
+    # Update the expected value of MAMBA_USER for the
+    # _entrypoint.sh consistency check.
+    echo "${NEW_MAMBA_USER}" > "/etc/arg_mamba_user" && \
+    :
+
+ENV MAMBA_USER=$NEW_MAMBA_USER
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+
+USER $MAMBA_USER
+
+COPY --chown=$MAMBA_USER:$MAMBA_USER ../environment.yml /tmp/environment.yml
+RUN micromamba install --yes -n base --file /tmp/environment.yml && \
+    micromamba clean --all --yes
+ARG MAMBA_DOCKERFILE_ACTIVATE=1
+
+RUN micromamba install -n base -c conda-forge pip
+
+ADD docker /tmp/
+WORKDIR /tmp
+
+RUN git clone https://github.com/dcs4cop/xcube.git
+WORKDIR /tmp/xcube
+RUN pip install -e .
+
+WORKDIR /tmp/
+RUN git clone https://github.com/dcs4cop/xcube-geodb.git
+WORKDIR /tmp/xcube-geodb
+RUN pip install -e .
+
+# Copy files for xcube_geodb_openeo source install
+COPY --chown=$MAMBA_USER:$MAMBA_USER ../setup.py /tmp/setup.py
+COPY --chown=$MAMBA_USER:$MAMBA_USER ../xcube_geodb_openeo /tmp/xcube_geodb_openeo
+
+# Switch into /tmp to install xcube-geodb-openeo
+WORKDIR /tmp
+RUN python setup.py install
+
+CMD ["python", "-m", "xcube.cli.main", "--loglevel=DETAIL", "--traceback", "serve", "-vvv", "-c", "/etc/config/config.yml"]

docker/README.md

@@ -0,0 +1,23 @@
+# Docker builds for the xcube-geoDB-openEO Backend
+
+The backend image is automatically built on push to master, and on release.
+The image is built and uploaded to _quay.io_ during the default GitHub workflow
+in the step `build-docker-image`.
+
+It can be run locally using docker like so:
+
+```bash
+docker run -p 8080:8080 -env-file env.list quay.io/bcdev/xcube-geodb-openeo
+```
+
+where `env.list` contains the following values:
+```
+postgrest_url:  <geoDB-PostGREST server url>
+postgrest_port: <geoDB-PostGREST server port>
+client_id:      <your client id>
+client_secret:  <your client secret>
+auth_domain:    <your auth domain>
+```
+
+This starts the geoDB-openEO server instance accessible through port 8080, e.g.
+at `localhost:8080/processes`

environment.yml

@@ -4,21 +4,20 @@ channels:
   - defaults
 dependencies:
   # Python
-  - python >=3.8,<3.10
+  - python >=3.8
   # Required
-  - click >=8.0
-  - deprecated >=1.2
-  - pyjwt >=1.7
-  - pyproj >=3.0,<3.3  # tests fail with 3.3.0, missing database on Windows
-  - pyyaml >=5.4
-  - requests >=2.25
-  - requests-oauthlib >=1.3
-  - setuptools >=41.0
-  - shapely >=1.6
-  - tornado >=6.0
-  - flask >=2.0
+  - geopandas
+  - pandas
+  - shapely
+  - xcube >= 1.1.1
+  - xcube_geodb >= 1.0.8
+  - geojson
+  - yaml
+  - ipython
+  - urllib3
+  - openeo
+  - requests
   # Testing
-  - flake8 >=3.7
   - pytest >=4.4
   - pytest-cov >=2.6
-  - requests-mock >=1.8.0
+  - requests-mock >=1.8

how-to-deploy.md

@@ -0,0 +1,104 @@
+# xcube-geodb-openeo deployment
+
+This document is intended to show all necessary steps to deploy the geoDB-
+openEO server to the BC K8s stage cluster.
+
+There is quite a large number of actors which have to be correctly configured
+for the service to work:
+- a GitHub action inside this repository
+- a Dockerfile inside this repository
+- a Helm chart, residing within the bc-org k8s-configs repository
+- BC's internet provider IONOS
+- the bc-org k8s-namespaces repository
+- ArgoCD
+
+### GitHub action
+
+A GitHub action has been set up at `.github/workflows/workflow.yaml`
+This workflow:
+- runs the unit-level tests
+- builds a docker image using the steps
+    1) check out (of xcube-geodb-openeo)
+    2) get tag of release or branch to set it to the docker image
+    3) retrieve the version of geoDB-openEO
+    4) deployment phase (dev/stage/prod/ignore), to determine if docker image
+       actually shall be built
+    5) Build docker image
+        - name: `bcdev/xcube-geodb-openeo-server`, tag: fetched in previous
+          step 2
+- Uploads docker image to [quay.io](https://quay.io/repository/bcdev/xcube-geodb-openeo-server)
+    - uses
+        - `secrets.QUAY_REG_USERNAME`
+        - `secrets.QUAY_REG_PASSWORD`
+- can be configured with the following variables:
+  - `SKIP_UNITTESTS:` "0" or "1"; if set to "1", unit tests are skipped
+  - `FORCE_DOCKER_BUILD`: "1"; if set to "1", a docker image will be built and 
+    uploaded to quay.io, regardless of the tag or release
+- The GH action does no change on the helm chart. This may be added later.
+
+### Dockerfile
+
+The Dockerfile is based on miniconda. It
+1) activates the base environment
+2) clones the xcube-GitHub repository
+3) checks out the branch that contains the server
+4) and installs it into the base environment
+5) then installs the current repository, i.e. xcube-geodb-openeo, into the base
+   environment
+
+In its `CMD` call, it passes the file `/etc/config/config.yml` to the starting
+server.
+
+### Helm chart
+
+The helm chart for the xcube-geodb-openeo-server is located in [k8s-configs](https://github.com/bc-org/k8s-configs/tree/main/xcube-geodb-openeo/helm).
+It consists of:
+1) templates for
+    - config-map (`api-config-map.yaml`), where the config is specified, and 
+      configured to reside in a file `config.yml`
+    - deployment (`api-deployment.yaml`), where the container is defined
+      (i.e. the docker image), credentials are put into the environment, and
+      the config map is written into a file at `/etc/config`
+    - ingress (`api-ingress.yaml`), where the exposure to the public internet
+      is configured
+    - service (`api-service.yaml`), where port and protocol are configured
+2) a values file used to fill in the templates. Currently in use:
+   `values-dev.yaml`.
+   In this file, the following values are set:
+    - the actual Docker image value
+    - the ingress encryption method
+    - the host name
+
+### IONOS
+
+Needed to add a CNAME, so the server can be reached on a human-readable URL.
+This has been done using these
+[directions](https://github.com/bc-org/k8s-configs/blob/main/howtos/How_to_add_new_CNAME_record.md).
+
+### k8s-namespaces
+
+Used to store secrets securely in a deployment, instead of putting them
+insecurely into the config map. Usage: every time the values change in
+[values-xcube-geodb.yaml](https://github.com/bc-org/k8s-namespaces/blob/main/helm/namespaces/values-xcube-geodb.yaml),
+run the [workflow](https://github.com/bc-org/k8s-namespaces/actions/workflows/create-xcube-geodb-namespaces-workflow.yaml)
+manually. Then, the namespace on the K8s cluster on AWS is updated.
+
+The values inside that file will change if a secret is added, changed or
+removed. Currently, these are the secret credentials for the geodb-access.
+
+The workflow is defined [here](https://github.com/bc-org/k8s-namespaces/blob/main/.github/workflows/create-xcube-geodb-namespaces-workflow.yaml).
+Note that there are different ones on the same level; this one is the one used
+for xcube-geodb-openeo, because xcube-geodb-openeo uses the xcube-geodb
+namespace, as configured in `values-dev.yaml` in k8s-configs.
+
+### ArgoCD-Deployment
+
+The purpose of the argoCD deployment is to take the helm chart and deploy it to
+BCs AWS K8s. It can be found [here](https://argocd.management.brockmann-consult.de/applications/geodb-openeo).
+
+The relevant configuration values are:
+- Cluster: `xc-dev-v2`
+- Namespace: `xc-geodb`
+- Using helm chart from
+    - `[email protected]:bc-org/k8s-configs.git`
+    - path `xcube-geodb-openeo/helm`

xcube_geodb_openeo/server/api.py → notebooks/credentials-example.py

@@ -19,6 +19,5 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-from ..backend.catalog import API_VERSION
-
-API_URL_PREFIX = f'/api/v{API_VERSION}'
+client_id = ''
+client_secret = ''