Add E2E tests, runAsUser, add comments for OpenShift UID issue.

.circleci/config.yml

@@ -0,0 +1,53 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine.
+# See: https://circleci.com/docs/2.0/configuration-reference
+version: 2.1
+
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
+jobs:
+  arm64-e2e:
+    machine:
+      image: ubuntu-2004:current
+    resource_class: arm.medium
+    steps:
+      - checkout
+      - run: docker version
+      - restore_cache:
+          key: cache-{{ checksum ".circleci/config.yml" }}
+      - run: go env
+      - run: go install sigs.k8s.io/[email protected]
+      - save_cache:
+          key: cache-{{ checksum ".circleci/config.yml" }}
+          paths:
+            - ~/go/bin
+            - ~/go/pkg/mod
+            - ~/.cache/go-build
+      - run: sudo snap install kubectl --classic
+      - run: kind version
+      - run: kind create cluster
+      - run: kubectl config use-context kind-kind
+      - run: kubectl create -f folding-cpu.yaml
+      # patch to one replica
+      - run: kubectl patch deployment fah-cpu -p "{\"spec\":{\"replicas\":1}}"
+      # patch to use image we just built
+      - run: |
+          if [ -n "$CIRCLE_PR_USERNAME" ]; then
+            kubectl patch deployment fah-cpu -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"fah-cpu\",\"image\":\"ghcr.io/$CIRCLE_PR_USERNAME/$CIRCLE_PR_REPONAME:$CIRCLE_BRANCH\"}]}}}}";
+            kubectl patch deployment fah-cpu -p "{\"spec\":{\"template\":{\"spec\":{\"initContainers\":[{\"name\":\"copy-config\",\"image\":\"ghcr.io/$CIRCLE_PR_USERNAME/$CIRCLE_PR_REPONAME:$CIRCLE_BRANCH\"}]}}}}";
+          else
+            kubectl patch deployment fah-cpu -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"fah-cpu\",\"image\":\"ghcr.io/$CIRCLE_USERNAME/$CIRCLE_PROJECT_REPONAME:$CIRCLE_BRANCH\"}]}}}}";
+            kubectl patch deployment fah-cpu -p "{\"spec\":{\"template\":{\"spec\":{\"initContainers\":[{\"name\":\"copy-config\",\"image\":\"ghcr.io/$CIRCLE_USERNAME/$CIRCLE_PROJECT_REPONAME:$CIRCLE_BRANCH\"}]}}}}";
+          fi
+      - run: kubectl get deployment/fah-cpu -o yaml
+      - run: kubectl rollout status deployment fah-cpu --timeout=300s; kubectl describe deployment fah-cpu; kubectl describe pods -l app=fah-cpu
+      - run: kubectl get deployment/fah-cpu -o yaml
+      - run: kubectl get pods -oyaml
+      #  print out uname to verify architecture in pod's container is amd64
+      - run: kubectl exec deployment/fah-cpu -- /bin/sh -c "uname -a"
+
+# Invoke jobs via workflows
+# See: https://circleci.com/docs/2.0/configuration-reference/#workflows
+workflows:
+  arm64-e2e-workflow:
+    jobs:
+      - arm64-e2e

.github/workflows/docker-publish.yml

@@ -1,4 +1,4 @@
-name: Docker
+name: Docker Publish & K8S E2E
 
 # This workflow uses actions that are not certified by GitHub.
 # They are provided by a third-party and are governed by
@@ -9,11 +9,11 @@ on:
   schedule:
     - cron: '29 3 * * *'
   push:
-    branches: [ "master" ]
+    branches: ['*']
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
   pull_request:
-    branches: [ "master" ]
+    branches: ['*']
 
 env:
   # Use docker.io for Docker Hub if empty
@@ -70,26 +70,33 @@ jobs:
 
       # Extract metadata (tags, labels) for Docker
       # https://github.com/docker/metadata-action
-      - name: Extract Docker metadata
+      - name: Extract Docker metadata for GitHub Container Registry
         id: meta
         uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        if: ${{ github.event_name != 'pull_request' }}
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
+
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
-      - name: Build and push Docker image
+      - name: Build and push Docker image and push to GitHub Container Registry
         id: build-and-push
         uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        if: ${{ github.event_name != 'pull_request' }}
         with:
           context: .
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
           platforms: linux/amd64,linux/arm64
 
+      - name: PR - Sleep for 30s. Assume using image from fork branch which should be available in 30s (with caching)
+        if: ${{ github.event_name == 'pull_request' }}
+        run: sleep 30
+
 
       # Sign the resulting Docker image digest except on PRs.
       # This will only write to the public Rekor transparency log when the Docker
@@ -103,3 +110,96 @@ jobs:
         # This step uses the identity token to provision an ephemeral certificate
         # against the sigstore community Fulcio instance.
         run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
+
+  k8s-e2e:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '^1.17' # The Go version to download (if necessary) and use.
+      - uses: satackey/[email protected]
+      # Ignore the failure of a step and avoid terminating the job.
+        continue-on-error: true
+
+      # Workaround: https://github.com/docker/build-push-action/issues/461
+      - run: docker version
+      - run: go install sigs.k8s.io/[email protected]
+      - run: sudo snap install kubectl --classic
+      - run: kind version
+      - run: kind create cluster
+      - run: kubectl cluster-info --context kind-kind
+      - run: kubectl create -f folding-cpu.yaml
+        # patch to one replica
+      - run: kubectl patch deployment fah-cpu -p '{"spec":{"replicas":1}}'
+      - name: Patch to use branch image
+        run: |
+          kubectl patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"containers":[{"name":"fah-cpu","image":"${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}"}]}}}}'
+          kubectl patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"copy-config","image":"${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}"}]}}}}'
+        if: ${{ github.event_name != 'pull_request' }}
+        # patch to use PR image
+      - name: PR - Patch to use image
+        run: |
+          kubectl patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"containers":[{"name":"fah-cpu","image":"${{ env.REGISTRY }}/${{ github.event.pull_request.head.repo.full_name }}:${{ github.head_ref }}"}]}}}}'
+          kubectl patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"copy-config","image":"${{ env.REGISTRY }}/${{ github.event.pull_request.head.repo.full_name }}:${{ github.head_ref }}"}]}}}}'
+        if: ${{ github.event_name == 'pull_request' }}
+      # wait for deployment to be ready
+      - run: kubectl rollout status deployment fah-cpu --timeout=300s; kubectl describe deployment fah-cpu; kubectl describe pods -l app=fah-cpu
+        continue-on-error: true
+      - run: kubectl get deployment/fah-cpu -o yaml
+      - run: kubectl get pods -oyaml
+      - name: print out uname to verify architecture in pod's container is amd64
+        run: kubectl exec deployment/fah-cpu -- /bin/sh -c "uname -a"
+      - run: kind delete cluster
+        continue-on-error: true
+  # k8s-e2e-arm64: Covered by CircleCI/Travis since it does not require slow ARM64 emulation.
+
+  docker-test:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: linux/arm64
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+      - run: docker run --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} uname -a
+        if: ${{ github.event_name != 'pull_request' }}
+      - run: docker run --rm ${{ env.REGISTRY }}/${{ github.event.pull_request.head.repo.full_name }}:${{ github.head_ref }} uname -a
+        if: ${{ github.event_name == 'pull_request' }}
+  docker-arm64-test:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: linux/arm64
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+      - run: docker run --platform=linux/arm64 --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} uname -a
+        if: ${{ github.event_name != 'pull_request' }}
+      - run: docker run --platform=linux/arm64 --rm ${{ env.REGISTRY }}/${{ github.event.pull_request.head.repo.full_name }}:${{ github.head_ref }} uname -a
+        if: ${{ github.event_name == 'pull_request' }}

.travis.yml

@@ -0,0 +1,47 @@
+os: linux
+services: docker
+dist: jammy
+language: go
+go: stable
+cache:
+  directories:
+    - ~/go/bin
+    - ~/go/pkg/mod
+    - ~/.cache/go-build
+script:
+- docker version
+- go install sigs.k8s.io/[email protected]
+- sudo snap install kubectl --classic
+- kind version
+- kind create cluster
+- kubectl config use-context kind-kind
+- kubectl create -f folding-cpu.yaml
+# patch to one replica
+- kubectl patch deployment fah-cpu -p "{\"spec\":{\"replicas\":1}}"
+# patch to use image we just built
+- |
+  if [ "${TRAVIS_EVENT_TYPE}" == "pull_request" ]; then
+    kubectl patch deployment fah-cpu -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"fah-cpu\",\"image\":\"ghcr.io/${TRAVIS_PULL_REQUEST_SLUG}:${TRAVIS_PULL_REQUEST_BRANCH}\"}]}}}}";
+    kubectl patch deployment fah-cpu -p "{\"spec\":{\"template\":{\"spec\":{\"initContainers\":[{\"name\":\"copy-config\",\"image\":\"ghcr.io/${TRAVIS_PULL_REQUEST_SLUG}:${TRAVIS_PULL_REQUEST_BRANCH}\"}]}}}}";
+  fi
+- |
+  if [ "${TRAVIS_EVENT_TYPE}" == "push" ]; then
+    kubectl patch deployment fah-cpu -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"fah-cpu\",\"image\":\"ghcr.io/${TRAVIS_REPO_SLUG}:${TRAVIS_BRANCH}\"}]}}}}";
+    kubectl patch deployment fah-cpu -p "{\"spec\":{\"template\":{\"spec\":{\"initContainers\":[{\"name\":\"copy-config\",\"image\":\"ghcr.io/${TRAVIS_REPO_SLUG}:${TRAVIS_BRANCH}\"}]}}}}";
+  fi
+- kubectl get deployment/fah-cpu -o yaml
+- kubectl rollout status deployment fah-cpu --timeout=300s; kubectl describe deployment fah-cpu; kubectl describe pods -l app=fah-cpu
+- kubectl get deployment/fah-cpu -o yaml
+- kubectl get pods -oyaml
+#  print out uname to verify architecture in pod's container is amd64
+- kubectl exec deployment/fah-cpu -- /bin/sh -c "uname -a"
+
+jobs:
+  include:
+    - stage: travis-ci
+      arch: arm64-graviton2
+      virt: vm
+      group: edge
+    # amd64 covered by GitHub Actions, travis credits are hard to come by you know :)
+    # PS. It's free for open source, but you have to email them to get credit refills.
+    # - arch: amd64

README.md

@@ -1,5 +1,5 @@
-# k8s-fah
-**Run folding@home on Kubernetes**.
+# k8s-fah [![Docker](https://github.com/richstokes/k8s-fah/actions/workflows/docker-publish.yml/badge.svg)](https://github.com/richstokes/k8s-fah/actions/workflows/docker-publish.yml) [![Travis-CI Status](https://app.travis-ci.com/richstokes/k8s-fah.svg?branch=master)](https://app.travis-ci.com/richstokes/k8s-fah) [![CircleCI](https://dl.circleci.com/status-badge/img/gh/richstokes/k8s-fah/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/richstokes/k8s-fah/tree/master)
+**Run folding@home on Kubernetes**. 
 
 The folding@home project [added support](https://foldingathome.org/2020/02/27/foldinghome-takes-up-the-fight-against-covid-19-2019-ncov/) for the Corona virus (2019-nCoV). 
 

folding-cpu.yaml

@@ -61,6 +61,11 @@ spec:
           # Make the container harder to break out of or exploit
           securityContext:
             runAsNonRoot: true
+            # UID needs to be defined to not default to root on kubernetes.
+            # If you are using OpenShift, patch to remove runAsUser
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"containers":[{"name":"fah-cpu","securityContext":{"runAsUser":null}}]}}}}'
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"copy-config","securityContext":{"runAsUser":null}}]}}}}'
+            runAsUser: 1234
             readOnlyRootFilesystem: true
             allowPrivilegeEscalation: false
           volumeMounts:
@@ -78,7 +83,7 @@ spec:
       initContainers:
         - name: copy-config
           imagePullPolicy: Always
-          image: "richstokes20/fah-covid:latest"
+          image: "ghcr.io/richstokes/k8s-fah:master"
           command:
             - "sh"
             - "-c"
@@ -89,6 +94,11 @@ spec:
             # - "/var/lib/fahclient/config.xml"
           securityContext:
             runAsNonRoot: true
+            # UID needs to be defined to not default to root on kubernetes.
+            # If you are using OpenShift, patch to remove runAsUser
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"containers":[{"name":"fah-cpu","securityContext":{"runAsUser":null}}]}}}}'
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"copy-config","securityContext":{"runAsUser":null}}]}}}}'
+            runAsUser: 1234
             readOnlyRootFilesystem: true
             allowPrivilegeEscalation: false
           volumeMounts:

folding-daemonset.yaml

@@ -51,6 +51,10 @@ spec:
           # Make the container harder to break out of or exploit
           securityContext:
             runAsNonRoot: true
+            # UID needs to be defined to not default to root on kubernetes.
+            # If you are using OpenShift, patch to remove runAsUser
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"containers":[{"name":"fah-cpu","securityContext":{"runAsUser":null}}]}}}}'
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"copy-config","securityContext":{"runAsUser":null}}]}}}}'
             runAsUser: 1234
             readOnlyRootFilesystem: true
             allowPrivilegeEscalation: false
@@ -69,7 +73,7 @@ spec:
       initContainers:
         - name: copy-config
           imagePullPolicy: Always
-          image: "richstokes20/fah-covid:latest"
+          image: "ghcr.io/richstokes/k8s-fah:master"
           command:
             - "sh"
             - "-c"
@@ -80,6 +84,10 @@ spec:
             # - "/var/lib/fahclient/config.xml"
           securityContext:
             runAsNonRoot: true
+            # UID needs to be defined to not default to root on kubernetes.
+            # If you are using OpenShift, patch to remove runAsUser
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"containers":[{"name":"fah-cpu","securityContext":{"runAsUser":null}}]}}}}'
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"copy-config","securityContext":{"runAsUser":null}}]}}}}'
             runAsUser: 1234
             readOnlyRootFilesystem: true
             allowPrivilegeEscalation: false

folding-gpu-cpu.yaml

@@ -63,6 +63,10 @@ spec:
           # Make the container harder to break out of or exploit
           securityContext:
             runAsNonRoot: true
+            # UID needs to be defined to not default to root on kubernetes.
+            # If you are using OpenShift, patch to remove runAsUser
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"containers":[{"name":"fah-cpu","securityContext":{"runAsUser":null}}]}}}}'
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"copy-config","securityContext":{"runAsUser":null}}]}}}}'
             runAsUser: 1234
             readOnlyRootFilesystem: true
             allowPrivilegeEscalation: false
@@ -81,7 +85,7 @@ spec:
       initContainers:
         - name: copy-config
           imagePullPolicy: Always
-          image: "richstokes20/fah-covid:latest"
+          image: "ghcr.io/richstokes/k8s-fah:master"
           command:
             - "sh"
             - "-c"
@@ -92,6 +96,10 @@ spec:
             # - "/var/lib/fahclient/config.xml"
           securityContext:
             runAsNonRoot: true
+            # UID needs to be defined to not default to root on kubernetes.
+            # If you are using OpenShift, patch to remove runAsUser
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"containers":[{"name":"fah-cpu","securityContext":{"runAsUser":null}}]}}}}'
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"copy-config","securityContext":{"runAsUser":null}}]}}}}'
             runAsUser: 1234
             readOnlyRootFilesystem: true
             allowPrivilegeEscalation: false

folding-gpu.yaml

@@ -60,6 +60,10 @@ spec:
           # Make the container harder to break out of or exploit
           securityContext:
             runAsNonRoot: true
+            # UID needs to be defined to not default to root on kubernetes.
+            # If you are using OpenShift, patch to remove runAsUser
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"containers":[{"name":"fah-cpu","securityContext":{"runAsUser":null}}]}}}}'
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"copy-config","securityContext":{"runAsUser":null}}]}}}}'
             runAsUser: 1234
             readOnlyRootFilesystem: true
             allowPrivilegeEscalation: false
@@ -78,7 +82,7 @@ spec:
       initContainers:
         - name: copy-config
           imagePullPolicy: Always
-          image: "richstokes20/fah-covid:latest"
+          image: "ghcr.io/richstokes/k8s-fah:master"
           command:
             - "sh"
             - "-c"
@@ -89,6 +93,10 @@ spec:
             # - "/var/lib/fahclient/config.xml"
           securityContext:
             runAsNonRoot: true
+            # UID needs to be defined to not default to root on kubernetes.
+            # If you are using OpenShift, patch to remove runAsUser
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"containers":[{"name":"fah-cpu","securityContext":{"runAsUser":null}}]}}}}'
+            # oc patch deployment fah-cpu -p '{"spec":{"template":{"spec":{"initContainers":[{"name":"copy-config","securityContext":{"runAsUser":null}}]}}}}'
             runAsUser: 1234
             readOnlyRootFilesystem: true
             allowPrivilegeEscalation: false