feat: golang switch

Switches the project from Python to Golang. This commit includes the following changes: - validation mode - redis caching - resource validation mode - notary: support for all TUF keys - unified "*" trustRoot option - update of cosign to 2.2.3
sse-secure-systems · Mar 15, 2024 · 67fccbd · 67fccbd
1 parent 8711fbc
commit 67fccbd
Show file tree

Hide file tree

Showing 589 changed files with 20,450 additions and 15,057 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,13 +1,15 @@
-# ignore Markdown files
+# ignore docs
 *.md
+docs/
 # ignore tests
-connaisseur/tests/
-# ignore img
-img/
-# ignore coverage files
-connaisseur/coverage.xml
-connaisseur/coverage.txt
-# ignore pycache
-connaisseur/__pycache__/
-# ignore venv folder
-venv/
+tests/
+test/
+# ignore tools
+tools/
+# ignore charts
+charts/
+helm/
+# ignore misc
+LICENSE
+Makefile
+mkdocs.yml
diff --git a/.kube-linter/config.yaml → .github/.kube-linter/config.yaml b/.kube-linter/config.yaml → .github/.kube-linter/config.yaml
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -15,4 +15,3 @@ Fixes #
 - [ ] Added tests (if necessary)
 - [ ] Extended README/Documentation (if necessary)
 - [ ] Adjusted versions of image and Helm chart in `Chart.yaml` (if necessary)
-
diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
@@ -61,10 +61,11 @@ runs:
       uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0
       with:
         push: true
-        tags: ${{ steps.tags.outputs.tags }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        file: build/Dockerfile
         labels: ${{ inputs.image_labels }}
-        file: docker/Dockerfile
-        build-args: COSIGN_VERSION=${{ inputs.cosign_version }}
+        tags: ${{ steps.tags.outputs.tags }}
         sbom: false  # Duplicates SBOMs manually created below
         provenance: false  #TODO: Set to false, as resulting format is not OCI (GHCR) compliant (https://github.com/docker/build-push-action/issues/820) and causes problems with GHCR and e.g. image deletion (https://github.com/snok/container-retention-policy/issues/63)
     - name: Create SBOM
@@ -88,9 +89,9 @@ runs:
     - name: Verify build data
       id: verify
       run: |
-        mkdir build
-        cosign public-key --key env://COSIGN_PRIVATE_KEY > build/cosign.pub
-        PUBLIC_KEY="$(cat build/cosign.pub)"
+        mkdir ci
+        cosign public-key --key env://COSIGN_PRIVATE_KEY > ci/cosign.pub
+        PUBLIC_KEY="$(cat ci/cosign.pub)"
         cosign tree ${TAGS}
         PUBLIC_KEY=${PUBLIC_KEY} cosign verify --key env://PUBLIC_KEY ${TAGS}
         PUBLIC_KEY=${PUBLIC_KEY} cosign verify --key env://PUBLIC_KEY --attachment sbom ${TAGS}
@@ -109,14 +110,14 @@ runs:
       uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
       with:
         name: cosign.pub
-        path: build/cosign.pub
+        path: ci/cosign.pub
     - name: Show build and signature information
       run: |
-        CONFIGURE="yq '. *+ load(\"tests/integration/var-img.yaml\")' tests/integration/ghcr-values.yaml > ghcr.yaml &&\n\t IMAGE=\"${{ inputs.image_registry }}/${{ inputs.image_repo }}\" TAG=\"${{ inputs.image_tag }}\" IMAGEPULLSECRET=\"<ImagePullSecret name>\" envsubst < ghcr.yaml > update &&\n\t yq '. *+ load(\"update\")' -i helm/values.yaml &&\n\t rm ghcr.yaml update"
+        CONFIGURE="yq '. *+ load(\"test/integration/var-img.yaml\")' test/integration/ghcr-values.yaml > ghcr.yaml &&\n\t IMAGE=\"${{ inputs.image_registry }}/${{ inputs.image_repo }}\" TAG=\"${{ inputs.image_tag }}\" IMAGEPULLSECRET=\"<ImagePullSecret name>\" envsubst < ghcr.yaml > update &&\n\t yq '. *+ load(\"update\")' -i charts/connaisseur/values.yaml &&\n\t rm ghcr.yaml update"
         CONFIGURE=$(printf -- "${CONFIGURE}")
         PUBLIC_KEY="${{ steps.verify.outputs.public_key }}"
         PUBLIC_KEY="$(printf -- "${PUBLIC_KEY//'<br>'/'\n'}")"
-        HELM_PATCH="yq e '.kubernetes.deployment.image.repository = \"${{ inputs.image_registry }}/${{ inputs.image_repo }}\"' -i helm/values.yaml\nyq e '.kubernetes.deployment.image.tag = \"${{ inputs.image_tag }}\"' -i helm/values.yaml"
+        HELM_PATCH="yq e '.kubernetes.deployment.image.repository = \"${{ inputs.image_registry }}/${{ inputs.image_repo }}\"' -i charts/connaisseur/values.yaml\nyq e '.kubernetes.deployment.image.tag = \"${{ inputs.image_tag }}\"' -i charts/connaisseur/values.yaml"
         HELM_PATCH=$(printf -- "${HELM_PATCH}")
         echo "# :building_construction: Build Information" >> ${GITHUB_STEP_SUMMARY}
         echo "<table>" >> ${GITHUB_STEP_SUMMARY}

diff --git a/.github/actions/context/action.yaml b/.github/actions/context/action.yaml
@@ -10,9 +10,6 @@ inputs:
     required: false
     default: "${{ github.repository }}"
 outputs:
-  cosign_version:
-    description: "Cosign version used for building Connaisseur image"
-    value: ${{ steps.get_context.outputs.COSIGN_VERSION }}
   chart_version:
     description: "Connaisseur Helm chart version"
     value: ${{ steps.get_context.outputs.CHART_VERSION }}
@@ -53,24 +50,23 @@ runs:
       id: get_chart_version
       uses: mikefarah/yq@47f4f8c7939f887e851b35f14def6741b8f5396e # v4.31.2
       with:
-        cmd: yq '.version' helm/Chart.yaml
+        cmd: yq '.version' charts/connaisseur/Chart.yaml
     - name: Get app version
       id: get_app_version
       uses: mikefarah/yq@47f4f8c7939f887e851b35f14def6741b8f5396e # v4.31.2
       with:
-        cmd: yq '.appVersion' helm/Chart.yaml
+        cmd: yq '.appVersion' charts/connaisseur/Chart.yaml
     - name: Get original image
       id: get_original_image_repository
       uses: mikefarah/yq@47f4f8c7939f887e851b35f14def6741b8f5396e # v4.31.2
       with:
-        cmd: yq '.kubernetes.deployment.image.repository' helm/values.yaml
+        cmd: yq '.kubernetes.deployment.image.repository' charts/connaisseur/values.yaml
     - name: Get context
       id: get_context
       run: |
         GHREF=${{ github.ref }}
         echo "github.ref is: ${GHREF}"
         CHART_VERSION=${{ steps.get_chart_version.outputs.result }}
-        COSIGN_VERSION=$(grep -Eo '^COSIGN_VERSION = .*' Makefile | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+')
         CONFIGURED_IMAGE_REPO=${{ steps.get_original_image_repository.outputs.result }}
         ORIGINAL_REGISTRY=$(echo "${CONFIGURED_IMAGE_REPO}" | cut -d "/" -f 1)
         ORIGINAL_REPO=$(echo "${CONFIGURED_IMAGE_REPO}" | cut -d "/" -f 2- | cut -d ":" -f 1)
@@ -84,7 +80,6 @@ runs:
           BUILD_REPO="${BUILD_REPO}-test"
         fi
 
-        echo COSIGN_VERSION=${COSIGN_VERSION} >> ${GITHUB_OUTPUT}
         echo CHART_VERSION=${CHART_VERSION} >> ${GITHUB_OUTPUT}
         echo ORIGINAL_REGISTRY=${ORIGINAL_REGISTRY} >> ${GITHUB_OUTPUT}
         echo ORIGINAL_REPO=${ORIGINAL_REPO} >> ${GITHUB_OUTPUT}
@@ -122,8 +117,7 @@ runs:
         echo "# :clipboard: Context" >> ${GITHUB_STEP_SUMMARY}
         echo "<table>" >> ${GITHUB_STEP_SUMMARY}
         echo "<tr><th>Build Context</th><th>Value</th></tr>" >> ${GITHUB_STEP_SUMMARY}
-        echo "<tr><td>Cosign version</td><td><code>${{ steps.get_context.outputs.COSIGN_VERSION }}</code></td></tr>" >> ${GITHUB_STEP_SUMMARY}
-        echo "<tr><td>Helm chart version</td><td><code>${{ steps.get_context.outputs.CHART_VERSION }}</td></tr>" >> ${GITHUB_STEP_SUMMARY}
+        echo "<tr><td>Helm chart version</td><td><code>${{ steps.get_context.outputs.CHART_VERSION }}</code></td></tr>" >> ${GITHUB_STEP_SUMMARY}
         echo "<tr><td>Original registry</td><td><code>${{ steps.get_context.outputs.ORIGINAL_REGISTRY }}</code></td></tr>" >> ${GITHUB_STEP_SUMMARY}
         echo "<tr><td>Original repository</td><td><code>${{ steps.get_context.outputs.ORIGINAL_REPO }}</code></td></tr>" >> ${GITHUB_STEP_SUMMARY}
         echo "<tr><td>Original tag</td><td><code>${{ steps.get_context.outputs.ORIGINAL_TAG }}</code></td></tr>" >> ${GITHUB_STEP_SUMMARY}
@@ -140,7 +134,6 @@ runs:
         echo "<details><summary>:pushpin: Context Variables References</summary>" >> ${GITHUB_STEP_SUMMARY}
         echo "(<context> job must run in workflow and <code>needs: [context]</code> mut be set for job)" >> ${GITHUB_STEP_SUMMARY}
         echo "<ul>" >> ${GITHUB_STEP_SUMMARY}
-        echo "<li>Cosign version: <pre><code>&#x24;&#x7b;&#x7b; needs.context.outputs.cosign_version }}</code></pre></li>" >> ${GITHUB_STEP_SUMMARY}
         echo "<li>Helm chart version: <pre><code>&#x24;&#x7b;&#x7b; needs.context.outputs.chart_version }}</code></pre></li>" >> ${GITHUB_STEP_SUMMARY}
         echo "<li>Original registry: <pre><code>&#x24;&#x7b;&#x7b; needs.context.outputs.original_registry &#x7d;&#x7d;</code></pre></li>" >> ${GITHUB_STEP_SUMMARY}
         echo "<li>Original repository: <pre><code>&#x24;&#x7b;&#x7b; needs.context.outputs.original_repo &#x7d;&#x7d;</code></pre></li>" >> ${GITHUB_STEP_SUMMARY}
@@ -157,4 +150,3 @@ runs:
         echo "" >> ${GITHUB_STEP_SUMMARY}
         echo "Let's start building :rocket:" >> ${GITHUB_STEP_SUMMARY}
       shell: bash
-
diff --git a/.github/actions/k3s-cluster/action.yaml b/.github/actions/k3s-cluster/action.yaml
@@ -71,4 +71,3 @@ runs:
         kubectl wait --for=condition=complete --timeout=300s job/helm-install-traefik -n kube-system || true
         kubectl rollout status --watch --timeout 300s deployment/traefik -n kube-system
       shell: bash
-
diff --git a/.github/actions/k8s-version-config/action.yaml b/.github/actions/k8s-version-config/action.yaml
@@ -19,7 +19,7 @@ runs:
     - name: Adjust Configuration
       run: |
         if [[ $(echo "${{ inputs.k8s-version }}" | tail -c 3) -lt "19" ]]; then
-          yq e 'del(.kubernetes.deployment.securityContext.seccompProfile)' -i helm/values.yaml
-          yq e '.kubernetes.deployment.annotations."seccomp.security.alpha.kubernetes.io/pod" = "runtime/default"' -i helm/values.yaml
+          yq e 'del(.kubernetes.deployment.securityContext.seccompProfile)' -i charts/connaisseur/values.yaml
+          yq e '.kubernetes.deployment.annotations."seccomp.security.alpha.kubernetes.io/pod" = "runtime/default"' -i charts/connaisseur/values.yaml
         fi
       shell: bash
diff --git a/.github/actions/safety/action.yaml b/.github/actions/safety/action.yaml
diff --git a/.github/actions/setup-notary/action.yaml b/.github/actions/setup-notary/action.yaml
@@ -15,7 +15,7 @@ runs:
       shell: bash
     - name: Trust root cert of notary instance
       run: |
-        sudo cp ./tests/data/notary_service_container/server/ca.crt /usr/local/share/ca-certificates/notary_root_ca.crt
+        sudo cp ./test/integration/notary_service_container/server/ca.crt /usr/local/share/ca-certificates/notary_root_ca.crt
         sudo update-ca-certificates
       shell: bash
     - name: Append notary ip to /etc/hosts
@@ -24,9 +24,9 @@ runs:
       shell: bash
     - name: Configure notary client
       run: |
-        ./tests/integration/notary_init.sh
+        ./test/integration/notary_init.sh
         docker pull docker.io/securesystemsengineering/testimage:self-hosted-notary-signed
         DIGEST=$(docker images --digests | grep self-hosted-notary-signed | awk '{print $3}')
         export DIGEST_WITHOUT_PREFIX=$(echo ${DIGEST#sha256:})
-        ./tests/integration/notary_addhash.sh ${DIGEST_WITHOUT_PREFIX}
+        ./test/integration/notary_addhash.sh ${DIGEST_WITHOUT_PREFIX}
       shell: bash
diff --git a/.github/actions/trivy-config/action.yaml b/.github/actions/trivy-config/action.yaml
@@ -1,5 +1,9 @@
 name: trivy-config
 description: 'Run Trivy on config'
+inputs:
+  output:
+    description: 'Trivy output either "sarif" (GITHUB_TOKEN with security-events:write) or print results as "table" and fail on error'
+    required: false
 runs:
   using: "composite"
   steps:
@@ -10,23 +14,40 @@ runs:
     - name: Render Helm charts
       run: |
         mkdir deployment
-        helm template helm > deployment/deployment.yaml
+        helm template charts/connaisseur > deployment/deployment.yaml
       shell: bash
     - name: Scan deployment.yaml
       uses: aquasecurity/trivy-action@fbd16365eb88e12433951383f5e99bd901fc618f # v0.12.0
+      if: inputs.output == 'table'
+      with:
+        scan-type: "config"
+        scan-ref: "deployment"
+        format: 'table'
+    - name: Scan Dockerfiles
+      uses: aquasecurity/trivy-action@fbd16365eb88e12433951383f5e99bd901fc618f # v0.12.0
+      if: inputs.output == 'table'
+      with:
+        scan-type: "config"
+        scan-ref: "build"
+        format: 'table'
+    - name: Scan deployment.yaml
+      uses: aquasecurity/trivy-action@fbd16365eb88e12433951383f5e99bd901fc618f # v0.12.0
+      if: inputs.output == 'sarif'
       with:
         scan-type: "config"
         scan-ref: "deployment"
         format: 'sarif'
         output: 'reports/trivy-k8s-results.sarif'
     - name: Scan Dockerfiles
       uses: aquasecurity/trivy-action@fbd16365eb88e12433951383f5e99bd901fc618f # v0.12.0
+      if: inputs.output == 'sarif'
       with:
         scan-type: "config"
-        scan-ref: "docker"
+        scan-ref: "build"
         format: 'sarif'
         output: 'reports/trivy-docker-results.sarif'
     - name: Upload
       uses: github/codeql-action/upload-sarif@32dc499307d133bb5085bae78498c0ac2cf762d5 # v2.2.5
+      if: inputs.output == 'sarif'
       with:
         sarif_file: 'reports'
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -1,19 +1,18 @@
 version: 2
 updates:
-  - package-ecosystem: "pip"
+  - package-ecosystem: "gomod"
     directory: "/"
     schedule:
       interval: "daily"
     commit-message:
       prefix: "update"
-    insecure-external-code-execution: "deny"
     target-branch: "develop"
     groups:
-      pip-packages:
+      gomod-packages:
         patterns:
           - "*"
   - package-ecosystem: "docker"
-    directory: "/docker"
+    directory: "/build"
     schedule:
       interval: "daily"
     commit-message:
@@ -34,4 +33,3 @@ updates:
       gh-actions-packages:
         patterns:
           - "*"
-
diff --git a/.github/workflows/.reusable-build.yml b/.github/workflows/.reusable-build.yml
@@ -1,17 +1,15 @@
 name: build
 
+permissions: {} 
+
 on:
   workflow_call:
     inputs:
-      build:
-        description: "Run actual build job?"
-        type: boolean
-        required: false
-        default: true
+      skip:
+        description: "Want to skip running certain jobs 'none', 'non-required', 'all'?"
+        type: string
+        default: "none"
     outputs:
-      cosign_version: 
-        description: "Cosign version used for building Connaisseur image"
-        value: ${{ jobs.context.outputs.cosign_version }}
       cosign_public_key: 
         description: "Cosign public key used for signing Connaisseur image"
         value: ${{ jobs.build.outputs.cosign_public_key }}
@@ -49,19 +47,17 @@ on:
         description: "Repository- and workflow-specific build labels"
         value: ${{ jobs.context.outputs.build_labels }}
 
-permissions: {}
-
 jobs:
   context:
     runs-on: ubuntu-latest
+    if: inputs.skip != 'all'
     permissions: {}
     outputs:
-      cosign_version: ${{ steps.get_context.outputs.cosign_version }}
       chart_version: ${{ steps.get_context.outputs.chart_version }}
       original_registry: ${{ steps.get_context.outputs.original_registry }}
       original_repo: ${{ steps.get_context.outputs.original_repo }}
-      original_tag: ${{ steps.get_context.outputs.original_tag }}
       original_image: ${{ steps.get_context.outputs.original_image }}
+      original_tag: ${{ steps.get_context.outputs.original_tag }}
       build_registry: ${{ steps.get_context.outputs.build_registry }}
       build_repo: ${{ steps.get_context.outputs.build_repo }}
       build_tag: ${{ steps.get_context.outputs.build_tag }}
@@ -77,7 +73,9 @@ jobs:
 
   build:
     runs-on: ubuntu-latest
-    if: inputs.build
+    if: |
+      inputs.skip != 'non-required' &&
+      inputs.skip != 'all'
     needs: [context]
     permissions:
       packages: write
@@ -97,6 +95,5 @@ jobs:
           image_labels: ${{ needs.context.outputs.build_labels }}
           repo_owner: ${{ github.repository_owner }}
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-          cosign_version: ${{ needs.context.outputs.cosign_version }}
           cosign_private_key: ${{ secrets.COSIGN_PRIVATE_KEY }}
           cosign_password: ${{ secrets.COSIGN_PASSWORD }}