Migrate Tekton Chains to an ExternalSecret

In order to facilitate backup/restore of a cluster, the signing secret
is now stored externally and deployed on the cluster using an ExternalSecret.

The original behavior is maintained for local development.

rh-pre-commit.version: 2.1.0
rh-pre-commit.check-secrets: ENABLED

developer/openshift/apps/pipeline-service.sh

@@ -17,10 +17,10 @@ setup_work_dir() {
   setup_tekton_results | indent
 }
 
-configure_argocd_apps(){
+configure_argocd_apps() {
   echo -n "- Updating source repository to '${GIT_URL}/tree/$GIT_REF': "
   # Patch the url/branch to target the expected repository/branch
-  yq --inplace ".resources[1] = \"$GIT_URL/developer/openshift/gitops/argocd?ref=$GIT_REF\"" "$manifests_dir/kustomization.yaml"
+  yq --inplace ".resources[0] = \"$GIT_URL/developer/openshift/gitops/argocd?ref=$GIT_REF\"" "$manifests_dir/kustomization.yaml"
   yq ".patches[] | .path" "$manifests_dir/kustomization.yaml" | while read -r patch; do
     yq --inplace ".spec.source.repoURL = \"$GIT_URL\", .spec.source.targetRevision = \"$GIT_REF\"" "$manifests_dir/$patch"
   done
@@ -42,7 +42,7 @@ get_tekton_results_credentials() {
     TEKTON_RESULTS_DATABASE_PASSWORD="$(yq ".tekton_results_db.password // \"$(openssl rand -base64 20)\"" "$CONFIG")"
     TEKTON_RESULTS_S3_USER="$(yq '.tekton_results_s3.user // "minio"' "$CONFIG")"
     TEKTON_RESULTS_S3_PASSWORD="$(yq ".tekton_results_s3.password // \"$(openssl rand -base64 20)\"" "$CONFIG")"
-    cat << EOF > "$tekton_results_credentials"
+    cat <<EOF >"$tekton_results_credentials"
 ---
 db_password: $TEKTON_RESULTS_DATABASE_PASSWORD
 db_user: $TEKTON_RESULTS_DATABASE_USER
@@ -56,7 +56,7 @@ EOF
   TEKTON_RESULTS_S3_PASSWORD="$(yq ".s3_password" "$tekton_results_credentials")"
 }
 
-patch_tekton_results_manifests(){
+patch_tekton_results_manifests() {
   yq --inplace "
     .data.[\"db.password\"]=\"$(echo -n "$TEKTON_RESULTS_DATABASE_PASSWORD" | base64)\",
     .data.[\"db.user\"]=\"$(echo -n "$TEKTON_RESULTS_DATABASE_USER" | base64)\"
@@ -65,7 +65,8 @@ patch_tekton_results_manifests(){
     .data.aws_access_key_id=\"$(echo -n "$TEKTON_RESULTS_S3_USER" | base64)\",
     .data.aws_secret_access_key=\"$(echo -n "$TEKTON_RESULTS_S3_PASSWORD" | base64)\"
   " "$WORK_DIR/environment/compute/tekton-results/tekton-results-s3-secret.yaml"
-  string_data="$(cat <<EOF | base64
+  string_data="$(
+    cat <<EOF | base64
 export MINIO_ROOT_USER="$TEKTON_RESULTS_S3_USER"
 export MINIO_ROOT_PASSWORD="$TEKTON_RESULTS_S3_PASSWORD"
 export MINIO_STORAGE_CLASS_STANDARD="EC:2"

developer/openshift/gitops/local/kustomization.yaml

@@ -2,8 +2,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - https://github.com/owner/repository.git/path/gitops/argocd?ref=branch # Keep this item first in the list.
+  - openshift-pipelines
   - tekton-results
-  - https://github.com/owner/repository.git/path/gitops/argocd?ref=branch
 patches:
   - path: patch-pipeline-service.yaml
   - path: patch-pipeline-service-storage.yaml

developer/openshift/gitops/local/openshift-pipelines/chains-secrets-config.yaml

@@ -0,0 +1,55 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: tekton-chains-signing-secret
+  namespace: openshift-pipelines
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation  # Delete the job so it can be recreated and updated during the sync.
+spec:
+  template:
+    spec:
+      containers:
+        - name: chains-secret-generation
+          image: quay.io/redhat-appstudio/appstudio-utils:dbbdd82734232e6289e8fbae5b4c858481a7c057
+          imagePullPolicy: Always
+          command:
+            - /bin/bash
+            - -c
+            - |
+              set -o errexit
+              set -o nounset
+              set -o pipefail
+
+              namespace="openshift-pipelines"
+              secret="signing-secrets"
+
+              if [ "$(kubectl get secret "$secret" -n "$namespace" -o jsonpath='{.data}' --ignore-not-found --allow-missing-template-keys)" != "" ]; then
+                echo "Signing secret exists and is non-empty."
+              else
+                # Delete secret/signing-secrets if already exists since by default cosign creates immutable secrets
+                kubectl delete secrets "$secret" -n "$namespace" --ignore-not-found=true
+
+                # To make this run conveniently without user input let's create a random password
+                RANDOM_PASS=$( openssl rand -base64 30 )
+
+                # Generate the key pair secret directly in the cluster.
+                echo "Generating k8s secret/$secret in $namespace with key-pair"
+                env COSIGN_PASSWORD=$RANDOM_PASS cosign generate-key-pair "k8s://$namespace/$secret"
+              fi
+          resources:
+            requests:
+              cpu: 10m
+              memory: 10Mi
+            limits:
+              cpu: 100m
+              memory: 250Mi
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+      dnsPolicy: ClusterFirst
+      restartPolicy: OnFailure
+      terminationGracePeriodSeconds: 30
+      serviceAccount: chains-secrets-admin
+      serviceAccountName: chains-secrets-admin

developer/openshift/gitops/local/openshift-pipelines/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - chains-secrets-config.yaml
+  - namespace.yaml

developer/openshift/gitops/local/openshift-pipelines/namespace.yaml

@@ -0,0 +1,10 @@
+# Solves https://issues.redhat.com/browse/PLNSRVCE-1620
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openshift-pipelines
+  labels:
+    argocd.argoproj.io/managed-by: openshift-gitops
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"

operator/gitops/argocd/pipeline-service/openshift-pipelines/kustomization.yaml

@@ -9,9 +9,9 @@ resources:
   - tekton-config.yaml
   - config-logging.yaml
   - chains-service-monitor.yaml
+  - public-key-secret.yaml
   - bugfix-pac-gitauth-secrets.yaml
   # Manually add ConfigMap and Service until PLNSRVCE-1359 is fixed
   - chains-observability-service.yaml
   - chains-public-key-viewer.yaml
-  - chains-secrets-config.yaml
   - namespace.yaml

operator/gitops/argocd/pipeline-service/openshift-pipelines/chains-secrets-config.yaml → operator/gitops/argocd/pipeline-service/openshift-pipelines/public-key-secret.yaml

@@ -46,7 +46,7 @@ subjects:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: tekton-chains-signing-secret
+  name: tekton-chains-public-key
   namespace: openshift-pipelines
   annotations:
     argocd.argoproj.io/sync-wave: "1"
@@ -55,7 +55,7 @@ spec:
   template:
     spec:
       containers:
-        - name: chains-secret-generation
+        - name: chains-public-key-generation
           image: quay.io/redhat-appstudio/appstudio-utils:dbbdd82734232e6289e8fbae5b4c858481a7c057
           imagePullPolicy: Always
           command:
@@ -69,30 +69,9 @@ spec:
               namespace="openshift-pipelines"
               secret="signing-secrets"
 
-              cd /tmp
-
-              if [ "$(kubectl get secret "$secret" -n "$namespace" -o jsonpath='{.data}' --ignore-not-found --allow-missing-template-keys)" != "" ]; then
-                echo "Signing secret exists and is non-empty."
-              else
-                # Delete secret/signing-secrets if already exists since by default cosign creates immutable secrets
-                kubectl delete secrets "$secret" -n "$namespace" --ignore-not-found=true
-
-                # To make this run conveniently without user input let's create a random password
-                RANDOM_PASS=$( openssl rand -base64 30 )
-
-                # Generate the key pair secret directly in the cluster.
-                # The secret should be created as immutable.
-                echo "Generating k8s secret/$secret in $namespace with key-pair"
-                env COSIGN_PASSWORD=$RANDOM_PASS cosign generate-key-pair "k8s://$namespace/$secret"
-              fi
-
-              # If the secret is not marked as immutable, make it so.
-              if [ "$(kubectl get secret "$secret" -n "$namespace" -o jsonpath='{.immutable}')" != "true" ]; then
-                echo "Making secret immutable"
-                kubectl patch secret "$secret" -n "$namespace" --dry-run=client -o yaml \
-                  --patch='{"immutable": true}' \
-                | kubectl apply -f -
-              fi
+              while [ "$(kubectl get secret "$secret" -n "$namespace" -o jsonpath='{.data}' --ignore-not-found --allow-missing-template-keys)" == "" ]; do
+                echo "Signing secret is empty."
+              done
 
               echo "Generating/updating the secret with the public key"
               kubectl create secret generic public-key \

operator/test/test.sh

@@ -89,7 +89,7 @@ setup_test() {
 wait_for_pipeline() {
   if ! kubectl wait --for=condition=succeeded "$1" -n "$2" --timeout 300s >"$DEBUG_OUTPUT"; then
     echo "[ERROR] Pipeline failed to complete successful" >&2
-    kubectl get pipelineruns "$1" -n "$2" >"$DEBUG_OUTPUT"
+    kubectl get "$1" -n "$2" >"$DEBUG_OUTPUT"
     exit 1
   fi
 }
@@ -160,11 +160,6 @@ test_chains() {
     echo "[ERROR] Secret does not exist" >&2
     exit 1
   fi
-  if [ "$(kubectl get secret signing-secrets -n openshift-pipelines -o jsonpath='{.immutable}')" != "true" ]; then
-    echo "Failed"
-    echo "[ERROR] Secret is not immutable" >&2
-    exit 1
-  fi
   echo "OK"
 
   # Trigger the pipeline
@@ -231,20 +226,6 @@ test_chains() {
     exit 1
   fi
 
-  # TODO: Reactivate on step 2/3 of the migration.
-  # This test is not critical until we ask EC to use the openshift-pipelines namespace.
-  # echo -n "  - Public key migration: "
-  # pipeline_name=$(kubectl create -f "$SCRIPT_DIR/manifests/test/tekton-chains/public-key-migration.yaml" -n "$NAMESPACE" | cut -d' ' -f1)
-  # wait_for_pipeline "$pipeline_name" "$NAMESPACE"
-  # if [ "$(kubectl get "$pipeline_name" -n "$NAMESPACE" \
-  #   -o 'jsonpath={.status.conditions[0].reason}')" = "Succeeded" ]; then
-  #   echo "OK"
-  # else
-  #   echo "Failed"
-  #   echo "[ERROR] Public key is not accessible" >&2
-  #   exit 1
-  # fi
-
   echo -n "  - Metrics: "
   prName="$(kubectl create -n "$NAMESPACE" -f "$SCRIPT_DIR/manifests/test/tekton-chains/tekton-chains-metrics.yaml" | awk '{print $1}')"
   wait_for_pipeline "$prName" "$NAMESPACE"