OCPQE-27908: refactor step registry structure and add unmanged tls cert step

ci-operator/config/quay/quay-tests/quay-quay-tests-master__ocp-417-quay.yaml

@@ -88,6 +88,25 @@ tests:
     - ref: quay-tests-qbo-qe-test
     - ref: quay-tests-cso-qe-test
     workflow: firewatch-cucushift-installer-rehearse-aws-ipi
+- as: quay-e2e-tests-quay313-ocp417-unmanged-tls
+  cron: 0 23 * * 6
+  steps:
+    cluster_profile: aws-qe
+    env:
+      BASE_DOMAIN: qe.devcluster.openshift.com
+      COMPUTE_NODE_TYPE: m5.2xlarge
+      ODF_OPERATOR_CHANNEL: stable-4.17
+      QUAY_INDEX_IMAGE_BUILD: brew.registry.redhat.io/rh-osbs/iib:886942
+      QUAY_OPERATOR_CHANNEL: stable-3.13
+      QUAY_OPERATOR_SOURCE: brew-operator-catalog
+    test:
+    - ref: quay-tests-enable-quay-catalogsource
+    - ref: quay-tests-provisioning-storage-odf
+    - ref: quay-tests-provisioning-tls
+    - ref: quay-tests-deploy-quay-operator
+    - ref: quay-tests-deploy-registry-unmanged-tls
+    - ref: quay-tests-test-quay-e2e
+    workflow: cucushift-installer-rehearse-aws-ipi
 zz_generated_metadata:
   branch: master
   org: quay

ci-operator/config/quay/quay-tests/quay-quay-tests-master__quay-operator-test.yaml

@@ -40,6 +40,7 @@ tests:
       QUAY_OPERATOR_CHANNEL: stable-3.13
       QUAY_OPERATOR_TESTCASE: Quay-High|Quay-Medium
     test:
+    - ref: quay-tests-provisioning-storage-odf
     - ref: quay-tests-provisioning-aws-unmanaged-component
     - ref: quay-tests-quay-operator-test
     workflow: quay-tests-cucushift-installer-rehearse-aws-ipi-operator

ci-operator/config/quay/quay-tests/quay-quay-tests-master__quay-upgrade.yaml

@@ -53,6 +53,7 @@ tests:
       QUAY_UPGRADE_TESTCASE: Quay-Upgrade-High|Quay-Upgrade-Medium
       QUAYREGISTRY_QUAY_VERSION: "3.13"
     test:
+    - ref: quay-tests-provisioning-storage-odf
     - ref: quay-tests-quay-upgrade
     workflow: cucushift-installer-rehearse-aws-ipi
   timeout: 8h0m0s

ci-operator/jobs/quay/quay-tests/quay-quay-tests-master-periodics.yaml

@@ -825,6 +825,88 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build03
+  cron: 0 23 * * 6
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: master
+    org: quay
+    repo: quay-tests
+  labels:
+    ci-operator.openshift.io/cloud: aws
+    ci-operator.openshift.io/cloud-cluster-profile: aws-qe
+    ci-operator.openshift.io/variant: ocp-417-quay
+    ci.openshift.io/generator: prowgen
+    job-release: "4.17"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-quay-quay-tests-master-ocp-417-quay-quay-e2e-tests-quay313-ocp417-unmanged-tls
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --oauth-token-path=/usr/local/github-credentials/oauth
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=quay-e2e-tests-quay313-ocp417-unmanged-tls
+      - --variant=ocp-417-quay
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /usr/local/github-credentials
+        name: github-credentials-openshift-ci-robot-private-git-cloner
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: github-credentials-openshift-ci-robot-private-git-cloner
+      secret:
+        secretName: github-credentials-openshift-ci-robot-private-git-cloner
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build03
   cron: 0 12 * * 2

ci-operator/step-registry/quay-tests/deploy-quay-operator/OWNERS

@@ -0,0 +1,3 @@
+approvers:
+- LiZhang19817
+- dongboyan77

ci-operator/step-registry/quay-tests/deploy-quay-operator/quay-tests-deploy-quay-operator-commands.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+QUAY_OPERATOR_CHANNEL="$QUAY_OPERATOR_CHANNEL"
+QUAY_OPERATOR_SOURCE="$QUAY_OPERATOR_SOURCE"
+
+#Deploy Quay Operator to OCP namespace '${QUAYNAMESPACE}'
+cat <<EOF | oc apply -f -
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ${QUAYNAMESPACE}
+EOF
+
+cat <<EOF | oc apply -f -
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  name: quay
+  namespace: ${QUAYNAMESPACE}
+spec:
+  targetNamespaces:
+  - ${QUAYNAMESPACE}
+EOF
+
+SUB=$(
+    cat <<EOF | oc apply -f - -o jsonpath='{.metadata.name}'
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: quay-operator
+  namespace: ${QUAYNAMESPACE}
+spec:
+  installPlanApproval: Automatic
+  name: quay-operator
+  channel: $QUAY_OPERATOR_CHANNEL
+  source: $QUAY_OPERATOR_SOURCE
+  sourceNamespace: openshift-marketplace
+EOF
+)
+
+echo "The Quay Operator subscription is $SUB"
+
+for _ in {1..60}; do
+    CSV=$(oc -n ${QUAYNAMESPACE} get subscription "$SUB" -o jsonpath='{.status.installedCSV}' || true)
+    if [[ -n "$CSV" ]]; then
+        if [[ "$(oc -n ${QUAYNAMESPACE} get csv "$CSV" -o jsonpath='{.status.phase}')" == "Succeeded" ]]; then
+            echo "Quay ClusterServiceVersion \"$CSV\" is ready"
+            break
+        fi
+    fi
+    sleep 10
+done
+echo "Quay Operator is deployed successfully"
+

ci-operator/step-registry/quay-tests/deploy-quay-operator/quay-tests-deploy-quay-operator-ref.metadata.json

@@ -0,0 +1,9 @@
+{
+	"path": "quay-tests/deploy-quay-operator/quay-tests-deploy-quay-operator-ref.yaml",
+	"owners": {
+		"approvers": [
+			"LiZhang19817",
+			"dongboyan77"
+		]
+	}
+}

ci-operator/step-registry/quay-tests/deploy-quay-operator/quay-tests-deploy-quay-operator-ref.yaml

@@ -0,0 +1,24 @@
+ref:
+  as: quay-tests-deploy-quay-operator
+  cli: latest
+  from_image:
+    name: quay-test-console
+    namespace: ci
+    tag: latest
+  commands: quay-tests-deploy-quay-operator-commands.sh
+  resources:
+    requests:
+      cpu: 10m
+      memory: 100Mi
+  documentation: |-
+    Deploy Quay Operator
+  env:
+  - name: QUAY_OPERATOR_CHANNEL
+    documentation: The quay operator channel
+    default: "stable-3.13"
+  - name: QUAY_OPERATOR_SOURCE
+    documentation: The quay operator source
+    default: "redhat-operators"
+  - name: QUAYNAMESPACE
+    documentation: The Quay installed namespace
+    default: "quay-enterprise"  

ci-operator/step-registry/quay-tests/deploy-registry-unmanged-tls/OWNERS

@@ -0,0 +1,3 @@
+approvers:
+- LiZhang19817
+- dongboyan77

ci-operator/step-registry/quay-tests/deploy-registry-unmanged-tls/quay-tests-deploy-registry-unmanged-tls-commands.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+#Get the credentials and Email of new Quay User
+QUAY_USERNAME=$(cat /var/run/quay-qe-quay-secret/username)
+QUAY_PASSWORD=$(cat /var/run/quay-qe-quay-secret/password)
+QUAY_EMAIL=$(cat /var/run/quay-qe-quay-secret/email)
+
+echo "Create registry ${QUAYREGISTRY} in ns ${QUAYNAMESPACE}"
+
+#create secret bundle with odf/noobaa
+cat >>config.yaml <<EOF
+CREATE_PRIVATE_REPO_ON_PUSH: true
+CREATE_NAMESPACE_ON_PUSH: true
+FEATURE_EXTENDED_REPOSITORY_NAMES: true
+FEATURE_QUOTA_MANAGEMENT: true
+FEATURE_AUTO_PRUNE: true
+FEATURE_PROXY_CACHE: true
+FEATURE_USER_INITIALIZE: true
+PERMANENTLY_DELETE_TAGS: true
+RESET_CHILD_MANIFEST_EXPIRATION: true
+FEATURE_PROXY_STORAGE: true
+IGNORE_UNKNOWN_MEDIATYPES: true
+FEATURE_UI_V2: true
+FEATURE_SUPERUSERS_FULL_ACCESS: true
+SUPER_USERS:
+  - quay
+FEATURE_ANONYMOUS_ACCESS: true
+BROWSER_API_CALLS_XHR_ONLY: false
+FEATURE_USERNAME_CONFIRMATION: false
+AUTHENTICATION_TYPE: Database
+FEATURE_LISTEN_IP_VERSION: IPv4
+REPO_MIRROR_ROLLBACK: false
+AUTOPRUNE_TASK_RUN_MINIMUM_INTERVAL_MINUTES: 1
+DEFAULT_TAG_EXPIRATION: 2w
+TAG_EXPIRATION_OPTIONS:
+  - 2w
+  - 1d
+EOF
+
+# Create secret bundle upon env variable TLS, by default it is false.
+# tls certs get from $SHARED_DIR folder
+if [[ "$TLS" == "true" ]]; then
+  oc create secret generic -n "${QUAYNAMESPACE}" --from-file config.yaml=./config.yaml config-bundle-secret
+elif [[ "$TLS" = "false" ]]; then
+  oc create secret generic -n "${QUAYNAMESPACE}" --from-file config.yaml=./config.yaml --from-file ssl.cert="$SHARED_DIR"/ssl.cert \
+    --from-file ssl.key="$SHARED_DIR"/ssl.key --from-file extra_ca_cert_build_cluster.crt="$SHARED_DIR"/build_cluster.crt \
+    config-bundle-secret
+fi
+
+#Deploy Quay registry, here disable monitoring component
+echo "Creating Quay registry..." >&2
+cat <<EOF | oc apply -f -
+apiVersion: quay.redhat.com/v1
+kind: QuayRegistry
+metadata:
+  name: ${QUAYREGISTRY}
+  namespace: ${QUAYNAMESPACE}
+spec:
+  configBundleSecret: config-bundle-secret
+  components:
+  - kind: objectstorage
+    managed: true
+  - kind: monitoring
+    managed: false
+  - kind: horizontalpodautoscaler
+    managed: true
+  - kind: quay
+    managed: true
+  - kind: mirror
+    managed: true
+  - kind: clair
+    managed: true
+  - kind: tls
+    managed: ${TLS}
+  - kind: route
+    managed: true
+EOF
+
+for i in {1..60}; do
+  if [[ "$(oc -n ${QUAYNAMESPACE} get quayregistry ${QUAYREGISTRY} -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' || true)" == "True" ]]; then
+    echo "Quay is in ready status" >&2
+    oc -n ${QUAYNAMESPACE} get quayregistries -o yaml >"$ARTIFACT_DIR/quayregistries.yaml"
+    oc get quayregistry ${QUAYREGISTRY} -n ${QUAYNAMESPACE} -o jsonpath='{.status.registryEndpoint}' >"$SHARED_DIR"/quayroute || true
+    quay_route=$(oc get quayregistry ${QUAYREGISTRY} -n ${QUAYNAMESPACE} -o jsonpath='{.status.registryEndpoint}') || true
+    echo "Quay Route is $quay_route"
+    curl -k $quay_route/api/v1/discovery | jq > "$SHARED_DIR"/quay_api_discovery
+    cp "$SHARED_DIR"/quay_api_discovery "$ARTIFACT_DIR"/quay_api_discovery || true
+
+    curl -k -X POST $quay_route/api/v1/user/initialize --header 'Content-Type: application/json' \
+      --data '{ "username": "'$QUAY_USERNAME'", "password": "'$QUAY_PASSWORD'", "email": "'$QUAY_EMAIL'", "access_token": true }' | jq '.access_token' | tr -d '"' | tr -d '\n' >"$SHARED_DIR"/quay_oauth2_token || true
+
+    exit 0
+  fi
+  sleep 15
+  echo "Wait for quay registry ready $((i*15))s"
+done
+echo "Timed out waiting for Quay to become ready afer 15 mins" >&2

ci-operator/step-registry/quay-tests/deploy-registry-unmanged-tls/quay-tests-deploy-registry-unmanged-tls-ref.metadata.json

@@ -0,0 +1,9 @@
+{
+	"path": "quay-tests/deploy-registry-unmanged-tls/quay-tests-deploy-registry-unmanged-tls-ref.yaml",
+	"owners": {
+		"approvers": [
+			"LiZhang19817",
+			"dongboyan77"
+		]
+	}
+}

ci-operator/step-registry/quay-tests/deploy-registry-unmanged-tls/quay-tests-deploy-registry-unmanged-tls-ref.yaml

@@ -0,0 +1,31 @@
+ref:
+  as: quay-tests-deploy-registry-unmanged-tls
+  cli: latest
+  from_image:
+    name: quay-test-console
+    namespace: ci
+    tag: latest
+  commands: quay-tests-deploy-registry-unmanged-tls-commands.sh
+  resources:
+    requests:
+      cpu: 10m
+      memory: 100Mi
+  credentials:
+  - namespace: test-credentials
+    name: quay-qe-aws-secret
+    mount_path: /var/run/quay-qe-aws-secret
+  - namespace: test-credentials
+    name: quay-qe-quay-secret
+    mount_path: /var/run/quay-qe-quay-secret
+  documentation: |-
+    Deploy Quay registry with unmanged tls component
+  env:
+  - name: TLS
+    documentation: The quay registry tls comoponent managed status - "false" or "true"
+    default: "false"
+  - name: QUAYREGISTRY
+    documentation: The quay registry name
+    default: "quay"
+  - name: QUAYNAMESPACE
+    documentation: The Quay installed namespace
+    default: "quay-enterprise"    

ci-operator/step-registry/quay-tests/provisioning/OWNERS

@@ -0,0 +1,3 @@
+approvers:
+- LiZhang19817
+- dongboyan77

ci-operator/step-registry/quay-tests/provisioning/storage/OWNERS

@@ -0,0 +1,3 @@
+approvers:
+- LiZhang19817
+- dongboyan77

ci-operator/step-registry/quay-tests/provisioning/storage/odf/OWNERS

@@ -0,0 +1,3 @@
+approvers:
+- LiZhang19817
+- dongboyan77