From 75a8f5319da92b3bb14863023d5ec04c2612106b Mon Sep 17 00:00:00 2001 From: Tomas Nozicka Date: Fri, 28 Jun 2024 14:44:50 +0200 Subject: [PATCH] Sync --- .../grafana/v1alpha1/deployment.yaml | 14 +++---- deploy/manager/dev/50_scyllacluster.yaml | 6 +++ deploy/manager/prod/50_scyllacluster.yaml | 6 +++ deploy/operator.yaml | 41 +++++++++++++------ deploy/operator/00_clusterrole_def.yaml | 32 +++++++++------ ..._scyllacluster_member_clusterrole_def.yaml | 9 ++++ examples/eks/nodeconfig-alpha.yaml | 2 +- examples/gke/nodeconfig-alpha.yaml | 2 +- .../10_haproxy-ingress.role.yaml | 13 ++++++ ...inding.yaml => 20_clusterrolebinding.yaml} | 0 .../20_haproxy-ingress.rolebinding.yaml | 12 ++++++ ...ng.yaml => 20_prometheus.rolebinding.yaml} | 0 ...oy.yaml => 50_haproxy-ingress.deploy.yaml} | 0 ...=> 50_ingress-default-backend.deploy.yaml} | 0 ....deploy.yaml => 50_prometheus.deploy.yaml} | 0 .../10_provisioner_clusterrole.yaml | 8 ++++ hack/ci-deploy.sh | 10 +++-- hack/run-e2e-remote.sh | 12 ++++++ helm/scylla-manager/values.yaml | 6 +++ .../templates/clusterrole_def.yaml | 32 +++++++++------ .../scyllacluster_member_clusterrole_def.yaml | 9 ++++ helm/scylla/values.yaml | 6 +++ pkg/controller/nodeconfig/resource.go | 20 ++++++++- .../fixture/scylla/scyllacluster.yaml.tmpl | 6 +++ test/e2e/framework/framework.go | 21 ++++++++++ 25 files changed, 218 insertions(+), 49 deletions(-) create mode 100644 examples/third-party/haproxy-ingress/10_haproxy-ingress.role.yaml rename examples/third-party/haproxy-ingress/{10_clusterrolebinding.yaml => 20_clusterrolebinding.yaml} (100%) create mode 100644 examples/third-party/haproxy-ingress/20_haproxy-ingress.rolebinding.yaml rename examples/third-party/haproxy-ingress/{10_prometheus.rolebinding.yaml => 20_prometheus.rolebinding.yaml} (100%) rename examples/third-party/haproxy-ingress/{10_haproxy-ingress.deploy.yaml => 50_haproxy-ingress.deploy.yaml} (100%) rename examples/third-party/haproxy-ingress/{10_ingress-default-backend.deploy.yaml => 50_ingress-default-backend.deploy.yaml} (100%) rename examples/third-party/haproxy-ingress/{10_prometheus.deploy.yaml => 50_prometheus.deploy.yaml} (100%) create mode 100755 hack/run-e2e-remote.sh diff --git a/assets/monitoring/grafana/v1alpha1/deployment.yaml b/assets/monitoring/grafana/v1alpha1/deployment.yaml index 8f15596d38b..961c3671f92 100644 --- a/assets/monitoring/grafana/v1alpha1/deployment.yaml +++ b/assets/monitoring/grafana/v1alpha1/deployment.yaml @@ -123,9 +123,9 @@ spec: securityContext: allowPrivilegeEscalation: false privileged: false - runAsNonRoot: true - runAsUser: 472 - runAsGroup: 472 +# runAsNonRoot: true +# runAsUser: 472 +# runAsGroup: 472 capabilities: drop: - ALL @@ -158,9 +158,9 @@ spec: emptyDir: sizeLimit: 100Mi securityContext: - runAsNonRoot: true - runAsUser: 472 - runAsGroup: 472 - fsGroup: 472 +# runAsNonRoot: true +# runAsUser: 472 +# runAsGroup: 472 +# fsGroup: 472 seccompProfile: type: RuntimeDefault diff --git a/deploy/manager/dev/50_scyllacluster.yaml b/deploy/manager/dev/50_scyllacluster.yaml index 0abffc3a2ba..a772c480397 100644 --- a/deploy/manager/dev/50_scyllacluster.yaml +++ b/deploy/manager/dev/50_scyllacluster.yaml @@ -25,3 +25,9 @@ spec: requests: cpu: 10m memory: 100Mi + placement: + tolerations: + - key: role + operator: Equal + value: scylla-clusters + effect: NoSchedule diff --git a/deploy/manager/prod/50_scyllacluster.yaml b/deploy/manager/prod/50_scyllacluster.yaml index f0b8e49bd5d..6091ca20303 100644 --- a/deploy/manager/prod/50_scyllacluster.yaml +++ b/deploy/manager/prod/50_scyllacluster.yaml @@ -25,3 +25,9 @@ spec: requests: cpu: 1 memory: 200Mi + placement: + tolerations: + - key: role + operator: Equal + value: scylla-clusters + effect: NoSchedule diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 31a9c75ebcf..4e33a91dcd3 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -61,6 +61,12 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - update - apiGroups: - "" resources: @@ -96,6 +102,7 @@ rules: resources: - statefulsets - daemonsets + - daemonsets/finalizers - deployments verbs: - create @@ -115,7 +122,9 @@ rules: - scylla.scylladb.com resources: - scyllaclusters + - scyllaclusters/finalizers - scylladbmonitorings + - scylladbmonitorings/finalizers verbs: - create - delete @@ -139,6 +148,7 @@ rules: - "" resources: - configmaps + - configmaps/finalizers verbs: - create - delete @@ -175,6 +185,8 @@ rules: - scylla.scylladb.com resources: - nodeconfigs + - nodeconfigs/status + - nodeconfigs/finalizers verbs: - create - delete @@ -210,18 +222,6 @@ rules: - patch - update - watch -- apiGroups: - - scylla.scylladb.com - resources: - - nodeconfigs/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - batch resources: @@ -284,6 +284,14 @@ rules: - patch - update - delete +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use --- apiVersion: v1 @@ -5094,6 +5102,7 @@ rules: - "" resources: - configmaps + - configmaps/finalizers verbs: - get - list @@ -5123,6 +5132,14 @@ rules: - scyllaclusters verbs: - get +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/operator/00_clusterrole_def.yaml b/deploy/operator/00_clusterrole_def.yaml index a819a2f397a..823a28cdb59 100644 --- a/deploy/operator/00_clusterrole_def.yaml +++ b/deploy/operator/00_clusterrole_def.yaml @@ -51,6 +51,12 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - update - apiGroups: - "" resources: @@ -86,6 +92,7 @@ rules: resources: - statefulsets - daemonsets + - daemonsets/finalizers - deployments verbs: - create @@ -105,7 +112,9 @@ rules: - scylla.scylladb.com resources: - scyllaclusters + - scyllaclusters/finalizers - scylladbmonitorings + - scylladbmonitorings/finalizers verbs: - create - delete @@ -129,6 +138,7 @@ rules: - "" resources: - configmaps + - configmaps/finalizers verbs: - create - delete @@ -165,6 +175,8 @@ rules: - scylla.scylladb.com resources: - nodeconfigs + - nodeconfigs/status + - nodeconfigs/finalizers verbs: - create - delete @@ -200,18 +212,6 @@ rules: - patch - update - watch -- apiGroups: - - scylla.scylladb.com - resources: - - nodeconfigs/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - batch resources: @@ -274,3 +274,11 @@ rules: - patch - update - delete +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use diff --git a/deploy/operator/00_scyllacluster_member_clusterrole_def.yaml b/deploy/operator/00_scyllacluster_member_clusterrole_def.yaml index 4058a7f0d81..f49ff22f3b5 100644 --- a/deploy/operator/00_scyllacluster_member_clusterrole_def.yaml +++ b/deploy/operator/00_scyllacluster_member_clusterrole_def.yaml @@ -24,6 +24,7 @@ rules: - "" resources: - configmaps + - configmaps/finalizers verbs: - get - list @@ -53,3 +54,11 @@ rules: - scyllaclusters verbs: - get +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use diff --git a/examples/eks/nodeconfig-alpha.yaml b/examples/eks/nodeconfig-alpha.yaml index 8f5cfa95fe1..20a9c7bd9b3 100644 --- a/examples/eks/nodeconfig-alpha.yaml +++ b/examples/eks/nodeconfig-alpha.yaml @@ -9,7 +9,7 @@ spec: type: xfs mounts: - device: /dev/md/nvmes - mountPoint: /mnt/persistent-volumes + mountPoint: /var/mnt/persistent-volumes unsupportedOptions: - prjquota raids: diff --git a/examples/gke/nodeconfig-alpha.yaml b/examples/gke/nodeconfig-alpha.yaml index c14d2d3ac19..f44534f69c7 100644 --- a/examples/gke/nodeconfig-alpha.yaml +++ b/examples/gke/nodeconfig-alpha.yaml @@ -9,7 +9,7 @@ spec: type: xfs mounts: - device: /dev/md/nvmes - mountPoint: /mnt/persistent-volumes + mountPoint: /var/mnt/persistent-volumes unsupportedOptions: - prjquota raids: diff --git a/examples/third-party/haproxy-ingress/10_haproxy-ingress.role.yaml b/examples/third-party/haproxy-ingress/10_haproxy-ingress.role.yaml new file mode 100644 index 00000000000..01787839162 --- /dev/null +++ b/examples/third-party/haproxy-ingress/10_haproxy-ingress.role.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: haproxy-ingress +rules: +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use diff --git a/examples/third-party/haproxy-ingress/10_clusterrolebinding.yaml b/examples/third-party/haproxy-ingress/20_clusterrolebinding.yaml similarity index 100% rename from examples/third-party/haproxy-ingress/10_clusterrolebinding.yaml rename to examples/third-party/haproxy-ingress/20_clusterrolebinding.yaml diff --git a/examples/third-party/haproxy-ingress/20_haproxy-ingress.rolebinding.yaml b/examples/third-party/haproxy-ingress/20_haproxy-ingress.rolebinding.yaml new file mode 100644 index 00000000000..ebe89868052 --- /dev/null +++ b/examples/third-party/haproxy-ingress/20_haproxy-ingress.rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: haproxy-ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: haproxy-ingress +subjects: +- kind: ServiceAccount + name: haproxy-ingress + namespace: haproxy-ingress diff --git a/examples/third-party/haproxy-ingress/10_prometheus.rolebinding.yaml b/examples/third-party/haproxy-ingress/20_prometheus.rolebinding.yaml similarity index 100% rename from examples/third-party/haproxy-ingress/10_prometheus.rolebinding.yaml rename to examples/third-party/haproxy-ingress/20_prometheus.rolebinding.yaml diff --git a/examples/third-party/haproxy-ingress/10_haproxy-ingress.deploy.yaml b/examples/third-party/haproxy-ingress/50_haproxy-ingress.deploy.yaml similarity index 100% rename from examples/third-party/haproxy-ingress/10_haproxy-ingress.deploy.yaml rename to examples/third-party/haproxy-ingress/50_haproxy-ingress.deploy.yaml diff --git a/examples/third-party/haproxy-ingress/10_ingress-default-backend.deploy.yaml b/examples/third-party/haproxy-ingress/50_ingress-default-backend.deploy.yaml similarity index 100% rename from examples/third-party/haproxy-ingress/10_ingress-default-backend.deploy.yaml rename to examples/third-party/haproxy-ingress/50_ingress-default-backend.deploy.yaml diff --git a/examples/third-party/haproxy-ingress/10_prometheus.deploy.yaml b/examples/third-party/haproxy-ingress/50_prometheus.deploy.yaml similarity index 100% rename from examples/third-party/haproxy-ingress/10_prometheus.deploy.yaml rename to examples/third-party/haproxy-ingress/50_prometheus.deploy.yaml diff --git a/hack/.ci/manifests/namespaces/local-csi-driver/10_provisioner_clusterrole.yaml b/hack/.ci/manifests/namespaces/local-csi-driver/10_provisioner_clusterrole.yaml index c211a270b2f..87adf9dd592 100644 --- a/hack/.ci/manifests/namespaces/local-csi-driver/10_provisioner_clusterrole.yaml +++ b/hack/.ci/manifests/namespaces/local-csi-driver/10_provisioner_clusterrole.yaml @@ -3,6 +3,14 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: scylladb:csi-external-provisioner rules: +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use - apiGroups: - "" resources: diff --git a/hack/ci-deploy.sh b/hack/ci-deploy.sh index 17b8e45a0b2..4e6b1fe7d04 100755 --- a/hack/ci-deploy.sh +++ b/hack/ci-deploy.sh @@ -31,7 +31,7 @@ cp ./examples/third-party/haproxy-ingress/*.yaml "${DEPLOY_DIR}/haproxy-ingress" cp ./examples/common/cert-manager.yaml "${DEPLOY_DIR}/" for f in $( find "${DEPLOY_DIR}"/ -type f -name '*.yaml' ); do - sed -i -E -e "s~docker.io/scylladb/scylla-operator(:|@sha256:)[^ ]*~${OPERATOR_IMAGE_REF}~" "${f}" + sed -i -E -e "s~docker.io/scylladb/scylla-operator:[^ ]*~${OPERATOR_IMAGE_REF}~" "${f}" done yq e --inplace '.spec.template.spec.containers[0].args += ["--qps=200", "--burst=400"]' "${DEPLOY_DIR}/operator/50_operator.deployment.yaml" @@ -41,8 +41,8 @@ if [[ -n ${SCYLLA_OPERATOR_FEATURE_GATES+x} ]]; then yq e --inplace '.spec.template.spec.containers[0].args += "--feature-gates="+ strenv(SCYLLA_OPERATOR_FEATURE_GATES)' "${DEPLOY_DIR}/operator/50_operator.deployment.yaml" fi -kubectl_create -n prometheus-operator -f "${DEPLOY_DIR}/prometheus-operator" -kubectl_create -n haproxy-ingress -f "${DEPLOY_DIR}/haproxy-ingress" +kubectl_create -n=prometheus-operator -f="${DEPLOY_DIR}/prometheus-operator" +kubectl_create -n=haproxy-ingress -f="${DEPLOY_DIR}/haproxy-ingress" kubectl_create -f "${DEPLOY_DIR}"/cert-manager.yaml # Wait for cert-manager @@ -69,6 +69,7 @@ if [[ -z "${SO_CSI_DRIVER_PATH:-}" ]]; then echo "Skipping CSI driver creation" else kubectl_create -n=local-csi-driver -f="${SO_CSI_DRIVER_PATH}" + kubectl -n=local-csi-driver rollout status -f="${SO_CSI_DRIVER_PATH}" fi if [[ -n "${SO_SCYLLACLUSTER_STORAGECLASS_NAME}" ]]; then @@ -90,3 +91,6 @@ kubectl wait --for condition=established crd/nodeconfigs.scylla.scylladb.com kubectl wait --for condition=established crd/scyllaoperatorconfigs.scylla.scylladb.com kubectl wait --for condition=established crd/scylladbmonitorings.scylla.scylladb.com kubectl wait --for condition=established $( find "${DEPLOY_DIR}/prometheus-operator/" -name '*.crd.yaml' -printf '-f=%p\n' ) + +kubectl -n=haproxy-ingress rollout status -f="${DEPLOY_DIR}/haproxy-ingress" +kubectl -n=haproxy-ingress rollout status -f="${DEPLOY_DIR}/prometheus-operator" diff --git a/hack/run-e2e-remote.sh b/hack/run-e2e-remote.sh new file mode 100755 index 00000000000..e570938dd7e --- /dev/null +++ b/hack/run-e2e-remote.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2024 ScyllaDB +# + +set -euExo pipefail +shopt -s inherit_errexit + +source "$( dirname "${BASH_SOURCE[0]}" )/.ci/lib/e2e.sh" + +KUBECONFIG="${KUBECONFIGS[0]}" apply-e2e-workarounds +KUBECONFIG="${KUBECONFIGS[0]}" run-e2e diff --git a/helm/scylla-manager/values.yaml b/helm/scylla-manager/values.yaml index dcf26b76339..554ffe8ff47 100644 --- a/helm/scylla-manager/values.yaml +++ b/helm/scylla-manager/values.yaml @@ -90,6 +90,12 @@ scylla: requests: cpu: 1 memory: 200Mi + placement: + tolerations: + - key: role + operator: Equal + value: scylla-clusters + effect: NoSchedule # Whether to create Prometheus ServiceMonitor serviceMonitor: diff --git a/helm/scylla-operator/templates/clusterrole_def.yaml b/helm/scylla-operator/templates/clusterrole_def.yaml index a819a2f397a..823a28cdb59 100644 --- a/helm/scylla-operator/templates/clusterrole_def.yaml +++ b/helm/scylla-operator/templates/clusterrole_def.yaml @@ -51,6 +51,12 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - update - apiGroups: - "" resources: @@ -86,6 +92,7 @@ rules: resources: - statefulsets - daemonsets + - daemonsets/finalizers - deployments verbs: - create @@ -105,7 +112,9 @@ rules: - scylla.scylladb.com resources: - scyllaclusters + - scyllaclusters/finalizers - scylladbmonitorings + - scylladbmonitorings/finalizers verbs: - create - delete @@ -129,6 +138,7 @@ rules: - "" resources: - configmaps + - configmaps/finalizers verbs: - create - delete @@ -165,6 +175,8 @@ rules: - scylla.scylladb.com resources: - nodeconfigs + - nodeconfigs/status + - nodeconfigs/finalizers verbs: - create - delete @@ -200,18 +212,6 @@ rules: - patch - update - watch -- apiGroups: - - scylla.scylladb.com - resources: - - nodeconfigs/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - batch resources: @@ -274,3 +274,11 @@ rules: - patch - update - delete +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use diff --git a/helm/scylla-operator/templates/scyllacluster_member_clusterrole_def.yaml b/helm/scylla-operator/templates/scyllacluster_member_clusterrole_def.yaml index 4058a7f0d81..f49ff22f3b5 100644 --- a/helm/scylla-operator/templates/scyllacluster_member_clusterrole_def.yaml +++ b/helm/scylla-operator/templates/scyllacluster_member_clusterrole_def.yaml @@ -24,6 +24,7 @@ rules: - "" resources: - configmaps + - configmaps/finalizers verbs: - get - list @@ -53,3 +54,11 @@ rules: - scyllaclusters verbs: - get +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use diff --git a/helm/scylla/values.yaml b/helm/scylla/values.yaml index 4c04f771c70..60e71be5a7b 100644 --- a/helm/scylla/values.yaml +++ b/helm/scylla/values.yaml @@ -74,6 +74,12 @@ racks: requests: cpu: 1 memory: 4Gi + placement: + tolerations: + - key: role + operator: Equal + value: scylla-clusters + effect: NoSchedule # Whether to create Prometheus ServiceMonitor serviceMonitor: diff --git a/pkg/controller/nodeconfig/resource.go b/pkg/controller/nodeconfig/resource.go index e4e5b7ab6e3..01165310044 100644 --- a/pkg/controller/nodeconfig/resource.go +++ b/pkg/controller/nodeconfig/resource.go @@ -85,6 +85,11 @@ func NodeConfigClusterRole() *rbacv1.ClusterRole { Resources: []string{"daemonsets"}, Verbs: []string{"get", "list", "watch"}, }, + { + APIGroups: []string{"apps"}, + Resources: []string{"daemonsets/finalizers"}, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, { APIGroups: []string{"batch"}, Resources: []string{"jobs"}, @@ -100,6 +105,12 @@ func NodeConfigClusterRole() *rbacv1.ClusterRole { Resources: []string{"nodeconfigs/status"}, Verbs: []string{"update"}, }, + { + APIGroups: []string{"security.openshift.io"}, + ResourceNames: []string{"privileged"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + }, }, } } @@ -113,7 +124,14 @@ func makePerftuneRole() *rbacv1.Role { naming.NodeConfigNameLabel: naming.NodeConfigAppName, }, }, - Rules: []rbacv1.PolicyRule{}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{"privileged"}, + Verbs: []string{"use"}, + }, + }, } } diff --git a/test/e2e/fixture/scylla/scyllacluster.yaml.tmpl b/test/e2e/fixture/scylla/scyllacluster.yaml.tmpl index db0aa0ca776..15ef33bacfc 100644 --- a/test/e2e/fixture/scylla/scyllacluster.yaml.tmpl +++ b/test/e2e/fixture/scylla/scyllacluster.yaml.tmpl @@ -35,3 +35,9 @@ spec: limits: cpu: 1 memory: 1Gi + placement: + tolerations: + - key: role + operator: Equal + value: scylla-clusters + effect: NoSchedule diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index d039f563f7d..576c333625f 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -249,6 +249,27 @@ func CreateUserNamespace(ctx context.Context, clusterName string, labels map[str }, metav1.CreateOptions{}) o.Expect(err).NotTo(o.HaveOccurred()) + // Grant it permission needed for ScyllaCLusters + _, err = adminClient.RbacV1().RoleBindings(ns.Name).Create(ctx, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: userSA.Name + "-scyllacluster-member", + }, + Subjects: []rbacv1.Subject{ + { + APIGroup: corev1.GroupName, + Kind: rbacv1.ServiceAccountKind, + Namespace: userSA.Namespace, + Name: userSA.Name, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: "scyllacluster-member", + }, + }, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + // Create a service account token Secret for the user ServiceAccount. userSATokenSecret, err := adminClient.CoreV1().Secrets(ns.Name).Create(ctx, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{