From a974e2241ef5320dd970514a0ecc0a0c43dcd31c Mon Sep 17 00:00:00 2001 From: Daniel Fan Date: Sun, 23 Jun 2024 15:52:27 -0400 Subject: [PATCH] Add webhook to replicate operandrequest in partial watched namespace (#1059) * Add webhook to replicate operandrequest in partial watched namespace Signed-off-by: Daniel Fan * update setup-envtest Signed-off-by: Daniel Fan * Add test cases Signed-off-by: Daniel Fan * Update GetFilteredOpreqSpec name Signed-off-by: Daniel Fan * wait for OperandRegistry before annotating OperandRequest Signed-off-by: Daniel Fan --------- Signed-off-by: Daniel Fan --- ...fecycle-manager.clusterserviceversion.yaml | 19 + config/manager/manager.yaml | 6 + config/rbac/role.yaml | 13 + controllers/constant/constant.go | 8 + controllers/util/util.go | 65 ++++ .../operandrequest_mutating_webhook.go | 258 +++++++++++++ .../operandrequest_mutating_webhook_test.go | 216 +++++++++++ controllers/webhooks/operator_webhook.go | 365 ++++++++++++++++++ controllers/webhooks/rules.go | 73 ++++ controllers/webhooks/webhook_register.go | 182 +++++++++ controllers/webhooks/webhookreconciler.go | 335 ++++++++++++++++ go.mod | 3 + go.sum | 3 + main.go | 40 ++ 14 files changed, 1586 insertions(+) create mode 100644 controllers/webhooks/operandrequestreplication/operandrequest_mutating_webhook.go create mode 100644 controllers/webhooks/operandrequestreplication/operandrequest_mutating_webhook_test.go create mode 100644 controllers/webhooks/operator_webhook.go create mode 100644 controllers/webhooks/rules.go create mode 100644 controllers/webhooks/webhook_register.go create mode 100644 controllers/webhooks/webhookreconciler.go diff --git a/bundle/manifests/operand-deployment-lifecycle-manager.clusterserviceversion.yaml b/bundle/manifests/operand-deployment-lifecycle-manager.clusterserviceversion.yaml index 59cddd30..4a3cc04c 100644 --- a/bundle/manifests/operand-deployment-lifecycle-manager.clusterserviceversion.yaml +++ b/bundle/manifests/operand-deployment-lifecycle-manager.clusterserviceversion.yaml @@ -573,6 +573,7 @@ spec: - get - list - watch + - update - apiGroups: - operator.ibm.com resources: @@ -618,6 +619,18 @@ spec: - patch - update - watch + - apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch serviceAccountName: operand-deployment-lifecycle-manager deployments: - label: @@ -705,9 +718,15 @@ spec: privileged: false readOnlyRootFilesystem: true runAsNonRoot: true + volumeMounts: + - mountPath: /etc/ssl/certs/webhook + name: webhook-certs serviceAccount: operand-deployment-lifecycle-manager serviceAccountName: operand-deployment-lifecycle-manager terminationGracePeriodSeconds: 10 + volumes: + - emptyDir: {} + name: webhook-certs permissions: - rules: - apiGroups: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ae6aa089..e40b2586 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -92,5 +92,11 @@ spec: privileged: false readOnlyRootFilesystem: true runAsNonRoot: true + volumeMounts: + - mountPath: /etc/ssl/certs/webhook + name: webhook-certs terminationGracePeriodSeconds: 10 serviceAccount: operand-deployment-lifecycle-manager + volumes: + - emptyDir: {} + name: webhook-certs diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 507ba0e6..73420c2c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -10,6 +10,7 @@ rules: - get - list - watch + - update apiGroups: - operator.ibm.com resources: @@ -62,6 +63,18 @@ rules: - patch - update - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/controllers/constant/constant.go b/controllers/constant/constant.go index cb536c94..c3b0ae3d 100644 --- a/controllers/constant/constant.go +++ b/controllers/constant/constant.go @@ -21,6 +21,11 @@ import ( ) const ( + //OperatorName is the name of operator + OperatorName string = "operand-deployment-lifecycle-manager" + + //CSVName is the name of Operand Deployment Lifecycle Manager CSV + CSVName string = "operand-deployment-lifecycle-manager.v1.21.13" //ClusterOperatorNamespace is the namespace of cluster operators ClusterOperatorNamespace string = "openshift-operators" @@ -52,6 +57,9 @@ const ( //FindOperandRegistry is the key for checking if the OperandRegistry is found FindOperandRegistry string = "operator.ibm.com/operandregistry-is-not-found" + //OdlmManagedLabel is the label used to label the webhook managed by ODLM + OdlmManagedLabel string = "operator.ibm.com/managedBy-odlm" + //HashedData is the key for checking the checksum of data section HashedData string = "hashedData" diff --git a/controllers/util/util.go b/controllers/util/util.go index ad53be14..180f673a 100644 --- a/controllers/util/util.go +++ b/controllers/util/util.go @@ -17,6 +17,8 @@ package util import ( + "context" + "errors" "os" "sort" "strconv" @@ -24,7 +26,10 @@ import ( "sync" "time" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" ) // GetOperatorNamespace returns the Namespace of the operator @@ -70,6 +75,23 @@ func GetoperatorCheckerMode() bool { return false } +func GetPartialWatchNamespace() string { + ns, found := os.LookupEnv("PARTIAL_WATCH_NAMESPACE") + if !found { + return "" + } + return ns +} + +func PartialWatchNamespaceEnabled() bool { + // If it is not found, it is enabled by default + isEnabled, found := os.LookupEnv("ENABLE_PARTIAL_WATCH_NAMESPACE") + if !found || isEnabled == "true" { + return true + } + return false +} + // ResourceExists returns true if the given resource kind exists // in the given api groupversion func ResourceExists(dc discovery.DiscoveryInterface, apiGroupVersion, kind string) (bool, error) { @@ -188,3 +210,46 @@ func Contains(list []string, s string) bool { } return false } + +// returns error, roleName, roleUID +func GetClusterRoleDetails(kube client.Client, ns string, csvName string) (string, types.UID, error) { + existingResource := &rbacv1.ClusterRoleList{} + opts := []client.ListOption{ + client.MatchingLabels(map[string]string{ + "olm.owner.namespace": ns, + "olm.owner": csvName, + }), + } + err := kube.List(context.TODO(), existingResource, opts...) + if err != nil { + return "", "", err + } + switch len(existingResource.Items) { + case 0: + return "", "", errors.New("unable to find ClusterRole for operator " + csvName) + default: + // 1 or more ClusterRole returned so index first one + return existingResource.Items[0].Name, existingResource.Items[0].UID, nil + } +} + +func GetClusterRole(kube client.Client, ns string, csvName string) (*rbacv1.ClusterRole, error) { + existingResource := &rbacv1.ClusterRoleList{} + opts := []client.ListOption{ + client.MatchingLabels(map[string]string{ + "olm.owner.namespace": ns, + "olm.owner": csvName, + }), + } + err := kube.List(context.TODO(), existingResource, opts...) + if err != nil { + return nil, err + } + switch len(existingResource.Items) { + case 0: + return nil, errors.New("unable to find ClusterRole for operator " + csvName) + default: + // 1 or more ClusterRole returned so index first one + return &existingResource.Items[0], nil + } +} diff --git a/controllers/webhooks/operandrequestreplication/operandrequest_mutating_webhook.go b/controllers/webhooks/operandrequestreplication/operandrequest_mutating_webhook.go new file mode 100644 index 00000000..8255c478 --- /dev/null +++ b/controllers/webhooks/operandrequestreplication/operandrequest_mutating_webhook.go @@ -0,0 +1,258 @@ +// +// Copyright 2022 IBM Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package operandrequest + +import ( + "context" + "net/http" + "strings" + "time" + + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + odlm "github.com/IBM/operand-deployment-lifecycle-manager/api/v1alpha1" +) + +// OperandRequestDefaulter points to correct RegistryNamespace +type Defaulter struct { + decoder *admission.Decoder + Client client.Client + OperatorNs string +} + +func (r *Defaulter) Handle(ctx context.Context, req admission.Request) admission.Response { + klog.Infof("Webhook is invoked by OperandRequest %s/%s", req.AdmissionRequest.Namespace, req.AdmissionRequest.Name) + + if req.AdmissionRequest.Operation == admissionv1.Create { + return admission.Allowed("") + } + + isDeleting := req.AdmissionRequest.Operation == admissionv1.Delete + + if req.AdmissionRequest.Operation == admissionv1.Update { + opreq := &odlm.OperandRequest{} + + if err := r.decoder.Decode(req, opreq); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + // Check if the OperandRequest is being deleted + if opreq.DeletionTimestamp != nil { + isDeleting = true + } else { + // check if the newObject contains annotation "operand.ibm.com/watched-by-odlm-in-$OperatorNs" key and it is not empty value nor false + if opreq.Annotations == nil || opreq.Annotations["operand.ibm.com/watched-by-odlm-in-"+r.OperatorNs] == "false" || opreq.Annotations["operand.ibm.com/watched-by-odlm-in-"+r.OperatorNs] == "" { + klog.Warningf("OperandRequest %s is not watched by ODLM in the %s namespace", opreq.Name, r.OperatorNs) + return admission.Allowed("") + } + + // get the new OperandRequest spec with the valid operands + if newOpreqSpec, err := GetFilteredOpreqSpec(ctx, r.Client, opreq, r.OperatorNs); err != nil { + klog.Errorf("Failed to get new OperandRequest spec: %v", err) + return admission.Errored(http.StatusInternalServerError, err) + } else if newOpreqSpec == nil || len(newOpreqSpec.Requests) == 0 { + klog.Infof("No valid operand found for OperandRequest %s in the %s namespace to replicate, deleting it", opreq.Name, r.OperatorNs) + isDeleting = true + } else { + existingOpreq := &odlm.OperandRequest{} + // get the OperandRequest from the r.OperatorNs namespace + opreqKey := types.NamespacedName{ + Name: opreq.Name + "-from-" + opreq.Namespace, + Namespace: r.OperatorNs, + } + if err := r.Client.Get(ctx, opreqKey, existingOpreq); err != nil { + if errors.IsNotFound(err) { + // if the OperandRequest is not found, create a new OperandRequest in the r.OperatorNs namespace + klog.Infof("OperandRequest %s not found in the %s namespace", opreq.Name, r.OperatorNs) + existingOpreq.Name = opreq.Name + "-from-" + opreq.Namespace + existingOpreq.Namespace = r.OperatorNs + existingOpreq.Spec = *newOpreqSpec + + // Add label "operand.ibm.com/opreq-replicated-from-$opreq.Namespace" to the OperandRequest + if existingOpreq.Labels == nil { + existingOpreq.Labels = make(map[string]string) + } + existingOpreq.Labels["operand.ibm.com/opreq-replicated-from-"+opreq.Namespace] = "true" + + if err = r.Client.Create(ctx, existingOpreq); err != nil { + klog.Errorf("Failed to replicate OperandRequest %s in the %s namespace: %v", existingOpreq.Name, r.OperatorNs, err) + return admission.Errored(http.StatusInternalServerError, err) + } + klog.Infof("OperandRequest %s is replicated in the %s namespace", existingOpreq.Name, r.OperatorNs) + return admission.Allowed("") + } + klog.Errorf("Failed to get OperandRequest %s in the %s namespace: %v", existingOpreq.Name, r.OperatorNs, err) + return admission.Errored(http.StatusInternalServerError, err) + } + // update the existing OperandRequest in the r.OperatorNs namespace + existingOpreq.Spec = *newOpreqSpec + if err := r.Client.Update(ctx, existingOpreq); err != nil { + klog.Errorf("Failed to update OperandRequest %s in the %s namespace: %v", existingOpreq.Name, r.OperatorNs, err) + return admission.Errored(http.StatusInternalServerError, err) + } + klog.Infof("OperandRequest %s is updated in the %s namespace", existingOpreq.Name, r.OperatorNs) + } + } + } + + if isDeleting { + // invoke a delete action on the OperandRequest named opreq.Name-from-opreq.Namespace in the r.OperatorNs namespace + existingOpreq := &odlm.OperandRequest{} + // get the OperandRequest from the r.OperatorNs namespace + opreqKey := types.NamespacedName{ + Name: req.Name + "-from-" + req.Namespace, + Namespace: r.OperatorNs, + } + if err := r.Client.Get(ctx, opreqKey, existingOpreq); err != nil { + if errors.IsNotFound(err) { + klog.Infof("OperandRequest %s not found in the %s namespace", existingOpreq.Name, r.OperatorNs) + return admission.Allowed("") + } + klog.Errorf("Failed to get OperandRequest %s in the %s namespace: %v", existingOpreq.Name, r.OperatorNs, err) + return admission.Errored(http.StatusInternalServerError, err) + } + + if err := r.Client.Delete(ctx, existingOpreq); err != nil { + klog.Errorf("Failed to delete OperandRequest %s in the %s namespace: %v", existingOpreq.Name, r.OperatorNs, err) + return admission.Errored(http.StatusInternalServerError, err) + } + klog.Infof("OperandRequest %s is deleted in the %s namespace", existingOpreq.Name, r.OperatorNs) + return admission.Allowed("") + } + + return admission.Allowed("") +} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *Defaulter) Default(instance *odlm.OperandRequest) { +} + +func GetFilteredOpreqSpec(ctx context.Context, kube client.Client, opreq *odlm.OperandRequest, operatorNs string) (*odlm.OperandRequestSpec, error) { + newOpreqSpec := &odlm.OperandRequestSpec{} + for _, req := range opreq.Spec.Requests { + // get the OperandRegistry from the registry and registryNamespace + registryKey := types.NamespacedName{ + Name: req.Registry, + Namespace: operatorNs, + } + registry := &odlm.OperandRegistry{} + if err := kube.Get(ctx, registryKey, registry); err != nil { + if errors.IsNotFound(err) { + klog.Warningf("OperandRegistry %s not found in the %s namespace", registryKey.Name, registryKey.Namespace) + continue + } + klog.Errorf("Failed to get OperandRegistry %s in the %s namespace: %v", registryKey.Name, registryKey.Namespace, err) + return nil, err + } + newOpreqSpecRequest := odlm.Request{Registry: req.Registry, RegistryNamespace: operatorNs} + for _, op := range req.Operands { + opt := registry.GetOperator(op.Name) + if opt != nil { + newOpreqSpecRequest.Operands = append(newOpreqSpecRequest.Operands, op) + } + } + if len(newOpreqSpecRequest.Operands) > 0 { + newOpreqSpec.Requests = append(newOpreqSpec.Requests, newOpreqSpecRequest) + } + } + return newOpreqSpec, nil +} + +func (r *Defaulter) InjectDecoder(decoder *admission.Decoder) error { + r.decoder = decoder + return nil +} + +func AddAnnotationToOperandRequests(kube client.Client, partialWatchNamespace, operatorNamespace string) { + for { + // wait for OperandRegistry common-service in operatorNamespace to be created + registryKey := types.NamespacedName{ + Name: "common-service", + Namespace: operatorNamespace, + } + registry := &odlm.OperandRegistry{} + if err := kube.Get(context.TODO(), registryKey, registry); err != nil { + klog.Warningf("Failed to get OperandRegistry common-service in the %s namespace: %v, retrying...", operatorNamespace, err) + time.Sleep(5 * time.Second) + continue + } + + isFinished := true + sleepTime := 5 * time.Second + // Get all OperandRequests in the partial watch namespace + for _, ns := range strings.Split(partialWatchNamespace, ",") { + opreqList := &odlm.OperandRequestList{} + if err := kube.List(context.TODO(), opreqList, &client.ListOptions{Namespace: ns}); err != nil { + isFinished = false + klog.Warningf("Failed to list OperandRequests in the %s namespace: %v", ns, err) + continue + } + + for _, opreq := range opreqList.Items { + opreq := opreq + annotations := opreq.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + // if annotation value is false, then skip it + if annotations["operand.ibm.com/watched-by-odlm-in-"+operatorNamespace] == "false" { + continue + } + // Always refresh the annotation value to trigger the webhook replication + annotations["operand.ibm.com/watched-by-odlm-in-"+operatorNamespace] = time.Now().Format(time.RFC3339) + opreq.SetAnnotations(annotations) + if err := kube.Update(context.Background(), &opreq); err != nil { + isFinished = false + klog.Warningf("Failed to add annotation to OperandRequest %s/%s: %v", ns, opreq.GetName(), err) + continue + } + + // Check if there are valid operand in the OperandRequest requiring replication + if newOpreqSpec, err := GetFilteredOpreqSpec(context.TODO(), kube, &opreq, operatorNamespace); err != nil { + isFinished = false + klog.Errorf("Failed to get new OperandRequest spec: %v", err) + continue + } else if newOpreqSpec == nil || len(newOpreqSpec.Requests) == 0 { + klog.Infof("No valid operand found for OperandRequest %s in the %s namespace to replicate, skipping it", opreq.Name, operatorNamespace) + } else { + // check the OperandRequest opreq.Name-from-opreq.Namespace is replicated in operatorNamespace + opreqKey := types.NamespacedName{ + Namespace: operatorNamespace, + Name: opreq.GetName() + "-from-" + ns, + } + opreqInOperatorNs := &odlm.OperandRequest{} + if err := kube.Get(context.TODO(), opreqKey, opreqInOperatorNs); err != nil { + isFinished = false + klog.Warningf("Failed to get OperandRequest %s in the %s namespace: %v", opreqKey.Name, operatorNamespace, err) + continue + } + } + } + } + if isFinished { + klog.Info("Successfully added annotation to all OperandRequests in the partial watch namespace") + break + } else { + time.Sleep(sleepTime) + } + } +} diff --git a/controllers/webhooks/operandrequestreplication/operandrequest_mutating_webhook_test.go b/controllers/webhooks/operandrequestreplication/operandrequest_mutating_webhook_test.go new file mode 100644 index 00000000..4017a339 --- /dev/null +++ b/controllers/webhooks/operandrequestreplication/operandrequest_mutating_webhook_test.go @@ -0,0 +1,216 @@ +// +// Copyright 2022 IBM Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package operandrequest + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + odlm "github.com/IBM/operand-deployment-lifecycle-manager/api/v1alpha1" +) + +var ( + isolatedNs = "cp4i" + operatorNamespace = "ibm-common-services" + opreqName = "example-service" + + specInIsolatedNs = odlm.OperandRequestSpec{ + Requests: []odlm.Request{ + { + Registry: "common-service", + RegistryNamespace: isolatedNs, + Operands: []odlm.Operand{{Name: "ibm-iam-operator"}}, + }, + { + Registry: "standalone-registry", + RegistryNamespace: isolatedNs, + Operands: []odlm.Operand{{Name: "ibm-zen-operator"}}, + }, + { + Registry: "common-service", + RegistryNamespace: isolatedNs, + Operands: []odlm.Operand{{Name: "keycloak-operator"}, {Name: "ibm-zen-operator"}}, + }, + }, + } + + opreqInIsolatedNs = &odlm.OperandRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: isolatedNs, + Name: opreqName, + Annotations: map[string]string{ + "operand.ibm.com/watched-by-odlm-in-" + operatorNamespace: "2024-06-22T23:51:36Z", + }, + }, + Spec: specInIsolatedNs, + } + + registryInOperatorNs = &odlm.OperandRegistry{ + ObjectMeta: metav1.ObjectMeta{ + Name: "common-service", + Namespace: operatorNamespace, + }, + Spec: odlm.OperandRegistrySpec{ + Operators: []odlm.Operator{ + { + Name: "ibm-zen-operator", + }, + { + Name: "ibm-iam-operator", + }, + }, + }, + } + + opreqSpecInOperatorNs = &odlm.OperandRequestSpec{ + Requests: []odlm.Request{ + { + Registry: "common-service", + RegistryNamespace: operatorNamespace, + Operands: []odlm.Operand{{Name: "ibm-iam-operator"}}, + }, + { + Registry: "common-service", + RegistryNamespace: operatorNamespace, + Operands: []odlm.Operand{{Name: "ibm-zen-operator"}}, + }, + }, + } +) + +func TestGetFilteredOpreqSpec(t *testing.T) { + ctx := context.TODO() + + scheme := runtime.NewScheme() + utilruntime.Must(odlm.AddToScheme(scheme)) + kube := fake.NewClientBuilder().WithScheme(scheme).Build() + + opreq := opreqInIsolatedNs.DeepCopy() + + registry := registryInOperatorNs.DeepCopy() + + err := kube.Create(ctx, registry) + assert.NoError(t, err) + + newOpreqSpec, err := GetFilteredOpreqSpec(ctx, kube, opreq, operatorNamespace) + assert.NoError(t, err) + + expectedOpreqSpec := opreqSpecInOperatorNs.DeepCopy() + assert.Equal(t, expectedOpreqSpec, newOpreqSpec) +} +func TestHandle(t *testing.T) { + ctx := context.TODO() + + scheme := runtime.NewScheme() + utilruntime.Must(odlm.AddToScheme(scheme)) + kube := fake.NewClientBuilder().WithScheme(scheme).Build() + + defaulter := &Defaulter{ + Client: kube, + OperatorNs: operatorNamespace, + } + + // Create an OperandRequest in the isolated namespace + opreq := opreqInIsolatedNs.DeepCopy() + rawOpreq, err := json.Marshal(opreq) + assert.NoError(t, err) + + // Create an existing OperandRequest in the operatorNamespace + existingOpreq := &odlm.OperandRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: operatorNamespace, + Name: opreqName + "-from-" + isolatedNs, + }, + Spec: odlm.OperandRequestSpec{ + Requests: []odlm.Request{ + { + Registry: "common-service", + RegistryNamespace: operatorNamespace, + Operands: []odlm.Operand{{Name: "ibm-zen-operator"}}, + }, + }, + }, + } + err = kube.Create(ctx, existingOpreq) + assert.NoError(t, err) + + registry := registryInOperatorNs.DeepCopy() + + err = kube.Create(ctx, registry) + assert.NoError(t, err) + + // Create an admission.Request + admissionReq := admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: isolatedNs, + Name: opreqName, + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: rawOpreq, + }, + }, + } + + decoder, err := admission.NewDecoder(kube.Scheme()) + assert.NoError(t, err) + + err = defaulter.InjectDecoder(decoder) + assert.NoError(t, err) + + resp := defaulter.Handle(ctx, admissionReq) + assert.True(t, resp.Allowed) + + // Assert the existing OperandRequest is updated + err = kube.Get(ctx, types.NamespacedName{Name: existingOpreq.Name, Namespace: existingOpreq.Namespace}, existingOpreq) + assert.NoError(t, err) + + expectedOpreqSpec := opreqSpecInOperatorNs.DeepCopy() + assert.Equal(t, *expectedOpreqSpec, existingOpreq.Spec) + + // Remove the existing OperandRequest + err = kube.Delete(ctx, existingOpreq) + assert.NoError(t, err) + + resp = defaulter.Handle(ctx, admissionReq) + assert.True(t, resp.Allowed) + + // Assert a new OperandRequest is created + newOpreq := &odlm.OperandRequest{} + err = kube.Get(ctx, types.NamespacedName{Name: opreqName + "-from-" + isolatedNs, Namespace: operatorNamespace}, newOpreq) + assert.NoError(t, err) + + assert.Equal(t, *expectedOpreqSpec, newOpreq.Spec) + + // Call the Handle function with an delete operation + admissionReq.Operation = admissionv1.Delete + resp = defaulter.Handle(ctx, admissionReq) + assert.True(t, resp.Allowed) + + // Assert the new OperandRequest is deleted + err = kube.Get(ctx, types.NamespacedName{Name: newOpreq.Name, Namespace: newOpreq.Namespace}, newOpreq) + assert.Error(t, err) +} diff --git a/controllers/webhooks/operator_webhook.go b/controllers/webhooks/operator_webhook.go new file mode 100644 index 00000000..99ebecd3 --- /dev/null +++ b/controllers/webhooks/operator_webhook.go @@ -0,0 +1,365 @@ +// +// Copyright 2022 IBM Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package webhooks + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" + "sigs.k8s.io/controller-runtime/pkg/builder" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/IBM/operand-deployment-lifecycle-manager/controllers/constant" + "github.com/IBM/operand-deployment-lifecycle-manager/controllers/util" +) + +// CSWebhookConfig contains the data and logic to setup the webhooks +// server of a given Manager implementation, and to reconcile webhook configuration +// CRs pointing to the server. +type CSWebhookConfig struct { + scheme *runtime.Scheme + + Port int + CertDir string + CAConfigMap string + + Webhooks []CSWebhook +} + +// CSWebhook acts as a single source of truth for validating webhooks +// managed by the operator. It's data are used both for registering the +// endpoint to the webhook server and to reconcile the ValidatingWebhookConfiguration +// that points to the server. +type CSWebhook struct { + // Name of the webhookConfiguration. + Name string + + // Name of the webhook. + WebhookName string + + // Rule for the webhook to be triggered + Rule RuleWithOperations + + // Register for the webhook into the server + Register WebhookRegister + + // NsSelector for add namespaceselector to the admission webhook + NsSelector v1.LabelSelector +} + +const ( + operatorPodServiceName = "operand-deployment-lifecycle-manager" + operatorPodPort = 8443 + servicePort = 443 + mountedCertDir = "/etc/ssl/certs/webhook" + caConfigMap = "operand-deployment-lifecycle-manager-webhook-ca" + caConfigMapAnnotation = "service.beta.openshift.io/inject-cabundle" + caServiceAnnotation = "service.beta.openshift.io/serving-cert-secret-name" + caCertificateName = "odlm-webhook-cert" +) + +var labels = map[string]string{ + constant.OdlmManagedLabel: "true", + "app.kubernetes.io/instance": constant.OperatorName, + "app.kubernetes.io/managed-by": constant.OperatorName, + "app.kubernetes.io/name": constant.OperatorName, + "name": constant.OperatorName, +} + +// Config is a global instance. The same instance is needed in order to use the +// same configuration for the webhooks server that's run at startup and the +// reconciliation of the ValidatingWebhookConfiguration CRs +var Config *CSWebhookConfig = &CSWebhookConfig{ + // Port that the webhook service is pointing to + Port: operatorPodPort, + + // Mounted as a volume from the secret generated from Openshift + CertDir: mountedCertDir, + + // Name of the config map where the CA certificate is injected + CAConfigMap: caConfigMap, + + // List of webhooks to configure + Webhooks: []CSWebhook{}, +} + +// SetupServer sets up the webhook server managed by mgr with the settings from +// webhookConfig. It sets the port and cert dir based on the settings and +// registers the Validator implementations from each webhook from webhookConfig.Webhooks +func (webhookConfig *CSWebhookConfig) SetupServer(mgr manager.Manager, namespace string) error { + // Create a new client to reconcile the Service. `mgr.GetClient()` can't + // be used as it relies on the cache that hasn't been initialized yet + client, err := k8sclient.New(mgr.GetConfig(), k8sclient.Options{ + Scheme: mgr.GetScheme(), + }) + if err != nil { + return err + } + + // Create the service pointing to the operator pod + if err := webhookConfig.ReconcileService(context.TODO(), client, nil, namespace); err != nil { + return err + } + // Get the secret with the certificates for the service + if err := webhookConfig.setupCerts(context.TODO(), client, namespace); err != nil { + return err + } + + webhookServer := mgr.GetWebhookServer() + webhookServer.Port = webhookConfig.Port + webhookServer.CertDir = webhookConfig.CertDir + + webhookConfig.scheme = mgr.GetScheme() + + bldr := builder.WebhookManagedBy(mgr) + + for _, webhook := range webhookConfig.Webhooks { + bldr = webhook.Register.RegisterToBuilder(bldr) + if err := webhook.Register.RegisterToServer(webhookConfig.scheme, webhookServer); err != nil { + return err + } + } + + if err := bldr.Complete(); err != nil { + return err + } + + return nil +} + +// Reconcile reconciles a `ValidationWebhookConfiguration` object for each webhook +// in `webhookConfig.Webhooks`, using the rules and the path as it's generated +// by controller-runtime webhook builder. +// It reconciles a Service that exposes the webhook server +// A ownerRef to the owner parameter is set on the reconciled resources. This +// parameter is optional, if `nil` is passed, no ownerReference will be set +func (webhookConfig *CSWebhookConfig) Reconcile(ctx context.Context, client k8sclient.Client, owner ownerutil.Owner) error { + + namespace := util.GetOperatorNamespace() + + // Reconcile the Service + if err := webhookConfig.ReconcileService(ctx, client, owner, namespace); err != nil { + return err + } + + // Create (if it doesn't exist) the config map where the CA certificate is + // injected + caConfigMap := &corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Name: webhookConfig.CAConfigMap, + Namespace: namespace, + Annotations: map[string]string{ + caConfigMapAnnotation: "true", + }, + }, + } + if owner != nil { + ownerutil.EnsureOwner(caConfigMap, owner) + } + + klog.Info("Creating operand deployment lifecycle manager webhook CA ConfigMap") + if err := client.Create(ctx, caConfigMap); err != nil && !errors.IsAlreadyExists(err) { + klog.Error(err) + return err + } + + // Wait for the config map to be injected with the CA + caBundle, err := webhookConfig.waitForCAInConfigMap(ctx, client, namespace) + if err != nil { + klog.Error(err) + return err + } + + // Reconcile the webhooks + for _, webhook := range webhookConfig.Webhooks { + reconciler, err := webhook.Register.GetReconciler(webhookConfig.scheme) + if err != nil { + return err + } + + reconciler.SetName(webhook.Name) + reconciler.SetWebhookName(webhook.WebhookName) + reconciler.SetRule(webhook.Rule) + reconciler.SetNsSelector(webhook.NsSelector) + klog.Infof("Reconciling webhook %s", webhook.Name) + if err := reconciler.Reconcile(ctx, client, caBundle); err != nil { + return err + } + } + + return nil +} + +// ReconcileService creates or updates the service that points to the Pod +func (webhookConfig *CSWebhookConfig) ReconcileService(ctx context.Context, client k8sclient.Client, owner ownerutil.Owner, namespace string) error { + + klog.Info("Reconciling operand deployment lifecycle manager webhook service") + // Get the service. If it's not found, create it + service := &corev1.Service{} + if err := client.Get(ctx, k8sclient.ObjectKey{ + Namespace: namespace, + Name: operatorPodServiceName, + }, service); err != nil { + if !errors.IsNotFound(err) { + return err + } + + return createService(ctx, client, owner, namespace) + } + + // If the existing service has a different .spec.clusterIP value, delete it + if service.Spec.ClusterIP != "None" { + if err := client.Delete(ctx, service); err != nil { + return err + } + } + + return createService(ctx, client, owner, namespace) +} + +func createService(ctx context.Context, client k8sclient.Client, owner ownerutil.Owner, namespace string) error { + klog.Info("Creating operand deployment lifecycle manager webhook service") + + service := &corev1.Service{ + ObjectMeta: v1.ObjectMeta{ + Name: operatorPodServiceName, + Namespace: namespace, + Labels: labels, + }, + } + _, err := controllerutil.CreateOrUpdate(ctx, client, service, func() error { + if owner != nil { + ownerutil.EnsureOwner(service, owner) + } + + if service.Annotations == nil { + service.Annotations = map[string]string{} + } + service.Annotations[caServiceAnnotation] = caCertificateName + service.Spec.ClusterIP = "None" + service.Spec.Selector = map[string]string{ + "name": constant.OperatorName, + } + service.Spec.Ports = []corev1.ServicePort{ + { + Protocol: corev1.ProtocolTCP, + Port: int32(servicePort), + TargetPort: intstr.FromInt(operatorPodPort), + }, + } + + return nil + }) + if err != nil { + klog.Error(err) + } + return err +} + +// setupCerts waits for the secret created for the operator Service to exist, and +// when it's ready, extracts the certificates and saves them in webhookConfig.CertDir +func (webhookConfig *CSWebhookConfig) setupCerts(ctx context.Context, client k8sclient.Client, namespace string) error { + // Wait for the secret to te created + secret := &corev1.Secret{} + err := wait.PollImmediate(time.Second*1, time.Second*30, func() (bool, error) { + err := client.Get(ctx, k8sclient.ObjectKey{Namespace: namespace, Name: caCertificateName}, secret) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + + return true, nil + }) + if err != nil { + return err + } + + // Save the key + if err := webhookConfig.saveCertFromSecret(secret.Data, "tls.key"); err != nil { + return err + } + // Save the cert + return webhookConfig.saveCertFromSecret(secret.Data, "tls.crt") +} + +func (webhookConfig *CSWebhookConfig) waitForCAInConfigMap(ctx context.Context, client k8sclient.Client, namespace string) ([]byte, error) { + klog.Info("Waiting for operand deployment lifecycle manager webhook CA generated") + + var caBundle []byte + + err := wait.PollImmediate(time.Second, time.Second*30, func() (bool, error) { + caConfigMap := &corev1.ConfigMap{} + if err := client.Get(ctx, + k8sclient.ObjectKey{Name: webhookConfig.CAConfigMap, Namespace: namespace}, + caConfigMap, + ); err != nil { + if errors.IsNotFound(err) { + return false, nil + } + + return false, err + } + + result, ok := caConfigMap.Data["service-ca.crt"] + + if !ok { + return false, nil + } + + caBundle = []byte(result) + return true, nil + }) + + return caBundle, err +} + +// AddWebhook adds a webhook configuration to a webhookSettings. This must be done before +// starting the server as it registers the endpoints for the validation +func (webhookConfig *CSWebhookConfig) AddWebhook(webhook CSWebhook) { + webhookConfig.Webhooks = append(webhookConfig.Webhooks, webhook) +} + +func (webhookConfig *CSWebhookConfig) saveCertFromSecret(secretData map[string][]byte, fileName string) error { + value, ok := secretData[fileName] + if !ok { + return fmt.Errorf("secret does not contain key %s", fileName) + } + + // Save the key + f, err := os.Create(fmt.Sprintf("%s/%s", webhookConfig.CertDir, fileName)) + if err != nil { + return err + } + defer f.Close() + + _, err = f.Write(value) + return err +} diff --git a/controllers/webhooks/rules.go b/controllers/webhooks/rules.go new file mode 100644 index 00000000..88fad3da --- /dev/null +++ b/controllers/webhooks/rules.go @@ -0,0 +1,73 @@ +// +// Copyright 2022 IBM Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package webhooks + +import admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + +// The `RuleWithOperations` and `Rule` types redefine the original ones from +// k8s.io/api/admissionregistration/v1 in order to allow to define methods +// to build the rule as a fluent interface. + +type RuleWithOperations struct { + Operations []admissionregistrationv1.OperationType + Rule +} + +type Rule struct { + APIGroups []string + APIVersions []string + Resources []string + Scope admissionregistrationv1.ScopeType +} + +func NewRule() RuleWithOperations { + return RuleWithOperations{} +} + +func (rule RuleWithOperations) OneResource(apiGroup, apiVersion, resource string) RuleWithOperations { + rule.APIGroups = []string{apiGroup} + rule.APIVersions = []string{apiVersion} + rule.Resources = []string{resource} + + return rule +} + +func (rule RuleWithOperations) NamespacedScope() RuleWithOperations { + rule.Scope = admissionregistrationv1.NamespacedScope + + return rule +} + +func (rule RuleWithOperations) ForCreate() RuleWithOperations { + rule.Operations = append(rule.Operations, admissionregistrationv1.Create) + return rule +} + +func (rule RuleWithOperations) ForUpdate() RuleWithOperations { + rule.Operations = append(rule.Operations, admissionregistrationv1.Update) + return rule +} + +func (rule RuleWithOperations) ForDelete() RuleWithOperations { + rule.Operations = append(rule.Operations, admissionregistrationv1.Delete) + return rule +} + +func (rule RuleWithOperations) ForAll() RuleWithOperations { + rule.Operations = append(rule.Operations, admissionregistrationv1.OperationAll) + return rule +} diff --git a/controllers/webhooks/webhook_register.go b/controllers/webhooks/webhook_register.go new file mode 100644 index 00000000..9e57a7b4 --- /dev/null +++ b/controllers/webhooks/webhook_register.go @@ -0,0 +1,182 @@ +// +// Copyright 2022 IBM Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package webhooks + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// WebhookRegister knows how the register a webhook into the server. Either by +// regstering to the WebhookBuilder or directly to the webhook server. +type WebhookRegister interface { + RegisterToBuilder(blrd *builder.WebhookBuilder) *builder.WebhookBuilder + RegisterToServer(scheme *runtime.Scheme, srv *webhook.Server) error + + GetReconciler(scheme *runtime.Scheme) (WebhookReconciler, error) +} + +// ObjectWebhookRegister registers objects that implement either the `Validator` +// interface or the `Defaulting` interface into the WebhookBuilder +type ObjectWebhookRegister struct { + Object runtime.Object +} + +type valueForType struct { + validating string + mutating string +} + +// WebhookRegisterFor creates a WebhookRegister for a given object, validating +// beforehand that the object implements either the `Defaulter` of `Validator` +// interfaces +func WebhookRegisterFor(object runtime.Object) (*ObjectWebhookRegister, error) { + _, isDefaulter := object.(admission.Defaulter) + _, isValidator := object.(admission.Validator) + + if isDefaulter || isValidator { + return &ObjectWebhookRegister{object}, nil + } + + return nil, fmt.Errorf("object %v does not implement Defaulter or Validator interface", object) +} + +// RegisterToBuilder adds the object into the builder, which registers the webhook +// for the object into the webhook server +func (vwr ObjectWebhookRegister) RegisterToBuilder(bldr *builder.WebhookBuilder) *builder.WebhookBuilder { + return bldr.For(vwr.Object) +} + +// RegisterToServer does nothing, as the register is done by the builder +func (vwr ObjectWebhookRegister) RegisterToServer(_ *runtime.Scheme, _ *webhook.Server) {} + +// GetReconciler creates a reconciler according to the implementation of vwr.Object. +// The object can implement the `Validator` or `Defaulter` interfaces, and if both +// interfaces are implemented, two webhook configurations must be reconciled, as +// two endpoints will be registered in the webhook server +func (vwr ObjectWebhookRegister) GetReconciler(scheme *runtime.Scheme) (WebhookReconciler, error) { + paths, err := vwr.getPaths(scheme) + if err != nil { + return nil, err + } + + reconcilers := []WebhookReconciler{} + + if paths.mutating != "" { + reconcilers = append(reconcilers, &MutatingWebhookReconciler{ + Path: paths.mutating, + }) + } + + if paths.validating != "" { + reconcilers = append(reconcilers, &ValidatingWebhookReconciler{ + Path: paths.validating, + }) + } + + return &CompositeWebhookReconciler{ + Reconcilers: reconcilers, + }, nil +} + +// getPaths retrieves the paths for the webhook as implemented at controller-runtime/pkg/builder/webhook.go +// in order to match the path registered under the hood by the WebhookBuilder +func (vwr ObjectWebhookRegister) getPaths(scheme *runtime.Scheme) (*valueForType, error) { + gvk, err := apiutil.GVKForObject(vwr.Object, scheme) + if err != nil { + return nil, err + } + + result := &valueForType{} + + _, isDefaulter := vwr.Object.(admission.Defaulter) + if isDefaulter { + result.mutating = generatePath("mutate", gvk) + } + + _, isValidator := vwr.Object.(admission.Validator) + if isValidator { + result.validating = generatePath("validate", gvk) + } + + return result, nil +} + +func generatePath(prefix string, gvk schema.GroupVersionKind) string { + path := fmt.Sprintf("/%s-", prefix) + strings.Replace(gvk.Group, ".", "-", -1) + "-" + + gvk.Version + "-" + strings.ToLower(gvk.Kind) + + return path +} + +// WebhookType represents the type of webhook configuration to reconcile. Can +// be ValidatingType or MutatingType +type WebhookType string + +// ValidatingType indicates that a ValidatingWebhookConfiguration must be +// reconciled +const ValidatingType = "Validating" + +// MutatingType indicates that a MutatingWebhookConfiguration must be reconciled +const MutatingType = "Mutating" + +// AdmissionWebhookRegister registers a given webhook into a specific path. +// This allows a more low level alternative to the WebhookBuilder, as it can +// directly get access the the AdmissionReview object sent to the webhook. +type AdmissionWebhookRegister struct { + Type WebhookType + Hook *admission.Webhook + Path string +} + +// RegisterToBuilder does not mutate the WebhookBuilder +func (awr AdmissionWebhookRegister) RegisterToBuilder(bldr *builder.WebhookBuilder) *builder.WebhookBuilder { + return bldr +} + +// RegisterToServer regsiters the webhook to the path of `awr` +func (awr AdmissionWebhookRegister) RegisterToServer(scheme *runtime.Scheme, srv *webhook.Server) error { + err := awr.Hook.InjectScheme(scheme) + if err != nil { + return err + } + srv.Register(awr.Path, awr.Hook) + return nil +} + +// GetReconciler creates a reconciler for awr's given Path and Type +func (awr AdmissionWebhookRegister) GetReconciler(_ *runtime.Scheme) (WebhookReconciler, error) { + switch awr.Type { + case ValidatingType: + return &ValidatingWebhookReconciler{ + Path: awr.Path, + }, nil + case MutatingType: + return &MutatingWebhookReconciler{ + Path: awr.Path, + }, nil + } + + return nil, fmt.Errorf("unsupported type for AdmissionWebhookRegister: %s", awr.Type) +} diff --git a/controllers/webhooks/webhookreconciler.go b/controllers/webhooks/webhookreconciler.go new file mode 100644 index 00000000..1166804a --- /dev/null +++ b/controllers/webhooks/webhookreconciler.go @@ -0,0 +1,335 @@ +// +// Copyright 2022 IBM Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package webhooks + +import ( + "context" + "strings" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/IBM/operand-deployment-lifecycle-manager/controllers/constant" + "github.com/IBM/operand-deployment-lifecycle-manager/controllers/util" + operandrequestwebhook "github.com/IBM/operand-deployment-lifecycle-manager/controllers/webhooks/operandrequestreplication" +) + +// WebhookReconciler knows how to reconcile webhook configuration CRs +type WebhookReconciler interface { + SetName(name string) + SetWebhookName(webhookName string) + SetRule(rule RuleWithOperations) + SetNsSelector(selector v1.LabelSelector) + Reconcile(ctx context.Context, client k8sclient.Client, caBundle []byte) error +} + +type CompositeWebhookReconciler struct { + Reconcilers []WebhookReconciler +} + +func (reconciler *CompositeWebhookReconciler) SetName(name string) { + for _, innerReconciler := range reconciler.Reconcilers { + innerReconciler.SetName(name) + } +} + +func (reconciler *CompositeWebhookReconciler) SetWebhookName(webhookName string) { + for _, innerReconciler := range reconciler.Reconcilers { + innerReconciler.SetWebhookName(webhookName) + } +} + +func (reconciler *CompositeWebhookReconciler) SetRule(rule RuleWithOperations) { + for _, innerReconciler := range reconciler.Reconcilers { + innerReconciler.SetRule(rule) + } +} + +func (reconciler *CompositeWebhookReconciler) SetNsSelector(selector v1.LabelSelector) { + for _, innerReconciler := range reconciler.Reconcilers { + innerReconciler.SetNsSelector(selector) + } +} + +func (reconciler *CompositeWebhookReconciler) Reconcile(ctx context.Context, client k8sclient.Client, caBundle []byte) error { + for _, innerReconciler := range reconciler.Reconcilers { + if err := innerReconciler.Reconcile(ctx, client, caBundle); err != nil { + return err + } + } + + return nil +} + +type ValidatingWebhookReconciler struct { + Path string + name string + webhookName string + rule RuleWithOperations + NameSpaceSelector v1.LabelSelector +} + +type MutatingWebhookReconciler struct { + Path string + name string + webhookName string + rule RuleWithOperations + NameSpaceSelector v1.LabelSelector +} + +// Reconcile MutatingWebhookConfiguration +func (reconciler *MutatingWebhookReconciler) Reconcile(ctx context.Context, client k8sclient.Client, caBundle []byte) error { + var ( + sideEffects = admissionregistrationv1.SideEffectClassNone + port = int32(servicePort) + matchPolicy = admissionregistrationv1.Exact + ignorePolicy = admissionregistrationv1.Ignore + timeoutSeconds = int32(10) + labels = map[string]string{ + constant.OdlmManagedLabel: "true", + "app.kubernetes.io/instance": constant.OperatorName, + "app.kubernetes.io/managed-by": constant.OperatorName, + "app.kubernetes.io/name": constant.OperatorName, + "name": constant.OperatorName, + } + ) + + namespace := util.GetOperatorNamespace() + roleName, roleUID, err := util.GetClusterRoleDetails(client, namespace, constant.CSVName) + if err != nil { + return err + } + + cr := &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: v1.ObjectMeta{ + Name: reconciler.name, + OwnerReferences: []v1.OwnerReference{ + { + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRole", + Name: roleName, + UID: roleUID, + }, + }, + Labels: labels, + }, + } + + klog.Infof("Creating/Updating MutatingWebhook %s", reconciler.name) + _, err = controllerutil.CreateOrUpdate(ctx, client, cr, func() error { + cr.Webhooks = []admissionregistrationv1.MutatingWebhook{ + { + Name: reconciler.webhookName, + SideEffects: &sideEffects, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + CABundle: caBundle, + Service: &admissionregistrationv1.ServiceReference{ + Namespace: namespace, + Name: operatorPodServiceName, + Path: &reconciler.Path, + Port: &port, + }, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: reconciler.rule.Operations, + Rule: admissionregistrationv1.Rule{ + APIGroups: reconciler.rule.APIGroups, + APIVersions: reconciler.rule.APIVersions, + Resources: reconciler.rule.Resources, + Scope: &reconciler.rule.Scope, + }, + }, + }, + MatchPolicy: &matchPolicy, + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &ignorePolicy, + TimeoutSeconds: &timeoutSeconds, + }, + } + for index := range cr.Webhooks { + cr.Webhooks[index].NamespaceSelector = &reconciler.NameSpaceSelector + } + return nil + }) + if err != nil { + klog.Error(err) + } + return err +} + +// Reconcile ValidatingWebhookConfiguration +func (reconciler *ValidatingWebhookReconciler) Reconcile(ctx context.Context, client k8sclient.Client, caBundle []byte) error { + var ( + sideEffects = admissionregistrationv1.SideEffectClassNone + port = int32(servicePort) + matchPolicy = admissionregistrationv1.Exact + failurePolicy = admissionregistrationv1.Fail + timeoutSeconds = int32(10) + labels = map[string]string{ + constant.OdlmManagedLabel: "true", + "app.kubernetes.io/instance": constant.OperatorName, + "app.kubernetes.io/managed-by": constant.OperatorName, + "app.kubernetes.io/name": constant.OperatorName, + "name": constant.OperatorName, + } + ) + + namespace := util.GetOperatorNamespace() + roleName, roleUID, err := util.GetClusterRoleDetails(client, namespace, constant.CSVName) + if err != nil { + return err + } + + cr := &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: v1.ObjectMeta{ + Name: reconciler.name, + OwnerReferences: []v1.OwnerReference{ + { + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRole", + Name: roleName, + UID: roleUID, + }, + }, + Labels: labels, + }, + } + + webhookLabel := make(map[string]string) + webhookLabel[constant.OdlmManagedLabel] = "true" + + klog.Infof("Creating/Updating ValidatingWebhook %s", reconciler.name) + _, err = controllerutil.CreateOrUpdate(ctx, client, cr, func() error { + cr.Webhooks = []admissionregistrationv1.ValidatingWebhook{ + { + Name: reconciler.webhookName, + SideEffects: &sideEffects, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + CABundle: caBundle, + Service: &admissionregistrationv1.ServiceReference{ + Namespace: namespace, + Name: operatorPodServiceName, + Path: &reconciler.Path, + Port: &port, + }, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: reconciler.rule.Operations, + Rule: admissionregistrationv1.Rule{ + APIGroups: reconciler.rule.APIGroups, + APIVersions: reconciler.rule.APIVersions, + Resources: reconciler.rule.Resources, + Scope: &reconciler.rule.Scope, + }, + }, + }, + MatchPolicy: &matchPolicy, + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failurePolicy, + TimeoutSeconds: &timeoutSeconds, + }, + } + for index := range cr.Webhooks { + cr.Webhooks[index].NamespaceSelector = &reconciler.NameSpaceSelector + } + return nil + }) + if err != nil { + klog.Error(err) + } + return err +} + +func (reconciler *ValidatingWebhookReconciler) SetName(name string) { + reconciler.name = name +} + +func (reconciler *MutatingWebhookReconciler) SetName(name string) { + reconciler.name = name +} + +func (reconciler *ValidatingWebhookReconciler) SetWebhookName(webhookName string) { + reconciler.webhookName = webhookName +} + +func (reconciler *MutatingWebhookReconciler) SetWebhookName(webhookName string) { + reconciler.webhookName = webhookName +} + +func (reconciler *ValidatingWebhookReconciler) SetRule(rule RuleWithOperations) { + reconciler.rule = rule +} + +func (reconciler *MutatingWebhookReconciler) SetRule(rule RuleWithOperations) { + reconciler.rule = rule +} + +func (reconciler *MutatingWebhookReconciler) SetNsSelector(selector v1.LabelSelector) { + reconciler.NameSpaceSelector = selector +} + +func (reconciler *ValidatingWebhookReconciler) SetNsSelector(selector v1.LabelSelector) { + reconciler.NameSpaceSelector = selector +} + +func SetupWebhooks(mgr manager.Manager, client k8sclient.Client, operatorNs, partialWatchNs string) error { + + klog.Info("Creating odlm webhook configuration") + + nsLabelSelector := &v1.LabelSelector{} + nsLabelSelector.MatchExpressions = []v1.LabelSelectorRequirement{ + { + Key: "kubernetes.io/metadata.name", + Operator: v1.LabelSelectorOpIn, + Values: strings.Split(partialWatchNs, ","), + }, + } + Config.AddWebhook(CSWebhook{ + Name: "ibm-opreq-replication-webhook-" + operatorNs, + WebhookName: "ibm-cloudpak-operandrequest-replication.operator.ibm.com", + Rule: NewRule(). + OneResource("operator.ibm.com", "v1alpha1", "operandrequests"). + ForUpdate(). + ForDelete(). + NamespacedScope(), + Register: AdmissionWebhookRegister{ + Type: MutatingType, + Path: "/mutate-ibm-cp-operandrequest-replication", + Hook: &admission.Webhook{ + Handler: &operandrequestwebhook.Defaulter{ + Client: client, + OperatorNs: operatorNs, + }, + }, + }, + NsSelector: *nsLabelSelector, + }) + + klog.Info("setting up webhook server") + if err := Config.SetupServer(mgr, operatorNs); err != nil { + return err + } + + return nil +} diff --git a/go.mod b/go.mod index 6d11ad0c..632f61b3 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/operator-framework/operator-registry v1.13.6 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.12.2 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect @@ -73,6 +74,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.12.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect @@ -96,6 +98,7 @@ require ( k8s.io/apiextensions-apiserver v0.24.2 // indirect k8s.io/component-base v0.24.2 // indirect k8s.io/klog/v2 v2.70.1 // indirect + k8s.io/kube-aggregator v0.18.9 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect diff --git a/go.sum b/go.sum index ffeb36be..0da3f335 100644 --- a/go.sum +++ b/go.sum @@ -923,6 +923,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= @@ -1651,6 +1653,7 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-aggregator v0.18.9 h1:kqwbA15uygYfLfdMUlyBm/q3OHaYbnirFrg7tGUTVZk= k8s.io/kube-aggregator v0.18.9/go.mod h1:ik5Mf6JaP2M9XbWZR/AYgXx2Nj4EDBrHyakUx7C8cdw= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= diff --git a/main.go b/main.go index 57c49924..aa538784 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ package main import ( + "context" "flag" "os" "strings" @@ -33,6 +34,8 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/klog" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/healthz" cache "github.com/IBM/controller-filtered-cache/filteredcache" @@ -49,6 +52,8 @@ import ( deploy "github.com/IBM/operand-deployment-lifecycle-manager/controllers/operator" "github.com/IBM/operand-deployment-lifecycle-manager/controllers/operatorchecker" "github.com/IBM/operand-deployment-lifecycle-manager/controllers/util" + "github.com/IBM/operand-deployment-lifecycle-manager/controllers/webhooks" + opreqreplication "github.com/IBM/operand-deployment-lifecycle-manager/controllers/webhooks/operandrequestreplication" // +kubebuilder:scaffold:imports ) @@ -163,6 +168,41 @@ func main() { } } } + + partialWatchNamespace := util.GetPartialWatchNamespace() + if partialWatchNamespace != "" && util.PartialWatchNamespaceEnabled() { + operatorNamespace := util.GetOperatorNamespace() + cfg, err := config.GetConfig() + if err != nil { + klog.Error(err, "") + os.Exit(1) + } + kubeClient, err := client.New(cfg, client.Options{ + Scheme: scheme, + }) + if err != nil { + klog.Error(err, "") + os.Exit(1) + } + + if err := webhooks.SetupWebhooks(mgr, kubeClient, operatorNamespace, partialWatchNamespace); err != nil { + klog.Error(err, "Error setting up webhook server") + } + + clusterRole, err := util.GetClusterRole(kubeClient, operatorNamespace, constant.CSVName) + if err != nil { + klog.Errorf("unable to get cluster role: %v", err) + os.Exit(1) + } + + if err := webhooks.Config.Reconcile(context.TODO(), kubeClient, clusterRole); err != nil { + klog.Errorf("unable to create webhook configuration: %v", err) + os.Exit(1) + } + + // add annotation annotation "operand.ibm.com/watched-by-odlm-in-$OperatorNs" to all OperandRequests in partial watch namespace + go opreqreplication.AddAnnotationToOperandRequests(kubeClient, partialWatchNamespace, operatorNamespace) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {