Merge pull request #244 from VaishnaviHire/sync_upstream_2.9.0

Sync upstream 2.9.0

components/datasciencepipelines/datasciencepipelines.go

@@ -9,19 +9,23 @@ import (
 	"path/filepath"
 
 	operatorv1 "github.com/openshift/api/operator/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1"
 	"github.com/opendatahub-io/opendatahub-operator/v2/components"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring"
 )
 
 var (
-	ComponentName = "data-science-pipelines-operator"
-	Path          = deploy.DefaultManifestPath + "/" + ComponentName + "/base"
-	OverlayPath   = deploy.DefaultManifestPath + "/" + ComponentName + "/overlays"
+	ComponentName   = "data-science-pipelines-operator"
+	Path            = deploy.DefaultManifestPath + "/" + ComponentName + "/base"
+	OverlayPath     = deploy.DefaultManifestPath + "/" + ComponentName + "/overlays"
+	ArgoWorkflowCRD = "workflows.argoproj.io"
 )
 
 // Verifies that Dashboard implements ComponentInterface.
@@ -101,6 +105,10 @@ func (d *DataSciencePipelines) ReconcileComponent(ctx context.Context,
 				return err
 			}
 		}
+		// Check for existing Argo Workflows
+		if err := UnmanagedArgoWorkFlowExists(ctx, cli); err != nil {
+			return err
+		}
 	}
 
 	// new overlay
@@ -136,3 +144,21 @@ func (d *DataSciencePipelines) ReconcileComponent(ctx context.Context,
 
 	return nil
 }
+
+func UnmanagedArgoWorkFlowExists(ctx context.Context,
+	cli client.Client) error {
+	workflowCRD := &apiextensionsv1.CustomResourceDefinition{}
+	if err := cli.Get(ctx, client.ObjectKey{Name: ArgoWorkflowCRD}, workflowCRD); err != nil {
+		if apierrs.IsNotFound(err) {
+			return nil
+		}
+		return fmt.Errorf("failed to get existing Workflow CRD : %w", err)
+	}
+	// Verify if existing workflow is deployed by ODH
+	_, odhLabelExists := workflowCRD.Labels[labels.ODH.Component(ComponentName)]
+	if odhLabelExists {
+		return nil
+	}
+	return fmt.Errorf(" %v CRD already exists but not deployed by this operator. Remove existing Argo workflows or set datasciencepipelines to Removed to proceed ",
+		ArgoWorkflowCRD)
+}

controllers/datasciencecluster/datasciencecluster_controller.go

@@ -34,6 +34,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	netv1 "k8s.io/api/networking/v1"
 	authv1 "k8s.io/api/rbac/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -52,6 +53,7 @@ import (
 	dsc "github.com/opendatahub-io/opendatahub-operator/v2/apis/datasciencecluster/v1"
 	dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1"
 	"github.com/opendatahub-io/opendatahub-operator/v2/components"
+	"github.com/opendatahub-io/opendatahub-operator/v2/components/datasciencepipelines"
 	"github.com/opendatahub-io/opendatahub-operator/v2/components/trustyai"
 	"github.com/opendatahub-io/opendatahub-operator/v2/controllers/status"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster"
@@ -79,7 +81,7 @@ const (
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
-func (r *DataScienceClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { //nolint:gocyclo,maintidx
+func (r *DataScienceClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { //nolint:maintidx,gocyclo
 	r.Log.Info("Reconciling DataScienceCluster resources", "Request.Name", req.Name)
 
 	instances := &dsc.DataScienceClusterList{}
@@ -211,6 +213,22 @@ func (r *DataScienceClusterReconciler) Reconcile(ctx context.Context, req ctrl.R
 		}
 		return ctrl.Result{}, nil
 	}
+	// Check preconditions if this is an upgrade
+	if instance.Status.Phase == status.PhaseReady {
+		// Check for existence of Argo Workflows if DSP is
+		if instance.Spec.Components.DataSciencePipelines.ManagementState == v1.Managed {
+			if err := datasciencepipelines.UnmanagedArgoWorkFlowExists(ctx, r.Client); err != nil {
+				message := fmt.Sprintf("Failed upgrade: %v ", err.Error())
+				_, err = status.UpdateWithRetry(ctx, r.Client, instance, func(saved *dsc.DataScienceCluster) {
+					status.SetExistingArgoCondition(&saved.Status.Conditions, status.ArgoWorkflowExist, message)
+					status.SetErrorCondition(&saved.Status.Conditions, status.ArgoWorkflowExist, message)
+					saved.Status.InstalledComponents[datasciencepipelines.ComponentName] = false
+					saved.Status.Phase = status.PhaseError
+				})
+				return ctrl.Result{}, err
+			}
+		}
+	}
 
 	// Start reconciling
 	if instance.Status.Conditions == nil {
@@ -297,7 +315,12 @@ func (r *DataScienceClusterReconciler) reconcileSubComponent(ctx context.Context
 		instance = r.reportError(err, instance, "failed to reconcile "+componentName+" on DataScienceCluster")
 		instance, _ = status.UpdateWithRetry(ctx, r.Client, instance, func(saved *dsc.DataScienceCluster) {
 			if enabled {
-				status.SetComponentCondition(&saved.Status.Conditions, componentName, status.ReconcileFailed, fmt.Sprintf("Component reconciliation failed: %v", err), corev1.ConditionFalse)
+				if strings.Contains(err.Error(), datasciencepipelines.ArgoWorkflowCRD+" CRD already exists") {
+					status.SetExistingArgoCondition(&saved.Status.Conditions, status.ArgoWorkflowExist, fmt.Sprintf("Component update failed: %v", err))
+					saved.Status.InstalledComponents[datasciencepipelines.ComponentName] = false
+				} else {
+					status.SetComponentCondition(&saved.Status.Conditions, componentName, status.ReconcileFailed, fmt.Sprintf("Component reconciliation failed: %v", err), corev1.ConditionFalse)
+				}
 			} else {
 				status.SetComponentCondition(&saved.Status.Conditions, componentName, status.ReconcileFailed, fmt.Sprintf("Component removal failed: %v", err), corev1.ConditionFalse)
 			}
@@ -434,6 +457,8 @@ func (r *DataScienceClusterReconciler) SetupWithManager(mgr ctrl.Manager) error
 		Owns(&corev1.ServiceAccount{}, builder.WithPredicates(saPredicates)).
 		Watches(&source.Kind{Type: &dsci.DSCInitialization{}}, handler.EnqueueRequestsFromMapFunc(r.watchDataScienceClusterResources)).
 		Watches(&source.Kind{Type: &corev1.ConfigMap{}}, handler.EnqueueRequestsFromMapFunc(r.watchDataScienceClusterResources), builder.WithPredicates(configMapPredicates)).
+		Watches(&source.Kind{Type: &apiextensionsv1.CustomResourceDefinition{}}, handler.EnqueueRequestsFromMapFunc(r.watchDataScienceClusterResources),
+			builder.WithPredicates(argoWorkflowCRDPredicates)).
 		// this predicates prevents meaningless reconciliations from being triggered
 		WithEventFilter(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{})).
 		Complete(r)
@@ -455,6 +480,12 @@ func (r *DataScienceClusterReconciler) watchDataScienceClusterResources(a client
 		return nil
 	}
 
+	if a.GetObjectKind().GroupVersionKind().Kind == "CustomResourceDefinition" {
+		return []reconcile.Request{{
+			NamespacedName: types.NamespacedName{Name: requestName},
+		}}
+	}
+
 	// Trigger reconcile function when uninstall configmap is created
 	operatorNs, err := cluster.GetOperatorNamespace()
 	if err != nil {
@@ -472,3 +503,16 @@ func (r *DataScienceClusterReconciler) watchDataScienceClusterResources(a client
 	}
 	return nil
 }
+
+// argoWorkflowCRDPredicates filters the delete events to trigger reconcile when Argo Workflow CRD is deleted.
+var argoWorkflowCRDPredicates = predicate.Funcs{
+	DeleteFunc: func(e event.DeleteEvent) bool {
+		if e.Object.GetName() == datasciencepipelines.ArgoWorkflowCRD {
+			labels := e.Object.GetLabels()
+			if _, ok := labels["app.opendatahub.io/"+datasciencepipelines.ComponentName]; !ok {
+				return true
+			}
+		}
+		return false
+	},
+}

controllers/status/status.go

@@ -22,6 +22,7 @@ import (
 	conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1"
 	corev1 "k8s.io/api/core/v1"
 
+	"github.com/opendatahub-io/opendatahub-operator/v2/components/datasciencepipelines"
 	"github.com/opendatahub-io/opendatahub-operator/v2/components/trustyai"
 )
 
@@ -72,13 +73,15 @@ const (
 const (
 	CapabilityServiceMesh              conditionsv1.ConditionType = "CapabilityServiceMesh"
 	CapabilityServiceMeshAuthorization conditionsv1.ConditionType = "CapabilityServiceMeshAuthorization"
+	CapabilityDSPv2Argo                conditionsv1.ConditionType = "CapabilityDSPv2Argo"
 )
 
 const (
 	MissingOperatorReason string = "MissingOperator"
 	ConfiguredReason      string = "Configured"
 	RemovedReason         string = "Removed"
 	CapabilityFailed      string = "CapabilityFailed"
+	ArgoWorkflowExist     string = "ArgoWorkflowExist"
 )
 
 const (
@@ -149,7 +152,7 @@ func SetErrorCondition(conditions *[]conditionsv1.Condition, reason string, mess
 	})
 	conditionsv1.SetStatusCondition(conditions, conditionsv1.Condition{
 		Type:    conditionsv1.ConditionUpgradeable,
-		Status:  corev1.ConditionUnknown,
+		Status:  corev1.ConditionFalse,
 		Reason:  reason,
 		Message: message,
 	})
@@ -188,6 +191,7 @@ func SetCompleteCondition(conditions *[]conditionsv1.Condition, reason string, m
 		Reason:  reason,
 		Message: message,
 	})
+	conditionsv1.RemoveStatusCondition(conditions, CapabilityDSPv2Argo)
 }
 
 // SetComponentCondition appends Condition Type with const ReadySuffix for given component
@@ -224,3 +228,14 @@ func RemoveComponentCondition(conditions *[]conditionsv1.Condition, component st
 	}
 	conditionsv1.RemoveStatusCondition(conditions, conditionsv1.ConditionType(condType))
 }
+
+func SetExistingArgoCondition(conditions *[]conditionsv1.Condition, reason, message string) {
+	conditionsv1.SetStatusCondition(conditions, conditionsv1.Condition{
+		Type:    CapabilityDSPv2Argo,
+		Status:  corev1.ConditionFalse,
+		Reason:  reason,
+		Message: message,
+	})
+
+	SetComponentCondition(conditions, datasciencepipelines.ComponentName, ReconcileFailed, message, corev1.ConditionFalse)
+}

go.mod

@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/ghodss/yaml v1.0.0
-	github.com/go-logr/logr v1.2.4
+	github.com/go-logr/logr v1.4.1
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/onsi/ginkgo/v2 v2.12.1
 	github.com/onsi/gomega v1.27.10
@@ -15,9 +15,9 @@ require (
 	github.com/operator-framework/api v0.18.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0
-	github.com/spf13/afero v1.9.2
-	github.com/stretchr/testify v1.8.3
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
+	github.com/spf13/afero v1.10.0
+	github.com/stretchr/testify v1.8.4
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.28.3
 	k8s.io/apiextensions-apiserver v0.28.3
@@ -46,13 +46,13 @@ require (
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic v0.6.9 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.3.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -65,7 +65,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.16.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/rhobs/obo-prometheus-operator/pkg/apis/monitoring v0.61.1-rhobs1 // indirect
@@ -77,16 +77,16 @@ require (
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/oauth2 v0.11.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/oauth2 v0.17.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -101,10 +101,10 @@ require (
 
 replace (
 	github.com/go-yaml/yaml => github.com/go-yaml/yaml v2.2.8+incompatible
+	github.com/mattn/go-sqlite3 => github.com/mattn/go-sqlite3 v1.14.18
 	github.com/tektoncd/pipeline => github.com/tektoncd/pipeline v0.12.0
 	go.uber.org/multierr => go.uber.org/multierr v1.6.0
-	golang.org/x/image => golang.org/x/image v0.10.0
-	google.golang.org/grpc => google.golang.org/grpc v1.58.3
+	google.golang.org/grpc => google.golang.org/grpc v1.63.2
 	k8s.io/api => k8s.io/api v0.26.0
 	k8s.io/client-go => k8s.io/client-go v0.26.0
 	k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20230113010536-ace3e4a52ca1