From 177d4f3629732cb0c9aa1b4be1c2c1f1b9ce795c Mon Sep 17 00:00:00 2001 From: Denis Arsh Date: Mon, 2 Sep 2024 16:51:42 +0300 Subject: [PATCH 1/4] allow multiple instances of metastorageprovisioner --- api/v1alpha1/metastoragetypes.go | 2 ++ charts/metastorageprovisioner/templates/dep.yml | 2 +- .../mlops.cnvrg.io_metastorageprovisioners.yaml | 8 +++++--- .../mlops.cnvrg.io_metastorageprovisioners.yaml | 8 +++++--- controllers/metastorageprovider/controller.go | 15 +++++++++------ pkg/app/controlplane/tmpl/conf/rbac/ccp-role.tpl | 13 ++++++++++++- .../crds/mlops.cnvrg.io_cnvrgthirdparties.yaml | 3 +++ .../mlops.cnvrg.io_metastorageprovisioners.yaml | 8 +++++--- pkg/helm/helm.go | 3 ++- 9 files changed, 44 insertions(+), 18 deletions(-) diff --git a/api/v1alpha1/metastoragetypes.go b/api/v1alpha1/metastoragetypes.go index 7ced9113..d4161433 100644 --- a/api/v1alpha1/metastoragetypes.go +++ b/api/v1alpha1/metastoragetypes.go @@ -23,6 +23,8 @@ type MetaStorageProvisionerStatus string // MetaStorageProvisioner represents the storage provisioner to be installed // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.status` + type MetaStorageProvisioner struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/charts/metastorageprovisioner/templates/dep.yml b/charts/metastorageprovisioner/templates/dep.yml index 0bb2df25..61ff9372 100644 --- a/charts/metastorageprovisioner/templates/dep.yml +++ b/charts/metastorageprovisioner/templates/dep.yml @@ -21,7 +21,7 @@ spec: containers: - name: metastorageprovisioner imagePullPolicy: Always - image: "{{.Values.imageHub}}/cnvrg-operator:5.1.20" + image: "{{.Values.imageHub}}/cnvrg-operator:5.1.22" command: - /opt/app-root/cnvrg-metastorageprovisioner - start diff --git a/charts/mlops/crds/mlops.cnvrg.io_metastorageprovisioners.yaml b/charts/mlops/crds/mlops.cnvrg.io_metastorageprovisioners.yaml index 6c840380..948e9031 100644 --- a/charts/mlops/crds/mlops.cnvrg.io_metastorageprovisioners.yaml +++ b/charts/mlops/crds/mlops.cnvrg.io_metastorageprovisioners.yaml @@ -14,11 +14,13 @@ spec: singular: metastorageprovisioner scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.status + name: Status + type: string + name: v1alpha1 schema: openAPIV3Schema: - description: MetaStorageProvisioner represents the storage provisioner to - be installed properties: apiVersion: description: |- diff --git a/config/crd/bases/mlops.cnvrg.io_metastorageprovisioners.yaml b/config/crd/bases/mlops.cnvrg.io_metastorageprovisioners.yaml index 6c840380..948e9031 100644 --- a/config/crd/bases/mlops.cnvrg.io_metastorageprovisioners.yaml +++ b/config/crd/bases/mlops.cnvrg.io_metastorageprovisioners.yaml @@ -14,11 +14,13 @@ spec: singular: metastorageprovisioner scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.status + name: Status + type: string + name: v1alpha1 schema: openAPIV3Schema: - description: MetaStorageProvisioner represents the storage provisioner to - be installed properties: apiVersion: description: |- diff --git a/controllers/metastorageprovider/controller.go b/controllers/metastorageprovider/controller.go index d5723e34..7e5e050c 100644 --- a/controllers/metastorageprovider/controller.go +++ b/controllers/metastorageprovider/controller.go @@ -87,7 +87,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // readiness checker for NFS provisioner checker = func() (bool, error) { - deployment := types.NamespacedName{Name: nfsProvisionerReleaseName, Namespace: r.Namespace} + deployment := types.NamespacedName{Name: provisionerObject.Name, Namespace: r.Namespace} return CheckDeploymentReadiness(r.Client, deployment) } } @@ -115,7 +115,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // if not ready, requeue if !ready { r.EventLogger.WithMessage("waiting for deployment readiness").WithStatus(v1alpha1.Pending).Log(ctx) - return ctrl.Result{RequeueAfter: 5}, nil + return ctrl.Result{RequeueAfter: 30}, nil } // provisioner is ready @@ -138,7 +138,9 @@ func (r *Reconciler) nfsConfig(provisionerObject *v1alpha1.MetaStorageProvisione "path": provisionerObject.Spec.NFSProvisioner.NFSPath, } - chartConfig = r.ChartConfig(values) + values["fullnameOverride"] = provisionerObject.Name + + chartConfig = r.NFSChartConfig(provisionerObject.Name, values) return chartConfig } @@ -150,10 +152,11 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r *Reconciler) ChartConfig(values map[string]interface{}) installer.ChartConfig { +func (r *Reconciler) NFSChartConfig(name string, values map[string]interface{}) installer.ChartConfig { return installer.ChartConfig{ Namespace: r.Namespace, - ReleaseName: nfsProvisionerReleaseName, + ReleaseName: name, + ChartName: nfsProvisionerReleaseName, Url: fmt.Sprintf("https://kubernetes-sigs.github.io/%s", nfsProvisionerReleaseName), Values: values, Version: nfsProvisionerVersion, @@ -178,7 +181,7 @@ func (r *Reconciler) ReconcileDelete(ctx context.Context, provisioner *v1alpha1. logger := log.FromContext(ctx) r.EventLogger.WithMessage("deleting meta storage provisioner").WithStatus(v1alpha1.Deleting).Log(ctx) - chartConfig := r.ChartConfig(nil) + chartConfig := r.NFSChartConfig(provisioner.Name, nil) helm, err := installer.NewHelm(chartConfig, logger) if err != nil { return fmt.Errorf("error while creating helm installer: %w", err) diff --git a/pkg/app/controlplane/tmpl/conf/rbac/ccp-role.tpl b/pkg/app/controlplane/tmpl/conf/rbac/ccp-role.tpl index 30a20aff..88b5a735 100644 --- a/pkg/app/controlplane/tmpl/conf/rbac/ccp-role.tpl +++ b/pkg/app/controlplane/tmpl/conf/rbac/ccp-role.tpl @@ -263,4 +263,15 @@ rules: - list - patch - update - - watch \ No newline at end of file + - watch +- apiGroups: + - mlops.cnvrg.io + resources: + - metastorageprovisioners + verbs: + - get + - list + - watch + - create + - update + - delete \ No newline at end of file diff --git a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml index 9c2254c2..c1f90cd1 100644 --- a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml +++ b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml @@ -93,6 +93,9 @@ metadata: mlops.cnvrg.io/default-loader: "true" mlops.cnvrg.io/own: "false" mlops.cnvrg.io/updatable: "true" + mlops.cnvrg.io/default-loader: "true" + mlops.cnvrg.io/own: "false" + mlops.cnvrg.io/updatable: "true" controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: cnvrgthirdparties.mlops.cnvrg.io diff --git a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_metastorageprovisioners.yaml b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_metastorageprovisioners.yaml index 6c840380..948e9031 100644 --- a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_metastorageprovisioners.yaml +++ b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_metastorageprovisioners.yaml @@ -14,11 +14,13 @@ spec: singular: metastorageprovisioner scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.status + name: Status + type: string + name: v1alpha1 schema: openAPIV3Schema: - description: MetaStorageProvisioner represents the storage provisioner to - be installed properties: apiVersion: description: |- diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 4f0713c0..d260b195 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -31,6 +31,7 @@ type ChartConfig struct { Namespace string Pass string ReleaseName string + ChartName string Url string User string Values map[string]interface{} @@ -99,7 +100,7 @@ func (h *Helm) configureRegistryClient() error { func (h *Helm) loadChart(cpo *action.ChartPathOptions) (*chart.Chart, error) { cpo.Version = h.chartConfig.Version cpo.RepoURL = h.chartConfig.Url - cp, err := cpo.LocateChart(h.chartConfig.ReleaseName, h.envSettings) + cp, err := cpo.LocateChart(h.chartConfig.ChartName, h.envSettings) if err != nil { return nil, err } From 897850cff35984c76e3c22ae77bd08b21e2823ca Mon Sep 17 00:00:00 2001 From: Denis Arsh Date: Tue, 3 Sep 2024 10:11:07 +0300 Subject: [PATCH 2/4] remove redundant comment --- charts/mlops/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mlops/Chart.yaml b/charts/mlops/Chart.yaml index 00527892..eed4f903 100644 --- a/charts/mlops/Chart.yaml +++ b/charts/mlops/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: mlops description: A cnvrg.io MLOps platform Helm Chart type: application -version: 5.1.0 # NOTE: if changed should be updated in charts/metastorageprovisioner/Chart.yaml accordingly +version: 5.1.0 appVersion: 5.1.0 dependencies: - name: nomex From ca1a0668ff15217ae0ca1ed3fe384908b1acfbee Mon Sep 17 00:00:00 2001 From: Dmitry Kartsev Date: Sun, 29 Sep 2024 12:05:36 +0300 Subject: [PATCH 3/4] DEV-23991-watcher-ns: add missing ns parameter, fixing redirect URLs composition logic --- charts/mlops/templates/cap.yml | 17 +++++++++++++---- charts/mlops/values.yaml | 3 +-- pkg/app/sso/central.go | 9 +++++++-- pkg/app/sso/tmpl/proxy/dep.tpl | 2 ++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/charts/mlops/templates/cap.yml b/charts/mlops/templates/cap.yml index 9d3a9ac5..f8f6fc28 100644 --- a/charts/mlops/templates/cap.yml +++ b/charts/mlops/templates/cap.yml @@ -104,6 +104,11 @@ spec: baseConfig: jobsStorageClass: {{.Values.controlPlane.baseConfig.jobsStorageClass}} featureFlags: + {{- if eq .Values.platform "ocp"}} + OCP_ENABLED: "true" + {{- else }} + OCP_ENABLED: "false" + {{- end }} {{- range $fk, $fv := .Values.controlPlane.baseConfig.featureFlags }} {{$fk}}: "{{$fv}}" {{- end }} @@ -284,7 +289,11 @@ spec: credsRef: {{.Values.dbs.prom.grafana.credsRef}} networking: ingress: + {{- if eq .Values.platform "ocp"}} + type: "openshift" + {{- else}} type: {{.Values.networking.ingress.type}} + {{- end }} timeout: {{.Values.networking.ingress.timeout}} retriesAttempts: {{.Values.networking.ingress.retriesAttempts}} perTryTimeout: {{.Values.networking.ingress.perTryTimeout}} @@ -311,16 +320,16 @@ spec: enabled: {{.Values.sso.enabled}} version: {{.Values.sso.version}} pki: - enabled: {{.Values.sso.pki.enabled}} + enabled: {{.Values.sso.enabled}} rootCaSecret: {{.Values.sso.pki.rootCaSecret}} privateKeySecret: {{.Values.sso.pki.privateKeySecret}} publicKeySecret: {{.Values.sso.pki.publicKeySecret}} jwks: - enabled: {{.Values.sso.jwks.enabled}} + enabled: {{.Values.sso.enabled}} image: {{.Values.sso.jwks.image}} cacheImage: {{.Values.sso.jwks.cacheImage}} central: - enabled: {{.Values.sso.central.enabled}} + enabled: {{.Values.sso.enabled}} publicUrl: {{.Values.sso.central.publicUrl}} oauthProxyImage: {{.Values.sso.central.oauthProxyImage}} centralUiImage: {{.Values.sso.central.centralUiImage}} @@ -344,7 +353,7 @@ spec: cpu: "{{.Values.sso.central.limits.cpu}}" memory: "{{.Values.sso.central.limits.memory}}" proxy: - enabled: {{.Values.sso.proxy.enabled}} + enabled: {{.Values.sso.enabled}} image: {{.Values.sso.proxy.image}} address: {{.Values.sso.proxy.address}} readiness: {{.Values.sso.proxy.readiness}} diff --git a/charts/mlops/values.yaml b/charts/mlops/values.yaml index 68efc7ce..91a1e7e5 100644 --- a/charts/mlops/values.yaml +++ b/charts/mlops/values.yaml @@ -103,7 +103,6 @@ controlPlane: CNVRG_ENABLE_MOUNT_FOLDERS: false CNVRG_MOUNT_HOST_FOLDERS: false CNVRG_PROMETHEUS_METRICS: true - OCP_ENABLED: false sentryUrl: '' runJobsOnSelfCluster: '' agentCustomTag: '' @@ -340,7 +339,7 @@ sso: memory: 4Gi proxy: enabled: false - image: cnvrg-proxy:ns-watch-scope + image: cnvrg-proxy:4d10ded address: "" readiness: true requests: diff --git a/pkg/app/sso/central.go b/pkg/app/sso/central.go index d5f76c84..9b39aaf0 100644 --- a/pkg/app/sso/central.go +++ b/pkg/app/sso/central.go @@ -64,7 +64,11 @@ func (c *CentralStateManager) depData() map[string]interface{} { "Namespace": c.app.Namespace, "SsoDomainId": strings.Split(c.app.Spec.ClusterDomain, ".")[0], "Spec": c.app.Spec, - "AppUrl": fmt.Sprintf("%s://%s.%s", c.schema(), c.app.Spec.ControlPlane.WebApp.SvcName, c.app.Spec.ClusterDomain), + "AppUrl": fmt.Sprintf("%s://%s%s.%s", c.schema(), + c.app.Spec.Networking.ClusterDomainPrefix.Prefix, + c.app.Spec.ControlPlane.WebApp.SvcName, + c.app.Spec.ClusterDomain, + ), } } @@ -79,9 +83,10 @@ func (c *CentralStateManager) proxyCfgData() map[string]interface{} { "Provider": c.app.Spec.SSO.Central.Provider, "ClientId": c.app.Spec.SSO.Central.ClientID, "ClientSecret": c.app.Spec.SSO.Central.ClientSecret, - "RedirectUrl": fmt.Sprintf("%s://%s.%s/oauth2/callback", + "RedirectUrl": fmt.Sprintf("%s://%s%s.%s/oauth2/callback", c.schema(), c.app.Spec.SSO.Central.SvcName, + c.app.Spec.Networking.ClusterDomainPrefix.Prefix, c.app.Spec.ClusterDomain), "OidcIssuerURL": c.app.Spec.SSO.Central.OidcIssuerURL, "Scope": c.app.Spec.SSO.Central.Scope, diff --git a/pkg/app/sso/tmpl/proxy/dep.tpl b/pkg/app/sso/tmpl/proxy/dep.tpl index 8b9e470d..996fe8c8 100644 --- a/pkg/app/sso/tmpl/proxy/dep.tpl +++ b/pkg/app/sso/tmpl/proxy/dep.tpl @@ -66,6 +66,7 @@ spec: - start - --authz-addr=127.0.0.1:50052 - --ingress-type={{.Spec.Networking.Ingress.Type}} + - --namespace={{.Namespace }} ports: - containerPort: 8888 {{- if isTrue .Spec.SSO.Proxy.Readiness }} @@ -90,6 +91,7 @@ spec: - /opt/app-root/authz - start - --ingress-type={{.Spec.Networking.Ingress.Type}} + - --namespace={{.Namespace }} resources: limits: cpu: {{ .Spec.SSO.Proxy.Limits.Cpu }} From 54b07be78f61b57e5919d0a6005e14b6806c0364 Mon Sep 17 00:00:00 2001 From: Dmitry Kartsev Date: Sun, 29 Sep 2024 16:08:53 +0300 Subject: [PATCH 4/4] DEV-23991-watcher-ns: small fix for DEV-23991 --- pkg/app/sso/central.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/app/sso/central.go b/pkg/app/sso/central.go index 9b39aaf0..1c17f9d7 100644 --- a/pkg/app/sso/central.go +++ b/pkg/app/sso/central.go @@ -65,8 +65,8 @@ func (c *CentralStateManager) depData() map[string]interface{} { "SsoDomainId": strings.Split(c.app.Spec.ClusterDomain, ".")[0], "Spec": c.app.Spec, "AppUrl": fmt.Sprintf("%s://%s%s.%s", c.schema(), - c.app.Spec.Networking.ClusterDomainPrefix.Prefix, c.app.Spec.ControlPlane.WebApp.SvcName, + c.app.Spec.Networking.ClusterDomainPrefix.Prefix, c.app.Spec.ClusterDomain, ), }