diff --git a/README.md b/README.md index 861f9ce7..a3a375ee 100644 --- a/README.md +++ b/README.md @@ -21,4 +21,49 @@ helm install cnvrg cnvrg/mlops \ --set registry.user="" \ --set registry.password="" \ --set controlPlane.baseConfig.agentCustomTag="" -``` \ No newline at end of file +``` + +## Using external secret for SMTP server +It's an option to specify external secret for SMTP server credintials instead setting it in helm chart values or cnvrgapp CRD . +The parameter to reference the secret is `controlPlane.smtp.CredentialsSecretRef` and the keys in the secret should be `username` and `password`. + +```bash +helm install cnvrg cnvrg/mlops \ + --create-namespace -n cnvrg \ + --set controlPlane.smtp.credentialsSecretRef="SECRET-NAME" +``` +secret example +```bash +apiVersion: v1 +kind: Secret +metadata: + name: SECRET-NAME + namespace: cnvrg +type: Opaque +data: + username: YWRtaW4= + password: c2VjcmV0 +``` + +## Using external secret for OAuth2 client configuration + +It's an option to specify external secret for OAuth2 client configuration instead setting it in helm chart values or cnvrgapp CRD. The parameter to reference the secret is `sso.central.credentialsSecretRef` and the keys in the secret should be `clientId`, `clientSecret` + +```bash +helm install cnvrg cnvrg/mlops \ + --create-namespace -n cnvrg \ + --set sso.central.credentialsSecretRef="SECRET-NAME" +``` + +secret example +```bash +apiVersion: v1 +kind: Secret +metadata: + name: SECRET-NAME + namespace: cnvrg +type: Opaque +data: + clientId: YWRtaW4= + clientSecret: c2VjcmV0 +``` diff --git a/api/v1/app.go b/api/v1/app.go index d150d48e..8a7eaab0 100644 --- a/api/v1/app.go +++ b/api/v1/app.go @@ -122,13 +122,14 @@ type Ldap struct { } type SMTP struct { - Server string `json:"server,omitempty"` - Port int `json:"port,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - Domain string `json:"domain,omitempty"` - OpensslVerifyMode string `json:"opensslVerifyMode,omitempty"` - Sender string `json:"sender,omitempty"` + Server string `json:"server,omitempty"` + Port int `json:"port,omitempty"` + CredentialsSecretRef string `json:"credentialsSecretRef,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Domain string `json:"domain,omitempty"` + OpensslVerifyMode string `json:"opensslVerifyMode,omitempty"` + Sender string `json:"sender,omitempty"` } type ObjectStorage struct { @@ -407,6 +408,7 @@ type CentralSSO struct { EmailDomain []string `json:"emailDomain,omitempty"` ClientID string `json:"clientId,omitempty"` ClientSecret string `json:"clientSecret,omitempty"` + CredentialsSecretRef string `json:"credentialsSecretRef,omitempty"` OidcIssuerURL string `json:"oidcIssuerUrl,omitempty"` ServiceUrl string `json:"serviceUrl,omitempty"` Scope string `json:"scope,omitempty"` diff --git a/charts/mlops/crds/mlops.cnvrg.io_cnvrgapps.yaml b/charts/mlops/crds/mlops.cnvrg.io_cnvrgapps.yaml index 62cb324a..f86765b0 100644 --- a/charts/mlops/crds/mlops.cnvrg.io_cnvrgapps.yaml +++ b/charts/mlops/crds/mlops.cnvrg.io_cnvrgapps.yaml @@ -267,6 +267,8 @@ spec: type: object smtp: properties: + credentialsSecretRef: + type: string domain: type: string opensslVerifyMode: @@ -768,6 +770,8 @@ spec: type: string cookieDomain: type: string + credentialsSecretRef: + type: string emailDomain: items: type: string diff --git a/charts/mlops/templates/cap.yml b/charts/mlops/templates/cap.yml index f8f6fc28..156f0111 100644 --- a/charts/mlops/templates/cap.yml +++ b/charts/mlops/templates/cap.yml @@ -138,6 +138,7 @@ spec: port: {{.Values.controlPlane.smtp.port}} username: {{.Values.controlPlane.smtp.username}} password: {{.Values.controlPlane.smtp.password}} + credentialsSecretRef: {{.Values.controlPlane.smtp.credentialsSecretRef}} domain: {{.Values.controlPlane.smtp.domain}} opensslVerifyMode: {{.Values.controlPlane.smtp.opensslVerifyMode}} sender: {{.Values.controlPlane.smtp.sender}} @@ -338,6 +339,7 @@ spec: emailDomain: {{ toJson .Values.sso.central.emailDomain }} clientId: {{.Values.sso.central.clientId}} clientSecret: {{.Values.sso.central.clientSecret}} + credentialsSecretRef: {{.Values.sso.central.credentialsSecretRef}} oidcIssuerUrl: {{.Values.sso.central.oidcIssuerUrl}} serviceUrl: {{.Values.sso.central.serviceUrl}} scope: {{.Values.sso.central.scope}} diff --git a/charts/mlops/values.yaml b/charts/mlops/values.yaml index 5e95515d..974d852b 100644 --- a/charts/mlops/values.yaml +++ b/charts/mlops/values.yaml @@ -128,6 +128,7 @@ controlPlane: domain: '' opensslVerifyMode: '' sender: info@cnvrg.io + credentialsSecretRef: '' objectStorage: type: minio bucket: cnvrg-storage @@ -323,6 +324,7 @@ sso: - "*" clientId: '' clientSecret: '' + credentialsSecretRef: '' oidcIssuerUrl: '' serviceUrl: '' scope: openid email profile diff --git a/config/crd/bases/mlops.cnvrg.io_cnvrgapps.yaml b/config/crd/bases/mlops.cnvrg.io_cnvrgapps.yaml index 62cb324a..f86765b0 100644 --- a/config/crd/bases/mlops.cnvrg.io_cnvrgapps.yaml +++ b/config/crd/bases/mlops.cnvrg.io_cnvrgapps.yaml @@ -267,6 +267,8 @@ spec: type: object smtp: properties: + credentialsSecretRef: + type: string domain: type: string opensslVerifyMode: @@ -768,6 +770,8 @@ spec: type: string cookieDomain: type: string + credentialsSecretRef: + type: string emailDomain: items: type: string diff --git a/pkg/app/controlplane/controlplane.go b/pkg/app/controlplane/controlplane.go index a483831c..b09124d0 100644 --- a/pkg/app/controlplane/controlplane.go +++ b/pkg/app/controlplane/controlplane.go @@ -1,12 +1,15 @@ package controlplane import ( + "context" "embed" "fmt" mlopsv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" "github.com/AccessibleAI/cnvrg-operator/pkg/desired" "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -51,6 +54,59 @@ func (m *CpStateManager) LoadKiqs(kiqName string, hpa bool) error { return nil } +func (m *CpStateManager) renderSmtpConfigs() error { + assets := []string{"secret-smtp.tpl"} + f := &desired.LoadFilter{AssetName: assets} + smtp := desired.NewAssetsGroup(fs, m.RootPath()+"/conf/smtp", m.Log(), f) + if err := smtp.LoadAssets(); err != nil { + return err + } + + configData, err := m.smtpCfgData() + if err != nil { + return err + } + + if err = smtp.Render(configData); err != nil { + return err + } + + m.AddToState(smtp) + + return nil +} + +func (m *CpStateManager) smtpCfgData() (map[string]interface{}, error) { + var userName, password string + + if m.app.Spec.ControlPlane.SMTP.CredentialsSecretRef != "" { + secret := &corev1.Secret{} + if err := m.C.Get(context.Background(), types.NamespacedName{Name: m.app.Spec.ControlPlane.SMTP.CredentialsSecretRef, Namespace: m.app.Namespace}, secret); err != nil { + return nil, err + } + userName = string(secret.Data["username"]) + password = string(secret.Data["password"]) + } else { + userName = m.app.Spec.ControlPlane.SMTP.Username + password = m.app.Spec.ControlPlane.SMTP.Password + } + + d := map[string]interface{}{ + "Namespace": m.app.Namespace, + "Annotations": m.app.Spec.Annotations, + "Labels": m.app.Spec.Labels, + "Server": m.app.Spec.ControlPlane.SMTP.Server, + "Port": m.app.Spec.ControlPlane.SMTP.Port, + "Username": userName, + "Password": password, + "Domain": m.app.Spec.ControlPlane.SMTP.Domain, + "Sender": m.app.Spec.ControlPlane.SMTP.Sender, + "OpensslVerifyMode": m.app.Spec.ControlPlane.SMTP.OpensslVerifyMode, + } + + return d, nil +} + func (m *CpStateManager) Load() error { f := &desired.LoadFilter{DefaultLoader: true} @@ -122,6 +178,10 @@ func (m *CpStateManager) Load() error { } func (m *CpStateManager) Apply() error { + if err := m.renderSmtpConfigs(); err != nil { + return err + } + if err := m.Load(); err != nil { return err } diff --git a/pkg/app/controlplane/tmpl/conf/cm/secret-smtp.tpl b/pkg/app/controlplane/tmpl/conf/cm/secret-smtp.tpl deleted file mode 100644 index 49012273..00000000 --- a/pkg/app/controlplane/tmpl/conf/cm/secret-smtp.tpl +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: cp-smtp - namespace: {{ .Namespace }} - annotations: - mlops.cnvrg.io/default-loader: "true" - mlops.cnvrg.io/own: "true" - mlops.cnvrg.io/updatable: "true" - {{- range $k, $v := .Spec.Annotations }} - {{$k}}: "{{$v}}" - {{- end }} - labels: - {{- range $k, $v := .Spec.Labels }} - {{$k}}: "{{$v}}" - {{- end }} -data: - SMTP_SERVER: {{ .Spec.ControlPlane.SMTP.Server | b64enc }} - SMTP_PORT: {{ .Spec.ControlPlane.SMTP.Port | toString | b64enc }} - SMTP_USERNAME: {{ .Spec.ControlPlane.SMTP.Username | b64enc }} - SMTP_PASSWORD: {{ .Spec.ControlPlane.SMTP.Password | b64enc }} - SMTP_DOMAIN: {{ .Spec.ControlPlane.SMTP.Domain | b64enc }} - SMTP_OPENSSL_VERIFY_MODE: {{ .Spec.ControlPlane.SMTP.OpensslVerifyMode | b64enc }} - SMTP_SENDER: {{ .Spec.ControlPlane.SMTP.Sender | b64enc }} diff --git a/pkg/app/controlplane/tmpl/conf/smtp/secret-smtp.tpl b/pkg/app/controlplane/tmpl/conf/smtp/secret-smtp.tpl new file mode 100644 index 00000000..e5b33465 --- /dev/null +++ b/pkg/app/controlplane/tmpl/conf/smtp/secret-smtp.tpl @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Secret +metadata: + name: cp-smtp + namespace: {{ .Namespace }} + annotations: + mlops.cnvrg.io/default-loader: "true" + mlops.cnvrg.io/own: "true" + mlops.cnvrg.io/updatable: "true" + {{- range $k, $v := .Annotations }} + {{$k}}: "{{$v}}" + {{- end }} + labels: + {{- range $k, $v := .Labels }} + {{$k}}: "{{$v}}" + {{- end }} +data: + SMTP_SERVER: {{ .Server | b64enc }} + SMTP_PORT: {{ .Port | toString | b64enc }} + SMTP_USERNAME: {{ .Username | b64enc }} + SMTP_PASSWORD: {{ .Password | b64enc}} + SMTP_DOMAIN: {{ .Domain | b64enc}} + SMTP_OPENSSL_VERIFY_MODE: {{ .OpensslVerifyMode | b64enc }} + SMTP_SENDER: {{ .Sender | b64enc }} diff --git a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgapps.yaml b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgapps.yaml index fec46f69..35f7a07d 100644 --- a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgapps.yaml +++ b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgapps.yaml @@ -270,6 +270,8 @@ spec: type: object smtp: properties: + credentialsSecretRef: + type: string domain: type: string opensslVerifyMode: @@ -771,6 +773,8 @@ spec: type: string cookieDomain: type: string + credentialsSecretRef: + type: string emailDomain: items: type: string diff --git a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml index c1f90cd1..ba30cd76 100644 --- a/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml +++ b/pkg/app/controlplane/tmpl/crds/mlops.cnvrg.io_cnvrgthirdparties.yaml @@ -96,6 +96,12 @@ metadata: mlops.cnvrg.io/default-loader: "true" mlops.cnvrg.io/own: "false" mlops.cnvrg.io/updatable: "true" + mlops.cnvrg.io/default-loader: "true" + mlops.cnvrg.io/own: "false" + mlops.cnvrg.io/updatable: "true" + mlops.cnvrg.io/default-loader: "true" + mlops.cnvrg.io/own: "false" + mlops.cnvrg.io/updatable: "true" controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: cnvrgthirdparties.mlops.cnvrg.io diff --git a/pkg/app/sso/central.go b/pkg/app/sso/central.go index 1c17f9d7..c961967b 100644 --- a/pkg/app/sso/central.go +++ b/pkg/app/sso/central.go @@ -1,10 +1,12 @@ package sso import ( + "context" "fmt" mlopsv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" "github.com/AccessibleAI/cnvrg-operator/pkg/desired" "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "strings" @@ -12,7 +14,8 @@ import ( type CentralStateManager struct { *desired.AssetsStateManager - app *mlopsv1.CnvrgApp + app *mlopsv1.CnvrgApp + client client.Client } func NewCentralStateManager(app *mlopsv1.CnvrgApp, c client.Client, s *runtime.Scheme, log logr.Logger) desired.StateManager { @@ -21,6 +24,7 @@ func NewCentralStateManager(app *mlopsv1.CnvrgApp, c client.Client, s *runtime.S return &CentralStateManager{ AssetsStateManager: desired.NewAssetsStateManager(app, c, s, l, fs, fsRoot+"/central", f), app: app, + client: c, } } @@ -32,7 +36,12 @@ func (c *CentralStateManager) renderSsoConfigs() error { return err } - if err := cfg.Render(c.proxyCfgData()); err != nil { + configData, err := c.proxyCfgData() + if err != nil { + return err + } + + if err = cfg.Render(configData); err != nil { return err } @@ -72,17 +81,47 @@ func (c *CentralStateManager) depData() map[string]interface{} { } } -func (c *CentralStateManager) proxyCfgData() map[string]interface{} { +func (c *CentralStateManager) proxyCfgData() (map[string]interface{}, error) { var groups []string if c.app.Spec.SSO.Central.GroupsAuth { groups = append(groups, c.domainId()) } + + var clientId, clientSecret string + + // if credentials secret ref is set, get clientId and clientSecret from the secret + if c.app.Spec.SSO.Central.CredentialsSecretRef != "" { + credentialsSecret := &corev1.Secret{} + err := c.client.Get(context.Background(), client.ObjectKey{Namespace: c.app.Namespace, Name: c.app.Spec.SSO.Central.CredentialsSecretRef}, credentialsSecret) + if err != nil { + return nil, err + } + + if _, ok := credentialsSecret.Data["clientId"]; !ok { + return nil, fmt.Errorf("credentialSecretRef configured for SSO, but clientId not found in secret %s", c.app.Spec.SSO.Central.CredentialsSecretRef) + } + if _, ok := credentialsSecret.Data["clientSecret"]; !ok { + return nil, fmt.Errorf("credentialSecretRef configured for SSO, but clientSecret not found in secret %s", c.app.Spec.SSO.Central.CredentialsSecretRef) + } + + clientId = string(credentialsSecret.Data["clientId"]) + clientSecret = string(credentialsSecret.Data["clientSecret"]) + } + + if c.app.Spec.SSO.Central.ClientID != "" { + clientId = c.app.Spec.SSO.Central.ClientID + } + + if c.app.Spec.SSO.Central.ClientSecret != "" { + clientSecret = c.app.Spec.SSO.Central.ClientSecret + } + d := map[string]interface{}{ "Namespace": c.app.Namespace, "EmailDomain": c.app.Spec.SSO.Central.EmailDomain, "Provider": c.app.Spec.SSO.Central.Provider, - "ClientId": c.app.Spec.SSO.Central.ClientID, - "ClientSecret": c.app.Spec.SSO.Central.ClientSecret, + "ClientId": clientId, + "ClientSecret": clientSecret, "RedirectUrl": fmt.Sprintf("%s://%s%s.%s/oauth2/callback", c.schema(), c.app.Spec.SSO.Central.SvcName, @@ -97,7 +136,7 @@ func (c *CentralStateManager) proxyCfgData() map[string]interface{} { "ExtraJwtIssuer": c.jwksUrlWithAudience(), "Groups": groups, } - return d + return d, nil } func (c *CentralStateManager) domainId() string { diff --git a/pkg/app/sso/tmpl/central/dep.tpl b/pkg/app/sso/tmpl/central/dep.tpl index ec706ede..ddd91991 100644 --- a/pkg/app/sso/tmpl/central/dep.tpl +++ b/pkg/app/sso/tmpl/central/dep.tpl @@ -106,8 +106,8 @@ spec: {{- end }} volumes: - name: "proxy-config" - configMap: - name: "proxy-config" + secret: + secretName: "proxy-config" - name: "private-key" secret: secretName: {{ .Spec.SSO.Pki.PrivateKeySecret }} \ No newline at end of file diff --git a/pkg/app/sso/tmpl/central/proxycfg.tpl b/pkg/app/sso/tmpl/central/proxycfg.tpl index 277c0a48..6bfddcfc 100644 --- a/pkg/app/sso/tmpl/central/proxycfg.tpl +++ b/pkg/app/sso/tmpl/central/proxycfg.tpl @@ -1,4 +1,4 @@ -kind: ConfigMap +kind: Secret apiVersion: v1 metadata: name: proxy-config @@ -7,7 +7,7 @@ metadata: mlops.cnvrg.io/default-loader: "false" mlops.cnvrg.io/own: "true" mlops.cnvrg.io/updatable: "false" -data: +stringData: conf: |- http_address = "0.0.0.0:8080" upstreams = [