Fix certificate management due to missing volumemount (#2866)

* Fix certificate management due to missing volumemount

* Fix KB, ES, ES-GW certificatemanagement

* Fix KB, ES, ES-GW certificatemanagement

* Fix KB, ES, ES-GW certificatemanagement

pkg/controller/logstorage/kubecontrollers/esgateway.go

@@ -73,7 +73,12 @@ func (r *ESKubeControllersController) createESGateway(
 		r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger)
 		return err
 	}
-	gatewayDNSNames := dns.GetServiceDNSNames(render.ElasticsearchServiceName, helper.InstallNamespace(), r.clusterDomain)
+	// For legacy reasons, es-gateway is sitting behind two services: tigera-secure-es-http (where originally ES resided)
+	// and tigera-secure-es-gateway-http.
+	gatewayDNSNames := append(
+		dns.GetServiceDNSNames(render.ElasticsearchServiceName, helper.InstallNamespace(), r.clusterDomain),
+		dns.GetServiceDNSNames(esgateway.ServiceName, helper.InstallNamespace(), r.clusterDomain)...,
+	)
 	gatewayKeyPair, err := cm.GetKeyPair(r.client, render.TigeraElasticsearchGatewaySecret, helper.TruthNamespace(), gatewayDNSNames)
 	if err != nil {
 		r.status.SetDegraded(operatorv1.ResourceNotFound, "Error getting TLS certificate", err, log)

pkg/controller/logstorage/linseed/linseed_controller.go

@@ -311,13 +311,16 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile.
 		r.status.SetDegraded(operatorv1.ResourceNotFound, fmt.Sprintf("Waiting for Linseed key pair (%s/%s) to exist", helper.TruthNamespace(), render.TigeraLinseedSecret), err, reqLogger)
 		return reconcile.Result{}, nil
 	}
-	tokenKeyPair, err := cm.GetKeyPair(r.client, render.TigeraLinseedTokenSecret, helper.TruthNamespace(), []string{render.TigeraLinseedTokenSecret})
-	if err != nil {
-		r.status.SetDegraded(operatorv1.ResourceReadError, "Error getting Linseed token secret", err, reqLogger)
-		return reconcile.Result{}, err
-	} else if tokenKeyPair == nil {
-		r.status.SetDegraded(operatorv1.ResourceNotFound, fmt.Sprintf("Waiting for Linseed key pair (%s/%s) to exist", helper.TruthNamespace(), render.TigeraLinseedTokenSecret), err, reqLogger)
-		return reconcile.Result{}, nil
+	var tokenKeyPair certificatemanagement.KeyPairInterface
+	if managementCluster != nil {
+		tokenKeyPair, err = cm.GetKeyPair(r.client, render.TigeraLinseedTokenSecret, helper.TruthNamespace(), []string{render.TigeraLinseedTokenSecret})
+		if err != nil {
+			r.status.SetDegraded(operatorv1.ResourceReadError, "Error getting Linseed token secret", err, reqLogger)
+			return reconcile.Result{}, err
+		} else if tokenKeyPair == nil {
+			r.status.SetDegraded(operatorv1.ResourceNotFound, fmt.Sprintf("Waiting for Linseed key pair (%s/%s) to exist", helper.TruthNamespace(), render.TigeraLinseedTokenSecret), err, reqLogger)
+			return reconcile.Result{}, nil
+		}
 	}
 
 	// Query the trusted bundle from the namespace.

pkg/controller/logstorage/secrets/secret_controller.go

@@ -226,6 +226,12 @@ func (r *SecretSubController) Reconcile(ctx context.Context, request reconcile.R
 		return reconcile.Result{}, nil
 	}
 
+	managementCluster, err := utils.GetManagementCluster(ctx, r.client)
+	if err != nil {
+		r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger)
+		return reconcile.Result{}, err
+	}
+
 	// In a multi-tenant system, secrets are organized in the following way:
 	// - Each tenant has its own CA, in that tenant's namespace, used for signing tenant certificates.
 	// - Elasticsearch has its own CA and KeyPair, that is global.
@@ -308,7 +314,7 @@ func (r *SecretSubController) Reconcile(ctx context.Context, request reconcile.R
 	reqLogger.Info("Rendering secrets for Tigera ES components")
 
 	// Create secrets for Tigera components.
-	keyPairs, err := r.generateNamespacedSecrets(reqLogger, helper, appCM)
+	keyPairs, err := r.generateNamespacedSecrets(reqLogger, helper, appCM, managementCluster)
 	if err != nil {
 		// Status manager is handled already, so we can just return
 		return reconcile.Result{}, err
@@ -396,7 +402,7 @@ func (r *SecretSubController) generateClusterSecrets(log logr.Logger, kibana boo
 }
 
 // generateNamespacedSecrets creates keypairs for components that are provisioned per-tenant in a multi-tenant system.
-func (r *SecretSubController) generateNamespacedSecrets(log logr.Logger, helper octrl.NamespaceHelper, cm certificatemanager.CertificateManager) (*keyPairCollection, error) {
+func (r *SecretSubController) generateNamespacedSecrets(log logr.Logger, helper octrl.NamespaceHelper, cm certificatemanager.CertificateManager, managementCluster *operatorv1.ManagementCluster) (*keyPairCollection, error) {
 	// Start by collecting upstream certificates that we need to trust, before generating keypairs.
 	collection, err := r.collectUpstreamCerts(log, helper, cm)
 	if err != nil {
@@ -413,9 +419,12 @@ func (r *SecretSubController) generateNamespacedSecrets(log logr.Logger, helper
 		}
 		collection.keypairs = append(collection.keypairs, metricsServerKeyPair)
 
-		// ES gateway keypair.
-		gatewayDNSNames := dns.GetServiceDNSNames(render.ElasticsearchServiceName, helper.InstallNamespace(), r.clusterDomain)
-		gatewayDNSNames = append(gatewayDNSNames, dns.GetServiceDNSNames(esgateway.ServiceName, helper.InstallNamespace(), r.clusterDomain)...)
+		// For legacy reasons, es-gateway is sitting behind two services: tigera-secure-es-http (where originally ES resided)
+		// and tigera-secure-es-gateway-http.
+		gatewayDNSNames := append(
+			dns.GetServiceDNSNames(render.ElasticsearchServiceName, helper.InstallNamespace(), r.clusterDomain),
+			dns.GetServiceDNSNames(esgateway.ServiceName, helper.InstallNamespace(), r.clusterDomain)...,
+		)
 		gatewayKeyPair, err := cm.GetOrCreateKeyPair(r.client, render.TigeraElasticsearchGatewaySecret, helper.TruthNamespace(), gatewayDNSNames)
 		if err != nil {
 			r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating TLS certificate", err, log)
@@ -436,13 +445,15 @@ func (r *SecretSubController) generateNamespacedSecrets(log logr.Logger, helper
 	}
 	collection.keypairs = append(collection.keypairs, linseedKeyPair)
 
-	// Create a key pair for Linseed to use for tokens.
-	linseedTokenKP, err := cm.GetOrCreateKeyPair(r.client, render.TigeraLinseedTokenSecret, helper.TruthNamespace(), []string{render.TigeraLinseedTokenSecret})
-	if err != nil {
-		r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating TLS certificate", err, log)
-		return nil, err
+	if managementCluster != nil {
+		// Create a key pair for Linseed to use for tokens.
+		linseedTokenKP, err := cm.GetOrCreateKeyPair(r.client, render.TigeraLinseedTokenSecret, helper.TruthNamespace(), []string{render.TigeraLinseedTokenSecret})
+		if err != nil {
+			r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating TLS certificate", err, log)
+			return nil, err
+		}
+		collection.keypairs = append(collection.keypairs, linseedTokenKP)
 	}
-	collection.keypairs = append(collection.keypairs, linseedTokenKP)
 
 	return collection, nil
 }
@@ -533,7 +544,7 @@ func (c *elasticKeyPairCollection) component(bundle certificatemanagement.Truste
 		Namespace:      render.ElasticsearchNamespace,
 		TruthNamespace: common.OperatorNamespace(),
 		ServiceAccounts: []string{
-			render.ElasticsearchName,
+			render.ElasticsearchObjectName,
 		},
 		KeyPairOptions: []rcertificatemanagement.KeyPairOption{
 			// We do not want to delete the elastic keypair secret from the tigera-elasticsearch namespace when CertificateManagement is
@@ -547,7 +558,7 @@ func (c *elasticKeyPairCollection) component(bundle certificatemanagement.Truste
 func (c *elasticKeyPairCollection) kibanaComponent(bundle certificatemanagement.TrustedBundle) render.Component {
 	return rcertificatemanagement.CertificateManagement(&rcertificatemanagement.Config{
 		Namespace:       render.KibanaNamespace,
-		ServiceAccounts: []string{render.KibanaName},
+		ServiceAccounts: []string{render.KibanaObjectName},
 		KeyPairOptions: []rcertificatemanagement.KeyPairOption{
 			// We do not want to delete the secret from the tigera-elasticsearch when CertificateManagement is
 			// enabled. Instead, it will be replaced with a TLS secret that serves merely to pass ECK's validation

pkg/controller/logstorage/secrets/secret_controller_test.go

@@ -207,9 +207,6 @@ var _ = Describe("LogStorage Secrets controller", func() {
 
 			{Name: render.TigeraLinseedSecret, Namespace: common.OperatorNamespace()},
 			{Name: render.TigeraLinseedSecret, Namespace: render.ElasticsearchNamespace},
-
-			{Name: render.TigeraLinseedTokenSecret, Namespace: common.OperatorNamespace()},
-			{Name: render.TigeraLinseedTokenSecret, Namespace: render.ElasticsearchNamespace},
 		}
 		ExpectSecrets(ctx, cli, expected)
 	})

pkg/render/logstorage.go

@@ -90,7 +90,6 @@ const (
 	ESGatewayServiceName            = "tigera-secure-es-gateway-http"
 	ElasticsearchDefaultPort        = 9200
 	ElasticsearchInternalPort       = 9300
-	ElasticsearchOperatorUserSecret = "tigera-ee-operator-elasticsearch-access"
 	ElasticsearchAdminUserSecret    = "tigera-secure-es-elastic-user"
 	ElasticsearchLinseedUserSecret  = "tigera-ee-linseed-elasticsearch-user-secret"
 	ElasticsearchPolicyName         = networkpolicy.TigeraComponentPolicyPrefix + "elasticsearch-access"

pkg/render/logstorage/linseed/linseed.go

@@ -19,10 +19,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/tigera/operator/pkg/common"
-
-	"k8s.io/apiserver/pkg/authentication/serviceaccount"
-
 	"github.com/tigera/operator/pkg/ptr"
 
 	relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch"
@@ -33,10 +29,12 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	v3 "github.com/tigera/api/pkg/apis/projectcalico/v3"
 	operatorv1 "github.com/tigera/operator/api/v1"
+	"github.com/tigera/operator/pkg/common"
 	"github.com/tigera/operator/pkg/components"
 	"github.com/tigera/operator/pkg/render"
 	rcomponents "github.com/tigera/operator/pkg/render/common/components"
@@ -312,13 +310,20 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment {
 	if l.cfg.KeyPair.UseCertificateManagement() {
 		initContainers = append(initContainers, l.cfg.KeyPair.InitContainer(l.namespace))
 	}
-	if l.cfg.TokenKeyPair != nil && l.cfg.TokenKeyPair.UseCertificateManagement() {
-		initContainers = append(initContainers, l.cfg.TokenKeyPair.InitContainer(l.namespace))
-	}
 
 	annotations := l.cfg.TrustedBundle.HashAnnotations()
 	annotations[l.cfg.KeyPair.HashAnnotationKey()] = l.cfg.KeyPair.HashAnnotationValue()
+
 	if l.cfg.TokenKeyPair != nil {
+		envVars = append(envVars,
+			corev1.EnvVar{Name: "TOKEN_CONTROLLER_ENABLED", Value: "true"},
+			corev1.EnvVar{Name: "LINSEED_TOKEN_KEY", Value: l.cfg.TokenKeyPair.VolumeMountKeyFilePath()},
+		)
+		volumes = append(volumes, l.cfg.TokenKeyPair.Volume())
+		volumeMounts = append(volumeMounts, l.cfg.TokenKeyPair.VolumeMount(l.SupportedOSType()))
+		if l.cfg.TokenKeyPair.UseCertificateManagement() {
+			initContainers = append(initContainers, l.cfg.TokenKeyPair.InitContainer(l.namespace))
+		}
 		annotations[l.cfg.TokenKeyPair.HashAnnotationKey()] = l.cfg.TokenKeyPair.HashAnnotationValue()
 	}
 	podTemplate := &corev1.PodTemplateSpec{

pkg/render/logstorage/linseed/linseed_test.go

@@ -468,22 +468,41 @@ func compareResources(resources []client.Object, expectedResources []resourceTes
 func expectedVolumes(useCSR bool) []corev1.Volume {
 	var volumes []corev1.Volume
 	if useCSR {
-		volumes = append(volumes, corev1.Volume{
-			Name: render.TigeraLinseedSecret,
-			VolumeSource: corev1.VolumeSource{
-				EmptyDir: &corev1.EmptyDirVolumeSource{},
+		volumes = append(volumes,
+			corev1.Volume{
+				Name: render.TigeraLinseedSecret,
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
 			},
-		})
+			corev1.Volume{
+				Name: "tigera-secure-linseed-token-tls",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			},
+		)
 	} else {
-		volumes = append(volumes, corev1.Volume{
-			Name: render.TigeraLinseedSecret,
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName:  render.TigeraLinseedSecret,
-					DefaultMode: ptr.Int32ToPtr(420),
+		volumes = append(volumes,
+			corev1.Volume{
+				Name: render.TigeraLinseedSecret,
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName:  render.TigeraLinseedSecret,
+						DefaultMode: ptr.Int32ToPtr(420),
+					},
 				},
 			},
-		})
+			corev1.Volume{
+				Name: "tigera-secure-linseed-token-tls",
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName:  "tigera-secure-linseed-token-tls",
+						DefaultMode: ptr.Int32ToPtr(420),
+					},
+				},
+			},
+		)
 	}
 
 	volumes = append(volumes, corev1.Volume{
@@ -652,6 +671,14 @@ func expectedContainers() []corev1.Container {
 					Name:  "ELASTIC_CA",
 					Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt",
 				},
+				{
+					Name:  "TOKEN_CONTROLLER_ENABLED",
+					Value: "true",
+				},
+				{
+					Name:  "LINSEED_TOKEN_KEY",
+					Value: "/tigera-secure-linseed-token-tls/tls.key",
+				},
 			},
 			VolumeMounts: []corev1.VolumeMount{
 				{
@@ -664,6 +691,11 @@ func expectedContainers() []corev1.Container {
 					MountPath: "/tigera-secure-linseed-cert",
 					ReadOnly:  true,
 				},
+				{
+					Name:      "tigera-secure-linseed-token-tls",
+					MountPath: "/tigera-secure-linseed-token-tls",
+					ReadOnly:  true,
+				},
 			},
 		},
 	}