Skip to content

Commit

Permalink
Use internal manager tls certificate for es-proxy and linseed
Browse files Browse the repository at this point in the history 
communication
  • Loading branch information
@asincu
asincu committed Jul 14, 2023
1 parent f1da738 commit ce67ac6
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 31 deletions.
2 changes: 1 addition & 1 deletion pkg/controller/installation/core_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1060,7 +1060,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile
return reconcile.Result{}, err
}
var managerInternalTLSSecret certificatemanagement.KeyPairInterface
if instance.Spec.Variant == operator.TigeraSecureEnterprise && managementCluster != nil {
if instance.Spec.Variant == operator.TigeraSecureEnterprise {
dnsNames := append(dns.GetServiceDNSNames(render.ManagerServiceName, render.ManagerNamespace, r.clusterDomain), render.ManagerServiceIP)
managerInternalTLSSecret, err = certificateManager.GetOrCreateKeyPair(r.client, render.ManagerInternalTLSSecretName, common.OperatorNamespace(), dnsNames)
if err != nil {
Expand Down
30 changes: 15 additions & 15 deletions pkg/controller/manager/manager_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,9 +444,22 @@ func (r *ReconcileManager) Reconcile(ctx context.Context, request reconcile.Requ
return reconcile.Result{}, err
}

var tunnelSecret certificatemanagement.KeyPairInterface
var internalTrafficSecret certificatemanagement.KeyPairInterface
// We expect that the secret that holds the certificates for internal communication within the management
// K8S cluster is already created by the KubeControllers
internalTrafficSecret, err := certificateManager.GetKeyPair(r.client, render.ManagerInternalTLSSecretName, common.OperatorNamespace())
if internalTrafficSecret == nil {
r.status.SetDegraded(operatorv1.ResourceNotReady, fmt.Sprintf("Waiting for secret %s in namespace %s to be available", render.ManagerInternalTLSSecretName, common.OperatorNamespace()), nil, reqLogger)
return reconcile.Result{}, err
} else if err != nil {
r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error fetching TLS secret %s in namespace %s", render.ManagerInternalTLSSecretName, common.OperatorNamespace()), err, reqLogger)
return reconcile.Result{}, nil
}
// Es-proxy needs to trust Voltron for cross-cluster requests.
trustedBundle.AddCertificates(internalTrafficSecret)

var linseedVoltronSecret certificatemanagement.KeyPairInterface
var tunnelSecret certificatemanagement.KeyPairInterface

if managementCluster != nil {
preDefaultPatchFrom := client.MergeFrom(managementCluster.DeepCopy())
fillDefaults(managementCluster)
Expand Down Expand Up @@ -481,19 +494,6 @@ func (r *ReconcileManager) Reconcile(ctx context.Context, request reconcile.Requ
r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error fetching TLS secret %s in namespace %s", render.VoltronTunnelSecretName, common.OperatorNamespace()), err, reqLogger)
return reconcile.Result{}, nil
}

// We expect that the secret that holds the certificates for internal communication within the management
// K8S cluster is already created by the KubeControllers
internalTrafficSecret, err = certificateManager.GetKeyPair(r.client, render.ManagerInternalTLSSecretName, common.OperatorNamespace())
if internalTrafficSecret == nil {
r.status.SetDegraded(operatorv1.ResourceNotReady, fmt.Sprintf("Waiting for secret %s in namespace %s to be available", render.ManagerInternalTLSSecretName, common.OperatorNamespace()), nil, reqLogger)
return reconcile.Result{}, err
} else if err != nil {
r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error fetching TLS secret %s in namespace %s", render.ManagerInternalTLSSecretName, common.OperatorNamespace()), err, reqLogger)
return reconcile.Result{}, nil
}
// Es-proxy needs to trust Voltron for cross-cluster requests.
trustedBundle.AddCertificates(internalTrafficSecret)
}

keyValidatorConfig, err := utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.clusterDomain)
Expand Down
11 changes: 11 additions & 0 deletions pkg/controller/manager/manager_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ var _ = Describe("Manager controller tests", func() {
mockStatus.On("SetDegraded", operatorv1.ResourceNotReady, "Waiting for secret 'tigera-packetcapture-server-tls' to become available", mock.Anything, mock.Anything).Return().Maybe()
mockStatus.On("SetDegraded", operatorv1.ResourceNotReady, "Waiting for secret 'tigera-secure-linseed-cert' to become available", mock.Anything, mock.Anything).Return().Maybe()
mockStatus.On("SetDegraded", operatorv1.ResourceNotReady, "Waiting for secret 'calico-node-prometheus-tls' to become available", mock.Anything, mock.Anything).Return().Maybe()
mockStatus.On("SetDegraded", operatorv1.ResourceNotReady, "Waiting for secret internal-manager-tls in namespace tigera-operator to be available", mock.Anything, mock.Anything).Return().Maybe()
mockStatus.On("ReadyToMonitor")
mockStatus.On("SetMetaData", mock.Anything).Return()

Expand Down Expand Up @@ -185,6 +186,9 @@ var _ = Describe("Manager controller tests", func() {
queryServerKp, err := certificateManager.GetOrCreateKeyPair(c, render.ProjectCalicoAPIServerTLSSecretName(operatorv1.TigeraSecureEnterprise), common.OperatorNamespace(), []string{render.ProjectCalicoAPIServerTLSSecretName(operatorv1.TigeraSecureEnterprise)})
Expect(err).NotTo(HaveOccurred())
Expect(c.Create(ctx, queryServerKp.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred())
internalKp, err := certificateManager.GetOrCreateKeyPair(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace(), []string{render.ManagerInternalTLSSecretName})
Expect(err).NotTo(HaveOccurred())
Expect(c.Create(ctx, internalKp.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred())

Expect(c.Create(ctx, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -224,6 +228,9 @@ var _ = Describe("Manager controller tests", func() {
_, err = r.Reconcile(ctx, reconcile.Request{})
Expect(err).ShouldNot(HaveOccurred())

internalSecret := &corev1.Secret{}
Expect(c.Get(ctx, types.NamespacedName{Name: render.ManagerInternalTLSSecretName, Namespace: render.ManagerNamespace}, internalSecret)).ShouldNot(HaveOccurred())

// Verify that the existing cert didn't change
secret := &corev1.Secret{}
Expect(c.Get(ctx, types.NamespacedName{Name: render.ManagerTLSSecretName, Namespace: common.OperatorNamespace()}, secret)).ShouldNot(HaveOccurred())
Expand Down Expand Up @@ -324,6 +331,7 @@ var _ = Describe("Manager controller tests", func() {
mockStatus.On("SetDegraded", operatorv1.ResourceNotReady, "Waiting for secret 'calico-node-prometheus-tls' to become available", mock.Anything, mock.Anything).Return().Maybe()
mockStatus.On("SetDegraded", operatorv1.ResourceNotReady, "Waiting for secret 'tigera-packetcapture-server-tls' to become available", mock.Anything, mock.Anything).Return().Maybe()
mockStatus.On("SetDegraded", operatorv1.ResourceNotReady, "Waiting for secret 'tigera-secure-linseed-cert' to become available", mock.Anything, mock.Anything).Return().Maybe()
mockStatus.On("SetDegraded", operatorv1.ResourceNotReady, "Waiting for secret internal-manager-tls in namespace tigera-operator to be available", mock.Anything, mock.Anything).Return().Maybe()
mockStatus.On("RemoveCertificateSigningRequests", mock.Anything)
mockStatus.On("ReadyToMonitor")
mockStatus.On("SetMetaData", mock.Anything).Return()
Expand Down Expand Up @@ -406,6 +414,9 @@ var _ = Describe("Manager controller tests", func() {
queryServerKp, err := certificateManager.GetOrCreateKeyPair(c, render.ProjectCalicoAPIServerTLSSecretName(operatorv1.TigeraSecureEnterprise), common.OperatorNamespace(), []string{render.ProjectCalicoAPIServerTLSSecretName(operatorv1.TigeraSecureEnterprise)})
Expect(err).NotTo(HaveOccurred())
Expect(c.Create(ctx, queryServerKp.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred())
internalCertKp, err := certificateManager.GetOrCreateKeyPair(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace(), []string{render.ManagerInternalTLSSecretName})
Expect(err).NotTo(HaveOccurred())
Expect(c.Create(ctx, internalCertKp.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred())

Expect(c.Create(ctx, relasticsearch.NewClusterConfig("cluster", 1, 1, 1).ConfigMap())).NotTo(HaveOccurred())

Expand Down
13 changes: 7 additions & 6 deletions pkg/render/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,9 @@ func Manager(cfg *ManagerConfiguration) (Component, error) {
}
}

tlsSecrets = append(tlsSecrets)
tlsAnnotations[cfg.InternalTrafficSecret.HashAnnotationKey()] = cfg.InternalTrafficSecret.HashAnnotationValue()
if cfg.ManagementCluster != nil {
tlsAnnotations[cfg.InternalTrafficSecret.HashAnnotationKey()] = cfg.InternalTrafficSecret.HashAnnotationValue()
tlsAnnotations[cfg.TunnelSecret.HashAnnotationKey()] = cfg.InternalTrafficSecret.HashAnnotationValue()
}
return &managerComponent{
Expand Down Expand Up @@ -139,7 +140,7 @@ type ManagerConfiguration struct {
// KeyPair used for establishing mTLS tunnel with Guardian.
TunnelSecret certificatemanagement.KeyPairInterface

// TLS KeyPair used by Voltron within the cluster when operating as a management cluster.
// TLS KeyPair used by Voltron/EsProxy within the cluster when operating as a management cluster.
InternalTrafficSecret certificatemanagement.KeyPairInterface

// Certificate bundle used by the manager pod to verify certificates presented
Expand Down Expand Up @@ -311,10 +312,10 @@ func (c *managerComponent) managerVolumes() []corev1.Volume {
v := []corev1.Volume{
c.cfg.TLSKeyPair.Volume(),
c.cfg.TrustedCertBundle.Volume(),
c.cfg.InternalTrafficSecret.Volume(),
}
if c.cfg.ManagementCluster != nil {
v = append(v,
c.cfg.InternalTrafficSecret.Volume(),
c.cfg.TunnelSecret.Volume(),
c.cfg.VoltronLinseedKeyPair.Volume(),
)
Expand Down Expand Up @@ -516,9 +517,9 @@ func (c *managerComponent) voltronContainer() corev1.Container {
// managerEsProxyContainer returns the ES proxy container
func (c *managerComponent) managerEsProxyContainer() corev1.Container {
var keyPath, certPath string
if c.cfg.TLSKeyPair != nil {
if c.cfg.InternalTrafficSecret != nil {
// This should never be nil, but we check it anyway just to be safe.
keyPath, certPath = c.cfg.TLSKeyPair.VolumeMountKeyFilePath(), c.cfg.TLSKeyPair.VolumeMountCertificateFilePath()
keyPath, certPath = c.cfg.InternalTrafficSecret.VolumeMountKeyFilePath(), c.cfg.InternalTrafficSecret.VolumeMountCertificateFilePath()
}

env := []corev1.EnvVar{
Expand All @@ -531,7 +532,7 @@ func (c *managerComponent) managerEsProxyContainer() corev1.Container {

volumeMounts := append(
c.cfg.TrustedCertBundle.VolumeMounts(c.SupportedOSType()),
c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()),
c.cfg.InternalTrafficSecret.VolumeMount(c.SupportedOSType()),
)
if c.cfg.ManagementCluster != nil {
env = append(env, corev1.EnvVar{Name: "VOLTRON_CA_PATH", Value: certificatemanagement.TrustedCertBundleMountPath})
Expand Down
28 changes: 19 additions & 9 deletions pkg/render/manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,13 @@ var _ = Describe("Tigera Secure Manager rendering tests", func() {
deployment := rtest.GetResource(resources, "tigera-manager", render.ManagerNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment)

// deployment
Expect(deployment.Spec.Template.Spec.Volumes).To(HaveLen(2))
Expect(deployment.Spec.Template.Spec.Volumes).To(HaveLen(3))
Expect(deployment.Spec.Template.Spec.Volumes[0].Name).To(Equal(render.ManagerTLSSecretName))
Expect(deployment.Spec.Template.Spec.Volumes[0].Secret.SecretName).To(Equal(render.ManagerTLSSecretName))
Expect(deployment.Spec.Template.Spec.Volumes[1].Name).To(Equal("tigera-ca-bundle"))
Expect(deployment.Spec.Template.Spec.Volumes[1].VolumeSource.ConfigMap.Name).To(Equal("tigera-ca-bundle"))
Expect(deployment.Spec.Template.Spec.Volumes[2].Name).To(Equal(render.ManagerInternalTLSSecretName))
Expect(deployment.Spec.Template.Spec.Volumes[2].Secret.SecretName).To(Equal(render.ManagerInternalTLSSecretName))

Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(3))
manager := deployment.Spec.Template.Spec.Containers[0]
Expand Down Expand Up @@ -136,13 +138,15 @@ var _ = Describe("Tigera Secure Manager rendering tests", func() {
// es-proxy container
Expect(esProxy.Env).Should(ContainElements(
corev1.EnvVar{Name: "ELASTIC_INDEX_SUFFIX", Value: "clusterTestName"},
corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: "/internal-manager-tls/tls.crt"},
corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: "/internal-manager-tls/tls.key"},
))

Expect(esProxy.VolumeMounts).To(HaveLen(2))
Expect(esProxy.VolumeMounts[0].Name).To(Equal("tigera-ca-bundle"))
Expect(esProxy.VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs"))
Expect(esProxy.VolumeMounts[1].Name).To(Equal(render.ManagerTLSSecretName))
Expect(esProxy.VolumeMounts[1].MountPath).To(Equal("/manager-tls"))
Expect(esProxy.VolumeMounts[1].Name).To(Equal(render.ManagerInternalTLSSecretName))
Expect(esProxy.VolumeMounts[1].MountPath).To(Equal("/internal-manager-tls"))

Expect(*esProxy.SecurityContext.AllowPrivilegeEscalation).To(BeFalse())
Expect(*esProxy.SecurityContext.Privileged).To(BeFalse())
Expand Down Expand Up @@ -382,7 +386,7 @@ var _ = Describe("Tigera Secure Manager rendering tests", func() {
Expect(resources).To(HaveLen(expectedResourcesNumber))
d := rtest.GetResource(resources, "tigera-manager", render.ManagerNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment)
// tigera-manager volumes/volumeMounts checks.
Expect(d.Spec.Template.Spec.Volumes).To(HaveLen(2))
Expect(d.Spec.Template.Spec.Volumes).To(HaveLen(3))
Expect(d.Spec.Template.Spec.Containers).To(HaveLen(3))
Expect(d.Spec.Template.Spec.Containers[0].Env).To(ContainElement(oidcEnvVar))
})
Expand Down Expand Up @@ -493,8 +497,8 @@ var _ = Describe("Tigera Secure Manager rendering tests", func() {
Expect(esProxy.VolumeMounts).To(HaveLen(2))
Expect(esProxy.VolumeMounts[0].Name).To(Equal("tigera-ca-bundle"))
Expect(esProxy.VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs"))
Expect(esProxy.VolumeMounts[1].Name).To(Equal(render.ManagerTLSSecretName))
Expect(esProxy.VolumeMounts[1].MountPath).To(Equal("/manager-tls"))
Expect(esProxy.VolumeMounts[1].Name).To(Equal(render.ManagerInternalTLSSecretName))
Expect(esProxy.VolumeMounts[1].MountPath).To(Equal("/internal-manager-tls"))

Expect(len(voltron.VolumeMounts)).To(Equal(5))
Expect(voltron.VolumeMounts[0].Name).To(Equal("tigera-ca-bundle"))
Expand Down Expand Up @@ -654,6 +658,7 @@ var _ = Describe("Tigera Secure Manager rendering tests", func() {
})

var kp certificatemanagement.KeyPairInterface
var internalKp certificatemanagement.KeyPairInterface
var voltronLinseedKP certificatemanagement.KeyPairInterface
var bundle certificatemanagement.TrustedBundle

Expand All @@ -663,6 +668,8 @@ var _ = Describe("Tigera Secure Manager rendering tests", func() {
Expect(err).NotTo(HaveOccurred())
kp = certificatemanagement.NewKeyPair(secret, []string{""}, "")
Expect(err).NotTo(HaveOccurred())
internalKp = certificatemanagement.NewKeyPair(secret, []string{""}, "")
Expect(err).NotTo(HaveOccurred())
voltronLinseedKP = certificatemanagement.NewKeyPair(secret, []string{""}, "")
Expect(err).NotTo(HaveOccurred())
scheme := runtime.NewScheme()
Expand All @@ -685,6 +692,7 @@ var _ = Describe("Tigera Secure Manager rendering tests", func() {
ESLicenseType: render.ElasticsearchLicenseTypeUnknown,
Replicas: &replicas,
UsePSP: true,
InternalTrafficSecret: internalKp,
}
component, err := render.Manager(cfg)
Expect(err).To(BeNil(), "Expected Manager to create successfully %s", err)
Expand Down Expand Up @@ -761,9 +769,11 @@ var _ = Describe("Tigera Secure Manager rendering tests", func() {
csrInitContainer := deployment.Spec.Template.Spec.InitContainers[0]
Expect(csrInitContainer.Name).To(Equal(fmt.Sprintf("%v-key-cert-provisioner", render.ManagerTLSSecretName)))

Expect(len(deployment.Spec.Template.Spec.Volumes)).To(Equal(2))
Expect(len(deployment.Spec.Template.Spec.Volumes)).To(Equal(3))
Expect(deployment.Spec.Template.Spec.Volumes[0].Name).To(Equal(render.ManagerTLSSecretName))
Expect(deployment.Spec.Template.Spec.Volumes[0].Secret).To(BeNil())
Expect(deployment.Spec.Template.Spec.Volumes[2].Name).To(Equal(render.ManagerInternalTLSSecretName))
Expect(deployment.Spec.Template.Spec.Volumes[2].Secret).To(BeNil())
})

It("should not render PodAffinity when ControlPlaneReplicas is 1", func() {
Expand Down Expand Up @@ -884,14 +894,14 @@ func renderObjects(roc renderConfig) []client.Object {
if roc.managementCluster != nil {
tunnelSecret, err = certificateManager.GetOrCreateKeyPair(cli, render.VoltronTunnelSecretName, common.OperatorNamespace(), []string{render.ManagerInternalTLSSecretName})
Expect(err).NotTo(HaveOccurred())
internalTraffic, err = certificateManager.GetOrCreateKeyPair(cli, render.ManagerInternalTLSSecretName, common.OperatorNamespace(), []string{render.ManagerInternalTLSSecretName})
Expect(err).NotTo(HaveOccurred())
voltronLinseedKP, err = certificateManager.GetOrCreateKeyPair(cli, render.VoltronLinseedTLS, common.OperatorNamespace(), []string{render.ManagerInternalTLSSecretName})
Expect(err).NotTo(HaveOccurred())

}
managerTLS, err := certificateManager.GetOrCreateKeyPair(cli, render.ManagerTLSSecretName, common.OperatorNamespace(), []string{""})
Expect(err).NotTo(HaveOccurred())
internalTraffic, err = certificateManager.GetOrCreateKeyPair(cli, render.ManagerInternalTLSSecretName, common.OperatorNamespace(), []string{render.ManagerInternalTLSSecretName})
Expect(err).NotTo(HaveOccurred())

esConfigMap := relasticsearch.NewClusterConfig("clusterTestName", 1, 1, 1)
cfg := &render.ManagerConfiguration{
Expand Down

0 comments on commit ce67ac6

Please sign in to comment.