From 6ed4dd0dc513656b97d9bcd9200c2cf6c2eb7a3c Mon Sep 17 00:00:00 2001 From: Michael Edgar Date: Thu, 5 Dec 2024 15:09:56 -0500 Subject: [PATCH] Add reconciler test for truststores, remove ability to override envs Signed-off-by: Michael Edgar --- .../console/api/v1alpha1/spec/Value.java | 13 ++ .../console/dependents/ConsoleDeployment.java | 2 - .../console/dependents/ConsoleSecret.java | 24 +-- .../console/ConsoleReconcilerTest.java | 158 ++++++++++++++++++ 4 files changed, 184 insertions(+), 13 deletions(-) diff --git a/operator/src/main/java/com/github/streamshub/console/api/v1alpha1/spec/Value.java b/operator/src/main/java/com/github/streamshub/console/api/v1alpha1/spec/Value.java index a76ca1566..f7f44b323 100644 --- a/operator/src/main/java/com/github/streamshub/console/api/v1alpha1/spec/Value.java +++ b/operator/src/main/java/com/github/streamshub/console/api/v1alpha1/spec/Value.java @@ -1,5 +1,6 @@ package com.github.streamshub.console.api.v1alpha1.spec; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; @@ -18,6 +19,18 @@ public class Value { @JsonPropertyDescription("Reference to an external source to use for this value") private ValueReference valueFrom; + public Value() { + } + + private Value(String value) { + this.value = value; + } + + @JsonIgnore + public static Value of(String value) { + return value != null ? new Value(value) : null; + } + public String getValue() { return value; } diff --git a/operator/src/main/java/com/github/streamshub/console/dependents/ConsoleDeployment.java b/operator/src/main/java/com/github/streamshub/console/dependents/ConsoleDeployment.java index 520a73b1b..b05c1a568 100644 --- a/operator/src/main/java/com/github/streamshub/console/dependents/ConsoleDeployment.java +++ b/operator/src/main/java/com/github/streamshub/console/dependents/ConsoleDeployment.java @@ -97,8 +97,6 @@ protected Deployment desired(Console primary, Context context) { .editMatchingContainer(c -> "console-api".equals(c.getName())) .withImage(imageAPI) .addAllToVolumeMounts(getResourcesByType(trustResources, VolumeMount.class)) - // Remove first to avoid duplicates - .removeMatchingFromEnv(env -> envVars.stream().anyMatch(e -> env.getName().equals(e.getName()))) .addAllToEnv(envVars) .endContainer() .editMatchingContainer(c -> "console-ui".equals(c.getName())) diff --git a/operator/src/main/java/com/github/streamshub/console/dependents/ConsoleSecret.java b/operator/src/main/java/com/github/streamshub/console/dependents/ConsoleSecret.java index 0e98b5f68..f8df349f0 100644 --- a/operator/src/main/java/com/github/streamshub/console/dependents/ConsoleSecret.java +++ b/operator/src/main/java/com/github/streamshub/console/dependents/ConsoleSecret.java @@ -174,6 +174,7 @@ private void reconcileTrustStore( String secretName = instanceName(primary); String typeCode = truststore.getType().toString(); String volumeName = replaceNonAlphanumeric(sourcePrefix + sourceName, '-'); + String fileName = sourcePrefix + sourceName + "." + typeCode; @SuppressWarnings("unchecked") List volumes = (List) deploymentResources.computeIfAbsent(Volume.class, k -> new ArrayList<>()); @@ -184,7 +185,7 @@ private void reconcileTrustStore( .withSecretName(secretName) .addNewItem() .withKey(sourcePrefix + sourceName + ".content") - .withPath(sourcePrefix + sourceName + "." + typeCode) + .withPath(fileName) .endItem() .withDefaultMode(420) .endSecret() @@ -195,8 +196,8 @@ private void reconcileTrustStore( mounts.add(new VolumeMountBuilder() .withName(volumeName) - .withMountPath("/etc/ssl/" + sourcePrefix + sourceName + "." + typeCode) - .withSubPath(sourcePrefix + sourceName + "." + typeCode) + .withMountPath("/etc/ssl/" + fileName) + .withSubPath(fileName) .build()); String configTemplate = "quarkus.tls.\"" + bucketPrefix + "-%s\".trust-store.%s.%s"; @@ -212,7 +213,7 @@ private void reconcileTrustStore( vars.add(new EnvVarBuilder() .withName(toEnv(configTemplate.formatted(sourceName, typeCode, pathKey))) - .withValue("/etc/ssl/" + sourcePrefix + sourceName + "." + typeCode) + .withValue("/etc/ssl/" + fileName) .build()); } @@ -225,7 +226,7 @@ private void reconcileTrustStore( .build()); } - if (putMetricsTrustStoreValue(data, sourceName, "alias", truststore.getAlias())) { + if (putMetricsTrustStoreValue(data, sourceName, "alias", getValue(context, namespace, Value.of(truststore.getAlias())))) { vars.add(new EnvVarBuilder() .withName(toEnv(configTemplate.formatted(sourceName, typeCode, "alias"))) .withNewValueFrom() @@ -541,16 +542,17 @@ String getValue(Context context, String namespace, Value valueSpec) { .map(this::encodeString) .or(() -> Optional.ofNullable(valueSpec.getValueFrom()) .map(ValueReference::getConfigMapKeyRef) - .map(ref -> getValue(context, ConfigMap.class, namespace, ref.getName(), ref.getKey(), ref.getOptional(), ConfigMap::getData)) - .map(this::encodeString)) + .flatMap(ref -> getValue(context, ConfigMap.class, namespace, ref.getName(), ref.getKey(), ref.getOptional(), ConfigMap::getData) + .map(this::encodeString) + .or(() -> getValue(context, ConfigMap.class, namespace, ref.getName(), ref.getKey(), ref.getOptional(), ConfigMap::getBinaryData)))) .or(() -> Optional.ofNullable(valueSpec.getValueFrom()) .map(ValueReference::getSecretKeyRef) - .map(ref -> getValue(context, Secret.class, namespace, ref.getName(), ref.getKey(), ref.getOptional(), Secret::getData)) + .flatMap(ref -> getValue(context, Secret.class, namespace, ref.getName(), ref.getKey(), ref.getOptional(), Secret::getData)) /* No need to call encodeString, the value is already encoded from Secret */) .orElse(null); } - String getValue(Context context, + Optional getValue(Context context, Class sourceType, String namespace, String name, @@ -561,10 +563,10 @@ String getValue(Context context, S source = getResource(context, sourceType, namespace, name, Boolean.TRUE.equals(optional)); if (source != null) { - return dataProvider.apply(source).get(key); + return Optional.ofNullable(dataProvider.apply(source).get(key)); } - return null; + return Optional.empty(); } static T getResource( diff --git a/operator/src/test/java/com/github/streamshub/console/ConsoleReconcilerTest.java b/operator/src/test/java/com/github/streamshub/console/ConsoleReconcilerTest.java index 692233373..8cd0da87a 100644 --- a/operator/src/test/java/com/github/streamshub/console/ConsoleReconcilerTest.java +++ b/operator/src/test/java/com/github/streamshub/console/ConsoleReconcilerTest.java @@ -6,6 +6,8 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; import jakarta.inject.Inject; @@ -19,6 +21,8 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.github.streamshub.console.api.v1alpha1.Console; import com.github.streamshub.console.api.v1alpha1.ConsoleBuilder; +import com.github.streamshub.console.api.v1alpha1.spec.TrustStore; +import com.github.streamshub.console.api.v1alpha1.spec.metrics.MetricsSource; import com.github.streamshub.console.api.v1alpha1.spec.metrics.MetricsSource.Type; import com.github.streamshub.console.config.ConsoleConfig; import com.github.streamshub.console.config.PrometheusConfig; @@ -27,13 +31,17 @@ import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.ConfigMapBuilder; +import io.fabric8.kubernetes.api.model.EnvVar; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.NamespaceBuilder; import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.api.model.SecretBuilder; +import io.fabric8.kubernetes.api.model.Volume; +import io.fabric8.kubernetes.api.model.VolumeMount; import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinitionBuilder; import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.networking.v1.Ingress; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.openshift.api.model.Route; @@ -120,6 +128,7 @@ void setUp() throws Exception { var allDeployments = client.resources(Deployment.class).inAnyNamespace().withLabels(ConsoleResource.MANAGEMENT_LABEL); var allConfigMaps = client.resources(ConfigMap.class).inAnyNamespace().withLabels(ConsoleResource.MANAGEMENT_LABEL); var allSecrets = client.resources(Secret.class).inAnyNamespace().withLabels(ConsoleResource.MANAGEMENT_LABEL); + var allIngresses = client.resources(Ingress.class).inAnyNamespace().withLabels(ConsoleResource.MANAGEMENT_LABEL); allConsoles.delete(); allKafkas.delete(); @@ -127,6 +136,7 @@ void setUp() throws Exception { allDeployments.delete(); allConfigMaps.delete(); allSecrets.delete(); + allIngresses.delete(); await().atMost(LIMIT).untilAsserted(() -> { assertTrue(allConsoles.list().getItems().isEmpty()); @@ -135,6 +145,7 @@ void setUp() throws Exception { assertTrue(allDeployments.list().getItems().isEmpty()); assertTrue(allConfigMaps.list().getItems().isEmpty()); assertTrue(allSecrets.list().getItems().isEmpty()); + assertTrue(allIngresses.list().getItems().isEmpty()); }); operator.start(); @@ -833,6 +844,153 @@ void testConsoleReconciliationWithPrometheusEmptyAuthN() { assertThrows(KubernetesClientException.class, resourceClient::create); } + @Test + void testConsoleReconciliationWithTrustStores() { + Secret passwordSecret = new SecretBuilder() + .withNewMetadata() + .withName("my-secret") + .withNamespace("ns2") + .addToLabels(ConsoleResource.MANAGEMENT_LABEL) + .endMetadata() + .addToData("pass", Base64.getEncoder().encodeToString("0p3n535@m3".getBytes())) + .build(); + ConfigMap contentConfigMap = new ConfigMapBuilder() + .withNewMetadata() + .withName("my-configmap") + .withNamespace("ns2") + .addToLabels(ConsoleResource.MANAGEMENT_LABEL) + .endMetadata() + .addToData("truststore", "dummy-keystore") + .build(); + + client.resource(passwordSecret).create(); + client.resource(contentConfigMap).create(); + + Console consoleCR = new ConsoleBuilder() + .withMetadata(new ObjectMetaBuilder() + .withName("console-1") + .withNamespace("ns2") + .build()) + .withNewSpec() + .withHostname("example.com") + .addNewMetricsSource() + .withName("example-prometheus") + .withType(MetricsSource.Type.STANDALONE) + .withUrl("https://prometheus.example.com") + .withNewTrustStore() + .withType(TrustStore.Type.JKS) + .withNewPassword() + .withNewValueFrom() + .withNewSecretKeyRef("pass", "my-secret", Boolean.FALSE) + .endValueFrom() + .endPassword() + .withNewContent() + .withNewValueFrom() + .withNewConfigMapKeyRef("truststore", "my-configmap", Boolean.FALSE) + .endValueFrom() + .endContent() + .withAlias("cert-ca") + .endTrustStore() + .endMetricsSource() + .addNewSchemaRegistry() + .withName("example-registry") + .withUrl("https://example.com/apis/registry/v2") + .withNewTrustStore() + .withType(TrustStore.Type.PEM) + .withNewContent() + .withValue("---CERT---") + .endContent() + .endTrustStore() + .endSchemaRegistry() + .addNewKafkaCluster() + .withName(kafkaCR.getMetadata().getName()) + .withNamespace(kafkaCR.getMetadata().getNamespace()) + .withListener(kafkaCR.getSpec().getKafka().getListeners().get(0).getName()) + .withSchemaRegistry("example-registry") + .endKafkaCluster() + .endSpec() + .build(); + + client.resource(consoleCR).create(); + + await().ignoreException(NullPointerException.class).atMost(LIMIT).untilAsserted(() -> { + var console = client.resources(Console.class) + .inNamespace(consoleCR.getMetadata().getNamespace()) + .withName(consoleCR.getMetadata().getName()) + .get(); + assertEquals(1, console.getStatus().getConditions().size()); + var condition = console.getStatus().getConditions().get(0); + assertEquals("Ready", condition.getType()); + assertEquals("False", condition.getStatus()); + assertEquals("DependentsNotReady", condition.getReason()); + assertTrue(condition.getMessage().contains("ConsoleIngress")); + }); + + setConsoleIngressReady(consoleCR); + + await().ignoreException(NullPointerException.class).atMost(LIMIT).untilAsserted(() -> { + var console = client.resources(Console.class) + .inNamespace(consoleCR.getMetadata().getNamespace()) + .withName(consoleCR.getMetadata().getName()) + .get(); + assertEquals(1, console.getStatus().getConditions().size()); + var condition = console.getStatus().getConditions().get(0); + assertEquals("Ready", condition.getType()); + assertEquals("False", condition.getStatus()); + assertEquals("DependentsNotReady", condition.getReason()); + assertTrue(condition.getMessage().contains("ConsoleDeployment")); + }); + + var consoleDeployment = client.apps().deployments() + .inNamespace(consoleCR.getMetadata().getNamespace()) + .withName("console-1-console-deployment") + .get(); + + var podSpec = consoleDeployment.getSpec().getTemplate().getSpec(); + var containerSpecAPI = podSpec.getContainers().get(0); + + var volumes = podSpec.getVolumes().stream().collect(Collectors.toMap(Volume::getName, Function.identity())); + assertEquals(4, volumes.size()); // cache, config + 2 volumes for truststores + + var metricsVolName = "metrics-source-truststore-example-prometheus"; + var registryVolName = "schema-registry-truststore-example-registry"; + + var metricsVolume = volumes.get(metricsVolName); + assertEquals("metrics-source-truststore.example-prometheus.content", metricsVolume.getSecret().getItems().get(0).getKey()); + assertEquals("metrics-source-truststore.example-prometheus.jks", metricsVolume.getSecret().getItems().get(0).getPath()); + + var registryVolume = volumes.get(registryVolName); + assertEquals("schema-registry-truststore.example-registry.content", registryVolume.getSecret().getItems().get(0).getKey()); + assertEquals("schema-registry-truststore.example-registry.pem", registryVolume.getSecret().getItems().get(0).getPath()); + + var mounts = containerSpecAPI.getVolumeMounts().stream().collect(Collectors.toMap(VolumeMount::getName, Function.identity())); + assertEquals(3, mounts.size()); + + var metricsMount = mounts.get(metricsVolName); + var metricsMountPath = "/etc/ssl/metrics-source-truststore.example-prometheus.jks"; + assertEquals(metricsMountPath, metricsMount.getMountPath()); + assertEquals("metrics-source-truststore.example-prometheus.jks", metricsMount.getSubPath()); + + var registryMount = mounts.get(registryVolName); + var registryMountPath = "/etc/ssl/schema-registry-truststore.example-registry.pem"; + assertEquals(registryMountPath, registryMount.getMountPath()); + assertEquals("schema-registry-truststore.example-registry.pem", registryMount.getSubPath()); + + var envVars = containerSpecAPI.getEnv().stream().collect(Collectors.toMap(EnvVar::getName, Function.identity())); + + var metricsTrustPath = envVars.get("QUARKUS_TLS__METRICS_SOURCE_EXAMPLE_PROMETHEUS__TRUST_STORE_JKS_PATH"); + assertEquals(metricsMountPath, metricsTrustPath.getValue()); + var metricsAliasSource = envVars.get("QUARKUS_TLS__METRICS_SOURCE_EXAMPLE_PROMETHEUS__TRUST_STORE_JKS_ALIAS"); + assertEquals("console-1-console-secret", metricsAliasSource.getValueFrom().getSecretKeyRef().getName()); + assertEquals("metrics-source-truststore.example-prometheus.alias", metricsAliasSource.getValueFrom().getSecretKeyRef().getKey()); + var metricsPasswordSource = envVars.get("QUARKUS_TLS__METRICS_SOURCE_EXAMPLE_PROMETHEUS__TRUST_STORE_JKS_PASSWORD"); + assertEquals("console-1-console-secret", metricsPasswordSource.getValueFrom().getSecretKeyRef().getName()); + assertEquals("metrics-source-truststore.example-prometheus.password", metricsPasswordSource.getValueFrom().getSecretKeyRef().getKey()); + + var registryTrustPath = envVars.get("QUARKUS_TLS__SCHEMA_REGISTRY_EXAMPLE_REGISTRY__TRUST_STORE_PEM_CERTS"); + assertEquals(registryMountPath, registryTrustPath.getValue()); + } + // Utility private void assertConsoleConfig(Consumer assertion) {