Skip to content

Commit

Permalink
Add reconciler test for truststores, remove ability to override envs
Browse files Browse the repository at this point in the history 
Signed-off-by: Michael Edgar <[email protected]>
  • Loading branch information
@MikeEdgar
MikeEdgar committed Dec 10, 2024
1 parent 1c07721 commit 6ed4dd0
Show file tree
Hide file tree
Showing 4 changed files with 184 additions and 13 deletions.
13 changes: 13 additions & 0 deletions operator/src/main/java/com/github/streamshub/console/api/v1alpha1/spec/Value.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.github.streamshub.console.api.v1alpha1.spec;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
Expand All @@ -18,6 +19,18 @@ public class Value {
@JsonPropertyDescription("Reference to an external source to use for this value")
private ValueReference valueFrom;

public Value() {
}

private Value(String value) {
this.value = value;
}

@JsonIgnore
public static Value of(String value) {
return value != null ? new Value(value) : null;
}

public String getValue() {
return value;
}
Expand Down
2 changes: 0 additions & 2 deletions operator/src/main/java/com/github/streamshub/console/dependents/ConsoleDeployment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,6 @@ protected Deployment desired(Console primary, Context<Console> context) {
.editMatchingContainer(c -> "console-api".equals(c.getName()))
.withImage(imageAPI)
.addAllToVolumeMounts(getResourcesByType(trustResources, VolumeMount.class))
// Remove first to avoid duplicates
.removeMatchingFromEnv(env -> envVars.stream().anyMatch(e -> env.getName().equals(e.getName())))
.addAllToEnv(envVars)
.endContainer()
.editMatchingContainer(c -> "console-ui".equals(c.getName()))
Expand Down
24 changes: 13 additions & 11 deletions operator/src/main/java/com/github/streamshub/console/dependents/ConsoleSecret.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ private void reconcileTrustStore(
String secretName = instanceName(primary);
String typeCode = truststore.getType().toString();
String volumeName = replaceNonAlphanumeric(sourcePrefix + sourceName, '-');
String fileName = sourcePrefix + sourceName + "." + typeCode;

@SuppressWarnings("unchecked")
List<Volume> volumes = (List<Volume>) deploymentResources.computeIfAbsent(Volume.class, k -> new ArrayList<>());
Expand All @@ -184,7 +185,7 @@ private void reconcileTrustStore(
.withSecretName(secretName)
.addNewItem()
.withKey(sourcePrefix + sourceName + ".content")
.withPath(sourcePrefix + sourceName + "." + typeCode)
.withPath(fileName)
.endItem()
.withDefaultMode(420)
.endSecret()
Expand All @@ -195,8 +196,8 @@ private void reconcileTrustStore(

mounts.add(new VolumeMountBuilder()
.withName(volumeName)
.withMountPath("/etc/ssl/" + sourcePrefix + sourceName + "." + typeCode)
.withSubPath(sourcePrefix + sourceName + "." + typeCode)
.withMountPath("/etc/ssl/" + fileName)
.withSubPath(fileName)
.build());

String configTemplate = "quarkus.tls.\"" + bucketPrefix + "-%s\".trust-store.%s.%s";
Expand All @@ -212,7 +213,7 @@ private void reconcileTrustStore(

vars.add(new EnvVarBuilder()
.withName(toEnv(configTemplate.formatted(sourceName, typeCode, pathKey)))
.withValue("/etc/ssl/" + sourcePrefix + sourceName + "." + typeCode)
.withValue("/etc/ssl/" + fileName)
.build());
}

Expand All @@ -225,7 +226,7 @@ private void reconcileTrustStore(
.build());
}

if (putMetricsTrustStoreValue(data, sourceName, "alias", truststore.getAlias())) {
if (putMetricsTrustStoreValue(data, sourceName, "alias", getValue(context, namespace, Value.of(truststore.getAlias())))) {
vars.add(new EnvVarBuilder()
.withName(toEnv(configTemplate.formatted(sourceName, typeCode, "alias")))
.withNewValueFrom()
Expand Down Expand Up @@ -541,16 +542,17 @@ String getValue(Context<Console> context, String namespace, Value valueSpec) {
.map(this::encodeString)
.or(() -> Optional.ofNullable(valueSpec.getValueFrom())
.map(ValueReference::getConfigMapKeyRef)
.map(ref -> getValue(context, ConfigMap.class, namespace, ref.getName(), ref.getKey(), ref.getOptional(), ConfigMap::getData))
.map(this::encodeString))
.flatMap(ref -> getValue(context, ConfigMap.class, namespace, ref.getName(), ref.getKey(), ref.getOptional(), ConfigMap::getData)
.map(this::encodeString)
.or(() -> getValue(context, ConfigMap.class, namespace, ref.getName(), ref.getKey(), ref.getOptional(), ConfigMap::getBinaryData))))
.or(() -> Optional.ofNullable(valueSpec.getValueFrom())
.map(ValueReference::getSecretKeyRef)
.map(ref -> getValue(context, Secret.class, namespace, ref.getName(), ref.getKey(), ref.getOptional(), Secret::getData))
.flatMap(ref -> getValue(context, Secret.class, namespace, ref.getName(), ref.getKey(), ref.getOptional(), Secret::getData))
/* No need to call encodeString, the value is already encoded from Secret */)
.orElse(null);
}

<S extends HasMetadata> String getValue(Context<Console> context,
<S extends HasMetadata> Optional<String> getValue(Context<Console> context,
Class<S> sourceType,
String namespace,
String name,
Expand All @@ -561,10 +563,10 @@ <S extends HasMetadata> String getValue(Context<Console> context,
S source = getResource(context, sourceType, namespace, name, Boolean.TRUE.equals(optional));

if (source != null) {
return dataProvider.apply(source).get(key);
return Optional.ofNullable(dataProvider.apply(source).get(key));
}

return null;
return Optional.empty();
}

static <T extends HasMetadata> T getResource(
Expand Down
158 changes: 158 additions & 0 deletions operator/src/test/java/com/github/streamshub/console/ConsoleReconcilerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import java.util.Optional;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;

import jakarta.inject.Inject;

Expand All @@ -19,6 +21,8 @@
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.github.streamshub.console.api.v1alpha1.Console;
import com.github.streamshub.console.api.v1alpha1.ConsoleBuilder;
import com.github.streamshub.console.api.v1alpha1.spec.TrustStore;
import com.github.streamshub.console.api.v1alpha1.spec.metrics.MetricsSource;
import com.github.streamshub.console.api.v1alpha1.spec.metrics.MetricsSource.Type;
import com.github.streamshub.console.config.ConsoleConfig;
import com.github.streamshub.console.config.PrometheusConfig;
Expand All @@ -27,13 +31,17 @@

import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.api.model.SecretBuilder;
import io.fabric8.kubernetes.api.model.Volume;
import io.fabric8.kubernetes.api.model.VolumeMount;
import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinitionBuilder;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.openshift.api.model.Route;
Expand Down Expand Up @@ -120,13 +128,15 @@ void setUp() throws Exception {
var allDeployments = client.resources(Deployment.class).inAnyNamespace().withLabels(ConsoleResource.MANAGEMENT_LABEL);
var allConfigMaps = client.resources(ConfigMap.class).inAnyNamespace().withLabels(ConsoleResource.MANAGEMENT_LABEL);
var allSecrets = client.resources(Secret.class).inAnyNamespace().withLabels(ConsoleResource.MANAGEMENT_LABEL);
var allIngresses = client.resources(Ingress.class).inAnyNamespace().withLabels(ConsoleResource.MANAGEMENT_LABEL);

allConsoles.delete();
allKafkas.delete();
allKafkaUsers.delete();
allDeployments.delete();
allConfigMaps.delete();
allSecrets.delete();
allIngresses.delete();

await().atMost(LIMIT).untilAsserted(() -> {
assertTrue(allConsoles.list().getItems().isEmpty());
Expand All @@ -135,6 +145,7 @@ void setUp() throws Exception {
assertTrue(allDeployments.list().getItems().isEmpty());
assertTrue(allConfigMaps.list().getItems().isEmpty());
assertTrue(allSecrets.list().getItems().isEmpty());
assertTrue(allIngresses.list().getItems().isEmpty());
});

operator.start();
Expand Down Expand Up @@ -833,6 +844,153 @@ void testConsoleReconciliationWithPrometheusEmptyAuthN() {
assertThrows(KubernetesClientException.class, resourceClient::create);
}

@Test
void testConsoleReconciliationWithTrustStores() {
Secret passwordSecret = new SecretBuilder()
.withNewMetadata()
.withName("my-secret")
.withNamespace("ns2")
.addToLabels(ConsoleResource.MANAGEMENT_LABEL)
.endMetadata()
.addToData("pass", Base64.getEncoder().encodeToString("0p3n535@m3".getBytes()))
.build();
ConfigMap contentConfigMap = new ConfigMapBuilder()
.withNewMetadata()
.withName("my-configmap")
.withNamespace("ns2")
.addToLabels(ConsoleResource.MANAGEMENT_LABEL)
.endMetadata()
.addToData("truststore", "dummy-keystore")
.build();

client.resource(passwordSecret).create();
client.resource(contentConfigMap).create();

Console consoleCR = new ConsoleBuilder()
.withMetadata(new ObjectMetaBuilder()
.withName("console-1")
.withNamespace("ns2")
.build())
.withNewSpec()
.withHostname("example.com")
.addNewMetricsSource()
.withName("example-prometheus")
.withType(MetricsSource.Type.STANDALONE)
.withUrl("https://prometheus.example.com")
.withNewTrustStore()
.withType(TrustStore.Type.JKS)
.withNewPassword()
.withNewValueFrom()
.withNewSecretKeyRef("pass", "my-secret", Boolean.FALSE)
.endValueFrom()
.endPassword()
.withNewContent()
.withNewValueFrom()
.withNewConfigMapKeyRef("truststore", "my-configmap", Boolean.FALSE)
.endValueFrom()
.endContent()
.withAlias("cert-ca")
.endTrustStore()
.endMetricsSource()
.addNewSchemaRegistry()
.withName("example-registry")
.withUrl("https://example.com/apis/registry/v2")
.withNewTrustStore()
.withType(TrustStore.Type.PEM)
.withNewContent()
.withValue("---CERT---")
.endContent()
.endTrustStore()
.endSchemaRegistry()
.addNewKafkaCluster()
.withName(kafkaCR.getMetadata().getName())
.withNamespace(kafkaCR.getMetadata().getNamespace())
.withListener(kafkaCR.getSpec().getKafka().getListeners().get(0).getName())
.withSchemaRegistry("example-registry")
.endKafkaCluster()
.endSpec()
.build();

client.resource(consoleCR).create();

await().ignoreException(NullPointerException.class).atMost(LIMIT).untilAsserted(() -> {
var console = client.resources(Console.class)
.inNamespace(consoleCR.getMetadata().getNamespace())
.withName(consoleCR.getMetadata().getName())
.get();
assertEquals(1, console.getStatus().getConditions().size());
var condition = console.getStatus().getConditions().get(0);
assertEquals("Ready", condition.getType());
assertEquals("False", condition.getStatus());
assertEquals("DependentsNotReady", condition.getReason());
assertTrue(condition.getMessage().contains("ConsoleIngress"));
});

setConsoleIngressReady(consoleCR);

await().ignoreException(NullPointerException.class).atMost(LIMIT).untilAsserted(() -> {
var console = client.resources(Console.class)
.inNamespace(consoleCR.getMetadata().getNamespace())
.withName(consoleCR.getMetadata().getName())
.get();
assertEquals(1, console.getStatus().getConditions().size());
var condition = console.getStatus().getConditions().get(0);
assertEquals("Ready", condition.getType());
assertEquals("False", condition.getStatus());
assertEquals("DependentsNotReady", condition.getReason());
assertTrue(condition.getMessage().contains("ConsoleDeployment"));
});

var consoleDeployment = client.apps().deployments()
.inNamespace(consoleCR.getMetadata().getNamespace())
.withName("console-1-console-deployment")
.get();

var podSpec = consoleDeployment.getSpec().getTemplate().getSpec();
var containerSpecAPI = podSpec.getContainers().get(0);

var volumes = podSpec.getVolumes().stream().collect(Collectors.toMap(Volume::getName, Function.identity()));
assertEquals(4, volumes.size()); // cache, config + 2 volumes for truststores

var metricsVolName = "metrics-source-truststore-example-prometheus";
var registryVolName = "schema-registry-truststore-example-registry";

var metricsVolume = volumes.get(metricsVolName);
assertEquals("metrics-source-truststore.example-prometheus.content", metricsVolume.getSecret().getItems().get(0).getKey());
assertEquals("metrics-source-truststore.example-prometheus.jks", metricsVolume.getSecret().getItems().get(0).getPath());

var registryVolume = volumes.get(registryVolName);
assertEquals("schema-registry-truststore.example-registry.content", registryVolume.getSecret().getItems().get(0).getKey());
assertEquals("schema-registry-truststore.example-registry.pem", registryVolume.getSecret().getItems().get(0).getPath());

var mounts = containerSpecAPI.getVolumeMounts().stream().collect(Collectors.toMap(VolumeMount::getName, Function.identity()));
assertEquals(3, mounts.size());

var metricsMount = mounts.get(metricsVolName);
var metricsMountPath = "/etc/ssl/metrics-source-truststore.example-prometheus.jks";
assertEquals(metricsMountPath, metricsMount.getMountPath());
assertEquals("metrics-source-truststore.example-prometheus.jks", metricsMount.getSubPath());

var registryMount = mounts.get(registryVolName);
var registryMountPath = "/etc/ssl/schema-registry-truststore.example-registry.pem";
assertEquals(registryMountPath, registryMount.getMountPath());
assertEquals("schema-registry-truststore.example-registry.pem", registryMount.getSubPath());

var envVars = containerSpecAPI.getEnv().stream().collect(Collectors.toMap(EnvVar::getName, Function.identity()));

var metricsTrustPath = envVars.get("QUARKUS_TLS__METRICS_SOURCE_EXAMPLE_PROMETHEUS__TRUST_STORE_JKS_PATH");
assertEquals(metricsMountPath, metricsTrustPath.getValue());
var metricsAliasSource = envVars.get("QUARKUS_TLS__METRICS_SOURCE_EXAMPLE_PROMETHEUS__TRUST_STORE_JKS_ALIAS");
assertEquals("console-1-console-secret", metricsAliasSource.getValueFrom().getSecretKeyRef().getName());
assertEquals("metrics-source-truststore.example-prometheus.alias", metricsAliasSource.getValueFrom().getSecretKeyRef().getKey());
var metricsPasswordSource = envVars.get("QUARKUS_TLS__METRICS_SOURCE_EXAMPLE_PROMETHEUS__TRUST_STORE_JKS_PASSWORD");
assertEquals("console-1-console-secret", metricsPasswordSource.getValueFrom().getSecretKeyRef().getName());
assertEquals("metrics-source-truststore.example-prometheus.password", metricsPasswordSource.getValueFrom().getSecretKeyRef().getKey());

var registryTrustPath = envVars.get("QUARKUS_TLS__SCHEMA_REGISTRY_EXAMPLE_REGISTRY__TRUST_STORE_PEM_CERTS");
assertEquals(registryMountPath, registryTrustPath.getValue());
}

// Utility

private void assertConsoleConfig(Consumer<ConsoleConfig> assertion) {
Expand Down

0 comments on commit 6ed4dd0

Please sign in to comment.