diff --git a/config-vault/pom.xml b/config-vault/pom.xml new file mode 100644 index 00000000..eca5e8aa --- /dev/null +++ b/config-vault/pom.xml @@ -0,0 +1,39 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>io.scalecube</groupId> + <artifactId>config-parent</artifactId> + <version>0.3.2-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <artifactId>config-vault</artifactId> + <properties> + <enforcer.skip>true</enforcer.skip> + </properties> + + <dependencies> + <dependency> + <groupId>io.scalecube</groupId> + <artifactId>config</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.bettercloud</groupId> + <artifactId>vault-java-driver</artifactId> + <version>3.1.0</version> + </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>vault</artifactId> + <version>1.6.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>co.unruly</groupId> + <artifactId>java-8-matchers</artifactId> + <version>1.5</version> + <scope>test</scope> + </dependency> + </dependencies> +</project> \ No newline at end of file diff --git a/config-vault/src/main/java/io/scalecube/config/vault/VaultConfigSource.java b/config-vault/src/main/java/io/scalecube/config/vault/VaultConfigSource.java new file mode 100644 index 00000000..a734ae94 --- /dev/null +++ b/config-vault/src/main/java/io/scalecube/config/vault/VaultConfigSource.java @@ -0,0 +1,136 @@ +package io.scalecube.config.vault; + +import static java.util.Objects.requireNonNull; + +import io.scalecube.config.ConfigProperty; +import io.scalecube.config.ConfigSourceNotAvailableException; +import io.scalecube.config.source.ConfigSource; +import io.scalecube.config.source.LoadedConfigProperty; +import io.scalecube.config.utils.ThrowableUtil; + +import com.bettercloud.vault.EnvironmentLoader; +import com.bettercloud.vault.SslConfig; +import com.bettercloud.vault.Vault; +import com.bettercloud.vault.VaultConfig; +import com.bettercloud.vault.VaultException; +import com.bettercloud.vault.response.LogicalResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * This class is a {@link ConfigSource} implemented for Vault + * + * @see <a href="https://www.vaultproject.io/">Vault Project</a> + */ +public class VaultConfigSource implements ConfigSource { + + static final Logger LOGGER = LoggerFactory.getLogger(VaultConfigSource.class); + + private final Vault vault; + private final String secretsPath; + + /** + * Create a new {@link VaultConfigSource} with the given {@link Builder}. <br> + * + * @param builder configuration to create vault access with. + * + */ + VaultConfigSource(Builder builder) { + this.secretsPath = builder.secretsPath(); + vault = new Vault(builder.config); + } + + private void checkVaultStatus() throws VaultException { + if (vault.seal().sealStatus().getSealed()) { + throw new VaultException("Vault is sealed"); + } + Boolean initialized = vault.debug().health().getInitialized(); + if (!initialized) { + throw new VaultException("Vault not yet initialized"); + } + } + + @Override + public Map<String, ConfigProperty> loadConfig() { + try { + checkVaultStatus(); + LogicalResponse response = vault.logical().read(this.secretsPath); + return response.getData().entrySet().stream().map(LoadedConfigProperty::withNameAndValue) + .map(LoadedConfigProperty.Builder::build) + .collect(Collectors.toMap(LoadedConfigProperty::name, Function.identity())); + } catch (VaultException vaultException) { + LOGGER.warn("unable to load config properties", vaultException); + throw new ConfigSourceNotAvailableException(vaultException); + } + } + + /** + * This builder method is used internally for test purposes. please use it only for tests. Please note the following + * required environment variables are required. + * <ul> + * <li><code>VAULT_SECRETS_PATH</pre> is the path to use (defaults to <code>secret</code>)</li> + * <li><code>VAULT_TOKEN</code> is the {@link VaultConfig#token(String) token} to use</li> + * <li><code>VAULT_ADDR</code> is the {@link VaultConfig#address(String) address} of the vault (API)</li> + * </ul> + */ + public static Builder builder() { + return builder(new EnvironmentLoader()); + } + + /** + * This builder method is used internally for test purposes. please use it only for tests + * + * @param environmentLoader an {@link EnvironmentLoader} + */ + static Builder builder(EnvironmentLoader environmentLoader) { + return builder(environmentLoader.loadVariable("VAULT_ADDR"), + environmentLoader.loadVariable("VAULT_TOKEN"), + environmentLoader.loadVariable("VAULT_SECRETS_PATH")); + } + + public static Builder builder(String address, String token, String secretsPath) { + return new Builder(address, token, secretsPath); + } + + public static final class Builder { + + final VaultConfig config = new VaultConfig(); + private final String secretsPath; + + Builder(String address, String token, String secretsPath) { + config.address(requireNonNull(address, "Missing address")) + .token(requireNonNull(token, "Missing token")) + .sslConfig(new SslConfig()); + this.secretsPath = requireNonNull(secretsPath, "Missing secretsPath"); + } + + public Builder connectTimeout(int connectTimeout) { + config.openTimeout(connectTimeout); + return this; + } + + public Builder readTimeout(int readTimeout) { + config.readTimeout(readTimeout); + return this; + } + + public VaultConfigSource build() { + try { + this.config.build(); + return new VaultConfigSource(this); + } catch (VaultException propogateException) { + LOGGER.error("Unable to build " + VaultConfigSource.class.getSimpleName(), propogateException); + throw ThrowableUtil.propagate(propogateException); + } + } + + public String secretsPath() { + return secretsPath; + } + } +} diff --git a/config-vault/src/test/java/io/scalecube/config/vault/VaultConfigSourceTest.java b/config-vault/src/test/java/io/scalecube/config/vault/VaultConfigSourceTest.java new file mode 100644 index 00000000..ce35ec65 --- /dev/null +++ b/config-vault/src/test/java/io/scalecube/config/vault/VaultConfigSourceTest.java @@ -0,0 +1,328 @@ +package io.scalecube.config.vault; + +import static co.unruly.matchers.OptionalMatchers.contains; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.junit.Assume.assumeThat; +import static org.junit.Assume.assumeTrue; + +import io.scalecube.config.ConfigProperty; +import io.scalecube.config.ConfigRegistry; +import io.scalecube.config.ConfigRegistrySettings; +import io.scalecube.config.ConfigSourceNotAvailableException; +import io.scalecube.config.StringConfigProperty; + +import com.bettercloud.vault.EnvironmentLoader; +import com.bettercloud.vault.SslConfig; +import com.bettercloud.vault.Vault; +import com.bettercloud.vault.VaultConfig; +import com.bettercloud.vault.VaultException; + +import org.hamcrest.CoreMatchers; +import org.junit.Assert; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.testcontainers.containers.Container.ExecResult; +import org.testcontainers.containers.output.OutputFrame; +import org.testcontainers.containers.wait.LogMessageWaitStrategy; +import org.testcontainers.containers.wait.WaitStrategy; +import org.testcontainers.vault.VaultContainer; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class VaultConfigSourceTest { + + private static final String VAULT_IMAGE_NAME = "vault:0.9.6"; + private static final int VAULT_PORT = 8200; + private static final String VAULT_TOKEN = "my-root-token"; + /** + * the environment variable name for vault secret path + */ + private static final String VAULT_SECRETS_PATH = "VAULT_SECRETS_PATH"; + + // these 3 are actual values we would like to test with + private static final String VAULT_SECRETS_PATH1 = "secret/application/tenant1"; + private static final String VAULT_SECRETS_PATH2 = "secret/application/tenant2"; + private static final String VAULT_SECRETS_PATH3 = "secret/application2/tenant3"; + + static WaitStrategy VAULT_SERVER_STARTED = + new LogMessageWaitStrategy().withRegEx("==> Vault server started! Log data will stream in below:\n").withTimes(1); + + private final Pattern unsealKeyPattern = Pattern.compile("Unseal Key: ([a-z/0-9=A-Z]*)\n"); + + private Consumer<OutputFrame> waitingForUnsealKey(AtomicReference<String> unsealKey) { + return onFrame -> { + Matcher matcher = unsealKeyPattern.matcher(onFrame.getUtf8String()); + if (matcher.find()) { + unsealKey.set(matcher.group(1)); + } + }; + } + + @ClassRule + public static VaultContainer<?> vaultContainer = new VaultContainer<>() + .waitingFor(VAULT_SERVER_STARTED).withVaultToken(VAULT_TOKEN) + .withVaultPort(VAULT_PORT) + .withSecretInVault(VAULT_SECRETS_PATH1, "top_secret=password1", "db_password=dbpassword1") + .withSecretInVault(VAULT_SECRETS_PATH2, "top_secret=password2", "db_password=dbpassword2") + .withSecretInVault(VAULT_SECRETS_PATH3, "secret=password", "password=dbpassword"); + + EnvironmentLoader loader1, loader2, loader3; + + Map<String, String> commonEnvironmentVariables = new HashMap<>(); + + @Before + public void setUp() throws Exception { + commonEnvironmentVariables.put("VAULT_TOKEN", VAULT_TOKEN); + commonEnvironmentVariables.put("VAULT_ADDR", new StringBuilder("http://") + .append(vaultContainer.getContainerIpAddress()).append(':').append(VAULT_PORT).toString()); + + Map<String, String> tenant1 = new HashMap<>(commonEnvironmentVariables); + tenant1.put(VAULT_SECRETS_PATH, VAULT_SECRETS_PATH1); + this.loader1 = new MockEnvironmentLoader(tenant1); + + Map<String, String> tenant2 = new HashMap<>(commonEnvironmentVariables); + tenant2.put(VAULT_SECRETS_PATH, VAULT_SECRETS_PATH2); + this.loader2 = new MockEnvironmentLoader(tenant2); + + Map<String, String> tenant3 = new HashMap<>(commonEnvironmentVariables); + tenant3.put(VAULT_SECRETS_PATH, VAULT_SECRETS_PATH3); + this.loader3 = new MockEnvironmentLoader(tenant3); + + + } + + private class MockEnvironmentLoader extends EnvironmentLoader { + /** + * + */ + private static final long serialVersionUID = 7285747202838173640L; + private final Map<String, String> delegate; + + public MockEnvironmentLoader(Map<String, String> delegate) { + this.delegate = delegate; + } + + @Override + public String loadVariable(String name) { + return delegate.get(name); + } + } + + @Test + public void testFirstTenant() { + VaultConfigSource vaultConfigSource = VaultConfigSource.builder(loader1).build(); + Map<String, ConfigProperty> loadConfig = vaultConfigSource.loadConfig(); + ConfigProperty actual = loadConfig.get("top_secret"); + assertThat(actual, notNullValue()); + assertThat(actual.name(), equalTo("top_secret")); + assertThat(actual.valueAsString(""), equalTo("password1")); + } + + @Test + public void testSecondTenant() { + VaultConfigSource vaultConfigSource = VaultConfigSource.builder(loader2).build(); + Map<String, ConfigProperty> loadConfig = vaultConfigSource.loadConfig(); + ConfigProperty actual = loadConfig.get("top_secret"); + assertThat(actual, notNullValue()); + assertThat(actual.name(), equalTo("top_secret")); + assertThat(actual.valueAsString(""), equalTo("password2")); + } + + @Test + public void testMissingProperty() { + VaultConfigSource vaultConfigSource = VaultConfigSource.builder(loader3).build(); + Map<String, ConfigProperty> loadConfig = vaultConfigSource.loadConfig(); + assertThat(loadConfig.size(), not(0)); + ConfigProperty actual = loadConfig.get("top_secret"); + assertThat(actual, nullValue()); + } + + @Test(expected = ConfigSourceNotAvailableException.class) + public void testMissingTenant() { + EnvironmentLoader loader4; + Map<String, String> tenant4 = new HashMap<>(commonEnvironmentVariables); + tenant4.put(VAULT_SECRETS_PATH, "secrets/unknown/path"); + loader4 = new MockEnvironmentLoader(tenant4); + VaultConfigSource vaultConfigSource = VaultConfigSource.builder(loader4).build(); + Map<String, ConfigProperty> loadConfig = vaultConfigSource.loadConfig(); + assertThat(loadConfig.size(), equalTo(0)); + } + + @Test(expected = ConfigSourceNotAvailableException.class) + public void testInvalidAddress() { + Map<String, String> invalidAddress = new HashMap<>(); + invalidAddress.put("VAULT_ADDR", "http://invalid.host.local:8200"); + invalidAddress.put("VAULT_TOKEN", VAULT_TOKEN); + invalidAddress.put(VAULT_SECRETS_PATH, VAULT_SECRETS_PATH1); + + VaultConfigSource vaultConfigSource = VaultConfigSource.builder(new MockEnvironmentLoader(invalidAddress)).build(); + vaultConfigSource.loadConfig(); + + } + + @Test(expected = ConfigSourceNotAvailableException.class) + public void testInvalidToken() { + Map<String, String> invalidToken = new HashMap<>(commonEnvironmentVariables); + invalidToken.put("VAULT_TOKEN", "zzzzzz"); + invalidToken.put(VAULT_SECRETS_PATH, "secrets/unknown/path"); + + VaultConfigSource vaultConfigSource = VaultConfigSource.builder(new MockEnvironmentLoader(invalidToken)).build(); + vaultConfigSource.loadConfig(); + + } + + @Test + public void shouldWorkWhenRegistryIsReloadedAndVaultIsRunning() throws InterruptedException { + try (VaultContainer<?> vaultContainer2 = new VaultContainer<>(VAULT_IMAGE_NAME)) { + vaultContainer2.withVaultToken(VAULT_TOKEN).withVaultPort(8202) + .withSecretInVault(VAULT_SECRETS_PATH1, "top_secret=password1", "db_password=dbpassword1") + .waitingFor(VAULT_SERVER_STARTED) + .start(); + String address = new StringBuilder("http://") + .append(vaultContainer2.getContainerIpAddress()).append(':').append(8202).toString(); + ConfigRegistrySettings settings = ConfigRegistrySettings.builder() + .addLastSource("vault", VaultConfigSource.builder(address, VAULT_TOKEN, VAULT_SECRETS_PATH1).build()) + .reloadIntervalSec(1) + .build(); + ConfigRegistry configRegistry = ConfigRegistry.create(settings); + StringConfigProperty configProperty = configRegistry.stringProperty("top_secret"); + + assertThat(configProperty.value(), contains("password1")); + try { + ExecResult execResult = vaultContainer2.execInContainer("/bin/sh", "-c", + "vault write " + VAULT_SECRETS_PATH1 + " top_secret=new_password"); + assumeThat(execResult.getStdout(), CoreMatchers.containsString("Success")); + TimeUnit.SECONDS.sleep(2); + } catch (Exception ignoredException) { + Assert.fail("oops"); + } + assertThat(configProperty.value(), contains("new_password")); + } + } + + @Test + public void shouldWorkWhenRegistryIsReloadedAndVaultIsDown() { + String PASSWORD_PROPERTY_NAME = "password"; + String PASSWORD_PROPERTY_VALUE = "123456"; + String secret = PASSWORD_PROPERTY_NAME + "=" + PASSWORD_PROPERTY_VALUE; + try (VaultContainer<?> vaultContainer2 = new VaultContainer<>(VAULT_IMAGE_NAME)) { + vaultContainer2.withVaultToken(VAULT_TOKEN).withVaultPort(8203) + .withEnv("VAULT_DEV_ROOT_TOKEN_ID", (String) VAULT_TOKEN) + .withSecretInVault(VAULT_SECRETS_PATH1, secret) + .waitingFor(VAULT_SERVER_STARTED) + .start(); + + String address = new StringBuilder("http://") + .append(vaultContainer2.getContainerIpAddress()).append(':').append(8203).toString(); + + ConfigRegistrySettings settings = ConfigRegistrySettings.builder() + .addLastSource("vault", VaultConfigSource.builder(address, VAULT_TOKEN, VAULT_SECRETS_PATH1).build()) + .reloadIntervalSec(1) + .build(); + ConfigRegistry configRegistry = ConfigRegistry.create(settings); + StringConfigProperty configProperty = configRegistry.stringProperty(PASSWORD_PROPERTY_NAME); + configProperty.addValidator(Objects::nonNull); + + vaultContainer2.stop(); + assertFalse(vaultContainer2.isRunning()); + + try { + TimeUnit.SECONDS.sleep(2); + } catch (InterruptedException ignoredException) { + } + + assertThat(configProperty.value(), contains(PASSWORD_PROPERTY_VALUE)); + } + } + + + @Test + public void testSealed() throws Throwable { + try (VaultContainer<?> vaultContainerSealed = new VaultContainer<>()) { + vaultContainerSealed.withVaultToken(VAULT_TOKEN).withVaultPort(8204) + .waitingFor(VAULT_SERVER_STARTED) + .start(); + + String address = new StringBuilder("http://") + .append(vaultContainerSealed.getContainerIpAddress()).append(':').append(8204).toString(); + Vault vault = new Vault(new VaultConfig().address(address).token(VAULT_TOKEN).sslConfig(new SslConfig())); + + vault.seal().seal(); + assumeTrue("valut seal status",vault.seal().sealStatus().getSealed()); + + + Map<String, String> clientEnv = new HashMap<>(); + clientEnv.put("VAULT_TOKEN", "ROOT"); + clientEnv.put("VAULT_ADDR", address); + clientEnv.put(VAULT_SECRETS_PATH, VAULT_SECRETS_PATH1); + + VaultConfigSource.builder(new MockEnvironmentLoader(clientEnv)).build().loadConfig(); + Assert.fail("Negative test failed"); + } catch (ConfigSourceNotAvailableException expectedException) { + assertThat(expectedException.getCause(), instanceOf(VaultException.class)); + String message = expectedException.getCause().getMessage(); + assertThat(message, containsString("Vault is sealed")); + } + } + + @Test + public void shouldWorkWhenRegistryIsReloadedAndVaultIsUnSealed() throws InterruptedException { + AtomicReference<String> unsealKey = new AtomicReference<>(); + try (VaultContainer<?> sealdVaultContainer = new VaultContainer<>(VAULT_IMAGE_NAME)) { + sealdVaultContainer.withVaultToken(VAULT_TOKEN).withVaultPort(8205) + .withSecretInVault(VAULT_SECRETS_PATH1, "top_secret=password1", "db_password=dbpassword1") + .withLogConsumer(waitingForUnsealKey(unsealKey)).waitingFor(VAULT_SERVER_STARTED) + .start(); + assumeThat("unable to get unseal key", unsealKey.get(), notNullValue()); + String address = new StringBuilder("http://") + .append(sealdVaultContainer.getContainerIpAddress()).append(':').append(8205).toString(); + ConfigRegistrySettings settings = ConfigRegistrySettings.builder() + .addLastSource("vault", VaultConfigSource.builder(address, VAULT_TOKEN, VAULT_SECRETS_PATH1).build()) + .reloadIntervalSec(1) + .build(); + + ConfigRegistry configRegistry = ConfigRegistry.create(settings); + StringConfigProperty configProperty = configRegistry.stringProperty("top_secret"); + + assertThat("initial value of top_secret", configProperty.value(), contains("password1")); + Vault vault = new Vault(new VaultConfig().address(address).token(VAULT_TOKEN).sslConfig(new SslConfig())); + Map<String, Object> newValues = new HashMap<>(); + newValues.put(configProperty.name(), "new_password"); + + try { + vault.logical().write(VAULT_SECRETS_PATH1, newValues); + vault.seal().seal(); + assumeThat("valut seal status", vault.seal().sealStatus().getSealed(), is(true)); + } catch (VaultException vaultException) { + fail(vaultException.getMessage()); + } + TimeUnit.SECONDS.sleep(2); + assumeThat("new value was unexpectedly set", configProperty.value(), not(contains("new_password"))); + try { + vault.seal().unseal(unsealKey.get()); + assumeThat("valut seal status", vault.seal().sealStatus().getSealed(), is(false)); + } catch (VaultException vaultException) { + fail(vaultException.getMessage()); + } + TimeUnit.SECONDS.sleep(2); + assertThat(configProperty.value(), contains("new_password")); + } + } +} diff --git a/pom.xml b/pom.xml index 70a04509..b2a57259 100644 --- a/pom.xml +++ b/pom.xml @@ -80,6 +80,7 @@ <module>config-mongo</module> <module>config-http-server</module> <module>config-examples</module> + <module>config-vault</module> </modules> <dependencyManagement>