From af58b0c23da58b62f757a675841f5371766195e1 Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Fri, 1 May 2020 15:31:24 -0400 Subject: [PATCH 01/23] GcpEmulator parent class --- pom.xml | 1 + spring-cloud-gcp-data-spanner/pom.xml | 5 ++ .../core/it/SpannerTemplateEmulatorTests.java | 85 ++++++++++++++++++ spring-cloud-gcp-dependencies/pom.xml | 6 +- spring-cloud-gcp-pubsub-stream-binder/pom.xml | 5 ++ ...bSubMessageChannelBinderEmulatorTests.java | 3 +- spring-cloud-gcp-test-support/pom.xml | 32 +++++++ .../cloud/gcp/test/GcpEmulatorRule.java | 88 ++++++++----------- .../cloud/gcp/test/PubSubEmulatorRule.java | 27 ++++++ .../cloud/gcp/test/SpannerEmulatorRule.java | 27 ++++++ 10 files changed, 227 insertions(+), 52 deletions(-) create mode 100644 spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java create mode 100644 spring-cloud-gcp-test-support/pom.xml rename spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubEmulator.java => spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java (78%) create mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java create mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java diff --git a/pom.xml b/pom.xml index 68b7d02cde..78fd5fb14f 100644 --- a/pom.xml +++ b/pom.xml @@ -45,6 +45,7 @@ spring-cloud-gcp-bigquery spring-cloud-gcp-security-firebase spring-cloud-gcp-secretmanager + spring-cloud-gcp-test-support diff --git a/spring-cloud-gcp-data-spanner/pom.xml b/spring-cloud-gcp-data-spanner/pom.xml index 2add52327a..2d642acf7d 100644 --- a/spring-cloud-gcp-data-spanner/pom.xml +++ b/spring-cloud-gcp-data-spanner/pom.xml @@ -42,5 +42,10 @@ org.springframework spring-tx + + org.springframework.cloud + spring-cloud-gcp-test-support + test + diff --git a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java new file mode 100644 index 0000000000..61ac78a63a --- /dev/null +++ b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java @@ -0,0 +1,85 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.data.spanner.core.it; + +import java.util.Arrays; +import java.util.List; + +import com.google.cloud.spanner.Key; +import com.google.cloud.spanner.KeySet; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; + +import org.springframework.cloud.gcp.data.spanner.core.SpannerPageableQueryOptions; +import org.springframework.cloud.gcp.data.spanner.test.AbstractSpannerIntegrationTest; +import org.springframework.cloud.gcp.data.spanner.test.domain.Trade; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Integration tests that use many features of the Spanner Template. + * + * @author Balint Pato + * @author Chengyuan Zhao + */ +@RunWith(SpringRunner.class) +public class SpannerTemplateEmulatorTests extends AbstractSpannerIntegrationTest { + + /** + * used for checking exception messages and types. + */ + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + @Test + public void insertAndDeleteSequence() { + + this.spannerOperations.delete(Trade.class, KeySet.all()); + + assertThat(this.spannerOperations.count(Trade.class)).isEqualTo(0L); + + Trade trade = Trade.aTrade(null, 1); + this.spannerOperations.insert(trade); + assertThat(this.spannerOperations.count(Trade.class)).isEqualTo(1L); + + List trades = this.spannerOperations.queryAll(Trade.class, new SpannerPageableQueryOptions()); + + assertThat(trades).containsExactly(trade); + + Trade retrievedTrade = this.spannerOperations.read(Trade.class, + Key.of(trade.getId(), trade.getTraderId())); + assertThat(retrievedTrade).isEqualTo(trade); + + trades = this.spannerOperations.readAll(Trade.class); + + assertThat(trades).containsExactly(trade); + + Trade trade2 = Trade.aTrade(null, 1); + this.spannerOperations.insert(trade2); + + trades = this.spannerOperations.read(Trade.class, KeySet.newBuilder().addKey(Key.of(trade.getId(), trade.getTraderId())) + .addKey(Key.of(trade2.getId(), trade2.getTraderId())).build()); + + assertThat(trades).containsExactlyInAnyOrder(trade, trade2); + + this.spannerOperations.deleteAll(Arrays.asList(trade, trade2)); + assertThat(this.spannerOperations.count(Trade.class)).isEqualTo(0L); + } +} diff --git a/spring-cloud-gcp-dependencies/pom.xml b/spring-cloud-gcp-dependencies/pom.xml index 062964db1c..eae4737861 100644 --- a/spring-cloud-gcp-dependencies/pom.xml +++ b/spring-cloud-gcp-dependencies/pom.xml @@ -190,7 +190,11 @@ spring-cloud-gcp-starter-secretmanager ${project.version} - + + org.springframework.cloud + spring-cloud-gcp-test-support + ${project.version} + com.google.cloud.sql diff --git a/spring-cloud-gcp-pubsub-stream-binder/pom.xml b/spring-cloud-gcp-pubsub-stream-binder/pom.xml index e6da5ba688..995ad458a9 100644 --- a/spring-cloud-gcp-pubsub-stream-binder/pom.xml +++ b/spring-cloud-gcp-pubsub-stream-binder/pom.xml @@ -42,5 +42,10 @@ 3.1.0.BUILD-SNAPSHOT test + + org.springframework.cloud + spring-cloud-gcp-test-support + test + diff --git a/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java b/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java index c85a74bc95..07cb4d47fe 100644 --- a/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java +++ b/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java @@ -22,6 +22,7 @@ import org.springframework.cloud.gcp.stream.binder.pubsub.properties.PubSubConsumerProperties; import org.springframework.cloud.gcp.stream.binder.pubsub.properties.PubSubProducerProperties; +import org.springframework.cloud.gcp.test.PubSubEmulatorRule; import org.springframework.cloud.stream.binder.AbstractBinderTests; import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; import org.springframework.cloud.stream.binder.ExtendedProducerProperties; @@ -42,7 +43,7 @@ public class PubSubMessageChannelBinderEmulatorTests extends * The emulator instance, shared across tests. */ @ClassRule - public static PubSubEmulator emulator = new PubSubEmulator(); + public static PubSubEmulatorRule emulator = new PubSubEmulatorRule(); @Override protected PubSubTestBinder getBinder() { diff --git a/spring-cloud-gcp-test-support/pom.xml b/spring-cloud-gcp-test-support/pom.xml new file mode 100644 index 0000000000..22436a422e --- /dev/null +++ b/spring-cloud-gcp-test-support/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + spring-cloud-gcp + org.springframework.cloud + 1.2.3.BUILD-SNAPSHOT + + + + spring-cloud-gcp-test-support + Spring Cloud GCP Test Support + Provides tools for testing Spring Cloud GCP projects + + + junit + junit + + + + org.springframework + spring-jcl + compile + + + org.awaitility + awaitility + + + diff --git a/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubEmulator.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java similarity index 78% rename from spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubEmulator.java rename to spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java index db7a39026c..08285bab4c 100644 --- a/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubEmulator.java +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.cloud.gcp.stream.binder.pubsub; +package org.springframework.cloud.gcp.test; import java.io.BufferedReader; import java.io.IOException; @@ -33,11 +33,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.awaitility.Awaitility; +import org.junit.Assert; +import org.junit.Assume; import org.junit.rules.ExternalResource; -import static org.junit.Assert.fail; -import static org.junit.Assume.assumeTrue; - /** * Rule for instantiating and tearing down a Pub/Sub emulator instance. * @@ -48,16 +48,16 @@ * * @since 1.1 */ -public class PubSubEmulator extends ExternalResource { +abstract class GcpEmulatorRule extends ExternalResource { - private static final Path EMULATOR_CONFIG_DIR = Paths.get(System.getProperty("user.home")).resolve( - Paths.get(".config", "gcloud", "emulators", "pubsub")); + private final Path EMULATOR_CONFIG_DIR = Paths.get(System.getProperty("user.home")).resolve( + Paths.get(".config", "gcloud", "emulators", getEmulatorName())); private static final String ENV_FILE_NAME = "env.yaml"; - private static final Path EMULATOR_CONFIG_PATH = EMULATOR_CONFIG_DIR.resolve(ENV_FILE_NAME); + private final Path EMULATOR_CONFIG_PATH = EMULATOR_CONFIG_DIR.resolve(ENV_FILE_NAME); - private static final Log LOGGER = LogFactory.getLog(PubSubEmulator.class); + private static final Log LOGGER = LogFactory.getLog(GcpEmulatorRule.class); // Reference to emulator instance, for cleanup. private Process emulatorProcess; @@ -65,17 +65,7 @@ public class PubSubEmulator extends ExternalResource { // Hostname for cleanup, should always be localhost. private String emulatorHostPort; - // Conditional rule execution based on an environmental flag. - private boolean enableTests; - - public PubSubEmulator() { - if ("true".equals(System.getProperty("it.pubsub-emulator"))) { - this.enableTests = true; - } - else { - LOGGER.warn("PubSubEmulator rule disabled. Please enable with -Dit.pubsub-emulator."); - } - } + abstract String getGatingPropertyName(); /** * Launch an instance of pubsub emulator or skip all tests. @@ -87,7 +77,8 @@ public PubSubEmulator() { @Override protected void before() throws IOException, InterruptedException { - assumeTrue("PubSubEmulator rule disabled. Please enable with -Dit.pubsub-emulator.", this.enableTests); + Assume.assumeTrue("Emulator rule disabled. Please enable with -D" + getGatingPropertyName() + ".", + "true".equals(System.getProperty(getGatingPropertyName()))); startEmulator(); determineHostPort(); @@ -174,11 +165,11 @@ private void startEmulator() throws IOException, InterruptedException { } try { - this.emulatorProcess = new ProcessBuilder("gcloud", "beta", "emulators", "pubsub", "start") + this.emulatorProcess = new ProcessBuilder("gcloud", "beta", "emulators", getEmulatorName(), "start") .start(); } catch (IOException ex) { - fail("Gcloud not found; leaving host/port uninitialized."); + Assert.fail("Gcloud not found; leaving host/port uninitialized."); } if (configPresent) { @@ -186,7 +177,7 @@ private void startEmulator() throws IOException, InterruptedException { watchService.close(); } else { - createConfig(); + waitForConfigCreation(); } } @@ -197,7 +188,7 @@ private void startEmulator() throws IOException, InterruptedException { * @throws InterruptedException for interruption errors */ private void determineHostPort() throws IOException, InterruptedException { - Process envInitProcess = new ProcessBuilder("gcloud", "beta", "emulators", "pubsub", "env-init").start(); + Process envInitProcess = new ProcessBuilder("gcloud", "beta", "emulators", getEmulatorName(), "env-init").start(); try (BufferedReader br = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream()))) { @@ -207,21 +198,19 @@ private void determineHostPort() throws IOException, InterruptedException { } } + abstract String getEmulatorName(); + /** * Wait until a PubSub emulator configuration file is present. * Fail if the file does not appear after 10 seconds. * @throws InterruptedException which should interrupt the peaceful slumber and bubble up * to fail the test. */ - private void createConfig() throws InterruptedException { - int attempts = 10; - while (!Files.exists(EMULATOR_CONFIG_PATH) && --attempts >= 0) { - Thread.sleep(1000); - } - if (attempts < 0) { - fail( - "Emulator could not be configured due to missing env.yaml. Are PubSub and beta tools installed?"); - } + private void waitForConfigCreation() throws InterruptedException { + Awaitility.await("Emulator could not be configured due to missing env.yaml. Is the emulator installed?") + .atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .until(() -> Files.exists(EMULATOR_CONFIG_PATH)); } /** @@ -232,22 +221,21 @@ private void createConfig() throws InterruptedException { * to fail the test. */ private void updateConfig(WatchService watchService) throws InterruptedException { - int attempts = 10; - while (--attempts >= 0) { - WatchKey key = watchService.poll(100, TimeUnit.MILLISECONDS); - - if (key != null) { - Optional configFilePath = key.pollEvents().stream() - .map((event) -> (Path) event.context()) - .filter((path) -> ENV_FILE_NAME.equals(path.toString())) - .findAny(); - if (configFilePath.isPresent()) { - return; - } - } - } - - fail("Configuration file update could not be detected"); + Awaitility.await("Configuration file update could not be detected") + .atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .until(() -> { + WatchKey key = watchService.poll(100, TimeUnit.MILLISECONDS); + + if (key != null) { + Optional configFilePath = key.pollEvents().stream() + .map((event) -> (Path) event.context()) + .filter((path) -> ENV_FILE_NAME.equals(path.toString())) + .findAny(); + return configFilePath.isPresent(); + } + return false; + }); } /** diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java new file mode 100644 index 0000000000..d797bf8791 --- /dev/null +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java @@ -0,0 +1,27 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test; + +public class PubSubEmulatorRule extends GcpEmulatorRule { + String getGatingPropertyName() { + return "it.pubsub-emulator"; + } + + String getEmulatorName() { + return "pubsub"; + } +} diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java new file mode 100644 index 0000000000..7765b05b74 --- /dev/null +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java @@ -0,0 +1,27 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test; + +public class SpannerEmulatorRule extends GcpEmulatorRule { + String getGatingPropertyName() { + return "it.spanner-emulator"; + } + + String getEmulatorName() { + return "spanner"; + } +} From 616fd6c51b9627a2401d1e4b0278b481fa2b42fd Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Fri, 1 May 2020 16:11:12 -0400 Subject: [PATCH 02/23] create GcpEmulator parent class --- .../cloud/gcp/test/GcpEmulatorRule.java | 60 +++++++++++++++++-- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java index 08285bab4c..d1914ff30e 100644 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java @@ -26,10 +26,12 @@ import java.nio.file.StandardWatchEventKinds; import java.nio.file.WatchKey; import java.nio.file.WatchService; +import java.util.List; import java.util.Optional; import java.util.StringTokenizer; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -180,6 +182,11 @@ private void startEmulator() throws IOException, InterruptedException { waitForConfigCreation(); } + afterEmulatorStart(); + } + + protected void afterEmulatorStart(){ + //does nothing by default } /** @@ -188,13 +195,30 @@ private void startEmulator() throws IOException, InterruptedException { * @throws InterruptedException for interruption errors */ private void determineHostPort() throws IOException, InterruptedException { - Process envInitProcess = new ProcessBuilder("gcloud", "beta", "emulators", getEmulatorName(), "env-init").start(); + ProcessOutcome processOutcome = runSystemCommand(new String[] { "gcloud", "beta", "emulators", getEmulatorName(), "env-init" }); + if ( processOutcome.getOutput().size() < 1 ) { + Assert.fail("env-init command did not produce output"); + } + String emulatorInitString = processOutcome.getOutput().get(0); + this.emulatorHostPort = emulatorInitString.substring(emulatorInitString.indexOf('=') + 1); + } + + private ProcessOutcome runSystemCommand(String[] command) throws IOException, InterruptedException { + Process envInitProcess = new ProcessBuilder(command).start(); + try (BufferedReader brInput = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream())); + BufferedReader brError = new BufferedReader(new InputStreamReader(envInitProcess.getErrorStream())) + ) { + ProcessOutcome processOutcome = new ProcessOutcome(brInput.lines().collect(Collectors.toList()), + brError.lines().collect(Collectors.toList()), + envInitProcess.waitFor()); + if (processOutcome.status != 0) { + Assert.fail("Command execution failed: " + String.join(" ", command) + + "; output: " + processOutcome.getOutput() + + "; error: " + processOutcome.getErrors()); - try (BufferedReader br = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream()))) { - String emulatorInitString = br.readLine(); - envInitProcess.waitFor(); - this.emulatorHostPort = emulatorInitString.substring(emulatorInitString.indexOf('=') + 1); + } + return processOutcome; } } @@ -251,4 +275,30 @@ private void killProcess(String pid) { LOGGER.warn("Failed to clean up PID " + pid); } } + + static class ProcessOutcome { + private List output; + + private List errors; + + private int status; + + public ProcessOutcome(List output, List errors, int status) { + this.output = output; + this.errors = errors; + this.status = status; + } + + public List getOutput() { + return output; + } + + public List getErrors() { + return errors; + } + + public int getStatus() { + return status; + } + } } From c97d2a511a14f2c3ff6dca618324dae5e85d10e7 Mon Sep 17 00:00:00 2001 From: Elena Felder <41136058+elefeint@users.noreply.github.com> Date: Fri, 1 May 2020 17:05:07 -0400 Subject: [PATCH 03/23] run all necessary setup, working test --- .../core/it/SpannerTemplateEmulatorTests.java | 41 +++++-------------- .../cloud/gcp/test/GcpEmulatorRule.java | 21 +++++++--- .../cloud/gcp/test/SpannerEmulatorRule.java | 23 +++++++++++ 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java index 61ac78a63a..bccbc782db 100644 --- a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java +++ b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java @@ -21,14 +21,20 @@ import com.google.cloud.spanner.Key; import com.google.cloud.spanner.KeySet; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.gcp.data.spanner.core.SpannerOperations; import org.springframework.cloud.gcp.data.spanner.core.SpannerPageableQueryOptions; import org.springframework.cloud.gcp.data.spanner.test.AbstractSpannerIntegrationTest; import org.springframework.cloud.gcp.data.spanner.test.domain.Trade; +import org.springframework.cloud.gcp.test.PubSubEmulatorRule; +import org.springframework.cloud.gcp.test.SpannerEmulatorRule; import org.springframework.test.context.junit4.SpringRunner; import static org.assertj.core.api.Assertions.assertThat; @@ -43,43 +49,16 @@ public class SpannerTemplateEmulatorTests extends AbstractSpannerIntegrationTest { /** - * used for checking exception messages and types. + * The emulator instance, shared across tests. */ - @Rule - public ExpectedException expectedEx = ExpectedException.none(); + @ClassRule + public static SpannerEmulatorRule emulator = new SpannerEmulatorRule(); @Test - public void insertAndDeleteSequence() { - - this.spannerOperations.delete(Trade.class, KeySet.all()); - - assertThat(this.spannerOperations.count(Trade.class)).isEqualTo(0L); + public void insertSingleRow() { Trade trade = Trade.aTrade(null, 1); this.spannerOperations.insert(trade); assertThat(this.spannerOperations.count(Trade.class)).isEqualTo(1L); - - List trades = this.spannerOperations.queryAll(Trade.class, new SpannerPageableQueryOptions()); - - assertThat(trades).containsExactly(trade); - - Trade retrievedTrade = this.spannerOperations.read(Trade.class, - Key.of(trade.getId(), trade.getTraderId())); - assertThat(retrievedTrade).isEqualTo(trade); - - trades = this.spannerOperations.readAll(Trade.class); - - assertThat(trades).containsExactly(trade); - - Trade trade2 = Trade.aTrade(null, 1); - this.spannerOperations.insert(trade2); - - trades = this.spannerOperations.read(Trade.class, KeySet.newBuilder().addKey(Key.of(trade.getId(), trade.getTraderId())) - .addKey(Key.of(trade2.getId(), trade2.getTraderId())).build()); - - assertThat(trades).containsExactlyInAnyOrder(trade, trade2); - - this.spannerOperations.deleteAll(Arrays.asList(trade, trade2)); - assertThat(this.spannerOperations.count(Trade.class)).isEqualTo(0L); } } diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java index d1914ff30e..0365c2bde5 100644 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java @@ -171,7 +171,7 @@ private void startEmulator() throws IOException, InterruptedException { .start(); } catch (IOException ex) { - Assert.fail("Gcloud not found; leaving host/port uninitialized."); + throw new RuntimeException("Gcloud not found; leaving host/port uninitialized.", ex); } if (configPresent) { @@ -194,17 +194,23 @@ protected void afterEmulatorStart(){ * @throws IOException for IO errors * @throws InterruptedException for interruption errors */ - private void determineHostPort() throws IOException, InterruptedException { + private void determineHostPort() { ProcessOutcome processOutcome = runSystemCommand(new String[] { "gcloud", "beta", "emulators", getEmulatorName(), "env-init" }); if ( processOutcome.getOutput().size() < 1 ) { - Assert.fail("env-init command did not produce output"); + throw new RuntimeException("env-init command did not produce output"); } String emulatorInitString = processOutcome.getOutput().get(0); this.emulatorHostPort = emulatorInitString.substring(emulatorInitString.indexOf('=') + 1); } - private ProcessOutcome runSystemCommand(String[] command) throws IOException, InterruptedException { - Process envInitProcess = new ProcessBuilder(command).start(); + protected ProcessOutcome runSystemCommand(String[] command) { + + Process envInitProcess = null; + try { + envInitProcess = new ProcessBuilder(command).start(); + } catch (IOException e) { + throw new RuntimeException(e); + } try (BufferedReader brInput = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream())); BufferedReader brError = new BufferedReader(new InputStreamReader(envInitProcess.getErrorStream())) @@ -213,12 +219,15 @@ private ProcessOutcome runSystemCommand(String[] command) throws IOException, In brError.lines().collect(Collectors.toList()), envInitProcess.waitFor()); if (processOutcome.status != 0) { - Assert.fail("Command execution failed: " + String.join(" ", command) + throw new RuntimeException("Command execution failed: " + String.join(" ", command) + "; output: " + processOutcome.getOutput() + "; error: " + processOutcome.getErrors()); } return processOutcome; + } catch (IOException|InterruptedException e) { + // Allow the rule to fail. + throw new RuntimeException(e); } } diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java index 7765b05b74..e8ac3ac7e4 100644 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java @@ -16,6 +16,9 @@ package org.springframework.cloud.gcp.test; +/** + * TODO: send users to https://cloud.google.com/spanner/docs/emulator#using_the_gcloud_cli_with_the_emulator + */ public class SpannerEmulatorRule extends GcpEmulatorRule { String getGatingPropertyName() { return "it.spanner-emulator"; @@ -24,4 +27,24 @@ String getGatingPropertyName() { String getEmulatorName() { return "spanner"; } + + @Override + protected void afterEmulatorStart() { + ProcessOutcome switchToEmulator = runSystemCommand(new String[] { + "gcloud", "config", "configurations", "activate", "emulator"}); + + ProcessOutcome processOutcome = runSystemCommand(new String[] { + "gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator-config", "--description=\"Test Instance\"", "--nodes=1" }); + + + // TODO: check for System.getenv("SPANNER_EMULATOR_HOST"); -- if it does not exist, bail on the emulator rule + // because otherwise users would connect to real spanner + + // TODO: don't forget to kill the 2 spanner processes + } + + /* + * gcloud spanner instances create test-instance --config=emulator-config --description="Test Instance" --nodes=1 + * gcloud config configurations activate [emulator | default] + * */ } From e76057082ee82c2734f9f3ac9317e300976dc1cc Mon Sep 17 00:00:00 2001 From: Elena Felder <41136058+elefeint@users.noreply.github.com> Date: Tue, 5 May 2020 15:00:46 -0400 Subject: [PATCH 04/23] refactored to allow killing arbitrary commands --- .../cloud/gcp/test/GcpEmulatorRule.java | 91 +++++++++++-------- .../cloud/gcp/test/PubSubEmulatorRule.java | 29 +++++- .../cloud/gcp/test/SpannerEmulatorRule.java | 22 ++++- 3 files changed, 98 insertions(+), 44 deletions(-) diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java index 0365c2bde5..997f553762 100644 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java @@ -82,7 +82,9 @@ protected void before() throws IOException, InterruptedException { Assume.assumeTrue("Emulator rule disabled. Please enable with -D" + getGatingPropertyName() + ".", "true".equals(System.getProperty(getGatingPropertyName()))); + beforeEmulatorStart(); startEmulator(); + afterEmulatorStart(); determineHostPort(); } @@ -97,6 +99,11 @@ protected void before() throws IOException, InterruptedException { @Override protected void after() { findAndDestroyEmulator(); + afterEmulatorDestroyed(); + } + + protected void afterEmulatorDestroyed() { + // does nothing by default. } private void findAndDestroyEmulator() { @@ -107,45 +114,32 @@ private void findAndDestroyEmulator() { else { LOGGER.warn("Emulator process null after tests; nothing to terminate."); } + } - // find destory emulator process spawned by gcloud - if (this.emulatorHostPort == null) { - LOGGER.warn("Host/port null after the test."); - } - else { - int portSeparatorIndex = this.emulatorHostPort.lastIndexOf(":"); - if (portSeparatorIndex < 0) { - LOGGER.warn("Malformed host: " + this.emulatorHostPort); - return; - } + protected void killByCommand(String command) { + AtomicBoolean foundProcess = new AtomicBoolean(false); - String emulatorHost = this.emulatorHostPort.substring(0, portSeparatorIndex); - String emulatorPort = this.emulatorHostPort.substring(portSeparatorIndex + 1); - - AtomicBoolean foundEmulatorProcess = new AtomicBoolean(false); - String hostPortParams = String.format("--host=%s --port=%s", emulatorHost, emulatorPort); - try { - Process psProcess = new ProcessBuilder("ps", "-vx").start(); - - try (BufferedReader br = new BufferedReader(new InputStreamReader(psProcess.getInputStream()))) { - br.lines() - .filter((psLine) -> psLine.contains(hostPortParams)) - .map((psLine) -> new StringTokenizer(psLine).nextToken()) - .forEach((p) -> { - LOGGER.info("Found emulator process to kill: " + p); - this.killProcess(p); - foundEmulatorProcess.set(true); - }); - } - - if (!foundEmulatorProcess.get()) { - LOGGER.warn("Did not find the emualtor process to kill based on: " + hostPortParams); - } + try { + Process psProcess = new ProcessBuilder("ps", "-vx").start(); + + try (BufferedReader br = new BufferedReader(new InputStreamReader(psProcess.getInputStream()))) { + br.lines() + .filter((psLine) -> psLine.contains(command)) + .map((psLine) -> new StringTokenizer(psLine).nextToken()) + .forEach((p) -> { + LOGGER.info("Found " + command + " process to kill: " + p); + this.killProcess(p); + foundProcess.set(true); + }); } - catch (IOException ex) { - LOGGER.warn("Failed to cleanup: ", ex); + + if (!foundProcess.get()) { + LOGGER.warn("Did not find the emualtor process to kill based on: " + command); } } + catch (IOException ex) { + LOGGER.warn("Failed to cleanup: ", ex); + } } /** @@ -182,7 +176,11 @@ private void startEmulator() throws IOException, InterruptedException { waitForConfigCreation(); } - afterEmulatorStart(); + + } + + protected void beforeEmulatorStart(){ + //does nothing by default } protected void afterEmulatorStart(){ @@ -204,6 +202,10 @@ private void determineHostPort() { } protected ProcessOutcome runSystemCommand(String[] command) { + return runSystemCommand(command, true); + } + + protected ProcessOutcome runSystemCommand(String[] command, boolean failOnError) { Process envInitProcess = null; try { @@ -215,10 +217,12 @@ protected ProcessOutcome runSystemCommand(String[] command) { try (BufferedReader brInput = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream())); BufferedReader brError = new BufferedReader(new InputStreamReader(envInitProcess.getErrorStream())) ) { - ProcessOutcome processOutcome = new ProcessOutcome(brInput.lines().collect(Collectors.toList()), + ProcessOutcome processOutcome = new ProcessOutcome(command, + brInput.lines().collect(Collectors.toList()), brError.lines().collect(Collectors.toList()), envInitProcess.waitFor()); - if (processOutcome.status != 0) { + + if (failOnError && processOutcome.status != 0) { throw new RuntimeException("Command execution failed: " + String.join(" ", command) + "; output: " + processOutcome.getOutput() + "; error: " + processOutcome.getErrors()); @@ -276,9 +280,9 @@ private void updateConfig(WatchService watchService) throws InterruptedException * Failure is logged and ignored, as it is not critical to the tests' functionality. * @param pid presumably a valid PID. No checking done to validate. */ - private void killProcess(String pid) { + protected void killProcess(String pid) { try { - new ProcessBuilder("kill", pid).start(); + new ProcessBuilder("kill", "-9", pid).start(); } catch (IOException ex) { LOGGER.warn("Failed to clean up PID " + pid); @@ -286,13 +290,16 @@ private void killProcess(String pid) { } static class ProcessOutcome { + private String[] command; + private List output; private List errors; private int status; - public ProcessOutcome(List output, List errors, int status) { + public ProcessOutcome(String[] command, List output, List errors, int status) { + this.command = command; this.output = output; this.errors = errors; this.status = status; @@ -309,5 +316,9 @@ public List getErrors() { public int getStatus() { return status; } + + public String getCommandString() { + return String.join(" ", command); + } } } diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java index d797bf8791..ee2dda3f7c 100644 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java @@ -16,12 +16,39 @@ package org.springframework.cloud.gcp.test; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + public class PubSubEmulatorRule extends GcpEmulatorRule { - String getGatingPropertyName() { + private static final Log LOGGER = LogFactory.getLog(PubSubEmulatorRule.class); + + String getGatingPropertyName() { return "it.pubsub-emulator"; } String getEmulatorName() { return "pubsub"; } + + @Override + protected void afterEmulatorDestroyed() { + String hostPort = getEmulatorHostPort(); + + // find destory emulator process spawned by gcloud + if (hostPort == null) { + LOGGER.warn("Host/port null after the test."); + } else { + int portSeparatorIndex = hostPort.lastIndexOf(":"); + if (portSeparatorIndex < 0) { + LOGGER.warn("Malformed host: " + hostPort); + return; + } + + String emulatorHost = hostPort.substring(0, portSeparatorIndex); + String emulatorPort = hostPort.substring(portSeparatorIndex + 1); + + String hostPortParams = String.format("--host=%s --port=%s", emulatorHost, emulatorPort); + killByCommand(hostPortParams); + } + } } diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java index e8ac3ac7e4..2b496941f1 100644 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java @@ -16,6 +16,9 @@ package org.springframework.cloud.gcp.test; +import java.util.stream.Collectors; +import org.junit.Assume; + /** * TODO: send users to https://cloud.google.com/spanner/docs/emulator#using_the_gcloud_cli_with_the_emulator */ @@ -28,17 +31,30 @@ String getEmulatorName() { return "spanner"; } + @Override + protected void beforeEmulatorStart() { + String emulatorHost = System.getenv("SPANNER_EMULATOR_HOST"); + Assume.assumeFalse( + "Run this command prior to running an emulator test:\n$(gcloud beta emulators spanner env-init)", + emulatorHost == null || emulatorHost.isEmpty()); + } + @Override protected void afterEmulatorStart() { ProcessOutcome switchToEmulator = runSystemCommand(new String[] { "gcloud", "config", "configurations", "activate", "emulator"}); ProcessOutcome processOutcome = runSystemCommand(new String[] { - "gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator-config", "--description=\"Test Instance\"", "--nodes=1" }); + "gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator-config", "--description=\"Test Instance\"", "--nodes=1" }, + false); + if (processOutcome.getStatus() != 0) { + // don't set breakpoint here + this.killByCommand("cloud_spanner_emulator/emulator_main"); + throw new RuntimeException("Creating instance failed: " + + String.join("\n", processOutcome.getErrors())); + } - // TODO: check for System.getenv("SPANNER_EMULATOR_HOST"); -- if it does not exist, bail on the emulator rule - // because otherwise users would connect to real spanner // TODO: don't forget to kill the 2 spanner processes } From ddd4fb7e221eae029e014f1af4f611e668bca434 Mon Sep 17 00:00:00 2001 From: Elena Felder <41136058+elefeint@users.noreply.github.com> Date: Tue, 5 May 2020 17:02:54 -0400 Subject: [PATCH 05/23] refactor to be independent from junit and spring --- .../core/it/SpannerTemplateEmulatorTests.java | 24 +- .../test/IntegrationTestConfiguration.java | 3 + spring-cloud-gcp-test-support/pom.xml | 44 ++- .../gcp/test/AbstractEmulatorHelper.java | 290 ++++++++++++++++ .../cloud/gcp/test/GcpEmulatorRule.java | 324 ------------------ .../cloud/gcp/test/PubSubEmulatorHelper.java | 38 ++ .../cloud/gcp/test/PubSubEmulatorRule.java | 46 +-- .../cloud/gcp/test/SpannerEmulatorHelper.java | 50 +++ .../cloud/gcp/test/SpannerEmulatorRule.java | 39 +-- .../SpannerEmulatorSpringConfiguration.java | 37 ++ 10 files changed, 479 insertions(+), 416 deletions(-) create mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java delete mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java create mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java create mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java create mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java diff --git a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java index bccbc782db..8054f1e5fa 100644 --- a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java +++ b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java @@ -16,25 +16,13 @@ package org.springframework.cloud.gcp.data.spanner.core.it; -import java.util.Arrays; -import java.util.List; - -import com.google.cloud.spanner.Key; -import com.google.cloud.spanner.KeySet; -import org.junit.ClassRule; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.cloud.gcp.data.spanner.core.SpannerOperations; -import org.springframework.cloud.gcp.data.spanner.core.SpannerPageableQueryOptions; import org.springframework.cloud.gcp.data.spanner.test.AbstractSpannerIntegrationTest; import org.springframework.cloud.gcp.data.spanner.test.domain.Trade; -import org.springframework.cloud.gcp.test.PubSubEmulatorRule; -import org.springframework.cloud.gcp.test.SpannerEmulatorRule; +import org.springframework.cloud.gcp.test.SpannerEmulatorSpringConfiguration; +import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import static org.assertj.core.api.Assertions.assertThat; @@ -46,14 +34,9 @@ * @author Chengyuan Zhao */ @RunWith(SpringRunner.class) +@ContextConfiguration(classes = {SpannerEmulatorSpringConfiguration.class}) public class SpannerTemplateEmulatorTests extends AbstractSpannerIntegrationTest { - /** - * The emulator instance, shared across tests. - */ - @ClassRule - public static SpannerEmulatorRule emulator = new SpannerEmulatorRule(); - @Test public void insertSingleRow() { @@ -61,4 +44,5 @@ public void insertSingleRow() { this.spannerOperations.insert(trade); assertThat(this.spannerOperations.count(Trade.class)).isEqualTo(1L); } + } diff --git a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java index d4389da4ae..50e1062c72 100644 --- a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java +++ b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java @@ -26,6 +26,7 @@ import com.google.cloud.spanner.SpannerOptions; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.cloud.gcp.core.Credentials; import org.springframework.cloud.gcp.core.DefaultCredentialsProvider; import org.springframework.cloud.gcp.core.DefaultGcpProjectIdProvider; @@ -104,6 +105,7 @@ public TradeRepositoryTransactionalService tradeRepositoryTransactionalService() } @Bean + @ConditionalOnMissingBean public SpannerOptions spannerOptions() { return SpannerOptions.newBuilder().setProjectId(getProjectId()) .setSessionPoolOption(SessionPoolOptions.newBuilder().setMaxSessions(10).build()) @@ -116,6 +118,7 @@ public DatabaseId databaseId() { } @Bean + @ConditionalOnMissingBean public Spanner spanner(SpannerOptions spannerOptions) { return spannerOptions.getService(); } diff --git a/spring-cloud-gcp-test-support/pom.xml b/spring-cloud-gcp-test-support/pom.xml index 22436a422e..ca9dd8bbd3 100644 --- a/spring-cloud-gcp-test-support/pom.xml +++ b/spring-cloud-gcp-test-support/pom.xml @@ -14,19 +14,37 @@ Spring Cloud GCP Test Support Provides tools for testing Spring Cloud GCP projects - - junit - junit - - - org.springframework - spring-jcl - compile - - - org.awaitility - awaitility - + + + junit + junit + true + + + + org.springframework + spring-jcl + compile + + + org.awaitility + awaitility + + + + + org.springframework + spring-context + true + + + + + com.google.cloud + google-cloud-spanner + true + + diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java new file mode 100644 index 0000000000..af06574bda --- /dev/null +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java @@ -0,0 +1,290 @@ +package org.springframework.cloud.gcp.test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardWatchEventKinds; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.List; +import java.util.Optional; +import java.util.StringTokenizer; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.awaitility.Awaitility; + +public abstract class AbstractEmulatorHelper { + + private final Path EMULATOR_CONFIG_DIR = Paths.get(System.getProperty("user.home")).resolve( + Paths.get(".config", "gcloud", "emulators", getEmulatorName())); + + private static final String ENV_FILE_NAME = "env.yaml"; + + private final Path EMULATOR_CONFIG_PATH = EMULATOR_CONFIG_DIR.resolve(ENV_FILE_NAME); + + private static final Log LOGGER = LogFactory.getLog(AbstractEmulatorHelper.class); + + // Reference to emulator instance, for cleanup. + private Process emulatorProcess; + + // Hostname for cleanup, should always be localhost. + private String emulatorHostPort; + + abstract String getGatingPropertyName(); + + /** + * Launch an instance of pubsub emulator or skip all tests. + * If it.pubsub-emulator environmental property is off, all tests will be skipped through the failed assumption. + * If the property is on, any setup failure will trigger test failure. Failures during teardown are merely logged. + * @throws IOException if config file creation or directory watcher on existing file fails. + * @throws InterruptedException if process is stopped while waiting to retry. + */ + public void startEmulator() throws IOException, InterruptedException { + + beforeEmulatorStart(); + doStartEmulator(); + afterEmulatorStart(); + determineHostPort(); + } + + /** + * Shut down the two emulator processes. + * gcloud command is shut down through the direct process handle. java process is + * identified and shut down through shell commands. + * There should normally be only one process with that host/port combination, but if there + * are more, they will be cleaned up as well. + * Any failure is logged and ignored since it's not critical to the tests' operation. + */ + public void shutdownEmulator() { + findAndDestroyEmulator(); + afterEmulatorDestroyed(); + } + + protected void afterEmulatorDestroyed() { + // does nothing by default. + } + + private void findAndDestroyEmulator() { + // destroy gcloud process + if (this.emulatorProcess != null) { + this.emulatorProcess.destroy(); + } + else { + LOGGER.warn("Emulator process null after tests; nothing to terminate."); + } + } + + protected void killByCommand(String command) { + AtomicBoolean foundProcess = new AtomicBoolean(false); + + try { + Process psProcess = new ProcessBuilder("ps", "-vx").start(); + + try (BufferedReader br = new BufferedReader(new InputStreamReader(psProcess.getInputStream()))) { + br.lines() + .filter((psLine) -> psLine.contains(command)) + .peek(line -> System.out.println("found line after filter: " + line)) + .map((psLine) -> new StringTokenizer(psLine).nextToken()) + .forEach((p) -> { + LOGGER.info("Found " + command + " process to kill: " + p); + this.killProcess(p); + foundProcess.set(true); + }); + } + + if (!foundProcess.get()) { + LOGGER.warn("Did not find the emulator process to kill based on: " + command); + } + } + catch (IOException ex) { + LOGGER.warn("Failed to cleanup: ", ex); + } + } + + /** + * Return the already-started emulator's host/port combination when called from within a + * JUnit method. + * @return emulator host/port string or null if emulator setup failed. + */ + public String getEmulatorHostPort() { + return this.emulatorHostPort; + } + + private void doStartEmulator() throws IOException, InterruptedException { + boolean configPresent = Files.exists(EMULATOR_CONFIG_PATH); + WatchService watchService = null; + + if (configPresent) { + watchService = FileSystems.getDefault().newWatchService(); + EMULATOR_CONFIG_DIR.register(watchService, StandardWatchEventKinds.ENTRY_MODIFY); + } + + try { + this.emulatorProcess = new ProcessBuilder("gcloud", "beta", "emulators", getEmulatorName(), "start") + .start(); + } + catch (IOException ex) { + throw new RuntimeException("Gcloud not found; leaving host/port uninitialized.", ex); + } + + if (configPresent) { + updateConfig(watchService); + watchService.close(); + } + else { + waitForConfigCreation(); + } + + + } + + protected void beforeEmulatorStart(){ + //does nothing by default + } + + protected void afterEmulatorStart(){ + //does nothing by default + } + + /** + * Extract host/port from output of env-init command: "export PUBSUB_EMULATOR_HOST=localhost:8085". + * @throws IOException for IO errors + * @throws InterruptedException for interruption errors + */ + private void determineHostPort() { + ProcessOutcome processOutcome = runSystemCommand(new String[] { "gcloud", "beta", "emulators", getEmulatorName(), "env-init" }); + if ( processOutcome.getOutput().size() < 1 ) { + throw new RuntimeException("env-init command did not produce output"); + } + String emulatorInitString = processOutcome.getOutput().get(0); + this.emulatorHostPort = emulatorInitString.substring(emulatorInitString.indexOf('=') + 1); + } + + protected ProcessOutcome runSystemCommand(String[] command) { + return runSystemCommand(command, true); + } + + protected ProcessOutcome runSystemCommand(String[] command, boolean failOnError) { + + Process envInitProcess = null; + try { + envInitProcess = new ProcessBuilder(command).start(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + try (BufferedReader brInput = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream())); + BufferedReader brError = new BufferedReader(new InputStreamReader(envInitProcess.getErrorStream())) + ) { + ProcessOutcome processOutcome = new ProcessOutcome(command, + brInput.lines().collect(Collectors.toList()), + brError.lines().collect(Collectors.toList()), + envInitProcess.waitFor()); + + if (failOnError && processOutcome.status != 0) { + throw new RuntimeException("Command execution failed: " + String.join(" ", command) + + "; output: " + processOutcome.getOutput() + + "; error: " + processOutcome.getErrors()); + + } + return processOutcome; + } catch (IOException|InterruptedException e) { + throw new RuntimeException(e); + } + } + + abstract String getEmulatorName(); + + /** + * Wait until a PubSub emulator configuration file is present. + * Fail if the file does not appear after 10 seconds. + * @throws InterruptedException which should interrupt the peaceful slumber and bubble up + * to fail the test. + */ + private void waitForConfigCreation() throws InterruptedException { + Awaitility.await("Emulator could not be configured due to missing env.yaml. Is the emulator installed?") + .atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .until(() -> Files.exists(EMULATOR_CONFIG_PATH)); + } + + /** + * Wait until a PubSub emulator configuration file is updated. + * Fail if the file does not update after 1 second. + * @param watchService the watch-service to poll + * @throws InterruptedException which should interrupt the peaceful slumber and bubble up + * to fail the test. + */ + private void updateConfig(WatchService watchService) throws InterruptedException { + Awaitility.await("Configuration file update could not be detected") + .atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .until(() -> { + WatchKey key = watchService.poll(100, TimeUnit.MILLISECONDS); + + if (key != null) { + Optional configFilePath = key.pollEvents().stream() + .map((event) -> (Path) event.context()) + .filter((path) -> ENV_FILE_NAME.equals(path.toString())) + .findAny(); + return configFilePath.isPresent(); + } + return false; + }); + } + + /** + * Attempt to kill a process on best effort basis. + * Failure is logged and ignored, as it is not critical to the tests' functionality. + * @param pid presumably a valid PID. No checking done to validate. + */ + protected void killProcess(String pid) { + try { + new ProcessBuilder("kill", "-9", pid).start(); + } + catch (IOException ex) { + LOGGER.warn("Failed to clean up PID " + pid); + } + } + + static class ProcessOutcome { + private String[] command; + + private List output; + + private List errors; + + private int status; + + public ProcessOutcome(String[] command, List output, List errors, int status) { + this.command = command; + this.output = output; + this.errors = errors; + this.status = status; + } + + public List getOutput() { + return output; + } + + public List getErrors() { + return errors; + } + + public int getStatus() { + return status; + } + + public String getCommandString() { + return String.join(" ", command); + } + } + +} diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java deleted file mode 100644 index 997f553762..0000000000 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/GcpEmulatorRule.java +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright 2017-2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.gcp.test; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardWatchEventKinds; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import java.util.List; -import java.util.Optional; -import java.util.StringTokenizer; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.awaitility.Awaitility; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.rules.ExternalResource; - -/** - * Rule for instantiating and tearing down a Pub/Sub emulator instance. - * - * Tests can access the emulator's host/port combination by calling {@link #getEmulatorHostPort()} method. - * - * @author Elena Felder - * @author Mike Eltsufin - * - * @since 1.1 - */ -abstract class GcpEmulatorRule extends ExternalResource { - - private final Path EMULATOR_CONFIG_DIR = Paths.get(System.getProperty("user.home")).resolve( - Paths.get(".config", "gcloud", "emulators", getEmulatorName())); - - private static final String ENV_FILE_NAME = "env.yaml"; - - private final Path EMULATOR_CONFIG_PATH = EMULATOR_CONFIG_DIR.resolve(ENV_FILE_NAME); - - private static final Log LOGGER = LogFactory.getLog(GcpEmulatorRule.class); - - // Reference to emulator instance, for cleanup. - private Process emulatorProcess; - - // Hostname for cleanup, should always be localhost. - private String emulatorHostPort; - - abstract String getGatingPropertyName(); - - /** - * Launch an instance of pubsub emulator or skip all tests. - * If it.pubsub-emulator environmental property is off, all tests will be skipped through the failed assumption. - * If the property is on, any setup failure will trigger test failure. Failures during teardown are merely logged. - * @throws IOException if config file creation or directory watcher on existing file fails. - * @throws InterruptedException if process is stopped while waiting to retry. - */ - @Override - protected void before() throws IOException, InterruptedException { - - Assume.assumeTrue("Emulator rule disabled. Please enable with -D" + getGatingPropertyName() + ".", - "true".equals(System.getProperty(getGatingPropertyName()))); - - beforeEmulatorStart(); - startEmulator(); - afterEmulatorStart(); - determineHostPort(); - } - - /** - * Shut down the two emulator processes. - * gcloud command is shut down through the direct process handle. java process is - * identified and shut down through shell commands. - * There should normally be only one process with that host/port combination, but if there - * are more, they will be cleaned up as well. - * Any failure is logged and ignored since it's not critical to the tests' operation. - */ - @Override - protected void after() { - findAndDestroyEmulator(); - afterEmulatorDestroyed(); - } - - protected void afterEmulatorDestroyed() { - // does nothing by default. - } - - private void findAndDestroyEmulator() { - // destroy gcloud process - if (this.emulatorProcess != null) { - this.emulatorProcess.destroy(); - } - else { - LOGGER.warn("Emulator process null after tests; nothing to terminate."); - } - } - - protected void killByCommand(String command) { - AtomicBoolean foundProcess = new AtomicBoolean(false); - - try { - Process psProcess = new ProcessBuilder("ps", "-vx").start(); - - try (BufferedReader br = new BufferedReader(new InputStreamReader(psProcess.getInputStream()))) { - br.lines() - .filter((psLine) -> psLine.contains(command)) - .map((psLine) -> new StringTokenizer(psLine).nextToken()) - .forEach((p) -> { - LOGGER.info("Found " + command + " process to kill: " + p); - this.killProcess(p); - foundProcess.set(true); - }); - } - - if (!foundProcess.get()) { - LOGGER.warn("Did not find the emualtor process to kill based on: " + command); - } - } - catch (IOException ex) { - LOGGER.warn("Failed to cleanup: ", ex); - } - } - - /** - * Return the already-started emulator's host/port combination when called from within a - * JUnit method. - * @return emulator host/port string or null if emulator setup failed. - */ - public String getEmulatorHostPort() { - return this.emulatorHostPort; - } - - private void startEmulator() throws IOException, InterruptedException { - boolean configPresent = Files.exists(EMULATOR_CONFIG_PATH); - WatchService watchService = null; - - if (configPresent) { - watchService = FileSystems.getDefault().newWatchService(); - EMULATOR_CONFIG_DIR.register(watchService, StandardWatchEventKinds.ENTRY_MODIFY); - } - - try { - this.emulatorProcess = new ProcessBuilder("gcloud", "beta", "emulators", getEmulatorName(), "start") - .start(); - } - catch (IOException ex) { - throw new RuntimeException("Gcloud not found; leaving host/port uninitialized.", ex); - } - - if (configPresent) { - updateConfig(watchService); - watchService.close(); - } - else { - waitForConfigCreation(); - } - - - } - - protected void beforeEmulatorStart(){ - //does nothing by default - } - - protected void afterEmulatorStart(){ - //does nothing by default - } - - /** - * Extract host/port from output of env-init command: "export PUBSUB_EMULATOR_HOST=localhost:8085". - * @throws IOException for IO errors - * @throws InterruptedException for interruption errors - */ - private void determineHostPort() { - ProcessOutcome processOutcome = runSystemCommand(new String[] { "gcloud", "beta", "emulators", getEmulatorName(), "env-init" }); - if ( processOutcome.getOutput().size() < 1 ) { - throw new RuntimeException("env-init command did not produce output"); - } - String emulatorInitString = processOutcome.getOutput().get(0); - this.emulatorHostPort = emulatorInitString.substring(emulatorInitString.indexOf('=') + 1); - } - - protected ProcessOutcome runSystemCommand(String[] command) { - return runSystemCommand(command, true); - } - - protected ProcessOutcome runSystemCommand(String[] command, boolean failOnError) { - - Process envInitProcess = null; - try { - envInitProcess = new ProcessBuilder(command).start(); - } catch (IOException e) { - throw new RuntimeException(e); - } - - try (BufferedReader brInput = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream())); - BufferedReader brError = new BufferedReader(new InputStreamReader(envInitProcess.getErrorStream())) - ) { - ProcessOutcome processOutcome = new ProcessOutcome(command, - brInput.lines().collect(Collectors.toList()), - brError.lines().collect(Collectors.toList()), - envInitProcess.waitFor()); - - if (failOnError && processOutcome.status != 0) { - throw new RuntimeException("Command execution failed: " + String.join(" ", command) - + "; output: " + processOutcome.getOutput() - + "; error: " + processOutcome.getErrors()); - - } - return processOutcome; - } catch (IOException|InterruptedException e) { - // Allow the rule to fail. - throw new RuntimeException(e); - } - } - - abstract String getEmulatorName(); - - /** - * Wait until a PubSub emulator configuration file is present. - * Fail if the file does not appear after 10 seconds. - * @throws InterruptedException which should interrupt the peaceful slumber and bubble up - * to fail the test. - */ - private void waitForConfigCreation() throws InterruptedException { - Awaitility.await("Emulator could not be configured due to missing env.yaml. Is the emulator installed?") - .atMost(10, TimeUnit.SECONDS) - .pollInterval(100, TimeUnit.MILLISECONDS) - .until(() -> Files.exists(EMULATOR_CONFIG_PATH)); - } - - /** - * Wait until a PubSub emulator configuration file is updated. - * Fail if the file does not update after 1 second. - * @param watchService the watch-service to poll - * @throws InterruptedException which should interrupt the peaceful slumber and bubble up - * to fail the test. - */ - private void updateConfig(WatchService watchService) throws InterruptedException { - Awaitility.await("Configuration file update could not be detected") - .atMost(10, TimeUnit.SECONDS) - .pollInterval(100, TimeUnit.MILLISECONDS) - .until(() -> { - WatchKey key = watchService.poll(100, TimeUnit.MILLISECONDS); - - if (key != null) { - Optional configFilePath = key.pollEvents().stream() - .map((event) -> (Path) event.context()) - .filter((path) -> ENV_FILE_NAME.equals(path.toString())) - .findAny(); - return configFilePath.isPresent(); - } - return false; - }); - } - - /** - * Attempt to kill a process on best effort basis. - * Failure is logged and ignored, as it is not critical to the tests' functionality. - * @param pid presumably a valid PID. No checking done to validate. - */ - protected void killProcess(String pid) { - try { - new ProcessBuilder("kill", "-9", pid).start(); - } - catch (IOException ex) { - LOGGER.warn("Failed to clean up PID " + pid); - } - } - - static class ProcessOutcome { - private String[] command; - - private List output; - - private List errors; - - private int status; - - public ProcessOutcome(String[] command, List output, List errors, int status) { - this.command = command; - this.output = output; - this.errors = errors; - this.status = status; - } - - public List getOutput() { - return output; - } - - public List getErrors() { - return errors; - } - - public int getStatus() { - return status; - } - - public String getCommandString() { - return String.join(" ", command); - } - } -} diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java new file mode 100644 index 0000000000..5301743682 --- /dev/null +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java @@ -0,0 +1,38 @@ +package org.springframework.cloud.gcp.test; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class PubSubEmulatorHelper extends AbstractEmulatorHelper { + private static final Log LOGGER = LogFactory.getLog(PubSubEmulatorHelper.class); + + String getGatingPropertyName() { + return "it.pubsub-emulator"; + } + + String getEmulatorName() { + return "pubsub"; + } + + @Override + protected void afterEmulatorDestroyed() { + String hostPort = getEmulatorHostPort(); + + // find destory emulator process spawned by gcloud + if (hostPort == null) { + LOGGER.warn("Host/port null after the test."); + } else { + int portSeparatorIndex = hostPort.lastIndexOf(":"); + if (portSeparatorIndex < 0) { + LOGGER.warn("Malformed host: " + hostPort); + return; + } + + String emulatorHost = hostPort.substring(0, portSeparatorIndex); + String emulatorPort = hostPort.substring(portSeparatorIndex + 1); + + String hostPortParams = String.format("--host=%s --port=%s", emulatorHost, emulatorPort); + killByCommand(hostPortParams); + } + } +} diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java index ee2dda3f7c..e0a3a43685 100644 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java @@ -16,39 +16,29 @@ package org.springframework.cloud.gcp.test; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.junit.rules.ExternalResource; -public class PubSubEmulatorRule extends GcpEmulatorRule { - private static final Log LOGGER = LogFactory.getLog(PubSubEmulatorRule.class); +public class PubSubEmulatorRule extends ExternalResource { - String getGatingPropertyName() { - return "it.pubsub-emulator"; - } + private PubSubEmulatorHelper emulatorHelper = new PubSubEmulatorHelper(); - String getEmulatorName() { - return "pubsub"; + @Override + protected void before() throws Throwable { + emulatorHelper.startEmulator(); } @Override - protected void afterEmulatorDestroyed() { - String hostPort = getEmulatorHostPort(); - - // find destory emulator process spawned by gcloud - if (hostPort == null) { - LOGGER.warn("Host/port null after the test."); - } else { - int portSeparatorIndex = hostPort.lastIndexOf(":"); - if (portSeparatorIndex < 0) { - LOGGER.warn("Malformed host: " + hostPort); - return; - } - - String emulatorHost = hostPort.substring(0, portSeparatorIndex); - String emulatorPort = hostPort.substring(portSeparatorIndex + 1); - - String hostPortParams = String.format("--host=%s --port=%s", emulatorHost, emulatorPort); - killByCommand(hostPortParams); - } + protected void after() { + emulatorHelper.shutdownEmulator(); } + + /** + * Return the already-started emulator's host/port combination when called from within a + * JUnit method. + * @return emulator host/port string or null if emulator setup failed. + */ + public String getEmulatorHostPort() { + return emulatorHelper.getEmulatorHostPort(); + } + } diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java new file mode 100644 index 0000000000..a2068b3986 --- /dev/null +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java @@ -0,0 +1,50 @@ +package org.springframework.cloud.gcp.test; + +public class SpannerEmulatorHelper extends AbstractEmulatorHelper { + String getGatingPropertyName() { + return "it.spanner-emulator"; + } + + String getEmulatorName() { + return "spanner"; + } + + @Override + protected void beforeEmulatorStart() { + //String emulatorHost = System.getenv("SPANNER_EMULATOR_HOST"); + //Assume.assumeFalse( + // "Run this command prior to running an emulator test and set SPANNER_EMULATOR_HOST environment variable:\ngcloud beta emulators spanner env-init", + // emulatorHost == null || emulatorHost.isEmpty()); + } + + @Override + protected void afterEmulatorStart() { + ProcessOutcome switchToEmulator = runSystemCommand(new String[] { + "gcloud", "config", "configurations", "activate", "emulator"}); + + ProcessOutcome processOutcome = runSystemCommand(new String[] { + "gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator-config", "--description=\"Test Instance\"", "--nodes=1" }, + false); + + if (processOutcome.getStatus() != 0) { + // don't set breakpoint here + cleanupSpannerEmulator(); + throw new RuntimeException("Creating instance failed: " + + String.join("\n", processOutcome.getErrors())); + } + + + // TODO: don't forget to kill the 2 spanner processes + } + + @Override + protected void afterEmulatorDestroyed() { + cleanupSpannerEmulator(); + } + + private void cleanupSpannerEmulator() { + this.killByCommand("cloud_spanner_emulator/emulator_main"); + // this.killByCommand("cloud_spanner_emulator/emulator_gateway"); + } + +} diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java index 2b496941f1..63fa95d0e5 100644 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java @@ -16,51 +16,28 @@ package org.springframework.cloud.gcp.test; -import java.util.stream.Collectors; import org.junit.Assume; +import org.junit.rules.ExternalResource; /** * TODO: send users to https://cloud.google.com/spanner/docs/emulator#using_the_gcloud_cli_with_the_emulator */ -public class SpannerEmulatorRule extends GcpEmulatorRule { - String getGatingPropertyName() { - return "it.spanner-emulator"; - } +public class SpannerEmulatorRule extends ExternalResource { - String getEmulatorName() { - return "spanner"; - } + private SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(); @Override - protected void beforeEmulatorStart() { + protected void before() throws Throwable { String emulatorHost = System.getenv("SPANNER_EMULATOR_HOST"); Assume.assumeFalse( - "Run this command prior to running an emulator test:\n$(gcloud beta emulators spanner env-init)", + "Run this command prior to running an emulator test and set SPANNER_EMULATOR_HOST environment variable:\ngcloud beta emulators spanner env-init", emulatorHost == null || emulatorHost.isEmpty()); + emulatorHelper.startEmulator(); } @Override - protected void afterEmulatorStart() { - ProcessOutcome switchToEmulator = runSystemCommand(new String[] { - "gcloud", "config", "configurations", "activate", "emulator"}); - - ProcessOutcome processOutcome = runSystemCommand(new String[] { - "gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator-config", "--description=\"Test Instance\"", "--nodes=1" }, - false); - - if (processOutcome.getStatus() != 0) { - // don't set breakpoint here - this.killByCommand("cloud_spanner_emulator/emulator_main"); - throw new RuntimeException("Creating instance failed: " - + String.join("\n", processOutcome.getErrors())); - } - - - // TODO: don't forget to kill the 2 spanner processes + protected void after() { + emulatorHelper.shutdownEmulator(); } - /* - * gcloud spanner instances create test-instance --config=emulator-config --description="Test Instance" --nodes=1 - * gcloud config configurations activate [emulator | default] - * */ } diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java new file mode 100644 index 0000000000..217c450832 --- /dev/null +++ b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java @@ -0,0 +1,37 @@ +package org.springframework.cloud.gcp.test; + +import com.google.cloud.NoCredentials; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerOptions; +import java.io.IOException; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; + +@Configuration +@Order(-1000) +public class SpannerEmulatorSpringConfiguration { + + private SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(); + + @Bean + public SpannerOptions spannerOptions() { + return SpannerOptions.newBuilder() + .setCredentials(NoCredentials.getInstance()) + .setEmulatorHost("localhost:9010") + .build(); + } + + @Bean (destroyMethod = "") + public Spanner spanner(SpannerOptions spannerOptions) throws IOException, InterruptedException { + emulatorHelper.startEmulator(); + return spannerOptions.getService(); + } + + @EventListener + public void afterCloseEvent(ContextClosedEvent event) { + emulatorHelper.shutdownEmulator(); + } +} From 6e9f411e92aff2251ca3760044ed0e9d430f7778 Mon Sep 17 00:00:00 2001 From: Elena Felder <41136058+elefeint@users.noreply.github.com> Date: Tue, 5 May 2020 17:04:47 -0400 Subject: [PATCH 06/23] break travis config with a TODO --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index adea11401a..00e72e964f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,5 +53,6 @@ before_install: - source $HOME/google-cloud-sdk/path.bash.inc - gcloud components update --quiet - gcloud components install beta pubsub-emulator --quiet +TODO: add spanner emulator and configuration profile - gcloud config set project spring-cloud-gcp-ci From 2a5105559a681c63b7032f3f4959853b8809abae Mon Sep 17 00:00:00 2001 From: Elena Felder <41136058+elefeint@users.noreply.github.com> Date: Wed, 6 May 2020 15:33:23 -0400 Subject: [PATCH 07/23] rename test module --- pom.xml | 2 +- spring-cloud-gcp-data-spanner/pom.xml | 2 +- spring-cloud-gcp-dependencies/pom.xml | 2 +- spring-cloud-gcp-pubsub-stream-binder/pom.xml | 2 +- .../gcp/test/AbstractEmulatorHelper.java | 290 ---------------- .../cloud/gcp/test/PubSubEmulatorHelper.java | 38 --- .../cloud/gcp/test/SpannerEmulatorHelper.java | 50 --- .../SpannerEmulatorSpringConfiguration.java | 37 --- .../pom.xml | 2 +- .../gcp/test/AbstractEmulatorHelper.java | 310 ++++++++++++++++++ .../cloud/gcp/test/PubSubEmulatorHelper.java | 55 ++++ .../cloud/gcp/test/PubSubEmulatorRule.java | 2 +- .../cloud/gcp/test/SpannerEmulatorHelper.java | 67 ++++ .../cloud/gcp/test/SpannerEmulatorRule.java | 2 +- .../SpannerEmulatorSpringConfiguration.java | 55 ++++ 15 files changed, 494 insertions(+), 422 deletions(-) delete mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java delete mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java delete mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java delete mode 100644 spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java rename {spring-cloud-gcp-test-support => spring-cloud-gcp-test}/pom.xml (96%) create mode 100644 spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java create mode 100644 spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java rename {spring-cloud-gcp-test-support => spring-cloud-gcp-test}/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java (94%) create mode 100644 spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java rename {spring-cloud-gcp-test-support => spring-cloud-gcp-test}/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java (96%) create mode 100644 spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java diff --git a/pom.xml b/pom.xml index 78fd5fb14f..68b4de0954 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,7 @@ spring-cloud-gcp-bigquery spring-cloud-gcp-security-firebase spring-cloud-gcp-secretmanager - spring-cloud-gcp-test-support + spring-cloud-gcp-test diff --git a/spring-cloud-gcp-data-spanner/pom.xml b/spring-cloud-gcp-data-spanner/pom.xml index 2d642acf7d..425f511944 100644 --- a/spring-cloud-gcp-data-spanner/pom.xml +++ b/spring-cloud-gcp-data-spanner/pom.xml @@ -44,7 +44,7 @@ org.springframework.cloud - spring-cloud-gcp-test-support + spring-cloud-gcp-test test diff --git a/spring-cloud-gcp-dependencies/pom.xml b/spring-cloud-gcp-dependencies/pom.xml index eae4737861..8c7b9541ee 100644 --- a/spring-cloud-gcp-dependencies/pom.xml +++ b/spring-cloud-gcp-dependencies/pom.xml @@ -192,7 +192,7 @@ org.springframework.cloud - spring-cloud-gcp-test-support + spring-cloud-gcp-test ${project.version} diff --git a/spring-cloud-gcp-pubsub-stream-binder/pom.xml b/spring-cloud-gcp-pubsub-stream-binder/pom.xml index 995ad458a9..1f2b651f13 100644 --- a/spring-cloud-gcp-pubsub-stream-binder/pom.xml +++ b/spring-cloud-gcp-pubsub-stream-binder/pom.xml @@ -44,7 +44,7 @@ org.springframework.cloud - spring-cloud-gcp-test-support + spring-cloud-gcp-test test diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java deleted file mode 100644 index af06574bda..0000000000 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java +++ /dev/null @@ -1,290 +0,0 @@ -package org.springframework.cloud.gcp.test; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardWatchEventKinds; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import java.util.List; -import java.util.Optional; -import java.util.StringTokenizer; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.awaitility.Awaitility; - -public abstract class AbstractEmulatorHelper { - - private final Path EMULATOR_CONFIG_DIR = Paths.get(System.getProperty("user.home")).resolve( - Paths.get(".config", "gcloud", "emulators", getEmulatorName())); - - private static final String ENV_FILE_NAME = "env.yaml"; - - private final Path EMULATOR_CONFIG_PATH = EMULATOR_CONFIG_DIR.resolve(ENV_FILE_NAME); - - private static final Log LOGGER = LogFactory.getLog(AbstractEmulatorHelper.class); - - // Reference to emulator instance, for cleanup. - private Process emulatorProcess; - - // Hostname for cleanup, should always be localhost. - private String emulatorHostPort; - - abstract String getGatingPropertyName(); - - /** - * Launch an instance of pubsub emulator or skip all tests. - * If it.pubsub-emulator environmental property is off, all tests will be skipped through the failed assumption. - * If the property is on, any setup failure will trigger test failure. Failures during teardown are merely logged. - * @throws IOException if config file creation or directory watcher on existing file fails. - * @throws InterruptedException if process is stopped while waiting to retry. - */ - public void startEmulator() throws IOException, InterruptedException { - - beforeEmulatorStart(); - doStartEmulator(); - afterEmulatorStart(); - determineHostPort(); - } - - /** - * Shut down the two emulator processes. - * gcloud command is shut down through the direct process handle. java process is - * identified and shut down through shell commands. - * There should normally be only one process with that host/port combination, but if there - * are more, they will be cleaned up as well. - * Any failure is logged and ignored since it's not critical to the tests' operation. - */ - public void shutdownEmulator() { - findAndDestroyEmulator(); - afterEmulatorDestroyed(); - } - - protected void afterEmulatorDestroyed() { - // does nothing by default. - } - - private void findAndDestroyEmulator() { - // destroy gcloud process - if (this.emulatorProcess != null) { - this.emulatorProcess.destroy(); - } - else { - LOGGER.warn("Emulator process null after tests; nothing to terminate."); - } - } - - protected void killByCommand(String command) { - AtomicBoolean foundProcess = new AtomicBoolean(false); - - try { - Process psProcess = new ProcessBuilder("ps", "-vx").start(); - - try (BufferedReader br = new BufferedReader(new InputStreamReader(psProcess.getInputStream()))) { - br.lines() - .filter((psLine) -> psLine.contains(command)) - .peek(line -> System.out.println("found line after filter: " + line)) - .map((psLine) -> new StringTokenizer(psLine).nextToken()) - .forEach((p) -> { - LOGGER.info("Found " + command + " process to kill: " + p); - this.killProcess(p); - foundProcess.set(true); - }); - } - - if (!foundProcess.get()) { - LOGGER.warn("Did not find the emulator process to kill based on: " + command); - } - } - catch (IOException ex) { - LOGGER.warn("Failed to cleanup: ", ex); - } - } - - /** - * Return the already-started emulator's host/port combination when called from within a - * JUnit method. - * @return emulator host/port string or null if emulator setup failed. - */ - public String getEmulatorHostPort() { - return this.emulatorHostPort; - } - - private void doStartEmulator() throws IOException, InterruptedException { - boolean configPresent = Files.exists(EMULATOR_CONFIG_PATH); - WatchService watchService = null; - - if (configPresent) { - watchService = FileSystems.getDefault().newWatchService(); - EMULATOR_CONFIG_DIR.register(watchService, StandardWatchEventKinds.ENTRY_MODIFY); - } - - try { - this.emulatorProcess = new ProcessBuilder("gcloud", "beta", "emulators", getEmulatorName(), "start") - .start(); - } - catch (IOException ex) { - throw new RuntimeException("Gcloud not found; leaving host/port uninitialized.", ex); - } - - if (configPresent) { - updateConfig(watchService); - watchService.close(); - } - else { - waitForConfigCreation(); - } - - - } - - protected void beforeEmulatorStart(){ - //does nothing by default - } - - protected void afterEmulatorStart(){ - //does nothing by default - } - - /** - * Extract host/port from output of env-init command: "export PUBSUB_EMULATOR_HOST=localhost:8085". - * @throws IOException for IO errors - * @throws InterruptedException for interruption errors - */ - private void determineHostPort() { - ProcessOutcome processOutcome = runSystemCommand(new String[] { "gcloud", "beta", "emulators", getEmulatorName(), "env-init" }); - if ( processOutcome.getOutput().size() < 1 ) { - throw new RuntimeException("env-init command did not produce output"); - } - String emulatorInitString = processOutcome.getOutput().get(0); - this.emulatorHostPort = emulatorInitString.substring(emulatorInitString.indexOf('=') + 1); - } - - protected ProcessOutcome runSystemCommand(String[] command) { - return runSystemCommand(command, true); - } - - protected ProcessOutcome runSystemCommand(String[] command, boolean failOnError) { - - Process envInitProcess = null; - try { - envInitProcess = new ProcessBuilder(command).start(); - } catch (IOException e) { - throw new RuntimeException(e); - } - - try (BufferedReader brInput = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream())); - BufferedReader brError = new BufferedReader(new InputStreamReader(envInitProcess.getErrorStream())) - ) { - ProcessOutcome processOutcome = new ProcessOutcome(command, - brInput.lines().collect(Collectors.toList()), - brError.lines().collect(Collectors.toList()), - envInitProcess.waitFor()); - - if (failOnError && processOutcome.status != 0) { - throw new RuntimeException("Command execution failed: " + String.join(" ", command) - + "; output: " + processOutcome.getOutput() - + "; error: " + processOutcome.getErrors()); - - } - return processOutcome; - } catch (IOException|InterruptedException e) { - throw new RuntimeException(e); - } - } - - abstract String getEmulatorName(); - - /** - * Wait until a PubSub emulator configuration file is present. - * Fail if the file does not appear after 10 seconds. - * @throws InterruptedException which should interrupt the peaceful slumber and bubble up - * to fail the test. - */ - private void waitForConfigCreation() throws InterruptedException { - Awaitility.await("Emulator could not be configured due to missing env.yaml. Is the emulator installed?") - .atMost(10, TimeUnit.SECONDS) - .pollInterval(100, TimeUnit.MILLISECONDS) - .until(() -> Files.exists(EMULATOR_CONFIG_PATH)); - } - - /** - * Wait until a PubSub emulator configuration file is updated. - * Fail if the file does not update after 1 second. - * @param watchService the watch-service to poll - * @throws InterruptedException which should interrupt the peaceful slumber and bubble up - * to fail the test. - */ - private void updateConfig(WatchService watchService) throws InterruptedException { - Awaitility.await("Configuration file update could not be detected") - .atMost(10, TimeUnit.SECONDS) - .pollInterval(100, TimeUnit.MILLISECONDS) - .until(() -> { - WatchKey key = watchService.poll(100, TimeUnit.MILLISECONDS); - - if (key != null) { - Optional configFilePath = key.pollEvents().stream() - .map((event) -> (Path) event.context()) - .filter((path) -> ENV_FILE_NAME.equals(path.toString())) - .findAny(); - return configFilePath.isPresent(); - } - return false; - }); - } - - /** - * Attempt to kill a process on best effort basis. - * Failure is logged and ignored, as it is not critical to the tests' functionality. - * @param pid presumably a valid PID. No checking done to validate. - */ - protected void killProcess(String pid) { - try { - new ProcessBuilder("kill", "-9", pid).start(); - } - catch (IOException ex) { - LOGGER.warn("Failed to clean up PID " + pid); - } - } - - static class ProcessOutcome { - private String[] command; - - private List output; - - private List errors; - - private int status; - - public ProcessOutcome(String[] command, List output, List errors, int status) { - this.command = command; - this.output = output; - this.errors = errors; - this.status = status; - } - - public List getOutput() { - return output; - } - - public List getErrors() { - return errors; - } - - public int getStatus() { - return status; - } - - public String getCommandString() { - return String.join(" ", command); - } - } - -} diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java deleted file mode 100644 index 5301743682..0000000000 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.springframework.cloud.gcp.test; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -public class PubSubEmulatorHelper extends AbstractEmulatorHelper { - private static final Log LOGGER = LogFactory.getLog(PubSubEmulatorHelper.class); - - String getGatingPropertyName() { - return "it.pubsub-emulator"; - } - - String getEmulatorName() { - return "pubsub"; - } - - @Override - protected void afterEmulatorDestroyed() { - String hostPort = getEmulatorHostPort(); - - // find destory emulator process spawned by gcloud - if (hostPort == null) { - LOGGER.warn("Host/port null after the test."); - } else { - int portSeparatorIndex = hostPort.lastIndexOf(":"); - if (portSeparatorIndex < 0) { - LOGGER.warn("Malformed host: " + hostPort); - return; - } - - String emulatorHost = hostPort.substring(0, portSeparatorIndex); - String emulatorPort = hostPort.substring(portSeparatorIndex + 1); - - String hostPortParams = String.format("--host=%s --port=%s", emulatorHost, emulatorPort); - killByCommand(hostPortParams); - } - } -} diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java deleted file mode 100644 index a2068b3986..0000000000 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.springframework.cloud.gcp.test; - -public class SpannerEmulatorHelper extends AbstractEmulatorHelper { - String getGatingPropertyName() { - return "it.spanner-emulator"; - } - - String getEmulatorName() { - return "spanner"; - } - - @Override - protected void beforeEmulatorStart() { - //String emulatorHost = System.getenv("SPANNER_EMULATOR_HOST"); - //Assume.assumeFalse( - // "Run this command prior to running an emulator test and set SPANNER_EMULATOR_HOST environment variable:\ngcloud beta emulators spanner env-init", - // emulatorHost == null || emulatorHost.isEmpty()); - } - - @Override - protected void afterEmulatorStart() { - ProcessOutcome switchToEmulator = runSystemCommand(new String[] { - "gcloud", "config", "configurations", "activate", "emulator"}); - - ProcessOutcome processOutcome = runSystemCommand(new String[] { - "gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator-config", "--description=\"Test Instance\"", "--nodes=1" }, - false); - - if (processOutcome.getStatus() != 0) { - // don't set breakpoint here - cleanupSpannerEmulator(); - throw new RuntimeException("Creating instance failed: " - + String.join("\n", processOutcome.getErrors())); - } - - - // TODO: don't forget to kill the 2 spanner processes - } - - @Override - protected void afterEmulatorDestroyed() { - cleanupSpannerEmulator(); - } - - private void cleanupSpannerEmulator() { - this.killByCommand("cloud_spanner_emulator/emulator_main"); - // this.killByCommand("cloud_spanner_emulator/emulator_gateway"); - } - -} diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java b/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java deleted file mode 100644 index 217c450832..0000000000 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.springframework.cloud.gcp.test; - -import com.google.cloud.NoCredentials; -import com.google.cloud.spanner.Spanner; -import com.google.cloud.spanner.SpannerOptions; -import java.io.IOException; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.event.ContextClosedEvent; -import org.springframework.context.event.EventListener; -import org.springframework.core.annotation.Order; - -@Configuration -@Order(-1000) -public class SpannerEmulatorSpringConfiguration { - - private SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(); - - @Bean - public SpannerOptions spannerOptions() { - return SpannerOptions.newBuilder() - .setCredentials(NoCredentials.getInstance()) - .setEmulatorHost("localhost:9010") - .build(); - } - - @Bean (destroyMethod = "") - public Spanner spanner(SpannerOptions spannerOptions) throws IOException, InterruptedException { - emulatorHelper.startEmulator(); - return spannerOptions.getService(); - } - - @EventListener - public void afterCloseEvent(ContextClosedEvent event) { - emulatorHelper.shutdownEmulator(); - } -} diff --git a/spring-cloud-gcp-test-support/pom.xml b/spring-cloud-gcp-test/pom.xml similarity index 96% rename from spring-cloud-gcp-test-support/pom.xml rename to spring-cloud-gcp-test/pom.xml index ca9dd8bbd3..9bdaa0c548 100644 --- a/spring-cloud-gcp-test-support/pom.xml +++ b/spring-cloud-gcp-test/pom.xml @@ -10,7 +10,7 @@ - spring-cloud-gcp-test-support + spring-cloud-gcp-test Spring Cloud GCP Test Support Provides tools for testing Spring Cloud GCP projects diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java new file mode 100644 index 0000000000..39c932b6cb --- /dev/null +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java @@ -0,0 +1,310 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardWatchEventKinds; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.List; +import java.util.Optional; +import java.util.StringTokenizer; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.awaitility.Awaitility; + +abstract class AbstractEmulatorHelper { + + private final Path EMULATOR_CONFIG_DIR = Paths.get(System.getProperty("user.home")).resolve( + Paths.get(".config", "gcloud", "emulators", getEmulatorName())); + + private static final String ENV_FILE_NAME = "env.yaml"; + + private final Path EMULATOR_CONFIG_PATH = EMULATOR_CONFIG_DIR.resolve(ENV_FILE_NAME); + + private static final Log LOGGER = LogFactory.getLog(AbstractEmulatorHelper.class); + + // Reference to emulator instance, for cleanup. + private Process emulatorProcess; + + // Hostname for cleanup, should always be localhost. + private String emulatorHostPort; + + abstract String getGatingPropertyName(); + + /** + * Launch an instance of pubsub emulator or skip all tests. If it.pubsub-emulator + * environmental property is off, all tests will be skipped through the failed assumption. + * If the property is on, any setup failure will trigger test failure. Failures during + * teardown are merely logged. + * @throws IOException if config file creation or directory watcher on existing file + * fails. + * @throws InterruptedException if process is stopped while waiting to retry. + */ + public void startEmulator() throws IOException, InterruptedException { + + beforeEmulatorStart(); + doStartEmulator(); + afterEmulatorStart(); + determineHostPort(); + } + + /** + * Shut down the two emulator processes. gcloud command is shut down through the direct + * process handle. java process is identified and shut down through shell commands. There + * should normally be only one process with that host/port combination, but if there are + * more, they will be cleaned up as well. Any failure is logged and ignored since it's not + * critical to the tests' operation. + */ + public void shutdownEmulator() { + findAndDestroyEmulator(); + afterEmulatorDestroyed(); + } + + protected void afterEmulatorDestroyed() { + // does nothing by default. + } + + private void findAndDestroyEmulator() { + // destroy gcloud process + if (this.emulatorProcess != null) { + this.emulatorProcess.destroy(); + } + else { + LOGGER.warn("Emulator process null after tests; nothing to terminate."); + } + } + + protected void killByCommand(String command) { + AtomicBoolean foundProcess = new AtomicBoolean(false); + + try { + Process psProcess = new ProcessBuilder("ps", "-vx").start(); + + try (BufferedReader br = new BufferedReader(new InputStreamReader(psProcess.getInputStream()))) { + br.lines() + .filter((psLine) -> psLine.contains(command)) + .peek(line -> System.out.println("found line after filter: " + line)) + .map((psLine) -> new StringTokenizer(psLine).nextToken()) + .forEach((p) -> { + LOGGER.info("Found " + command + " process to kill: " + p); + this.killProcess(p); + foundProcess.set(true); + }); + } + + if (!foundProcess.get()) { + LOGGER.warn("Did not find the emulator process to kill based on: " + command); + } + } + catch (IOException ex) { + LOGGER.warn("Failed to cleanup: ", ex); + } + } + + /** + * Return the already-started emulator's host/port combination when called from within a + * JUnit method. + * @return emulator host/port string or null if emulator setup failed. + */ + public String getEmulatorHostPort() { + return this.emulatorHostPort; + } + + private void doStartEmulator() throws IOException, InterruptedException { + boolean configPresent = Files.exists(EMULATOR_CONFIG_PATH); + WatchService watchService = null; + + if (configPresent) { + watchService = FileSystems.getDefault().newWatchService(); + EMULATOR_CONFIG_DIR.register(watchService, StandardWatchEventKinds.ENTRY_MODIFY); + } + + try { + this.emulatorProcess = new ProcessBuilder("gcloud", "beta", "emulators", getEmulatorName(), "start") + .start(); + } + catch (IOException ex) { + throw new RuntimeException("Gcloud not found; leaving host/port uninitialized.", ex); + } + + if (configPresent) { + updateConfig(watchService); + watchService.close(); + } + else { + waitForConfigCreation(); + } + + } + + protected void beforeEmulatorStart() { + // does nothing by default + } + + protected void afterEmulatorStart() { + // does nothing by default + } + + /** + * Extract host/port from output of env-init command: "export + * PUBSUB_EMULATOR_HOST=localhost:8085". + * @throws IOException for IO errors + * @throws InterruptedException for interruption errors + */ + private void determineHostPort() { + ProcessOutcome processOutcome = runSystemCommand( + new String[] { "gcloud", "beta", "emulators", getEmulatorName(), "env-init" }); + if (processOutcome.getOutput().size() < 1) { + throw new RuntimeException("env-init command did not produce output"); + } + String emulatorInitString = processOutcome.getOutput().get(0); + this.emulatorHostPort = emulatorInitString.substring(emulatorInitString.indexOf('=') + 1); + } + + protected ProcessOutcome runSystemCommand(String[] command) { + return runSystemCommand(command, true); + } + + protected ProcessOutcome runSystemCommand(String[] command, boolean failOnError) { + + Process envInitProcess = null; + try { + envInitProcess = new ProcessBuilder(command).start(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + + try (BufferedReader brInput = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream())); + BufferedReader brError = new BufferedReader(new InputStreamReader(envInitProcess.getErrorStream()))) { + ProcessOutcome processOutcome = new ProcessOutcome(command, + brInput.lines().collect(Collectors.toList()), + brError.lines().collect(Collectors.toList()), + envInitProcess.waitFor()); + + if (failOnError && processOutcome.status != 0) { + throw new RuntimeException("Command execution failed: " + String.join(" ", command) + + "; output: " + processOutcome.getOutput() + + "; error: " + processOutcome.getErrors()); + + } + return processOutcome; + } + catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + abstract String getEmulatorName(); + + /** + * Wait until a PubSub emulator configuration file is present. Fail if the file does not + * appear after 10 seconds. + * @throws InterruptedException which should interrupt the peaceful slumber and bubble up + * to fail the test. + */ + private void waitForConfigCreation() throws InterruptedException { + Awaitility.await("Emulator could not be configured due to missing env.yaml. Is the emulator installed?") + .atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .until(() -> Files.exists(EMULATOR_CONFIG_PATH)); + } + + /** + * Wait until a PubSub emulator configuration file is updated. Fail if the file does not + * update after 1 second. + * @param watchService the watch-service to poll + * @throws InterruptedException which should interrupt the peaceful slumber and bubble up + * to fail the test. + */ + private void updateConfig(WatchService watchService) throws InterruptedException { + Awaitility.await("Configuration file update could not be detected") + .atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .until(() -> { + WatchKey key = watchService.poll(100, TimeUnit.MILLISECONDS); + + if (key != null) { + Optional configFilePath = key.pollEvents().stream() + .map((event) -> (Path) event.context()) + .filter((path) -> ENV_FILE_NAME.equals(path.toString())) + .findAny(); + return configFilePath.isPresent(); + } + return false; + }); + } + + /** + * Attempt to kill a process on best effort basis. Failure is logged and ignored, as it is + * not critical to the tests' functionality. + * @param pid presumably a valid PID. No checking done to validate. + */ + protected void killProcess(String pid) { + try { + new ProcessBuilder("kill", "-9", pid).start(); + } + catch (IOException ex) { + LOGGER.warn("Failed to clean up PID " + pid); + } + } + + static class ProcessOutcome { + private String[] command; + + private List output; + + private List errors; + + private int status; + + ProcessOutcome(String[] command, List output, List errors, int status) { + this.command = command; + this.output = output; + this.errors = errors; + this.status = status; + } + + public List getOutput() { + return output; + } + + public List getErrors() { + return errors; + } + + public int getStatus() { + return status; + } + + public String getCommandString() { + return String.join(" ", command); + } + } + +} diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java new file mode 100644 index 0000000000..ff486bbb17 --- /dev/null +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java @@ -0,0 +1,55 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class PubSubEmulatorHelper extends AbstractEmulatorHelper { + private static final Log LOGGER = LogFactory.getLog(PubSubEmulatorHelper.class); + + String getGatingPropertyName() { + return "it.pubsub-emulator"; + } + + String getEmulatorName() { + return "pubsub"; + } + + @Override + protected void afterEmulatorDestroyed() { + String hostPort = getEmulatorHostPort(); + + // find destory emulator process spawned by gcloud + if (hostPort == null) { + LOGGER.warn("Host/port null after the test."); + } + else { + int portSeparatorIndex = hostPort.lastIndexOf(":"); + if (portSeparatorIndex < 0) { + LOGGER.warn("Malformed host: " + hostPort); + return; + } + + String emulatorHost = hostPort.substring(0, portSeparatorIndex); + String emulatorPort = hostPort.substring(portSeparatorIndex + 1); + + String hostPortParams = String.format("--host=%s --port=%s", emulatorHost, emulatorPort); + killByCommand(hostPortParams); + } + } +} diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java similarity index 94% rename from spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java rename to spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java index e0a3a43685..fc13bcae58 100644 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java @@ -20,7 +20,7 @@ public class PubSubEmulatorRule extends ExternalResource { - private PubSubEmulatorHelper emulatorHelper = new PubSubEmulatorHelper(); + private PubSubEmulatorHelper emulatorHelper = new PubSubEmulatorHelper(); @Override protected void before() throws Throwable { diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java new file mode 100644 index 0000000000..de6bf55811 --- /dev/null +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java @@ -0,0 +1,67 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test; + +public class SpannerEmulatorHelper extends AbstractEmulatorHelper { + String getGatingPropertyName() { + return "it.spanner-emulator"; + } + + String getEmulatorName() { + return "spanner"; + } + + @Override + protected void beforeEmulatorStart() { + // String emulatorHost = System.getenv("SPANNER_EMULATOR_HOST"); + // Assume.assumeFalse( + // "Run this command prior to running an emulator test and set SPANNER_EMULATOR_HOST + // environment variable:\ngcloud beta emulators spanner env-init", + // emulatorHost == null || emulatorHost.isEmpty()); + } + + @Override + protected void afterEmulatorStart() { + ProcessOutcome switchToEmulator = runSystemCommand(new String[] { + "gcloud", "config", "configurations", "activate", "emulator" }); + + ProcessOutcome processOutcome = runSystemCommand(new String[] { + "gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator-config", + "--description=\"Test Instance\"", "--nodes=1" }, + false); + + if (processOutcome.getStatus() != 0) { + // don't set breakpoint here + cleanupSpannerEmulator(); + throw new RuntimeException("Creating instance failed: " + + String.join("\n", processOutcome.getErrors())); + } + + // TODO: don't forget to kill the 2 spanner processes + } + + @Override + protected void afterEmulatorDestroyed() { + cleanupSpannerEmulator(); + } + + private void cleanupSpannerEmulator() { + this.killByCommand("cloud_spanner_emulator/emulator_main"); + // this.killByCommand("cloud_spanner_emulator/emulator_gateway"); + } + +} diff --git a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java similarity index 96% rename from spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java rename to spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java index 63fa95d0e5..52931dd4bd 100644 --- a/spring-cloud-gcp-test-support/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java @@ -20,7 +20,7 @@ import org.junit.rules.ExternalResource; /** - * TODO: send users to https://cloud.google.com/spanner/docs/emulator#using_the_gcloud_cli_with_the_emulator + * TODO: send users to https://cloud.google.com/spanner/docs/emulator#using_the_gcloud_cli_with_the_emulator. */ public class SpannerEmulatorRule extends ExternalResource { diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java new file mode 100644 index 0000000000..c2527a1518 --- /dev/null +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java @@ -0,0 +1,55 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test; + +import java.io.IOException; + +import com.google.cloud.NoCredentials; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerOptions; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; + +@Configuration +@Order(-1000) +public class SpannerEmulatorSpringConfiguration { + + private SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(); + + @Bean + public SpannerOptions spannerOptions() { + return SpannerOptions.newBuilder() + .setCredentials(NoCredentials.getInstance()) + .setEmulatorHost("localhost:9010") + .build(); + } + + @Bean(destroyMethod = "") + public Spanner spanner(SpannerOptions spannerOptions) throws IOException, InterruptedException { + emulatorHelper.startEmulator(); + return spannerOptions.getService(); + } + + @EventListener + public void afterCloseEvent(ContextClosedEvent event) { + emulatorHelper.shutdownEmulator(); + } +} From 2ac9f7801b47c20105d6639d0d3fa33eefdf8cb8 Mon Sep 17 00:00:00 2001 From: Elena Felder <41136058+elefeint@users.noreply.github.com> Date: Wed, 6 May 2020 16:36:55 -0400 Subject: [PATCH 08/23] fixed incorrect default project ID making it into spanner object --- docs/src/main/asciidoc/test.adoc | 2 ++ .../gcp/test/AbstractEmulatorHelper.java | 4 ++- .../cloud/gcp/test/SpannerEmulatorHelper.java | 31 +++++++++++++------ .../SpannerEmulatorSpringConfiguration.java | 13 +++++--- 4 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 docs/src/main/asciidoc/test.adoc diff --git a/docs/src/main/asciidoc/test.adoc b/docs/src/main/asciidoc/test.adoc new file mode 100644 index 0000000000..13b6cafdc0 --- /dev/null +++ b/docs/src/main/asciidoc/test.adoc @@ -0,0 +1,2 @@ +TODO: document that user has to create profile in order for Spanner emulator to do +`gcloud config configurations activate emulator` diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java index 39c932b6cb..ba2da0c350 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java @@ -108,7 +108,6 @@ protected void killByCommand(String command) { try (BufferedReader br = new BufferedReader(new InputStreamReader(psProcess.getInputStream()))) { br.lines() .filter((psLine) -> psLine.contains(command)) - .peek(line -> System.out.println("found line after filter: " + line)) .map((psLine) -> new StringTokenizer(psLine).nextToken()) .forEach((p) -> { LOGGER.info("Found " + command + " process to kill: " + p); @@ -208,6 +207,9 @@ protected ProcessOutcome runSystemCommand(String[] command, boolean failOnError) envInitProcess.waitFor()); if (failOnError && processOutcome.status != 0) { + // clean up anything that got started up first + this.shutdownEmulator(); + throw new RuntimeException("Command execution failed: " + String.join(" ", command) + "; output: " + processOutcome.getOutput() + "; error: " + processOutcome.getErrors()); diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java index de6bf55811..338fa4bbb9 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java @@ -16,7 +16,19 @@ package org.springframework.cloud.gcp.test; +import org.junit.Assume; + public class SpannerEmulatorHelper extends AbstractEmulatorHelper { + private boolean checkEnvironmentFlag = true; + + public SpannerEmulatorHelper() { + // keep default value of checkEnvironmentFlag=true. + } + + public SpannerEmulatorHelper(boolean checkEnvironmentFlag) { + this.checkEnvironmentFlag = checkEnvironmentFlag; + } + String getGatingPropertyName() { return "it.spanner-emulator"; } @@ -27,11 +39,12 @@ String getEmulatorName() { @Override protected void beforeEmulatorStart() { - // String emulatorHost = System.getenv("SPANNER_EMULATOR_HOST"); - // Assume.assumeFalse( - // "Run this command prior to running an emulator test and set SPANNER_EMULATOR_HOST - // environment variable:\ngcloud beta emulators spanner env-init", - // emulatorHost == null || emulatorHost.isEmpty()); + if (this.checkEnvironmentFlag) { + String emulatorHost = System.getenv("SPANNER_EMULATOR_HOST"); + Assume.assumeFalse( + "Set SPANNER_EMULATOR_HOST environment variable prior to running emulator tests; copy output of this command and run it:\ngcloud beta emulators spanner env-init", + emulatorHost == null || emulatorHost.isEmpty()); + } } @Override @@ -40,7 +53,7 @@ protected void afterEmulatorStart() { "gcloud", "config", "configurations", "activate", "emulator" }); ProcessOutcome processOutcome = runSystemCommand(new String[] { - "gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator-config", + "gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator", "--description=\"Test Instance\"", "--nodes=1" }, false); @@ -50,18 +63,18 @@ protected void afterEmulatorStart() { throw new RuntimeException("Creating instance failed: " + String.join("\n", processOutcome.getErrors())); } - - // TODO: don't forget to kill the 2 spanner processes } @Override protected void afterEmulatorDestroyed() { cleanupSpannerEmulator(); + runSystemCommand(new String[] { + "gcloud", "config", "configurations", "activate", "default" }); } private void cleanupSpannerEmulator() { this.killByCommand("cloud_spanner_emulator/emulator_main"); - // this.killByCommand("cloud_spanner_emulator/emulator_gateway"); + this.killByCommand("cloud_spanner_emulator/gateway_main"); } } diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java index c2527a1518..c421bc602b 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java @@ -16,6 +16,7 @@ package org.springframework.cloud.gcp.test; +import com.google.cloud.ServiceOptions; import java.io.IOException; import com.google.cloud.NoCredentials; @@ -32,19 +33,23 @@ @Order(-1000) public class SpannerEmulatorSpringConfiguration { - private SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(); + private SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(false); @Bean - public SpannerOptions spannerOptions() { + public SpannerOptions spannerOptions() throws IOException, InterruptedException { + // starting the emulator will change the default project ID to that of the emulator config. + emulatorHelper.startEmulator(); return SpannerOptions.newBuilder() + .setProjectId(ServiceOptions.getDefaultProjectId()) .setCredentials(NoCredentials.getInstance()) .setEmulatorHost("localhost:9010") .build(); } + // the real destroy method would attempt to connect the already-shutdown emulator instance, + // causing tests to fail. @Bean(destroyMethod = "") - public Spanner spanner(SpannerOptions spannerOptions) throws IOException, InterruptedException { - emulatorHelper.startEmulator(); + public Spanner spanner(SpannerOptions spannerOptions) { return spannerOptions.getService(); } From 9f848dcd9014cece8d71bf711b38e4d50e494398 Mon Sep 17 00:00:00 2001 From: Elena Felder <41136058+elefeint@users.noreply.github.com> Date: Wed, 6 May 2020 16:37:29 -0400 Subject: [PATCH 09/23] unbreak travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 00e72e964f..adea11401a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,6 +53,5 @@ before_install: - source $HOME/google-cloud-sdk/path.bash.inc - gcloud components update --quiet - gcloud components install beta pubsub-emulator --quiet -TODO: add spanner emulator and configuration profile - gcloud config set project spring-cloud-gcp-ci From 4a8087610d6cd1879f1c7e7aac6e4b6649398805 Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Thu, 7 May 2020 16:20:53 -0400 Subject: [PATCH 10/23] PR comments --- docs/src/main/asciidoc/test.adoc | 110 ++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 2 deletions(-) diff --git a/docs/src/main/asciidoc/test.adoc b/docs/src/main/asciidoc/test.adoc index 13b6cafdc0..9b5c2be85f 100644 --- a/docs/src/main/asciidoc/test.adoc +++ b/docs/src/main/asciidoc/test.adoc @@ -1,2 +1,108 @@ -TODO: document that user has to create profile in order for Spanner emulator to do -`gcloud config configurations activate emulator` +== Emulators support +Our tools simplify starting and stopping Cloud PubSub and Cloud Spanner emulators when you test your application. + +In order to use it, you need to add this dependency into your `pom.xml` file: + +[source,xml] +---- + + org.springframework.cloud + spring-cloud-gcp-test + test + +---- + +Also, you need to have `gcloud` and the emulators installed and configured, as described later. + +=== Cloud PubSub Emulator +In order to use our Cloud PubSub Emulator helper tools, you would need to install the emulator first. +Follow the https://cloud.google.com/pubsub/docs/emulator[installation instructions]. + +==== JUnit 4 Class Rule +The rule starts the emulator process before the tests run and kills it afterwards. +You just need to add a rule into your test class to enable it. + +[source,java] +---- +import org.springframework.cloud.gcp.test.PubSubEmulatorRule; + +public class PubSubTests { + @ClassRule + public static PubSubEmulatorRule emulator = new PubSubEmulatorRule(); + + //your tests +} +---- + +==== Utility class +If you prefer controlling the emulator manually, you can use PubSubEmulatorHelper. + +[source,java] +---- + PubSubEmulatorHelper emulatorHelper = new PubSubEmulatorHelper(); + + emulatorHelper.startEmulator(); + + //your code + + emulatorHelper.shutdownEmulator(); +---- + +=== Cloud Spanner Emulator +In order to use our Cloud Spanner Emulator helper tools, you would need to install the emulator first. +Follow the https://cloud.google.com/spanner/docs/emulator[installation instructions]. +Make sure you create an emulator configuration and call it `emulator`. + +==== Spring Configuration +If you are testing your Spring application, you can use our configuration class. +It provides a bean of type Spanner, which starts the emulator before creating a client. +This configuration also stops the emulator when the Spring context is shut down. + +In order to use it, you need to add the SpannerEmulatorSpringConfiguration.class to your configuration classes list in `@ContextConfiguration`. + +[source,java] +---- +import org.junit.runner.RunWith; + +import org.springframework.cloud.gcp.test.SpannerEmulatorSpringConfiguration; + +@RunWith(SpringRunner.class) +@ContextConfiguration(classes = {SpannerEmulatorSpringConfiguration.class, YourSpringConfiguration.class}) +public class SpannerTemplateEmulatorTests { + //your tests +} +---- + +==== JUnit 4 Class Rule +If you don't use spring in your tests, you can use the class rule. + +NOTE: The rule does not work with SpringRunner! +See <> section instead. + +[source,java] +---- +import org.springframework.cloud.gcp.test.SpannerEmulatorRule; + +public class SpannerTests { + @ClassRule + public static SpannerEmulatorRule emulator = new SpannerEmulatorRule(); + + //your tests +} +---- + + + +==== Utility class +If you prefer controlling the emulator manually, you can use SpannerEmulatorHelper. + +[source,java] +---- + SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(); + + emulatorHelper.startEmulator(); + + //your code + + emulatorHelper.shutdownEmulator(); +---- From ef720e0918aa8d8a183ffe76f6a2e6b9af8427ac Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Thu, 7 May 2020 16:59:55 -0400 Subject: [PATCH 11/23] add docs and clean up the code --- spring-cloud-gcp-test/pom.xml | 1 - .../gcp/test/AbstractEmulatorHelper.java | 43 ++++--------------- .../cloud/gcp/test/PubSubEmulatorHelper.java | 9 +++- .../SpannerEmulatorSpringConfiguration.java | 2 +- 4 files changed, 18 insertions(+), 37 deletions(-) diff --git a/spring-cloud-gcp-test/pom.xml b/spring-cloud-gcp-test/pom.xml index 9bdaa0c548..a673e4e02c 100644 --- a/spring-cloud-gcp-test/pom.xml +++ b/spring-cloud-gcp-test/pom.xml @@ -25,7 +25,6 @@ org.springframework spring-jcl - compile org.awaitility diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java index ba2da0c350..266262f044 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java @@ -38,7 +38,7 @@ import org.awaitility.Awaitility; abstract class AbstractEmulatorHelper { - + //Path to the directory that should contain env file private final Path EMULATOR_CONFIG_DIR = Paths.get(System.getProperty("user.home")).resolve( Paths.get(".config", "gcloud", "emulators", getEmulatorName())); @@ -56,38 +56,19 @@ abstract class AbstractEmulatorHelper { abstract String getGatingPropertyName(); - /** - * Launch an instance of pubsub emulator or skip all tests. If it.pubsub-emulator - * environmental property is off, all tests will be skipped through the failed assumption. - * If the property is on, any setup failure will trigger test failure. Failures during - * teardown are merely logged. - * @throws IOException if config file creation or directory watcher on existing file - * fails. - * @throws InterruptedException if process is stopped while waiting to retry. - */ public void startEmulator() throws IOException, InterruptedException { - beforeEmulatorStart(); doStartEmulator(); afterEmulatorStart(); determineHostPort(); } - /** - * Shut down the two emulator processes. gcloud command is shut down through the direct - * process handle. java process is identified and shut down through shell commands. There - * should normally be only one process with that host/port combination, but if there are - * more, they will be cleaned up as well. Any failure is logged and ignored since it's not - * critical to the tests' operation. - */ public void shutdownEmulator() { findAndDestroyEmulator(); afterEmulatorDestroyed(); } - protected void afterEmulatorDestroyed() { - // does nothing by default. - } + protected abstract void afterEmulatorDestroyed(); private void findAndDestroyEmulator() { // destroy gcloud process @@ -95,7 +76,7 @@ private void findAndDestroyEmulator() { this.emulatorProcess.destroy(); } else { - LOGGER.warn("Emulator process null after tests; nothing to terminate."); + LOGGER.warn("Emulator process is null after tests; nothing to terminate."); } } @@ -134,7 +115,7 @@ public String getEmulatorHostPort() { return this.emulatorHostPort; } - private void doStartEmulator() throws IOException, InterruptedException { + private void doStartEmulator() throws IOException { boolean configPresent = Files.exists(EMULATOR_CONFIG_PATH); WatchService watchService = null; @@ -172,8 +153,6 @@ protected void afterEmulatorStart() { /** * Extract host/port from output of env-init command: "export * PUBSUB_EMULATOR_HOST=localhost:8085". - * @throws IOException for IO errors - * @throws InterruptedException for interruption errors */ private void determineHostPort() { ProcessOutcome processOutcome = runSystemCommand( @@ -225,12 +204,10 @@ protected ProcessOutcome runSystemCommand(String[] command, boolean failOnError) abstract String getEmulatorName(); /** - * Wait until a PubSub emulator configuration file is present. Fail if the file does not + * Wait until the emulator configuration file is present. Fail if the file does not * appear after 10 seconds. - * @throws InterruptedException which should interrupt the peaceful slumber and bubble up - * to fail the test. */ - private void waitForConfigCreation() throws InterruptedException { + private void waitForConfigCreation() { Awaitility.await("Emulator could not be configured due to missing env.yaml. Is the emulator installed?") .atMost(10, TimeUnit.SECONDS) .pollInterval(100, TimeUnit.MILLISECONDS) @@ -238,13 +215,11 @@ private void waitForConfigCreation() throws InterruptedException { } /** - * Wait until a PubSub emulator configuration file is updated. Fail if the file does not - * update after 1 second. + * Wait until the emulator configuration file is updated. Fail if the file does not + * update after 10 second. * @param watchService the watch-service to poll - * @throws InterruptedException which should interrupt the peaceful slumber and bubble up - * to fail the test. */ - private void updateConfig(WatchService watchService) throws InterruptedException { + private void updateConfig(WatchService watchService) { Awaitility.await("Configuration file update could not be detected") .atMost(10, TimeUnit.SECONDS) .pollInterval(100, TimeUnit.MILLISECONDS) diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java index ff486bbb17..a9a0215256 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java @@ -30,11 +30,18 @@ String getEmulatorName() { return "pubsub"; } + /** + * Shut down the emulator process. + * java process is identified and shut down through shell commands. There + * should normally be only one process with that host/port combination, but if there are + * more, they will be cleaned up as well. Any failure is logged and ignored since it's not + * critical to the tests' operation. + */ @Override protected void afterEmulatorDestroyed() { String hostPort = getEmulatorHostPort(); - // find destory emulator process spawned by gcloud + // find and destory emulator process spawned by gcloud if (hostPort == null) { LOGGER.warn("Host/port null after the test."); } diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java index c421bc602b..002282b264 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java @@ -16,10 +16,10 @@ package org.springframework.cloud.gcp.test; -import com.google.cloud.ServiceOptions; import java.io.IOException; import com.google.cloud.NoCredentials; +import com.google.cloud.ServiceOptions; import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.SpannerOptions; From fe11788dfbf3fb91ad9b2c4177efb4fb11f8309b Mon Sep 17 00:00:00 2001 From: Elena Felder <41136058+elefeint@users.noreply.github.com> Date: Fri, 8 May 2020 10:57:35 -0400 Subject: [PATCH 12/23] sonar feedback --- docs/src/main/asciidoc/test.adoc | 2 +- .../cloud/gcp/test/AbstractEmulatorHelper.java | 15 +++++++-------- .../cloud/gcp/test/PubSubEmulatorHelper.java | 2 +- .../cloud/gcp/test/SpannerEmulatorHelper.java | 2 +- .../cloud/gcp/test/SpannerEmulatorRule.java | 3 --- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/src/main/asciidoc/test.adoc b/docs/src/main/asciidoc/test.adoc index 9b5c2be85f..d047392d8b 100644 --- a/docs/src/main/asciidoc/test.adoc +++ b/docs/src/main/asciidoc/test.adoc @@ -51,7 +51,7 @@ If you prefer controlling the emulator manually, you can use PubSubEmulatorHelpe === Cloud Spanner Emulator In order to use our Cloud Spanner Emulator helper tools, you would need to install the emulator first. Follow the https://cloud.google.com/spanner/docs/emulator[installation instructions]. -Make sure you create an emulator configuration and call it `emulator`. +Make sure you also create an https://cloud.google.com/spanner/docs/emulator#using_the_gcloud_cli_with_the_emulator[emulator configuration] and call it `emulator`. ==== Spring Configuration If you are testing your Spring application, you can use our configuration class. diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java index 266262f044..00d1ae0733 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.FileSystems; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardWatchEventKinds; @@ -39,12 +38,12 @@ abstract class AbstractEmulatorHelper { //Path to the directory that should contain env file - private final Path EMULATOR_CONFIG_DIR = Paths.get(System.getProperty("user.home")).resolve( + private final Path emulatorConfigDir = Paths.get(System.getProperty("user.home")).resolve( Paths.get(".config", "gcloud", "emulators", getEmulatorName())); private static final String ENV_FILE_NAME = "env.yaml"; - private final Path EMULATOR_CONFIG_PATH = EMULATOR_CONFIG_DIR.resolve(ENV_FILE_NAME); + private final Path emulatorConfigPath = emulatorConfigDir.resolve(ENV_FILE_NAME); private static final Log LOGGER = LogFactory.getLog(AbstractEmulatorHelper.class); @@ -56,7 +55,7 @@ abstract class AbstractEmulatorHelper { abstract String getGatingPropertyName(); - public void startEmulator() throws IOException, InterruptedException { + public void startEmulator() throws IOException { beforeEmulatorStart(); doStartEmulator(); afterEmulatorStart(); @@ -116,12 +115,12 @@ public String getEmulatorHostPort() { } private void doStartEmulator() throws IOException { - boolean configPresent = Files.exists(EMULATOR_CONFIG_PATH); + boolean configPresent = emulatorConfigPath.toFile().exists(); WatchService watchService = null; if (configPresent) { watchService = FileSystems.getDefault().newWatchService(); - EMULATOR_CONFIG_DIR.register(watchService, StandardWatchEventKinds.ENTRY_MODIFY); + emulatorConfigDir.register(watchService, StandardWatchEventKinds.ENTRY_MODIFY); } try { @@ -157,7 +156,7 @@ protected void afterEmulatorStart() { private void determineHostPort() { ProcessOutcome processOutcome = runSystemCommand( new String[] { "gcloud", "beta", "emulators", getEmulatorName(), "env-init" }); - if (processOutcome.getOutput().size() < 1) { + if (processOutcome.getOutput().isEmpty()) { throw new RuntimeException("env-init command did not produce output"); } String emulatorInitString = processOutcome.getOutput().get(0); @@ -211,7 +210,7 @@ private void waitForConfigCreation() { Awaitility.await("Emulator could not be configured due to missing env.yaml. Is the emulator installed?") .atMost(10, TimeUnit.SECONDS) .pollInterval(100, TimeUnit.MILLISECONDS) - .until(() -> Files.exists(EMULATOR_CONFIG_PATH)); + .until(() -> emulatorConfigPath.toFile().exists()); } /** diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java index a9a0215256..8549ca1910 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java @@ -46,7 +46,7 @@ protected void afterEmulatorDestroyed() { LOGGER.warn("Host/port null after the test."); } else { - int portSeparatorIndex = hostPort.lastIndexOf(":"); + int portSeparatorIndex = hostPort.lastIndexOf(':'); if (portSeparatorIndex < 0) { LOGGER.warn("Malformed host: " + hostPort); return; diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java index 338fa4bbb9..917738eb4f 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java @@ -49,7 +49,7 @@ protected void beforeEmulatorStart() { @Override protected void afterEmulatorStart() { - ProcessOutcome switchToEmulator = runSystemCommand(new String[] { + runSystemCommand(new String[] { "gcloud", "config", "configurations", "activate", "emulator" }); ProcessOutcome processOutcome = runSystemCommand(new String[] { diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java index 52931dd4bd..e9afbcf3e2 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java @@ -19,9 +19,6 @@ import org.junit.Assume; import org.junit.rules.ExternalResource; -/** - * TODO: send users to https://cloud.google.com/spanner/docs/emulator#using_the_gcloud_cli_with_the_emulator. - */ public class SpannerEmulatorRule extends ExternalResource { private SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(); From 7b119e7fd25c8d8587400f935bd730af5ec3e00b Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Fri, 8 May 2020 15:30:41 -0400 Subject: [PATCH 13/23] Massive refactoring with Dmitry and Elena --- docs/src/main/asciidoc/test.adoc | 63 ++---- .../core/it/SpannerTemplateEmulatorTests.java | 2 +- ...bSubMessageChannelBinderEmulatorTests.java | 5 +- .../cloud/gcp/test/Emulator.java | 53 +++++ ...mulatorHelper.java => EmulatorDriver.java} | 192 ++++++++++-------- ...SubEmulatorRule.java => EmulatorRule.java} | 20 +- .../cloud/gcp/test/SpannerEmulatorHelper.java | 80 -------- .../cloud/gcp/test/SpannerEmulatorRule.java | 40 ---- .../PubSubEmulator.java} | 33 ++- .../gcp/test/spanner/SpannerEmulator.java | 63 ++++++ .../SpannerEmulatorSpringConfiguration.java | 9 +- 11 files changed, 277 insertions(+), 283 deletions(-) create mode 100644 spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/Emulator.java rename spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/{AbstractEmulatorHelper.java => EmulatorDriver.java} (70%) rename spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/{PubSubEmulatorRule.java => EmulatorRule.java} (71%) delete mode 100644 spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java delete mode 100644 spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java rename spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/{PubSubEmulatorHelper.java => pubsub/PubSubEmulator.java} (62%) create mode 100644 spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulator.java rename spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/{ => spanner}/SpannerEmulatorSpringConfiguration.java (87%) diff --git a/docs/src/main/asciidoc/test.adoc b/docs/src/main/asciidoc/test.adoc index d047392d8b..d5021c762a 100644 --- a/docs/src/main/asciidoc/test.adoc +++ b/docs/src/main/asciidoc/test.adoc @@ -14,40 +14,41 @@ In order to use it, you need to add this dependency into your `pom.xml` file: Also, you need to have `gcloud` and the emulators installed and configured, as described later. -=== Cloud PubSub Emulator -In order to use our Cloud PubSub Emulator helper tools, you would need to install the emulator first. -Follow the https://cloud.google.com/pubsub/docs/emulator[installation instructions]. - -==== JUnit 4 Class Rule +=== JUnit 4 Class Rule The rule starts the emulator process before the tests run and kills it afterwards. You just need to add a rule into your test class to enable it. [source,java] ---- -import org.springframework.cloud.gcp.test.PubSubEmulatorRule; +import org.springframework.cloud.gcp.test.EmulatorRule; +import org.springframework.cloud.gcp.test.pubsub.PubSubEmulator; public class PubSubTests { @ClassRule - public static PubSubEmulatorRule emulator = new PubSubEmulatorRule(); + public static EmulatorRule emulator = new EmulatorRule(new PubSubEmulator()); //your tests } ---- -==== Utility class -If you prefer controlling the emulator manually, you can use PubSubEmulatorHelper. +=== Utility class +If you prefer controlling the emulator manually, you can use the `EmulatorDriver`. [source,java] ---- - PubSubEmulatorHelper emulatorHelper = new PubSubEmulatorHelper(); + EmulatorDriver emulatorDriver = new EmulatorDriver(new PubSubEmulator()); - emulatorHelper.startEmulator(); + emulatorDriver.startEmulator(); //your code - emulatorHelper.shutdownEmulator(); + emulatorDriver.shutdownEmulator(); ---- +=== Cloud PubSub Emulator +In order to use our Cloud PubSub Emulator helper tools, you would need to install the emulator first. +Follow the https://cloud.google.com/pubsub/docs/emulator[installation instructions]. + === Cloud Spanner Emulator In order to use our Cloud Spanner Emulator helper tools, you would need to install the emulator first. Follow the https://cloud.google.com/spanner/docs/emulator[installation instructions]. @@ -58,7 +59,7 @@ If you are testing your Spring application, you can use our configuration class. It provides a bean of type Spanner, which starts the emulator before creating a client. This configuration also stops the emulator when the Spring context is shut down. -In order to use it, you need to add the SpannerEmulatorSpringConfiguration.class to your configuration classes list in `@ContextConfiguration`. +In order to use it, you need to add the `SpannerEmulatorSpringConfiguration.class` to your configuration classes list in `@ContextConfiguration`. [source,java] ---- @@ -71,38 +72,4 @@ import org.springframework.cloud.gcp.test.SpannerEmulatorSpringConfiguration; public class SpannerTemplateEmulatorTests { //your tests } ----- - -==== JUnit 4 Class Rule -If you don't use spring in your tests, you can use the class rule. - -NOTE: The rule does not work with SpringRunner! -See <> section instead. - -[source,java] ----- -import org.springframework.cloud.gcp.test.SpannerEmulatorRule; - -public class SpannerTests { - @ClassRule - public static SpannerEmulatorRule emulator = new SpannerEmulatorRule(); - - //your tests -} ----- - - - -==== Utility class -If you prefer controlling the emulator manually, you can use SpannerEmulatorHelper. - -[source,java] ----- - SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(); - - emulatorHelper.startEmulator(); - - //your code - - emulatorHelper.shutdownEmulator(); ----- +---- \ No newline at end of file diff --git a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java index 8054f1e5fa..6afbc0cb73 100644 --- a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java +++ b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java @@ -21,7 +21,7 @@ import org.springframework.cloud.gcp.data.spanner.test.AbstractSpannerIntegrationTest; import org.springframework.cloud.gcp.data.spanner.test.domain.Trade; -import org.springframework.cloud.gcp.test.SpannerEmulatorSpringConfiguration; +import org.springframework.cloud.gcp.test.spanner.SpannerEmulatorSpringConfiguration; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; diff --git a/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java b/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java index 07cb4d47fe..efbb89836a 100644 --- a/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java +++ b/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java @@ -22,7 +22,8 @@ import org.springframework.cloud.gcp.stream.binder.pubsub.properties.PubSubConsumerProperties; import org.springframework.cloud.gcp.stream.binder.pubsub.properties.PubSubProducerProperties; -import org.springframework.cloud.gcp.test.PubSubEmulatorRule; +import org.springframework.cloud.gcp.test.EmulatorRule; +import org.springframework.cloud.gcp.test.pubsub.PubSubEmulator; import org.springframework.cloud.stream.binder.AbstractBinderTests; import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; import org.springframework.cloud.stream.binder.ExtendedProducerProperties; @@ -43,7 +44,7 @@ public class PubSubMessageChannelBinderEmulatorTests extends * The emulator instance, shared across tests. */ @ClassRule - public static PubSubEmulatorRule emulator = new PubSubEmulatorRule(); + public static EmulatorRule emulator = new EmulatorRule(new PubSubEmulator()); @Override protected PubSubTestBinder getBinder() { diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/Emulator.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/Emulator.java new file mode 100644 index 0000000000..944fe89c2e --- /dev/null +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/Emulator.java @@ -0,0 +1,53 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test; + +/** + * An interface that each emulator needs to implement so that it can be used with the {@link EmulatorDriver}. + * @author Mike Eltsufin + * @author Elena Felder + * @author Dmitry Solomakha + * @since 1.2.3 + */ +public interface Emulator { + + /** + * The name of the emulator command in gcloud CLI. + */ + String getName(); + + /** + * The list of command fragments that match the emulator processes to be killed. + * @param hostPort THe emulator host-port. + */ + String[] getKillCommandFragments(String hostPort); + + /** + * Custom kill commands that need to run to stop the emulator. + */ + default String[][] getPostKillCommands() { + return new String[0][0]; + } + + /** + * Custom start commands that need after the emulator is started. + */ + default String[][] getPostStartCommands() { + return new String[0][0]; + } + +} diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java similarity index 70% rename from spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java rename to spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java index 00d1ae0733..e7e1abda15 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/AbstractEmulatorHelper.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java @@ -36,16 +36,21 @@ import org.apache.commons.logging.LogFactory; import org.awaitility.Awaitility; -abstract class AbstractEmulatorHelper { - //Path to the directory that should contain env file - private final Path emulatorConfigDir = Paths.get(System.getProperty("user.home")).resolve( - Paths.get(".config", "gcloud", "emulators", getEmulatorName())); - +/** + * The main class used to start and stop an emulator. + * + * @author Elena Felder + * @author Dmitry Solomakha + * @author Mike Eltsufin + * + * @since 1.2.3 + */ +public class EmulatorDriver { private static final String ENV_FILE_NAME = "env.yaml"; - private final Path emulatorConfigPath = emulatorConfigDir.resolve(ENV_FILE_NAME); + private static final Log LOGGER = LogFactory.getLog(EmulatorDriver.class); - private static final Log LOGGER = LogFactory.getLog(AbstractEmulatorHelper.class); + private Emulator emulator; // Reference to emulator instance, for cleanup. private Process emulatorProcess; @@ -53,23 +58,40 @@ abstract class AbstractEmulatorHelper { // Hostname for cleanup, should always be localhost. private String emulatorHostPort; - abstract String getGatingPropertyName(); + /** + * Creates and emulator driver based on the emulator definition. + * @param emulator An implementation of an {@link Emulator} interface. + */ + public EmulatorDriver(Emulator emulator) { + this.emulator = emulator; + } + /** + * Starts the emulator. + */ public void startEmulator() throws IOException { - beforeEmulatorStart(); doStartEmulator(); - afterEmulatorStart(); determineHostPort(); } + /** + * Stops the emulator. + */ public void shutdownEmulator() { - findAndDestroyEmulator(); - afterEmulatorDestroyed(); + destroyGcloudEmulatorProcess(); + executeEmulatorKillCommands(); } - protected abstract void afterEmulatorDestroyed(); + /** + * Return the already-started emulator's host/port combination when called from within a + * JUnit method. + * @return emulator host/port string or null if emulator setup failed. + */ + public String getEmulatorHostPort() { + return this.emulatorHostPort; + } - private void findAndDestroyEmulator() { + private void destroyGcloudEmulatorProcess() { // destroy gcloud process if (this.emulatorProcess != null) { this.emulatorProcess.destroy(); @@ -79,7 +101,19 @@ private void findAndDestroyEmulator() { } } - protected void killByCommand(String command) { + private void executeEmulatorKillCommands() { + // pre-kill + for (String[] command : emulator.getPostKillCommands()) { + runSystemCommand(command, false); + } + + // kill + for (String command : emulator.getKillCommandFragments(emulatorHostPort)) { + killByCommand(command); + } + } + + private void killByCommand(String command) { AtomicBoolean foundProcess = new AtomicBoolean(false); try { @@ -91,7 +125,7 @@ protected void killByCommand(String command) { .map((psLine) -> new StringTokenizer(psLine).nextToken()) .forEach((p) -> { LOGGER.info("Found " + command + " process to kill: " + p); - this.killProcess(p); + killProcess(p); foundProcess.set(true); }); } @@ -105,16 +139,11 @@ protected void killByCommand(String command) { } } - /** - * Return the already-started emulator's host/port combination when called from within a - * JUnit method. - * @return emulator host/port string or null if emulator setup failed. - */ - public String getEmulatorHostPort() { - return this.emulatorHostPort; - } - private void doStartEmulator() throws IOException { + Path emulatorConfigDir = Paths.get(System.getProperty("user.home")).resolve( + Paths.get(".config", "gcloud", "emulators", emulator.getName())); + Path emulatorConfigPath = emulatorConfigDir.resolve(ENV_FILE_NAME); + boolean configPresent = emulatorConfigPath.toFile().exists(); WatchService watchService = null; @@ -124,7 +153,7 @@ private void doStartEmulator() throws IOException { } try { - this.emulatorProcess = new ProcessBuilder("gcloud", "beta", "emulators", getEmulatorName(), "start") + this.emulatorProcess = new ProcessBuilder("gcloud", "beta", "emulators", emulator.getName(), "start") .start(); } catch (IOException ex) { @@ -132,21 +161,24 @@ private void doStartEmulator() throws IOException { } if (configPresent) { - updateConfig(watchService); + waitForConfigChange(watchService, ENV_FILE_NAME); watchService.close(); } else { - waitForConfigCreation(); + waitForConfigCreation(emulatorConfigPath); } - - } - - protected void beforeEmulatorStart() { - // does nothing by default } protected void afterEmulatorStart() { - // does nothing by default + for (String[] command : this.emulator.getPostStartCommands()) { + ProcessOutcome processOutcome = runSystemCommand(command, true); + + if (processOutcome.getStatus() != 0) { + shutdownEmulator(); + throw new RuntimeException("After emulator start command failed: " + + String.join("\n", processOutcome.getErrors())); + } + } } /** @@ -155,7 +187,7 @@ protected void afterEmulatorStart() { */ private void determineHostPort() { ProcessOutcome processOutcome = runSystemCommand( - new String[] { "gcloud", "beta", "emulators", getEmulatorName(), "env-init" }); + new String[] { "gcloud", "beta", "emulators", this.emulator.getName(), "env-init" }, true); if (processOutcome.getOutput().isEmpty()) { throw new RuntimeException("env-init command did not produce output"); } @@ -163,64 +195,51 @@ private void determineHostPort() { this.emulatorHostPort = emulatorInitString.substring(emulatorInitString.indexOf('=') + 1); } - protected ProcessOutcome runSystemCommand(String[] command) { - return runSystemCommand(command, true); - } - - protected ProcessOutcome runSystemCommand(String[] command, boolean failOnError) { + private static ProcessOutcome runSystemCommand(String[] command, boolean failOnError) { + ProcessOutcome processOutcome = runSystemCommand(command); + if (failOnError && processOutcome.getStatus() != 0) { + throw new RuntimeException("Command execution failed: " + String.join(" ", command) + + "; output: " + processOutcome.getOutput() + + "; error: " + processOutcome.getErrors()); - Process envInitProcess = null; - try { - envInitProcess = new ProcessBuilder(command).start(); - } - catch (IOException e) { - throw new RuntimeException(e); } + return processOutcome; + } - try (BufferedReader brInput = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream())); - BufferedReader brError = new BufferedReader(new InputStreamReader(envInitProcess.getErrorStream()))) { - ProcessOutcome processOutcome = new ProcessOutcome(command, - brInput.lines().collect(Collectors.toList()), - brError.lines().collect(Collectors.toList()), - envInitProcess.waitFor()); - - if (failOnError && processOutcome.status != 0) { - // clean up anything that got started up first - this.shutdownEmulator(); - - throw new RuntimeException("Command execution failed: " + String.join(" ", command) - + "; output: " + processOutcome.getOutput() - + "; error: " + processOutcome.getErrors()); - - } - return processOutcome; + /** + * Attempt to kill a process on best effort basis. Failure is logged and ignored, as it is + * not critical to the tests' functionality. + * @param pid presumably a valid PID. No checking done to validate. + */ + private static void killProcess(String pid) { + try { + new ProcessBuilder("kill", "-9", pid).start(); } - catch (IOException | InterruptedException e) { - throw new RuntimeException(e); + catch (IOException ex) { + LOGGER.error("Failed to clean up PID " + pid, ex); } } - abstract String getEmulatorName(); - /** * Wait until the emulator configuration file is present. Fail if the file does not * appear after 10 seconds. */ - private void waitForConfigCreation() { + private static void waitForConfigCreation(Path emulatorConfigPath) { Awaitility.await("Emulator could not be configured due to missing env.yaml. Is the emulator installed?") .atMost(10, TimeUnit.SECONDS) .pollInterval(100, TimeUnit.MILLISECONDS) .until(() -> emulatorConfigPath.toFile().exists()); } + /** * Wait until the emulator configuration file is updated. Fail if the file does not * update after 10 second. * @param watchService the watch-service to poll */ - private void updateConfig(WatchService watchService) { + private static void waitForConfigChange(WatchService watchService, String envFileName) { Awaitility.await("Configuration file update could not be detected") - .atMost(10, TimeUnit.SECONDS) + .atMost(20, TimeUnit.SECONDS) .pollInterval(100, TimeUnit.MILLISECONDS) .until(() -> { WatchKey key = watchService.poll(100, TimeUnit.MILLISECONDS); @@ -228,7 +247,7 @@ private void updateConfig(WatchService watchService) { if (key != null) { Optional configFilePath = key.pollEvents().stream() .map((event) -> (Path) event.context()) - .filter((path) -> ENV_FILE_NAME.equals(path.toString())) + .filter((path) -> envFileName.equals(path.toString())) .findAny(); return configFilePath.isPresent(); } @@ -236,21 +255,31 @@ private void updateConfig(WatchService watchService) { }); } - /** - * Attempt to kill a process on best effort basis. Failure is logged and ignored, as it is - * not critical to the tests' functionality. - * @param pid presumably a valid PID. No checking done to validate. - */ - protected void killProcess(String pid) { + private static ProcessOutcome runSystemCommand(String[] command) { + + Process envInitProcess = null; try { - new ProcessBuilder("kill", "-9", pid).start(); + envInitProcess = new ProcessBuilder(command).start(); } - catch (IOException ex) { - LOGGER.warn("Failed to clean up PID " + pid); + catch (IOException e) { + throw new RuntimeException(e); + } + + try (BufferedReader brInput = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream())); + BufferedReader brError = new BufferedReader(new InputStreamReader(envInitProcess.getErrorStream()))) { + ProcessOutcome processOutcome = new ProcessOutcome(command, + brInput.lines().collect(Collectors.toList()), + brError.lines().collect(Collectors.toList()), + envInitProcess.waitFor()); + + return processOutcome; + } + catch (IOException | InterruptedException e) { + throw new RuntimeException(e); } } - static class ProcessOutcome { + private static class ProcessOutcome { private String[] command; private List output; @@ -282,5 +311,4 @@ public String getCommandString() { return String.join(" ", command); } } - } diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRule.java similarity index 71% rename from spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java rename to spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRule.java index fc13bcae58..75ace04872 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorRule.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRule.java @@ -18,18 +18,28 @@ import org.junit.rules.ExternalResource; -public class PubSubEmulatorRule extends ExternalResource { +/** + * @author Elena Felder + * @author Dmitry Solomakha + * @author Mike Eltsufin + * @since 1.2.3 + */ +public class EmulatorRule extends ExternalResource { + + private EmulatorDriver emulatorDriver; - private PubSubEmulatorHelper emulatorHelper = new PubSubEmulatorHelper(); + public EmulatorRule(Emulator emulator) { + this.emulatorDriver = new EmulatorDriver(emulator); + } @Override protected void before() throws Throwable { - emulatorHelper.startEmulator(); + emulatorDriver.startEmulator(); } @Override protected void after() { - emulatorHelper.shutdownEmulator(); + emulatorDriver.shutdownEmulator(); } /** @@ -38,7 +48,7 @@ protected void after() { * @return emulator host/port string or null if emulator setup failed. */ public String getEmulatorHostPort() { - return emulatorHelper.getEmulatorHostPort(); + return emulatorDriver.getEmulatorHostPort(); } } diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java deleted file mode 100644 index 917738eb4f..0000000000 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorHelper.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2017-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.gcp.test; - -import org.junit.Assume; - -public class SpannerEmulatorHelper extends AbstractEmulatorHelper { - private boolean checkEnvironmentFlag = true; - - public SpannerEmulatorHelper() { - // keep default value of checkEnvironmentFlag=true. - } - - public SpannerEmulatorHelper(boolean checkEnvironmentFlag) { - this.checkEnvironmentFlag = checkEnvironmentFlag; - } - - String getGatingPropertyName() { - return "it.spanner-emulator"; - } - - String getEmulatorName() { - return "spanner"; - } - - @Override - protected void beforeEmulatorStart() { - if (this.checkEnvironmentFlag) { - String emulatorHost = System.getenv("SPANNER_EMULATOR_HOST"); - Assume.assumeFalse( - "Set SPANNER_EMULATOR_HOST environment variable prior to running emulator tests; copy output of this command and run it:\ngcloud beta emulators spanner env-init", - emulatorHost == null || emulatorHost.isEmpty()); - } - } - - @Override - protected void afterEmulatorStart() { - runSystemCommand(new String[] { - "gcloud", "config", "configurations", "activate", "emulator" }); - - ProcessOutcome processOutcome = runSystemCommand(new String[] { - "gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator", - "--description=\"Test Instance\"", "--nodes=1" }, - false); - - if (processOutcome.getStatus() != 0) { - // don't set breakpoint here - cleanupSpannerEmulator(); - throw new RuntimeException("Creating instance failed: " - + String.join("\n", processOutcome.getErrors())); - } - } - - @Override - protected void afterEmulatorDestroyed() { - cleanupSpannerEmulator(); - runSystemCommand(new String[] { - "gcloud", "config", "configurations", "activate", "default" }); - } - - private void cleanupSpannerEmulator() { - this.killByCommand("cloud_spanner_emulator/emulator_main"); - this.killByCommand("cloud_spanner_emulator/gateway_main"); - } - -} diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java deleted file mode 100644 index e9afbcf3e2..0000000000 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorRule.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2017-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.gcp.test; - -import org.junit.Assume; -import org.junit.rules.ExternalResource; - -public class SpannerEmulatorRule extends ExternalResource { - - private SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(); - - @Override - protected void before() throws Throwable { - String emulatorHost = System.getenv("SPANNER_EMULATOR_HOST"); - Assume.assumeFalse( - "Run this command prior to running an emulator test and set SPANNER_EMULATOR_HOST environment variable:\ngcloud beta emulators spanner env-init", - emulatorHost == null || emulatorHost.isEmpty()); - emulatorHelper.startEmulator(); - } - - @Override - protected void after() { - emulatorHelper.shutdownEmulator(); - } - -} diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/pubsub/PubSubEmulator.java similarity index 62% rename from spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java rename to spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/pubsub/PubSubEmulator.java index 8549ca1910..865258dadf 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/PubSubEmulatorHelper.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/pubsub/PubSubEmulator.java @@ -14,34 +14,23 @@ * limitations under the License. */ -package org.springframework.cloud.gcp.test; +package org.springframework.cloud.gcp.test.pubsub; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -public class PubSubEmulatorHelper extends AbstractEmulatorHelper { - private static final Log LOGGER = LogFactory.getLog(PubSubEmulatorHelper.class); +import org.springframework.cloud.gcp.test.Emulator; - String getGatingPropertyName() { - return "it.pubsub-emulator"; - } +public class PubSubEmulator implements Emulator { + private static final Log LOGGER = LogFactory.getLog(PubSubEmulator.class); - String getEmulatorName() { + public String getName() { return "pubsub"; } - /** - * Shut down the emulator process. - * java process is identified and shut down through shell commands. There - * should normally be only one process with that host/port combination, but if there are - * more, they will be cleaned up as well. Any failure is logged and ignored since it's not - * critical to the tests' operation. - */ - @Override - protected void afterEmulatorDestroyed() { - String hostPort = getEmulatorHostPort(); - - // find and destory emulator process spawned by gcloud + public String[] getKillCommandFragments(String hostPort) { + + // find and destroy emulator process spawned by gcloud if (hostPort == null) { LOGGER.warn("Host/port null after the test."); } @@ -49,14 +38,16 @@ protected void afterEmulatorDestroyed() { int portSeparatorIndex = hostPort.lastIndexOf(':'); if (portSeparatorIndex < 0) { LOGGER.warn("Malformed host: " + hostPort); - return; } String emulatorHost = hostPort.substring(0, portSeparatorIndex); String emulatorPort = hostPort.substring(portSeparatorIndex + 1); String hostPortParams = String.format("--host=%s --port=%s", emulatorHost, emulatorPort); - killByCommand(hostPortParams); + + return new String[] { hostPortParams}; } + + return new String[0]; } } diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulator.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulator.java new file mode 100644 index 0000000000..10ddeafd1f --- /dev/null +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulator.java @@ -0,0 +1,63 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test.spanner; + +import org.junit.Assume; + +import org.springframework.cloud.gcp.test.Emulator; + +public class SpannerEmulator implements Emulator { + + public SpannerEmulator() { + } + + public SpannerEmulator(boolean checkEnvironmentFlag) { + if (checkEnvironmentFlag) { + String emulatorHost = System.getenv("SPANNER_EMULATOR_HOST"); + Assume.assumeFalse( + "Set SPANNER_EMULATOR_HOST environment variable prior to running emulator tests; copy output of this command and run it:\ngcloud beta emulators spanner env-init", + emulatorHost == null || emulatorHost.isEmpty()); + } + } + + public String getName() { + return "spanner"; + } + + @Override + public String[] getKillCommandFragments(String hostPort) { + return new String[] { + "cloud_spanner_emulator/emulator_main", + "cloud_spanner_emulator/gateway_main" }; + } + + @Override + public String[][] getPostStartCommands() { + return new String[][] { + {"gcloud", "config", "configurations", "activate", "emulator" }, + {"gcloud", "spanner", "instances", "create", "integration-instance", "--config=emulator", + "--description=\"Test Instance\"", "--nodes=1"} + }; + } + + @Override + public String[][] getPostKillCommands() { + return new String[][] { { + "gcloud", "config", "configurations", "activate", "default" } }; + } + +} diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java similarity index 87% rename from spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java rename to spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java index 002282b264..7a72176086 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/SpannerEmulatorSpringConfiguration.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.cloud.gcp.test; +package org.springframework.cloud.gcp.test.spanner; import java.io.IOException; @@ -23,6 +23,7 @@ import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.SpannerOptions; +import org.springframework.cloud.gcp.test.EmulatorDriver; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.event.ContextClosedEvent; @@ -33,12 +34,12 @@ @Order(-1000) public class SpannerEmulatorSpringConfiguration { - private SpannerEmulatorHelper emulatorHelper = new SpannerEmulatorHelper(false); + private EmulatorDriver emulatorDriver = new EmulatorDriver(new SpannerEmulator(false)); @Bean public SpannerOptions spannerOptions() throws IOException, InterruptedException { // starting the emulator will change the default project ID to that of the emulator config. - emulatorHelper.startEmulator(); + emulatorDriver.startEmulator(); return SpannerOptions.newBuilder() .setProjectId(ServiceOptions.getDefaultProjectId()) .setCredentials(NoCredentials.getInstance()) @@ -55,6 +56,6 @@ public Spanner spanner(SpannerOptions spannerOptions) { @EventListener public void afterCloseEvent(ContextClosedEvent event) { - emulatorHelper.shutdownEmulator(); + emulatorDriver.shutdownEmulator(); } } From 8af2f8a5e52a1075a6ab3d134815e624f030f591 Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Fri, 8 May 2020 19:28:19 -0400 Subject: [PATCH 14/23] add gating properties and improve docs --- docs/src/main/asciidoc/test.adoc | 7 +++++++ .../core/it/SpannerTemplateEmulatorTests.java | 10 ++++++++++ .../PubSubMessageChannelBinderEmulatorTests.java | 10 ++++++++++ .../spanner/SpannerEmulatorSpringConfiguration.java | 13 ++++++++++--- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/src/main/asciidoc/test.adoc b/docs/src/main/asciidoc/test.adoc index d5021c762a..9734a526a2 100644 --- a/docs/src/main/asciidoc/test.adoc +++ b/docs/src/main/asciidoc/test.adoc @@ -31,6 +31,9 @@ public class PubSubTests { } ---- +NOTE: The class rule doesn't work for SpannerEmulator. +See <> section instead. + === Utility class If you prefer controlling the emulator manually, you can use the `EmulatorDriver`. @@ -49,11 +52,15 @@ If you prefer controlling the emulator manually, you can use the `EmulatorDriver In order to use our Cloud PubSub Emulator helper tools, you would need to install the emulator first. Follow the https://cloud.google.com/pubsub/docs/emulator[installation instructions]. +Use `new PubSubEmulator()` as an argument for `EmulatorDriver` and `EmulatorRule`. + === Cloud Spanner Emulator In order to use our Cloud Spanner Emulator helper tools, you would need to install the emulator first. Follow the https://cloud.google.com/spanner/docs/emulator[installation instructions]. Make sure you also create an https://cloud.google.com/spanner/docs/emulator#using_the_gcloud_cli_with_the_emulator[emulator configuration] and call it `emulator`. +Use `new SpannerEmulator()` as an argument for `EmulatorDriver` + ==== Spring Configuration If you are testing your Spring application, you can use our configuration class. It provides a bean of type Spanner, which starts the emulator before creating a client. diff --git a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java index 6afbc0cb73..d9aa34da09 100644 --- a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java +++ b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java @@ -16,6 +16,7 @@ package org.springframework.cloud.gcp.data.spanner.core.it; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -26,6 +27,7 @@ import org.springframework.test.context.junit4.SpringRunner; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; /** * Integration tests that use many features of the Spanner Template. @@ -37,6 +39,14 @@ @ContextConfiguration(classes = {SpannerEmulatorSpringConfiguration.class}) public class SpannerTemplateEmulatorTests extends AbstractSpannerIntegrationTest { + @BeforeClass + public static void checkToRun() { + assumeThat(System.getProperty("it.spanner-emulator")) + .as("Spanner emulator tests are disabled. " + + "Please use '-Dit.spanner-emulator=true' to enable them. ") + .isEqualTo("true"); + } + @Test public void insertSingleRow() { diff --git a/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java b/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java index efbb89836a..67a5911854 100644 --- a/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java +++ b/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java @@ -16,6 +16,7 @@ package org.springframework.cloud.gcp.stream.binder.pubsub; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Ignore; import org.junit.Test; @@ -29,6 +30,8 @@ import org.springframework.cloud.stream.binder.ExtendedProducerProperties; import org.springframework.cloud.stream.binder.Spy; +import static org.assertj.core.api.Assumptions.assumeThat; + /** * Integration tests that require the Pub/Sub emulator to be installed. * @@ -39,6 +42,13 @@ public class PubSubMessageChannelBinderEmulatorTests extends AbstractBinderTests, ExtendedProducerProperties> { + @BeforeClass + public static void checkToRun() { + assumeThat(System.getProperty("it.pubsub-emulator")) + .as("PubSub emulator tests are disabled. " + + "Please use '-Dit.pubsub-emulator=true' to enable them. ") + .isEqualTo("true"); + } /** * The emulator instance, shared across tests. diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java index 7a72176086..86451366ab 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java @@ -29,7 +29,14 @@ import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; - +/** + * SpannerEmulatorSpringConfiguration should be used instead of JUnit class rule because spring context tries to close the connection when it is being destroyed. + * But the rule would already shut down the emulator by that time. That causes tests to hang. + * @author Mike Eltsufin + * @author Elena Felder + * @author Dmitry Solomakha + * @since 1.2.3 + */ @Configuration @Order(-1000) public class SpannerEmulatorSpringConfiguration { @@ -47,8 +54,8 @@ public SpannerOptions spannerOptions() throws IOException, InterruptedException .build(); } - // the real destroy method would attempt to connect the already-shutdown emulator instance, - // causing tests to fail. + // the real destroy method would attempt to connect to the already-shutdown emulator instance, + // causing tests to hang @Bean(destroyMethod = "") public Spanner spanner(SpannerOptions spannerOptions) { return spannerOptions.getService(); From 9d0f694b4dcb5ef183b900b098adc7d95f324b0a Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Tue, 12 May 2020 18:35:33 -0400 Subject: [PATCH 15/23] add tests --- spring-cloud-gcp-data-spanner/pom.xml | 6 - .../core/it/SpannerTemplateEmulatorTests.java | 58 ------- ...bSubMessageChannelBinderEmulatorTests.java | 2 - spring-cloud-gcp-test/pom.xml | 107 +++++++----- .../cloud/gcp/test/EmulatorDriver.java | 1 + .../SpannerEmulatorSpringConfiguration.java | 2 +- .../gcp/test/PubSubTemplateEmulatorTests.java | 95 +++++++++++ .../test/SpannerTemplateEmulatorTests.java | 160 +++++++++++++++++ .../gcp/test/SpannerTestConfiguration.java | 161 ++++++++++++++++++ 9 files changed, 481 insertions(+), 111 deletions(-) delete mode 100644 spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java create mode 100644 spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/PubSubTemplateEmulatorTests.java create mode 100644 spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTemplateEmulatorTests.java create mode 100644 spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java diff --git a/spring-cloud-gcp-data-spanner/pom.xml b/spring-cloud-gcp-data-spanner/pom.xml index 425f511944..1b3311b67f 100644 --- a/spring-cloud-gcp-data-spanner/pom.xml +++ b/spring-cloud-gcp-data-spanner/pom.xml @@ -9,7 +9,6 @@ org.springframework.cloud 1.2.3.BUILD-SNAPSHOT - org.springframework.cloud spring-cloud-gcp-data-spanner Spring Cloud GCP Cloud Spanner Module Spring Cloud GCP Cloud Spanner Module @@ -42,10 +41,5 @@ org.springframework spring-tx - - org.springframework.cloud - spring-cloud-gcp-test - test - diff --git a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java deleted file mode 100644 index d9aa34da09..0000000000 --- a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/core/it/SpannerTemplateEmulatorTests.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2017-2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.gcp.data.spanner.core.it; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.cloud.gcp.data.spanner.test.AbstractSpannerIntegrationTest; -import org.springframework.cloud.gcp.data.spanner.test.domain.Trade; -import org.springframework.cloud.gcp.test.spanner.SpannerEmulatorSpringConfiguration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assumptions.assumeThat; - -/** - * Integration tests that use many features of the Spanner Template. - * - * @author Balint Pato - * @author Chengyuan Zhao - */ -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = {SpannerEmulatorSpringConfiguration.class}) -public class SpannerTemplateEmulatorTests extends AbstractSpannerIntegrationTest { - - @BeforeClass - public static void checkToRun() { - assumeThat(System.getProperty("it.spanner-emulator")) - .as("Spanner emulator tests are disabled. " - + "Please use '-Dit.spanner-emulator=true' to enable them. ") - .isEqualTo("true"); - } - - @Test - public void insertSingleRow() { - - Trade trade = Trade.aTrade(null, 1); - this.spannerOperations.insert(trade); - assertThat(this.spannerOperations.count(Trade.class)).isEqualTo(1L); - } - -} diff --git a/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java b/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java index 67a5911854..343b74b1ab 100644 --- a/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java +++ b/spring-cloud-gcp-pubsub-stream-binder/src/test/java/org/springframework/cloud/gcp/stream/binder/pubsub/PubSubMessageChannelBinderEmulatorTests.java @@ -84,8 +84,6 @@ public void testClean() { @Test @Ignore("Looks like there is no Kryo support in SCSt") public void testSendPojoReceivePojoKryoWithStreamListener() { - } - } diff --git a/spring-cloud-gcp-test/pom.xml b/spring-cloud-gcp-test/pom.xml index a673e4e02c..320b399454 100644 --- a/spring-cloud-gcp-test/pom.xml +++ b/spring-cloud-gcp-test/pom.xml @@ -1,49 +1,68 @@ - 4.0.0 - - spring-cloud-gcp - org.springframework.cloud - 1.2.3.BUILD-SNAPSHOT - - - - spring-cloud-gcp-test - Spring Cloud GCP Test Support - Provides tools for testing Spring Cloud GCP projects - - - - - junit - junit - true - - - - org.springframework - spring-jcl - - - org.awaitility - awaitility - - - - - org.springframework - spring-context - true - - - - - com.google.cloud - google-cloud-spanner - true - + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + spring-cloud-gcp + org.springframework.cloud + 1.2.3.BUILD-SNAPSHOT + + + + spring-cloud-gcp-test + Spring Cloud GCP Test Support + Provides tools for testing Spring Cloud GCP projects + + + + + junit + junit + true + + + + org.springframework + spring-jcl + + + org.awaitility + awaitility + + + + + org.springframework + spring-context + true + + + + + com.google.cloud + google-cloud-spanner + true + + + + + org.springframework + spring-test + test + + + + org.springframework.cloud + spring-cloud-gcp-starter-pubsub + test + + + + org.springframework.cloud + spring-cloud-gcp-data-spanner + test + diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java index e7e1abda15..6466948836 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java @@ -71,6 +71,7 @@ public EmulatorDriver(Emulator emulator) { */ public void startEmulator() throws IOException { doStartEmulator(); + afterEmulatorStart(); determineHostPort(); } diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java index 86451366ab..7bd5477e3e 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java @@ -44,7 +44,7 @@ public class SpannerEmulatorSpringConfiguration { private EmulatorDriver emulatorDriver = new EmulatorDriver(new SpannerEmulator(false)); @Bean - public SpannerOptions spannerOptions() throws IOException, InterruptedException { + public SpannerOptions spannerOptions() throws IOException { // starting the emulator will change the default project ID to that of the emulator config. emulatorDriver.startEmulator(); return SpannerOptions.newBuilder() diff --git a/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/PubSubTemplateEmulatorTests.java b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/PubSubTemplateEmulatorTests.java new file mode 100644 index 0000000000..a172d23567 --- /dev/null +++ b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/PubSubTemplateEmulatorTests.java @@ -0,0 +1,95 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PubsubMessage; +import org.assertj.core.api.Assumptions; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.gcp.autoconfigure.core.GcpContextAutoConfiguration; +import org.springframework.cloud.gcp.autoconfigure.pubsub.GcpPubSubAutoConfiguration; +import org.springframework.cloud.gcp.autoconfigure.pubsub.GcpPubSubEmulatorAutoConfiguration; +import org.springframework.cloud.gcp.pubsub.PubSubAdmin; +import org.springframework.cloud.gcp.pubsub.core.PubSubTemplate; +import org.springframework.cloud.gcp.test.pubsub.PubSubEmulator; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Pub/Sub emulator tests. + * + * @author Dmitry Solomakha + */ +public class PubSubTemplateEmulatorTests { + + /** + * Emulator class rule. + */ + @ClassRule + public static EmulatorRule emulator = new EmulatorRule(new PubSubEmulator()); + + @BeforeClass + public static void enableTests() { + Assumptions.assumeThat(System.getProperty("it.emulator")) + .as("Spanner emulator tests are disabled. " + + "Please use '-Dit.emulator=true' to enable them. ") + .isEqualTo("true"); + } + + @Test + public void testCreatePublishPullNextAndDelete() { + ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withPropertyValues("spring.cloud.gcp.pubsub.subscriber.max-ack-extension-period=0") + .withPropertyValues("spring.cloud.gcp.pubsub.emulator-host=" + emulator.getEmulatorHostPort()) + .withConfiguration(AutoConfigurations.of(GcpContextAutoConfiguration.class, + GcpPubSubAutoConfiguration.class, GcpPubSubEmulatorAutoConfiguration.class)); + + contextRunner.run((context) -> { + PubSubAdmin pubSubAdmin = context.getBean(PubSubAdmin.class); + PubSubTemplate pubSubTemplate = context.getBean(PubSubTemplate.class); + + String topicName = "tarkus_" + UUID.randomUUID(); + String subscriptionName = "zatoichi_" + UUID.randomUUID(); + + assertThat(pubSubAdmin.getTopic(topicName)).isNull(); + assertThat(pubSubAdmin.getSubscription(subscriptionName)) + .isNull(); + pubSubAdmin.createTopic(topicName); + pubSubAdmin.createSubscription(subscriptionName, topicName); + + Map headers = new HashMap<>(); + headers.put("cactuar", "tonberry"); + headers.put("fujin", "raijin"); + pubSubTemplate.publish(topicName, "tatatatata", headers).get(); + PubsubMessage pubsubMessage = pubSubTemplate.pullNext(subscriptionName); + + assertThat(pubsubMessage.getData()).isEqualTo(ByteString.copyFromUtf8("tatatatata")); + assertThat(pubsubMessage.getAttributesCount()).isEqualTo(2); + assertThat(pubsubMessage.getAttributesOrThrow("cactuar")).isEqualTo("tonberry"); + assertThat(pubsubMessage.getAttributesOrThrow("fujin")).isEqualTo("raijin"); + }); + } +} diff --git a/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTemplateEmulatorTests.java b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTemplateEmulatorTests.java new file mode 100644 index 0000000000..4e6727c2d1 --- /dev/null +++ b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTemplateEmulatorTests.java @@ -0,0 +1,160 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Objects; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.assertj.core.api.Assertions; +import org.assertj.core.api.Assumptions; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.gcp.data.spanner.core.SpannerTemplate; +import org.springframework.cloud.gcp.data.spanner.core.admin.SpannerDatabaseAdminTemplate; +import org.springframework.cloud.gcp.data.spanner.core.admin.SpannerSchemaUtils; +import org.springframework.cloud.gcp.data.spanner.core.mapping.Column; +import org.springframework.cloud.gcp.data.spanner.core.mapping.Table; +import org.springframework.cloud.gcp.test.spanner.SpannerEmulatorSpringConfiguration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; + +/** + * Spanner emulator tests. + * + * @author Dmitry Solomakha + */ +@RunWith(SpringRunner.class) +@ContextConfiguration(classes = {SpannerEmulatorSpringConfiguration.class, SpannerTestConfiguration.class}) +public class SpannerTemplateEmulatorTests { + @Autowired + private SpannerTemplate spannerTemplate; + + @Autowired + private SpannerDatabaseAdminTemplate spannerDatabaseAdminTemplate; + + @Autowired + private SpannerSchemaUtils spannerSchemaUtils; + + private static final Log LOGGER = LogFactory.getLog(SpannerTemplateEmulatorTests.class); + + @BeforeClass + public static void checkToRun() { + Assumptions.assumeThat(System.getProperty("it.emulator")) + .as("Spanner emulator tests are disabled. " + + "Please use '-Dit.emulator=true' to enable them. ") + .isEqualTo("true"); + } + + @Test + public void insertSingleRow() { + createDatabaseWithSchema(); + Trade trade = Trade.aTrade(null, 1); + this.spannerTemplate.insert(trade); + Assertions.assertThat(this.spannerTemplate.count(Trade.class)).isEqualTo(1L); + } + + private void createDatabaseWithSchema() { + List createStatements = new ArrayList<>(this.spannerSchemaUtils + .getCreateTableDdlStringsForInterleavedHierarchy(Trade.class)); + + if (!this.spannerDatabaseAdminTemplate.databaseExists()) { + LOGGER.debug( + this.getClass() + " - Integration database created with schema: " + + createStatements); + this.spannerDatabaseAdminTemplate.executeDdlStrings(createStatements, true); + } + else { + LOGGER.debug( + this.getClass() + " - schema created: " + createStatements); + this.spannerDatabaseAdminTemplate.executeDdlStrings(createStatements, false); + } + } + + @Table + public static class Trade { + + @Column(nullable = false) + private int age; + + private Instant tradeTime; + + private Date tradeDate; + + private LocalDate tradeLocalDate; + + private LocalDateTime tradeLocalDateTime; + + private String action; + + private String symbol; + + + Trade(String symbol, List executionTimes) { + this.symbol = symbol; + } + + public static Trade aTrade() { + return aTrade(null, 0); + } + + static Trade aTrade(String customTraderId, int subTrades) { + Trade t = new Trade("ABCD", new ArrayList<>()); + + t.age = 8; + t.action = "BUY"; + t.tradeTime = Instant.ofEpochSecond(333); + t.tradeDate = Date.from(t.tradeTime); + t.tradeLocalDate = LocalDate.of(2015, 1, 1); + t.tradeLocalDateTime = LocalDateTime.of(2015, 1, 1, 2, 3, 4, 5); + + return t; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Trade trade = (Trade) o; + return age == trade.age && + Objects.equals(tradeTime, trade.tradeTime) && + Objects.equals(tradeDate, trade.tradeDate) && + Objects.equals(tradeLocalDate, trade.tradeLocalDate) && + Objects.equals(tradeLocalDateTime, trade.tradeLocalDateTime) && + Objects.equals(action, trade.action) && + Objects.equals(symbol, trade.symbol); + } + + @Override + public int hashCode() { + return Objects.hash(age, tradeTime, tradeDate, tradeLocalDate, tradeLocalDateTime, action, symbol); + } + } +} diff --git a/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java new file mode 100644 index 0000000000..11ac73f5b0 --- /dev/null +++ b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java @@ -0,0 +1,161 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.test; + +import java.io.IOException; + +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.DatabaseClient; +import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.SessionPoolOptions; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerOptions; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.cloud.gcp.core.Credentials; +import org.springframework.cloud.gcp.core.DefaultCredentialsProvider; +import org.springframework.cloud.gcp.core.DefaultGcpProjectIdProvider; +import org.springframework.cloud.gcp.data.spanner.core.SpannerMutationFactory; +import org.springframework.cloud.gcp.data.spanner.core.SpannerMutationFactoryImpl; +import org.springframework.cloud.gcp.data.spanner.core.SpannerTemplate; +import org.springframework.cloud.gcp.data.spanner.core.SpannerTransactionManager; +import org.springframework.cloud.gcp.data.spanner.core.admin.SpannerDatabaseAdminTemplate; +import org.springframework.cloud.gcp.data.spanner.core.admin.SpannerSchemaUtils; +import org.springframework.cloud.gcp.data.spanner.core.convert.ConverterAwareMappingSpannerEntityProcessor; +import org.springframework.cloud.gcp.data.spanner.core.convert.SpannerEntityProcessor; +import org.springframework.cloud.gcp.data.spanner.core.mapping.SpannerMappingContext; +import org.springframework.cloud.gcp.data.spanner.repository.config.EnableSpannerRepositories; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +/** + * Configuration for Spanner emulator test. + * + * @author Dmitry Solomakha + */ + +@Configuration +@EnableTransactionManagement +@EnableSpannerRepositories +public class SpannerTestConfiguration { + + private String databaseName = "testDatabase"; + + private String instanceId = "integration-instance"; + + @Bean + public String getDatabaseName() { + return this.databaseName; + } + + @Bean + public String getInstanceId() { + return this.instanceId; + } + + @Bean + public String getProjectId() { + return new DefaultGcpProjectIdProvider().getProjectId(); + } + + @Bean + public com.google.auth.Credentials getCredentials() { + try { + return new DefaultCredentialsProvider(Credentials::new).getCredentials(); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + @Bean + @ConditionalOnMissingBean + public SpannerOptions spannerOptions() { + return SpannerOptions.newBuilder().setProjectId(getProjectId()) + .setSessionPoolOption(SessionPoolOptions.newBuilder().setMaxSessions(10).build()) + .setCredentials(getCredentials()).build(); + } + + @Bean + public DatabaseId databaseId() { + return DatabaseId.of(getProjectId(), this.instanceId, this.databaseName); + } + + @Bean + @ConditionalOnMissingBean + public Spanner spanner(SpannerOptions spannerOptions) { + return spannerOptions.getService(); + } + + @Bean + public DatabaseClient spannerDatabaseClient(Spanner spanner, DatabaseId databaseId) { + return spanner.getDatabaseClient(databaseId); + } + + @Bean + public SpannerMappingContext spannerMappingContext() { + return new SpannerMappingContext(); + } + + @Bean + public SpannerTemplate spannerTemplate(DatabaseClient databaseClient, + SpannerMappingContext mappingContext, SpannerEntityProcessor spannerEntityProcessor, + SpannerMutationFactory spannerMutationFactory, + SpannerSchemaUtils spannerSchemaUtils) { + return new SpannerTemplate(() -> databaseClient, mappingContext, spannerEntityProcessor, + spannerMutationFactory, spannerSchemaUtils); + } + + @Bean + public SpannerEntityProcessor spannerConverter(SpannerMappingContext mappingContext) { + return new ConverterAwareMappingSpannerEntityProcessor(mappingContext); + } + + @Bean + public SpannerTransactionManager spannerTransactionManager( + DatabaseClient databaseClient) { + return new SpannerTransactionManager(() -> databaseClient); + } + + @Bean + public SpannerMutationFactory spannerMutationFactory( + SpannerEntityProcessor spannerEntityProcessor, + SpannerMappingContext spannerMappingContext, + SpannerSchemaUtils spannerSchemaUtils) { + return new SpannerMutationFactoryImpl(spannerEntityProcessor, + spannerMappingContext, spannerSchemaUtils); + } + + @Bean + public DatabaseAdminClient databaseAdminClient(Spanner spanner) { + return spanner.getDatabaseAdminClient(); + } + + @Bean + public SpannerSchemaUtils spannerSchemaUtils( + SpannerMappingContext spannerMappingContext, + SpannerEntityProcessor spannerEntityProcessor) { + return new SpannerSchemaUtils(spannerMappingContext, spannerEntityProcessor, true); + } + + @Bean + public SpannerDatabaseAdminTemplate spannerDatabaseAdminTemplate( + DatabaseAdminClient databaseAdminClient, DatabaseClient databaseClient, DatabaseId databaseId) { + return new SpannerDatabaseAdminTemplate(databaseAdminClient, () -> databaseClient, () -> databaseId); + } +} From d043c0eac05479e8175782a8a5cc6a29f74742cd Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Wed, 13 May 2020 15:59:01 -0400 Subject: [PATCH 16/23] address sonarqube issues --- .../cloud/gcp/test/EmulatorDriver.java | 30 ++++++++----------- .../gcp/test/EmulatorRuntimeException.java | 15 ++++++++++ 2 files changed, 28 insertions(+), 17 deletions(-) create mode 100644 spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java index 6466948836..2990f0615a 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java @@ -158,7 +158,7 @@ private void doStartEmulator() throws IOException { .start(); } catch (IOException ex) { - throw new RuntimeException("Gcloud not found; leaving host/port uninitialized.", ex); + throw new EmulatorRuntimeException("Gcloud not found; leaving host/port uninitialized.", ex); } if (configPresent) { @@ -176,7 +176,7 @@ protected void afterEmulatorStart() { if (processOutcome.getStatus() != 0) { shutdownEmulator(); - throw new RuntimeException("After emulator start command failed: " + throw new EmulatorRuntimeException("After emulator start command failed: " + String.join("\n", processOutcome.getErrors())); } } @@ -190,7 +190,7 @@ private void determineHostPort() { ProcessOutcome processOutcome = runSystemCommand( new String[] { "gcloud", "beta", "emulators", this.emulator.getName(), "env-init" }, true); if (processOutcome.getOutput().isEmpty()) { - throw new RuntimeException("env-init command did not produce output"); + throw new EmulatorRuntimeException("env-init command did not produce output"); } String emulatorInitString = processOutcome.getOutput().get(0); this.emulatorHostPort = emulatorInitString.substring(emulatorInitString.indexOf('=') + 1); @@ -199,7 +199,7 @@ private void determineHostPort() { private static ProcessOutcome runSystemCommand(String[] command, boolean failOnError) { ProcessOutcome processOutcome = runSystemCommand(command); if (failOnError && processOutcome.getStatus() != 0) { - throw new RuntimeException("Command execution failed: " + String.join(" ", command) + throw new EmulatorRuntimeException("Command execution failed: " + String.join(" ", command) + "; output: " + processOutcome.getOutput() + "; error: " + processOutcome.getErrors()); @@ -263,25 +263,26 @@ private static ProcessOutcome runSystemCommand(String[] command) { envInitProcess = new ProcessBuilder(command).start(); } catch (IOException e) { - throw new RuntimeException(e); + throw new EmulatorRuntimeException(e); } try (BufferedReader brInput = new BufferedReader(new InputStreamReader(envInitProcess.getInputStream())); BufferedReader brError = new BufferedReader(new InputStreamReader(envInitProcess.getErrorStream()))) { - ProcessOutcome processOutcome = new ProcessOutcome(command, + return new ProcessOutcome( brInput.lines().collect(Collectors.toList()), brError.lines().collect(Collectors.toList()), envInitProcess.waitFor()); - - return processOutcome; } - catch (IOException | InterruptedException e) { - throw new RuntimeException(e); + catch (IOException e) { + throw new EmulatorRuntimeException(e); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new EmulatorRuntimeException(e); } } private static class ProcessOutcome { - private String[] command; private List output; @@ -289,8 +290,7 @@ private static class ProcessOutcome { private int status; - ProcessOutcome(String[] command, List output, List errors, int status) { - this.command = command; + ProcessOutcome(List output, List errors, int status) { this.output = output; this.errors = errors; this.status = status; @@ -307,9 +307,5 @@ public List getErrors() { public int getStatus() { return status; } - - public String getCommandString() { - return String.join(" ", command); - } } } diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java new file mode 100644 index 0000000000..e8e6c756c1 --- /dev/null +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java @@ -0,0 +1,15 @@ +package org.springframework.cloud.gcp.test; + +public class EmulatorRuntimeException extends RuntimeException { + public EmulatorRuntimeException(String message) { + super(message); + } + + public EmulatorRuntimeException(String message, Throwable cause) { + super(message, cause); + } + + public EmulatorRuntimeException(Throwable cause) { + super(cause); + } +} From 946914afb6b2e1af2c147c84961b343f72832ff0 Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Wed, 13 May 2020 16:45:23 -0400 Subject: [PATCH 17/23] address sonarqube issues --- .../cloud/gcp/test/EmulatorRuntimeException.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java index e8e6c756c1..cb5cee60e1 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java @@ -1,3 +1,19 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.springframework.cloud.gcp.test; public class EmulatorRuntimeException extends RuntimeException { From df4c69caf7b165aa8c3a8dbce0ac85bafcb8d9e7 Mon Sep 17 00:00:00 2001 From: dmitry-s Date: Thu, 14 May 2020 17:54:08 -0400 Subject: [PATCH 18/23] Apply suggestions from code review Co-authored-by: Mike Eltsufin --- docs/src/main/asciidoc/test.adoc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/src/main/asciidoc/test.adoc b/docs/src/main/asciidoc/test.adoc index 9734a526a2..3170e97859 100644 --- a/docs/src/main/asciidoc/test.adoc +++ b/docs/src/main/asciidoc/test.adoc @@ -1,5 +1,5 @@ == Emulators support -Our tools simplify starting and stopping Cloud PubSub and Cloud Spanner emulators when you test your application. +Our tools simplify starting and stopping Cloud Pub/Sub and Cloud Spanner emulators when you test your applications locally. In order to use it, you need to add this dependency into your `pom.xml` file: @@ -12,10 +12,10 @@ In order to use it, you need to add this dependency into your `pom.xml` file: ---- -Also, you need to have `gcloud` and the emulators installed and configured, as described later. +Also, you need to have `gcloud` CLI and the emulators installed and configured, as described later. === JUnit 4 Class Rule -The rule starts the emulator process before the tests run and kills it afterwards. +The rule starts the emulator process before the tests run and stops it afterwards. You just need to add a rule into your test class to enable it. [source,java] @@ -31,7 +31,7 @@ public class PubSubTests { } ---- -NOTE: The class rule doesn't work for SpannerEmulator. +NOTE: The class rule doesn't work for `SpannerEmulator`. See <> section instead. === Utility class @@ -48,22 +48,22 @@ If you prefer controlling the emulator manually, you can use the `EmulatorDriver emulatorDriver.shutdownEmulator(); ---- -=== Cloud PubSub Emulator -In order to use our Cloud PubSub Emulator helper tools, you would need to install the emulator first. +=== Cloud Pub/Sub Emulator +In order to use our Cloud Pub/Sub Emulator helper tools, you would need to install the emulator first. Follow the https://cloud.google.com/pubsub/docs/emulator[installation instructions]. -Use `new PubSubEmulator()` as an argument for `EmulatorDriver` and `EmulatorRule`. +Use `new PubSubEmulator()` as an argument to `EmulatorDriver` and `EmulatorRule`. === Cloud Spanner Emulator In order to use our Cloud Spanner Emulator helper tools, you would need to install the emulator first. Follow the https://cloud.google.com/spanner/docs/emulator[installation instructions]. Make sure you also create an https://cloud.google.com/spanner/docs/emulator#using_the_gcloud_cli_with_the_emulator[emulator configuration] and call it `emulator`. -Use `new SpannerEmulator()` as an argument for `EmulatorDriver` +Use `new SpannerEmulator()` as an argument to `EmulatorDriver`. -==== Spring Configuration +==== Spanner Emulator Spring Configuration If you are testing your Spring application, you can use our configuration class. -It provides a bean of type Spanner, which starts the emulator before creating a client. +It provides a bean of type `Spanner`, which starts the emulator before creating a client. This configuration also stops the emulator when the Spring context is shut down. In order to use it, you need to add the `SpannerEmulatorSpringConfiguration.class` to your configuration classes list in `@ContextConfiguration`. @@ -79,4 +79,4 @@ import org.springframework.cloud.gcp.test.SpannerEmulatorSpringConfiguration; public class SpannerTemplateEmulatorTests { //your tests } ----- \ No newline at end of file +---- From aa31b9d18852a3a3d5599395631971c6d7afe1dd Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Thu, 14 May 2020 18:57:48 -0400 Subject: [PATCH 19/23] PR comments --- .../gcp/test/PubSubTemplateEmulatorTests.java | 6 +++++- .../cloud/gcp/test/SpannerTestConfiguration.java | 14 -------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/PubSubTemplateEmulatorTests.java b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/PubSubTemplateEmulatorTests.java index a172d23567..9fbe06c3bf 100644 --- a/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/PubSubTemplateEmulatorTests.java +++ b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/PubSubTemplateEmulatorTests.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.UUID; +import com.google.api.gax.rpc.TransportChannelProvider; import com.google.protobuf.ByteString; import com.google.pubsub.v1.PubsubMessage; import org.assertj.core.api.Assumptions; @@ -54,7 +55,7 @@ public class PubSubTemplateEmulatorTests { @BeforeClass public static void enableTests() { Assumptions.assumeThat(System.getProperty("it.emulator")) - .as("Spanner emulator tests are disabled. " + .as("Pub/Sub emulator tests are disabled. " + "Please use '-Dit.emulator=true' to enable them. ") .isEqualTo("true"); } @@ -68,6 +69,9 @@ public void testCreatePublishPullNextAndDelete() { GcpPubSubAutoConfiguration.class, GcpPubSubEmulatorAutoConfiguration.class)); contextRunner.run((context) -> { + TransportChannelProvider transportChannelProvider = context.getBean(TransportChannelProvider.class); + assertThat(transportChannelProvider.getTransportChannel().toString()).contains("target=dns:///localhost:8085"); + PubSubAdmin pubSubAdmin = context.getBean(PubSubAdmin.class); PubSubTemplate pubSubTemplate = context.getBean(PubSubTemplate.class); diff --git a/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java index 11ac73f5b0..1798213d3a 100644 --- a/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java +++ b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java @@ -83,25 +83,11 @@ public com.google.auth.Credentials getCredentials() { } } - @Bean - @ConditionalOnMissingBean - public SpannerOptions spannerOptions() { - return SpannerOptions.newBuilder().setProjectId(getProjectId()) - .setSessionPoolOption(SessionPoolOptions.newBuilder().setMaxSessions(10).build()) - .setCredentials(getCredentials()).build(); - } - @Bean public DatabaseId databaseId() { return DatabaseId.of(getProjectId(), this.instanceId, this.databaseName); } - @Bean - @ConditionalOnMissingBean - public Spanner spanner(SpannerOptions spannerOptions) { - return spannerOptions.getService(); - } - @Bean public DatabaseClient spannerDatabaseClient(Spanner spanner, DatabaseId databaseId) { return spanner.getDatabaseClient(databaseId); From dab420e03c34799178dfea5df4a86f30200fb529 Mon Sep 17 00:00:00 2001 From: dmitry-s Date: Fri, 15 May 2020 16:23:25 -0400 Subject: [PATCH 20/23] Apply suggestions from code review Co-authored-by: Elena Felder <41136058+elefeint@users.noreply.github.com> --- .../java/org/springframework/cloud/gcp/test/EmulatorDriver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java index 2990f0615a..45a2dbc74d 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java @@ -55,7 +55,7 @@ public class EmulatorDriver { // Reference to emulator instance, for cleanup. private Process emulatorProcess; - // Hostname for cleanup, should always be localhost. + // Hostname and port combination for cleanup; host should always be localhost. private String emulatorHostPort; /** From 4d9fd10c8fa163318899d95774184dc9e07c1bf2 Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Fri, 15 May 2020 16:31:08 -0400 Subject: [PATCH 21/23] PR comments --- .../spanner/test/IntegrationTestConfiguration.java | 2 -- .../org/springframework/cloud/gcp/test/Emulator.java | 4 +++- .../cloud/gcp/test/EmulatorDriver.java | 11 ++++++----- .../cloud/gcp/test/EmulatorRuntimeException.java | 2 +- .../spanner/SpannerEmulatorSpringConfiguration.java | 1 + 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java index 50e1062c72..10397cf986 100644 --- a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java +++ b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java @@ -105,7 +105,6 @@ public TradeRepositoryTransactionalService tradeRepositoryTransactionalService() } @Bean - @ConditionalOnMissingBean public SpannerOptions spannerOptions() { return SpannerOptions.newBuilder().setProjectId(getProjectId()) .setSessionPoolOption(SessionPoolOptions.newBuilder().setMaxSessions(10).build()) @@ -118,7 +117,6 @@ public DatabaseId databaseId() { } @Bean - @ConditionalOnMissingBean public Spanner spanner(SpannerOptions spannerOptions) { return spannerOptions.getService(); } diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/Emulator.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/Emulator.java index 944fe89c2e..6cb7318997 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/Emulator.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/Emulator.java @@ -32,12 +32,13 @@ public interface Emulator { /** * The list of command fragments that match the emulator processes to be killed. - * @param hostPort THe emulator host-port. + * @param hostPort The emulator host-port. */ String[] getKillCommandFragments(String hostPort); /** * Custom kill commands that need to run to stop the emulator. + * Each command is expected to be a string array. */ default String[][] getPostKillCommands() { return new String[0][0]; @@ -45,6 +46,7 @@ default String[][] getPostKillCommands() { /** * Custom start commands that need after the emulator is started. + * Each command is expected to be a string array. */ default String[][] getPostStartCommands() { return new String[0][0]; diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java index 2990f0615a..105ecd772c 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorDriver.java @@ -103,15 +103,16 @@ private void destroyGcloudEmulatorProcess() { } private void executeEmulatorKillCommands() { - // pre-kill - for (String[] command : emulator.getPostKillCommands()) { - runSystemCommand(command, false); - } - // kill for (String command : emulator.getKillCommandFragments(emulatorHostPort)) { killByCommand(command); } + + // post-kill + for (String[] command : emulator.getPostKillCommands()) { + runSystemCommand(command, false); + } + } private void killByCommand(String command) { diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java index cb5cee60e1..0344f8102b 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/EmulatorRuntimeException.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2019 the original author or authors. + * Copyright 2017-2020 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java index 7bd5477e3e..474bd1bec4 100644 --- a/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java +++ b/spring-cloud-gcp-test/src/main/java/org/springframework/cloud/gcp/test/spanner/SpannerEmulatorSpringConfiguration.java @@ -29,6 +29,7 @@ import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; + /** * SpannerEmulatorSpringConfiguration should be used instead of JUnit class rule because spring context tries to close the connection when it is being destroyed. * But the rule would already shut down the emulator by that time. That causes tests to hang. From 36cbc626cf517759356b2bf7ef73a6377a1a0bae Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Fri, 15 May 2020 16:35:09 -0400 Subject: [PATCH 22/23] PR comments --- spring-cloud-gcp-test/pom.xml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spring-cloud-gcp-test/pom.xml b/spring-cloud-gcp-test/pom.xml index 320b399454..cc3aa83bd5 100644 --- a/spring-cloud-gcp-test/pom.xml +++ b/spring-cloud-gcp-test/pom.xml @@ -22,10 +22,6 @@ true - - org.springframework - spring-jcl - org.awaitility awaitility @@ -38,6 +34,12 @@ true + + org.springframework + spring-jcl + true + + com.google.cloud From 10a10194769baf67e97c631540b7b94446e77d84 Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Mon, 18 May 2020 17:27:23 -0400 Subject: [PATCH 23/23] remove unused imports --- .../gcp/data/spanner/test/IntegrationTestConfiguration.java | 1 - .../cloud/gcp/test/SpannerTestConfiguration.java | 3 --- 2 files changed, 4 deletions(-) diff --git a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java index 10397cf986..d4389da4ae 100644 --- a/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java +++ b/spring-cloud-gcp-data-spanner/src/test/java/org/springframework/cloud/gcp/data/spanner/test/IntegrationTestConfiguration.java @@ -26,7 +26,6 @@ import com.google.cloud.spanner.SpannerOptions; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.cloud.gcp.core.Credentials; import org.springframework.cloud.gcp.core.DefaultCredentialsProvider; import org.springframework.cloud.gcp.core.DefaultGcpProjectIdProvider; diff --git a/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java index 1798213d3a..b501184d19 100644 --- a/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java +++ b/spring-cloud-gcp-test/src/test/java/org/springframework/cloud/gcp/test/SpannerTestConfiguration.java @@ -21,11 +21,8 @@ import com.google.cloud.spanner.DatabaseAdminClient; import com.google.cloud.spanner.DatabaseClient; import com.google.cloud.spanner.DatabaseId; -import com.google.cloud.spanner.SessionPoolOptions; import com.google.cloud.spanner.Spanner; -import com.google.cloud.spanner.SpannerOptions; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.cloud.gcp.core.Credentials; import org.springframework.cloud.gcp.core.DefaultCredentialsProvider; import org.springframework.cloud.gcp.core.DefaultGcpProjectIdProvider;