From f878aa9da74dfebc320ff264fc1c2666eefebfb2 Mon Sep 17 00:00:00 2001 From: xunliu Date: Mon, 14 Aug 2023 10:39:01 +0800 Subject: [PATCH 01/16] e2e integration test framework --- .github/workflows/integration.yml | 8 +- build.gradle.kts | 32 +- .../graviton/client/DTOConverters.java | 4 +- integration/build.gradle.kts | 32 ++ .../graviton/integration/e2e/MetalakeIT.java | 211 +++++++++++++ .../graviton/integration/util/AbstractIT.java | 71 +++++ .../integration/util/CommandExecutor.java | 63 ++++ .../integration/util/GravitonITUtils.java | 46 +++ .../integration/util/ProcessData.java | 286 ++++++++++++++++++ .../src/test/java/resources/log4j2.properties | 22 ++ settings.gradle.kts | 2 +- 11 files changed, 752 insertions(+), 25 deletions(-) create mode 100644 integration/build.gradle.kts create mode 100644 integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java create mode 100644 integration/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java create mode 100644 integration/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java create mode 100644 integration/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java create mode 100644 integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java create mode 100644 integration/src/test/java/resources/log4j2.properties diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f72e0789cba..70c0c8bd8d1 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -40,4 +40,10 @@ jobs: - name: Package Graviton run: | gradle build - gradle compileDistribution \ No newline at end of file + gradle compileDistribution + + - name: Graviton Integration Tests + run: | + ls -lt ./ + ls -lt distribution/package + gradle integrationTest \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index e4ad0d23da3..1a9f1a7cbd0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -40,8 +40,11 @@ subprojects { } tasks.configureEach { - useJUnitPlatform() - finalizedBy(tasks.getByName("jacocoTestReport")) + // Integration test module are tested sepatately + if (project.name != "integration") { + useJUnitPlatform() + finalizedBy(tasks.getByName("jacocoTestReport")) + } } tasks.withType { @@ -59,8 +62,10 @@ subprojects { tasks.withType { archiveFileName.set("${rootProject.name.lowercase(Locale.getDefault())}-${project.name}-$version.jar") - exclude("log4j2.properties") - exclude("test/**") + if (project.name != "integration") { + exclude("log4j2.properties") + exclude("test/**") + } } plugins.withType().configureEach { @@ -229,25 +234,10 @@ tasks { } } - // Print all dependencies of all subprojects, `./gradlew allDeps` - task("allDeps") { - doLast { - subprojects.forEach { project -> - println("Dependencies for project: ${project.name}") - project.configurations.forEach { configuration -> - configuration.allDependencies.forEach { dependency -> - println("- ${dependency.group}:${dependency.name}:${dependency.version}") - } - } - println() - } - } + task("integrationTest") { + dependsOn(":integration:integrationTest") } -// assemble { -// finalizedBy(assembleDistribution) -// } - clean { dependsOn(cleanDistribution) } diff --git a/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java b/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java index cbf4dad50f7..11695682c00 100644 --- a/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java +++ b/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java @@ -17,7 +17,7 @@ import com.datastrato.graviton.rel.SchemaChange; import com.datastrato.graviton.rel.TableChange; -class DTOConverters { +public class DTOConverters { private DTOConverters() {} static GravitonMetaLake toMetaLake(MetalakeDTO metalake, RESTClient client) { @@ -30,7 +30,7 @@ static GravitonMetaLake toMetaLake(MetalakeDTO metalake, RESTClient client) { .build(); } - static MetalakeUpdateRequest toMetalakeUpdateRequest(MetalakeChange change) { + public static MetalakeUpdateRequest toMetalakeUpdateRequest(MetalakeChange change) { if (change instanceof MetalakeChange.RenameMetalake) { return new MetalakeUpdateRequest.RenameMetalakeRequest( ((MetalakeChange.RenameMetalake) change).getNewName()); diff --git a/integration/build.gradle.kts b/integration/build.gradle.kts new file mode 100644 index 00000000000..adeb894d2f0 --- /dev/null +++ b/integration/build.gradle.kts @@ -0,0 +1,32 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ + +plugins { + `maven-publish` + id("java") + id("idea") + id("com.diffplug.spotless") +} + +dependencies { + implementation(project(":api")) + implementation(project(":common")) + implementation(project(":client-java")) + + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + testImplementation(libs.guava) + testImplementation(libs.commons.lang3) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.httpclient5) + testRuntimeOnly(libs.junit.jupiter.engine) +} + +tasks { + val integrationTest by creating(Test::class) { + useJUnitPlatform() + } +} \ No newline at end of file diff --git a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java new file mode 100644 index 00000000000..cd6f2c000d8 --- /dev/null +++ b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java @@ -0,0 +1,211 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.integration.e2e; + +import com.datastrato.graviton.MetalakeChange; +import com.datastrato.graviton.NameIdentifier; +import com.datastrato.graviton.client.DTOConverters; +import com.datastrato.graviton.client.ErrorHandlers; +import com.datastrato.graviton.client.GravitonMetaLake; +import com.datastrato.graviton.dto.MetalakeDTO; +import com.datastrato.graviton.dto.requests.MetalakeCreateRequest; +import com.datastrato.graviton.dto.requests.MetalakeUpdatesRequest; +import com.datastrato.graviton.dto.responses.DropResponse; +import com.datastrato.graviton.dto.responses.ErrorResponse; +import com.datastrato.graviton.dto.responses.MetalakeListResponse; +import com.datastrato.graviton.dto.responses.MetalakeResponse; +import com.datastrato.graviton.exceptions.MetalakeAlreadyExistsException; +import com.datastrato.graviton.exceptions.NoSuchMetalakeException; +import com.datastrato.graviton.integration.util.AbstractIT; +import com.datastrato.graviton.integration.util.GravitonITUtils; +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.apache.hc.core5.http.Method; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class MetalakeIT extends AbstractIT { + public static final Logger LOG = LoggerFactory.getLogger(MetalakeIT.class); + + public static String newMetalakeNameRESTful = GravitonITUtils.genRandomName(); + public static String newMetalakeNameAPI = GravitonITUtils.genRandomName(); + + static String reqPath = "api/metalakes"; + + static Consumer onError = ErrorHandlers.restErrorHandler(); + + @Order(1) + @Test + public void testCreateMetalakeRestful() { + MetalakeCreateRequest reqBody = + new MetalakeCreateRequest( + newMetalakeNameRESTful, "comment", ImmutableMap.of("key", "value")); + + MetalakeResponse successResponse = + doExecuteRequest(Method.POST, reqPath, reqBody, MetalakeResponse.class, onError, h -> {}); + LOG.info(successResponse.toString()); + Assertions.assertEquals(successResponse.getMetalake().name(), newMetalakeNameRESTful); + Assertions.assertEquals(successResponse.getMetalake().comment(), "comment"); + } + + @Order(2) + @Test + public void testListMetalakeRestful() { + MetalakeListResponse listResponse = + doExecuteRequest(Method.GET, reqPath, null, MetalakeListResponse.class, onError, h -> {}); + + List result = + Arrays.stream(listResponse.getMetalakes()) + .filter(metalakeDTO -> metalakeDTO.name().equals(newMetalakeNameRESTful)) + .collect(Collectors.toList()); + + Assertions.assertEquals(result.size(), 1); + + Assertions.assertEquals(result.get(0).name(), newMetalakeNameRESTful); + Assertions.assertEquals(result.get(0).comment(), "comment"); + } + + @Order(3) + @Test + public void testPutMetalakeRestful() { + String putMetalakeName = GravitonITUtils.genRandomName(); + String putReqPath = reqPath + File.separator + newMetalakeNameRESTful; + MetalakeChange[] changes1 = + new MetalakeChange[] { + MetalakeChange.rename(putMetalakeName), MetalakeChange.updateComment("newComment") + }; + + MetalakeUpdatesRequest reqUpdates1 = + new MetalakeUpdatesRequest( + Arrays.stream(changes1) + .map(DTOConverters::toMetalakeUpdateRequest) + .collect(Collectors.toList())); + + MetalakeResponse responseUpdates = + doExecuteRequest( + Method.PUT, putReqPath, reqUpdates1, MetalakeResponse.class, onError, h -> {}); + Assertions.assertEquals(responseUpdates.getMetalake().name(), putMetalakeName); + Assertions.assertEquals(responseUpdates.getMetalake().comment(), "newComment"); + + // Restore test record + putReqPath = reqPath + File.separator + putMetalakeName; + MetalakeChange[] changes2 = + new MetalakeChange[] {MetalakeChange.rename(newMetalakeNameRESTful)}; + + MetalakeUpdatesRequest reqUpdates2 = + new MetalakeUpdatesRequest( + Arrays.stream(changes2) + .map(DTOConverters::toMetalakeUpdateRequest) + .collect(Collectors.toList())); + + doExecuteRequest(Method.PUT, putReqPath, reqUpdates2, MetalakeResponse.class, onError, h -> {}); + } + + @Order(4) + @Test + public void testDropMetalakeRestful() { + DropResponse response = + doExecuteRequest( + Method.DELETE, + reqPath + File.separator + newMetalakeNameRESTful, + null, + DropResponse.class, + onError, + h -> {}); + Assertions.assertEquals(response.dropped(), true); + } + + @Order(5) + @Test + public void testCreateMetalakeAPI() { + GravitonMetaLake metaLake = + client.createMetalake( + NameIdentifier.parse(newMetalakeNameAPI), "comment", Collections.emptyMap()); + Assertions.assertEquals(newMetalakeNameAPI, metaLake.name()); + Assertions.assertEquals("comment", metaLake.comment()); + Assertions.assertEquals("graviton", metaLake.auditInfo().creator()); + + // Test metalake name already exists + Throwable excep = + Assertions.assertThrows( + MetalakeAlreadyExistsException.class, + () -> + client.createMetalake( + NameIdentifier.parse(newMetalakeNameAPI), "comment", Collections.emptyMap())); + Assertions.assertTrue(excep.getMessage().contains("already exists")); + } + + @Order(6) + @Test + public void testListMetalakeAPI() { + GravitonMetaLake[] metaLakes = client.listMetalakes(); + List result = + Arrays.stream(metaLakes) + .filter(metalakeDTO -> metalakeDTO.name().equals(newMetalakeNameAPI)) + .collect(Collectors.toList()); + + Assertions.assertEquals(result.size(), 1); + } + + @Order(7) + @Test + public void testLoadMetalakeAPI() { + GravitonMetaLake metaLake = client.loadMetalake(NameIdentifier.of(newMetalakeNameAPI)); + Assertions.assertEquals(metaLake.name(), newMetalakeNameAPI); + } + + @Order(8) + @Test + public void testAlterMetalakeAPI() { + String alterMetalakeName = GravitonITUtils.genRandomName(); + + MetalakeChange[] changes1 = + new MetalakeChange[] { + MetalakeChange.rename(alterMetalakeName), MetalakeChange.updateComment("newComment") + }; + GravitonMetaLake metaLake = + client.alterMetalake(NameIdentifier.of(newMetalakeNameAPI), changes1); + Assertions.assertEquals(alterMetalakeName, metaLake.name()); + Assertions.assertEquals("newComment", metaLake.comment()); + Assertions.assertEquals("graviton", metaLake.auditInfo().creator()); + + // Test return not found + Throwable excep = + Assertions.assertThrows( + NoSuchMetalakeException.class, + () -> client.alterMetalake(NameIdentifier.of(newMetalakeNameAPI + "mock"), changes1)); + Assertions.assertTrue(excep.getMessage().contains("does not exist")); + + // Restore test record + MetalakeChange[] changes2 = new MetalakeChange[] {MetalakeChange.rename(newMetalakeNameAPI)}; + client.alterMetalake(NameIdentifier.of(alterMetalakeName), changes2); + } + + @Order(9) + @Test + public void testDropMetalakeAPI() { + Assertions.assertTrue(client.dropMetalake(NameIdentifier.of(newMetalakeNameAPI))); + + // Test illegal metalake name identifier + Throwable excep1 = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + client.dropMetalake( + NameIdentifier.parse(newMetalakeNameAPI + "." + newMetalakeNameAPI))); + Assertions.assertTrue(excep1.getMessage().contains("namespace should be empty")); + } +} diff --git a/integration/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java b/integration/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java new file mode 100644 index 00000000000..0fe5b79ca5c --- /dev/null +++ b/integration/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java @@ -0,0 +1,71 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.integration.util; + +import com.datastrato.graviton.client.GravitonClient; +import com.datastrato.graviton.client.HTTPClient; +import com.datastrato.graviton.client.RESTClient; +import com.datastrato.graviton.dto.responses.ErrorResponse; +import com.datastrato.graviton.rest.RESTRequest; +import com.datastrato.graviton.rest.RESTResponse; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.Map; +import java.util.function.Consumer; +import org.apache.hc.core5.http.Method; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AbstractIT { + public static final Logger LOG = LoggerFactory.getLogger(AbstractIT.class); + private static RESTClient restClient; + + private static final int port = 8090; + private static final String URI = String.format("http://127.0.0.1:%d", port); + protected static GravitonClient client; + + @BeforeAll + public static void startUp() { + LOG.info("Starting up Graviton Server"); + GravitonITUtils.startGravitonServer(); + restClient = HTTPClient.builder(ImmutableMap.of()).uri(URI).build(); + + client = GravitonClient.builder("http://127.0.0.1:" + port).build(); + } + + @AfterAll + public static void tearDown() throws IOException { + restClient.close(); + GravitonITUtils.stopGravitonServer(); + LOG.info("Tearing down Graviton Server"); + } + + protected static T doExecuteRequest( + Method method, + String path, + RESTRequest body, + Class responseType, + Consumer onError, + Consumer> responseHeaders) { + Map headers = ImmutableMap.of(); + switch (method) { + case POST: + return restClient.post(path, body, responseType, headers, onError, responseHeaders); + case PUT: + return restClient.put(path, body, responseType, headers, onError, responseHeaders); + case GET: + return restClient.get(path, responseType, headers, onError); + case HEAD: + restClient.head(path, headers, onError); + return null; + case DELETE: + return restClient.delete(path, responseType, () -> headers, onError); + default: + throw new IllegalArgumentException(String.format("Invalid method: %s", method)); + } + } +} diff --git a/integration/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java b/integration/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java new file mode 100644 index 00000000000..6aeac9c1fba --- /dev/null +++ b/integration/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.integration.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// zeppelin-integration/src/test/java/org/apache/zeppelin/CommandExecutor.java +public class CommandExecutor { + public static final Logger LOG = LoggerFactory.getLogger(CommandExecutor.class); + + public enum IGNORE_ERRORS { + TRUE, + FALSE + } + + public static int NORMAL_EXIT = 0; + + private static IGNORE_ERRORS DEFAULT_BEHAVIOUR_ON_ERRORS = IGNORE_ERRORS.TRUE; + + public static Object executeCommandLocalHost( + String[] command, + boolean printToConsole, + ProcessData.TypesOfData type, + IGNORE_ERRORS ignore_errors) { + List subCommandsAsList = new ArrayList<>(Arrays.asList(command)); + String mergedCommand = StringUtils.join(subCommandsAsList, " "); + + LOG.info("Sending command \"" + mergedCommand + "\" to localhost"); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + Process process = null; + try { + process = processBuilder.start(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ProcessData dataOfProcess = new ProcessData(process, printToConsole); + Object outputOfProcess = dataOfProcess.getData(type); + int exit_code = dataOfProcess.getExitCodeValue(); + + if (!printToConsole) LOG.trace(outputOfProcess.toString()); + else LOG.debug(outputOfProcess.toString()); + if (ignore_errors == IGNORE_ERRORS.FALSE && exit_code != NORMAL_EXIT) { + LOG.error(String.format("Command '%s' failed with exit code %s", mergedCommand, exit_code)); + } + return outputOfProcess; + } + + public static Object executeCommandLocalHost( + String command, boolean printToConsole, ProcessData.TypesOfData type) { + return executeCommandLocalHost( + new String[] {"bash", "-c", command}, printToConsole, type, DEFAULT_BEHAVIOUR_ON_ERRORS); + } +} diff --git a/integration/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java b/integration/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java new file mode 100644 index 00000000000..8761e19d49e --- /dev/null +++ b/integration/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java @@ -0,0 +1,46 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.integration.util; + +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GravitonITUtils { + public static final Logger LOG = LoggerFactory.getLogger(GravitonITUtils.class); + + public static void startGravitonServer() { + CommandExecutor.executeCommandLocalHost( + "../distribution/package/bin/graviton.sh start", false, ProcessData.TypesOfData.OUTPUT); + // wait for server to start. + sleep(100, false); + } + + public static void stopGravitonServer() { + CommandExecutor.executeCommandLocalHost( + "../distribution/package/bin/graviton.sh stop", false, ProcessData.TypesOfData.OUTPUT); + // wait for server to stop. + sleep(100, false); + } + + public static void sleep(long millis, boolean logOutput) { + if (logOutput) { + LOG.info("Starting sleeping for " + (millis / 1000) + " seconds..."); + LOG.info("Caller: " + Thread.currentThread().getStackTrace()[2]); + } + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + LOG.error("Exception in WebDriverManager while getWebDriver ", e); + } + if (logOutput) { + LOG.info("Finished."); + } + } + + public static String genRandomName() { + return UUID.randomUUID().toString().replace("-", ""); + } +} diff --git a/integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java b/integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java new file mode 100644 index 00000000000..6ae9e144434 --- /dev/null +++ b/integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java @@ -0,0 +1,286 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.integration.util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// zeppelin-integration/src/test/java/org/apache/zeppelin/ProcessData.java +public class ProcessData { + public enum TypesOfData { + OUTPUT, + ERROR, + EXIT_CODE, + STREAMS_MERGED, + PROCESS_DATA_OBJECT + } + + public static final Logger LOG = LoggerFactory.getLogger(ProcessData.class); + + private Process checked_process; + private boolean printToConsole = false; + + public ProcessData( + Process connected_process, boolean printToConsole, int silenceTimeout, TimeUnit timeUnit) { + this.checked_process = connected_process; + this.printToConsole = printToConsole; + this.silenceTimeout = TimeUnit.MILLISECONDS.convert(silenceTimeout, timeUnit); + } + + public ProcessData(Process connected_process, boolean printToConsole, int silenceTimeoutSec) { + this.checked_process = connected_process; + this.printToConsole = printToConsole; + this.silenceTimeout = TimeUnit.MILLISECONDS.convert(silenceTimeoutSec, TimeUnit.SECONDS); + } + + public ProcessData(Process connected_process, boolean printToConsole) { + this.checked_process = connected_process; + this.printToConsole = printToConsole; + } + + public ProcessData(Process connected_process) { + this.checked_process = connected_process; + this.printToConsole = true; + } + + boolean returnCodeRetrieved = false; + + private String outPutStream = null; + private String errorStream = null; + private int returnCode; + private long silenceTimeout = 10 * 60 * 1000; + private final long unconditionalExitDelayMinutes = 30; + + public static boolean isRunning(Process process) { + try { + process.exitValue(); + return false; + } catch (IllegalThreadStateException e) { + return true; + } + } + + public Object getData(TypesOfData type) { + // TODO get rid of Pseudo-terminal will not be allocated because stdin is not a terminal. + switch (type) { + case OUTPUT: + { + return this.getOutPutStream(); + } + case ERROR: + { + return this.getErrorStream(); + } + case EXIT_CODE: + { + return this.getExitCodeValue(); + } + case STREAMS_MERGED: + { + return this.getOutPutStream() + "\n" + this.getErrorStream(); + } + case PROCESS_DATA_OBJECT: + { + this.getErrorStream(); + return this; + } + default: + { + throw new IllegalArgumentException("Data Type " + type + " not supported yet!"); + } + } + } + + public int getExitCodeValue() { + try { + if (!returnCodeRetrieved) { + this.checked_process.waitFor(); + this.returnCode = this.checked_process.exitValue(); + this.returnCodeRetrieved = true; + this.checked_process.destroy(); + } + } catch (Exception inter) { + throw new RuntimeException( + "Couldn't finish waiting for process " + this.checked_process + " termination", inter); + } + return this.returnCode; + } + + public String getOutPutStream() { + if (this.outPutStream == null) { + try { + buildOutputAndErrorStreamData(); + } catch (Exception e) { + throw new RuntimeException( + "Couldn't retrieve Output Stream data from process: " + this.checked_process.toString(), + e); + } + } + this.outPutStream = + this.outPutStream.replace( + "Pseudo-terminal will not be allocated because stdin is not a terminal.", ""); + this.errorStream = + this.errorStream.replace( + "Pseudo-terminal will not be allocated because stdin is not a terminal.", ""); + return this.outPutStream; + } + + public String getErrorStream() { + if (this.errorStream == null) { + try { + buildOutputAndErrorStreamData(); + } catch (Exception e) { + throw new RuntimeException( + "Couldn't retrieve Error Stream data from process: " + this.checked_process.toString(), + e); + } + } + this.outPutStream = + this.outPutStream.replace( + "Pseudo-terminal will not be allocated because stdin is not a terminal.", ""); + this.errorStream = + this.errorStream.replace( + "Pseudo-terminal will not be allocated because stdin is not a terminal.", ""); + return this.errorStream; + } + + public String toString() { + return String.format("[OUTPUT STREAM]\n%s\n", this.outPutStream) + + String.format("[ERROR STREAM]\n%s\n", this.errorStream) + + String.format("[EXIT CODE]\n%d", this.returnCode); + } + + private void buildOutputAndErrorStreamData() throws IOException { + StringBuilder sbInStream = new StringBuilder(); + StringBuilder sbErrorStream = new StringBuilder(); + + try { + InputStream in = this.checked_process.getInputStream(); + InputStream inErrors = this.checked_process.getErrorStream(); + BufferedReader inReader = new BufferedReader(new InputStreamReader(in)); + BufferedReader inReaderErrors = new BufferedReader(new InputStreamReader(inErrors)); + LOG.trace( + "Started retrieving data from streams of attached process: " + this.checked_process); + + // Store start time to be able to finish method if command hangs + long lastStreamDataTime = System.currentTimeMillis(); + + // Stop after 'unconditionalExitDelayMinutes' even if process is alive and sending output + long unconditionalExitTime = + System.currentTimeMillis() + + TimeUnit.MILLISECONDS.convert(unconditionalExitDelayMinutes, TimeUnit.MINUTES); + final int BUFFER_LEN = 300; + char[] charBuffer = + new char[BUFFER_LEN]; // Use char buffer to read output, size can be tuned. + boolean outputProduced = true; // Flag to check if previous iteration produced any output + + // Continue if process is alive or some output was produced on previous iteration + // and there may be still some data to read. + while (isRunning(this.checked_process) || outputProduced) { + outputProduced = false; + + // Some local commands can exit fast, but immediate stream reading will give no output and + // after iteration, 'while' condition will be false so we will not read out any output while + // it is still there, just need to wait for some time for it to appear in streams. + GravitonITUtils.sleep(100, false); + + StringBuilder tempSB = new StringBuilder(); + while (inReader.ready()) { + tempSB.setLength(0); // clean temporary StringBuilder + int readCount = + inReader.read(charBuffer, 0, BUFFER_LEN); // read up to 'BUFFER_LEN' chars to buffer + if (readCount < 1) { // if nothing read or error occurred + break; + } + tempSB.append(charBuffer, 0, readCount); + + sbInStream.append(tempSB); + if (tempSB.length() > 0) { + // set flag to know that we read something and there may be moire data, even if process + // already exited + outputProduced = true; + } + + // remember last time data was read from streams to be sure we are not looping infinitely + lastStreamDataTime = System.currentTimeMillis(); + } + + tempSB = new StringBuilder(); // Same, but for error stream + while (inReaderErrors.ready()) { + tempSB.setLength(0); + int readCount = inReaderErrors.read(charBuffer, 0, BUFFER_LEN); + if (readCount < 1) { + break; + } + tempSB.append(charBuffer, 0, readCount); + sbErrorStream.append(tempSB); + if (tempSB.length() > 0) { + outputProduced = true; + String temp = new String(tempSB); + temp = + temp.replaceAll( + "Pseudo-terminal will not be allocated because stdin is not a terminal.", ""); + // TODO : error stream output need to be improved, because it outputs downloading + // information. + if (printToConsole) { + if (!temp.trim().equals("")) { + if (temp.toLowerCase().contains("error") || temp.toLowerCase().contains("failed")) { + LOG.warn(temp.trim()); + } else { + LOG.debug(temp.trim()); + } + } + } + } + lastStreamDataTime = System.currentTimeMillis(); + } + + // Exit if silenceTimeout ms has passed from last stream read. Means process is alive but + // not sending any data. + if ((System.currentTimeMillis() - lastStreamDataTime > silenceTimeout) + || + // Exit unconditionally - guards against alive process continuously sending data. + (System.currentTimeMillis() > unconditionalExitTime)) { + + LOG.info( + "Conditions: " + + (System.currentTimeMillis() - lastStreamDataTime > silenceTimeout) + + " " + + (System.currentTimeMillis() > unconditionalExitTime)); + this.checked_process.destroy(); + try { + if ((System.currentTimeMillis() > unconditionalExitTime)) { + LOG.error( + "!Unconditional exit occured!\nsome process hag up for more than " + + unconditionalExitDelayMinutes + + " minutes."); + } + LOG.error("!##################################!"); + StringWriter sw = new StringWriter(); + Exception e = new Exception("Exited from buildOutputAndErrorStreamData by timeout"); + e.printStackTrace(new PrintWriter(sw)); // Get stack trace + LOG.error(String.valueOf(e), e); + } catch (Exception ignore) { + LOG.info("Exception in ProcessData while buildOutputAndErrorStreamData ", ignore); + } + break; + } + } + + in.close(); + inErrors.close(); + } finally { + this.outPutStream = sbInStream.toString(); + this.errorStream = sbErrorStream.toString(); + } + } +} diff --git a/integration/src/test/java/resources/log4j2.properties b/integration/src/test/java/resources/log4j2.properties new file mode 100644 index 00000000000..d4b8355d896 --- /dev/null +++ b/integration/src/test/java/resources/log4j2.properties @@ -0,0 +1,22 @@ +# +# Copyright 2023 Datastrato. +# This software is licensed under the Apache License version 2. +# + +# Set to debug or trace if log4j initialization is failing +status = warn + +# Name of the configuration +name = ConsoleLogConfigDemo + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Root logger level +rootLogger.level = debug + +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = consoleLogger \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 78ec653c0e5..0b6c37de384 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -4,4 +4,4 @@ */ rootProject.name = "graviton" -include("api", "client-java", "common", "core", "meta", "server", "catalog-hive") +include("api", "client-java", "common", "core", "meta", "server", "catalog-hive", "integration") From a0463cdf93796ae8662b562919b8fbc66a6108f9 Mon Sep 17 00:00:00 2001 From: xunliu Date: Mon, 14 Aug 2023 16:56:59 +0800 Subject: [PATCH 02/16] Add apache license --- .../integration/util/CommandExecutor.java | 16 ++++++++++++++-- .../graviton/integration/util/ProcessData.java | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/integration/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java b/integration/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java index 6aeac9c1fba..084e1129165 100644 --- a/integration/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java +++ b/integration/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java @@ -1,6 +1,18 @@ /* - * Copyright 2023 Datastrato. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.datastrato.graviton.integration.util; diff --git a/integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java b/integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java index 6ae9e144434..6edd5ac2212 100644 --- a/integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java +++ b/integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java @@ -1,6 +1,18 @@ /* - * Copyright 2023 Datastrato. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.datastrato.graviton.integration.util; From db86974a71a4e5ea64f3c7cdeeb07d6cd63acccf Mon Sep 17 00:00:00 2001 From: xunliu Date: Mon, 14 Aug 2023 22:17:51 +0800 Subject: [PATCH 03/16] fix --- .../com/datastrato/graviton/integration/e2e/MetalakeIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java index cd6f2c000d8..4f86a7961e6 100644 --- a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java +++ b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java @@ -58,7 +58,7 @@ public void testCreateMetalakeRestful() { doExecuteRequest(Method.POST, reqPath, reqBody, MetalakeResponse.class, onError, h -> {}); LOG.info(successResponse.toString()); Assertions.assertEquals(successResponse.getMetalake().name(), newMetalakeNameRESTful); - Assertions.assertEquals(successResponse.getMetalake().comment(), "comment"); + Assertions.assertEquals("comment", successResponse.getMetalake().comment()); } @Order(2) @@ -75,7 +75,7 @@ public void testListMetalakeRestful() { Assertions.assertEquals(result.size(), 1); Assertions.assertEquals(result.get(0).name(), newMetalakeNameRESTful); - Assertions.assertEquals(result.get(0).comment(), "comment"); + Assertions.assertEquals("comment", result.get(0).comment()); } @Order(3) @@ -98,7 +98,7 @@ public void testPutMetalakeRestful() { doExecuteRequest( Method.PUT, putReqPath, reqUpdates1, MetalakeResponse.class, onError, h -> {}); Assertions.assertEquals(responseUpdates.getMetalake().name(), putMetalakeName); - Assertions.assertEquals(responseUpdates.getMetalake().comment(), "newComment"); + Assertions.assertEquals("newComment", responseUpdates.getMetalake().comment()); // Restore test record putReqPath = reqPath + File.separator + putMetalakeName; From 454b9110a842686b6cc04a26247fa630416422a4 Mon Sep 17 00:00:00 2001 From: xunliu Date: Tue, 15 Aug 2023 09:49:28 +0800 Subject: [PATCH 04/16] Create default metalake before all test --- .../graviton/integration/e2e/MetalakeIT.java | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java index 4f86a7961e6..2039cae8f6e 100644 --- a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java +++ b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java @@ -22,13 +22,16 @@ import com.datastrato.graviton.integration.util.GravitonITUtils; import com.google.common.collect.ImmutableMap; import java.io.File; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.function.Consumer; import java.util.stream.Collectors; import org.apache.hc.core5.http.Method; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; @@ -47,21 +50,39 @@ public class MetalakeIT extends AbstractIT { static Consumer onError = ErrorHandlers.restErrorHandler(); - @Order(1) - @Test - public void testCreateMetalakeRestful() { + public static MetalakeResponse createDefMetalake(String metalakeName) { MetalakeCreateRequest reqBody = - new MetalakeCreateRequest( - newMetalakeNameRESTful, "comment", ImmutableMap.of("key", "value")); + new MetalakeCreateRequest(metalakeName, "comment", ImmutableMap.of("key", "value")); MetalakeResponse successResponse = doExecuteRequest(Method.POST, reqPath, reqBody, MetalakeResponse.class, onError, h -> {}); - LOG.info(successResponse.toString()); + return successResponse; + } + + @BeforeAll + public static void startUp() { + AbstractIT.startUp(); + + MetalakeResponse successResponse = createDefMetalake(newMetalakeNameRESTful); Assertions.assertEquals(successResponse.getMetalake().name(), newMetalakeNameRESTful); - Assertions.assertEquals("comment", successResponse.getMetalake().comment()); } - @Order(2) + @AfterAll + public static void tearDown() throws IOException { + DropResponse response = + doExecuteRequest( + Method.DELETE, + reqPath + File.separator + newMetalakeNameRESTful, + null, + DropResponse.class, + onError, + h -> {}); + Assertions.assertEquals(response.dropped(), true); + + AbstractIT.tearDown(); + } + + @Order(1) @Test public void testListMetalakeRestful() { MetalakeListResponse listResponse = @@ -78,7 +99,7 @@ public void testListMetalakeRestful() { Assertions.assertEquals("comment", result.get(0).comment()); } - @Order(3) + @Order(2) @Test public void testPutMetalakeRestful() { String putMetalakeName = GravitonITUtils.genRandomName(); @@ -116,20 +137,6 @@ public void testPutMetalakeRestful() { @Order(4) @Test - public void testDropMetalakeRestful() { - DropResponse response = - doExecuteRequest( - Method.DELETE, - reqPath + File.separator + newMetalakeNameRESTful, - null, - DropResponse.class, - onError, - h -> {}); - Assertions.assertEquals(response.dropped(), true); - } - - @Order(5) - @Test public void testCreateMetalakeAPI() { GravitonMetaLake metaLake = client.createMetalake( @@ -148,7 +155,7 @@ public void testCreateMetalakeAPI() { Assertions.assertTrue(excep.getMessage().contains("already exists")); } - @Order(6) + @Order(5) @Test public void testListMetalakeAPI() { GravitonMetaLake[] metaLakes = client.listMetalakes(); @@ -160,14 +167,14 @@ public void testListMetalakeAPI() { Assertions.assertEquals(result.size(), 1); } - @Order(7) + @Order(6) @Test public void testLoadMetalakeAPI() { GravitonMetaLake metaLake = client.loadMetalake(NameIdentifier.of(newMetalakeNameAPI)); Assertions.assertEquals(metaLake.name(), newMetalakeNameAPI); } - @Order(8) + @Order(7) @Test public void testAlterMetalakeAPI() { String alterMetalakeName = GravitonITUtils.genRandomName(); @@ -194,7 +201,7 @@ public void testAlterMetalakeAPI() { client.alterMetalake(NameIdentifier.of(alterMetalakeName), changes2); } - @Order(9) + @Order(8) @Test public void testDropMetalakeAPI() { Assertions.assertTrue(client.dropMetalake(NameIdentifier.of(newMetalakeNameAPI))); From 80fc8a1b456acab2ba34f87658ff665bc9fc2201 Mon Sep 17 00:00:00 2001 From: xunliu Date: Tue, 15 Aug 2023 17:41:31 +0800 Subject: [PATCH 05/16] auto call AbstractIT.startUp() functions. --- .../datastrato/graviton/integration/e2e/MetalakeIT.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java index 2039cae8f6e..6458b482fb7 100644 --- a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java +++ b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java @@ -60,15 +60,13 @@ public static MetalakeResponse createDefMetalake(String metalakeName) { } @BeforeAll - public static void startUp() { - AbstractIT.startUp(); - + private static void startup() { MetalakeResponse successResponse = createDefMetalake(newMetalakeNameRESTful); Assertions.assertEquals(successResponse.getMetalake().name(), newMetalakeNameRESTful); } @AfterAll - public static void tearDown() throws IOException { + private static void teardown() throws IOException { DropResponse response = doExecuteRequest( Method.DELETE, @@ -78,8 +76,6 @@ public static void tearDown() throws IOException { onError, h -> {}); Assertions.assertEquals(response.dropped(), true); - - AbstractIT.tearDown(); } @Order(1) From 0aeabb55c6f7efb764a45a340419e1d962c87afd Mon Sep 17 00:00:00 2001 From: xunliu Date: Tue, 15 Aug 2023 22:00:32 +0800 Subject: [PATCH 06/16] Remove LOG --- .../com/datastrato/graviton/integration/e2e/MetalakeIT.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java index 6458b482fb7..837d72fd3c9 100644 --- a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java +++ b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java @@ -36,13 +36,9 @@ import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class MetalakeIT extends AbstractIT { - public static final Logger LOG = LoggerFactory.getLogger(MetalakeIT.class); - public static String newMetalakeNameRESTful = GravitonITUtils.genRandomName(); public static String newMetalakeNameAPI = GravitonITUtils.genRandomName(); From 068ca23b251e9a55983f25e16e72442f79ed51e6 Mon Sep 17 00:00:00 2001 From: xunliu Date: Tue, 15 Aug 2023 22:03:36 +0800 Subject: [PATCH 07/16] Better var name --- .../graviton/integration/e2e/MetalakeIT.java | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java index 837d72fd3c9..7930b3e4dec 100644 --- a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java +++ b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java @@ -39,8 +39,8 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class MetalakeIT extends AbstractIT { - public static String newMetalakeNameRESTful = GravitonITUtils.genRandomName(); - public static String newMetalakeNameAPI = GravitonITUtils.genRandomName(); + public static String metalakeName_RESTful = GravitonITUtils.genRandomName(); + public static String metalakeName_API = GravitonITUtils.genRandomName(); static String reqPath = "api/metalakes"; @@ -57,8 +57,8 @@ public static MetalakeResponse createDefMetalake(String metalakeName) { @BeforeAll private static void startup() { - MetalakeResponse successResponse = createDefMetalake(newMetalakeNameRESTful); - Assertions.assertEquals(successResponse.getMetalake().name(), newMetalakeNameRESTful); + MetalakeResponse successResponse = createDefMetalake(metalakeName_RESTful); + Assertions.assertEquals(successResponse.getMetalake().name(), metalakeName_RESTful); } @AfterAll @@ -66,7 +66,7 @@ private static void teardown() throws IOException { DropResponse response = doExecuteRequest( Method.DELETE, - reqPath + File.separator + newMetalakeNameRESTful, + reqPath + File.separator + metalakeName_RESTful, null, DropResponse.class, onError, @@ -82,12 +82,12 @@ public void testListMetalakeRestful() { List result = Arrays.stream(listResponse.getMetalakes()) - .filter(metalakeDTO -> metalakeDTO.name().equals(newMetalakeNameRESTful)) + .filter(metalakeDTO -> metalakeDTO.name().equals(metalakeName_RESTful)) .collect(Collectors.toList()); Assertions.assertEquals(result.size(), 1); - Assertions.assertEquals(result.get(0).name(), newMetalakeNameRESTful); + Assertions.assertEquals(result.get(0).name(), metalakeName_RESTful); Assertions.assertEquals("comment", result.get(0).comment()); } @@ -95,7 +95,7 @@ public void testListMetalakeRestful() { @Test public void testPutMetalakeRestful() { String putMetalakeName = GravitonITUtils.genRandomName(); - String putReqPath = reqPath + File.separator + newMetalakeNameRESTful; + String putReqPath = reqPath + File.separator + metalakeName_RESTful; MetalakeChange[] changes1 = new MetalakeChange[] { MetalakeChange.rename(putMetalakeName), MetalakeChange.updateComment("newComment") @@ -115,8 +115,7 @@ public void testPutMetalakeRestful() { // Restore test record putReqPath = reqPath + File.separator + putMetalakeName; - MetalakeChange[] changes2 = - new MetalakeChange[] {MetalakeChange.rename(newMetalakeNameRESTful)}; + MetalakeChange[] changes2 = new MetalakeChange[] {MetalakeChange.rename(metalakeName_RESTful)}; MetalakeUpdatesRequest reqUpdates2 = new MetalakeUpdatesRequest( @@ -132,8 +131,8 @@ public void testPutMetalakeRestful() { public void testCreateMetalakeAPI() { GravitonMetaLake metaLake = client.createMetalake( - NameIdentifier.parse(newMetalakeNameAPI), "comment", Collections.emptyMap()); - Assertions.assertEquals(newMetalakeNameAPI, metaLake.name()); + NameIdentifier.parse(metalakeName_API), "comment", Collections.emptyMap()); + Assertions.assertEquals(metalakeName_API, metaLake.name()); Assertions.assertEquals("comment", metaLake.comment()); Assertions.assertEquals("graviton", metaLake.auditInfo().creator()); @@ -143,7 +142,7 @@ public void testCreateMetalakeAPI() { MetalakeAlreadyExistsException.class, () -> client.createMetalake( - NameIdentifier.parse(newMetalakeNameAPI), "comment", Collections.emptyMap())); + NameIdentifier.parse(metalakeName_API), "comment", Collections.emptyMap())); Assertions.assertTrue(excep.getMessage().contains("already exists")); } @@ -153,7 +152,7 @@ public void testListMetalakeAPI() { GravitonMetaLake[] metaLakes = client.listMetalakes(); List result = Arrays.stream(metaLakes) - .filter(metalakeDTO -> metalakeDTO.name().equals(newMetalakeNameAPI)) + .filter(metalakeDTO -> metalakeDTO.name().equals(metalakeName_API)) .collect(Collectors.toList()); Assertions.assertEquals(result.size(), 1); @@ -162,8 +161,8 @@ public void testListMetalakeAPI() { @Order(6) @Test public void testLoadMetalakeAPI() { - GravitonMetaLake metaLake = client.loadMetalake(NameIdentifier.of(newMetalakeNameAPI)); - Assertions.assertEquals(metaLake.name(), newMetalakeNameAPI); + GravitonMetaLake metaLake = client.loadMetalake(NameIdentifier.of(metalakeName_API)); + Assertions.assertEquals(metaLake.name(), metalakeName_API); } @Order(7) @@ -175,8 +174,7 @@ public void testAlterMetalakeAPI() { new MetalakeChange[] { MetalakeChange.rename(alterMetalakeName), MetalakeChange.updateComment("newComment") }; - GravitonMetaLake metaLake = - client.alterMetalake(NameIdentifier.of(newMetalakeNameAPI), changes1); + GravitonMetaLake metaLake = client.alterMetalake(NameIdentifier.of(metalakeName_API), changes1); Assertions.assertEquals(alterMetalakeName, metaLake.name()); Assertions.assertEquals("newComment", metaLake.comment()); Assertions.assertEquals("graviton", metaLake.auditInfo().creator()); @@ -185,18 +183,18 @@ public void testAlterMetalakeAPI() { Throwable excep = Assertions.assertThrows( NoSuchMetalakeException.class, - () -> client.alterMetalake(NameIdentifier.of(newMetalakeNameAPI + "mock"), changes1)); + () -> client.alterMetalake(NameIdentifier.of(metalakeName_API + "mock"), changes1)); Assertions.assertTrue(excep.getMessage().contains("does not exist")); // Restore test record - MetalakeChange[] changes2 = new MetalakeChange[] {MetalakeChange.rename(newMetalakeNameAPI)}; + MetalakeChange[] changes2 = new MetalakeChange[] {MetalakeChange.rename(metalakeName_API)}; client.alterMetalake(NameIdentifier.of(alterMetalakeName), changes2); } @Order(8) @Test public void testDropMetalakeAPI() { - Assertions.assertTrue(client.dropMetalake(NameIdentifier.of(newMetalakeNameAPI))); + Assertions.assertTrue(client.dropMetalake(NameIdentifier.of(metalakeName_API))); // Test illegal metalake name identifier Throwable excep1 = @@ -204,7 +202,7 @@ public void testDropMetalakeAPI() { IllegalArgumentException.class, () -> client.dropMetalake( - NameIdentifier.parse(newMetalakeNameAPI + "." + newMetalakeNameAPI))); + NameIdentifier.parse(metalakeName_API + "." + metalakeName_API))); Assertions.assertTrue(excep1.getMessage().contains("namespace should be empty")); } } From 765b20c092f8cb19ef198035b5fb91da06f07ba3 Mon Sep 17 00:00:00 2001 From: xunliu Date: Mon, 28 Aug 2023 14:22:13 +0800 Subject: [PATCH 08/16] change module name to integration-test --- build.gradle.kts | 6 +- .../graviton/client/DTOConverters.java | 2 +- .../build.gradle.kts | 1 + .../graviton/integration/e2e/MetalakeIT.java | 112 ++++++++++ .../graviton/integration/util/AbstractIT.java | 33 +++ .../integration/util/CommandExecutor.java | 0 .../integration/util/GravitonITUtils.java | 8 +- .../integration/util/ProcessData.java | 35 ++- .../src/test/java/resources/log4j2.properties | 0 .../graviton/integration/e2e/MetalakeIT.java | 208 ------------------ .../graviton/integration/util/AbstractIT.java | 71 ------ settings.gradle.kts | 2 +- 12 files changed, 174 insertions(+), 304 deletions(-) rename {integration => integration-test}/build.gradle.kts (90%) create mode 100644 integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java create mode 100644 integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java rename {integration => integration-test}/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java (100%) rename {integration => integration-test}/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java (83%) rename {integration => integration-test}/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java (91%) rename {integration => integration-test}/src/test/java/resources/log4j2.properties (100%) delete mode 100644 integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java delete mode 100644 integration/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java diff --git a/build.gradle.kts b/build.gradle.kts index 1a9f1a7cbd0..3b5bdb65b95 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -41,7 +41,7 @@ subprojects { tasks.configureEach { // Integration test module are tested sepatately - if (project.name != "integration") { + if (project.name != "integration-test") { useJUnitPlatform() finalizedBy(tasks.getByName("jacocoTestReport")) } @@ -62,7 +62,7 @@ subprojects { tasks.withType { archiveFileName.set("${rootProject.name.lowercase(Locale.getDefault())}-${project.name}-$version.jar") - if (project.name != "integration") { + if (project.name != "integration-test") { exclude("log4j2.properties") exclude("test/**") } @@ -235,7 +235,7 @@ tasks { } task("integrationTest") { - dependsOn(":integration:integrationTest") + dependsOn(":integration-test:integrationTest") } clean { diff --git a/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java b/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java index 11695682c00..cc100a6ca5e 100644 --- a/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java +++ b/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java @@ -30,7 +30,7 @@ static GravitonMetaLake toMetaLake(MetalakeDTO metalake, RESTClient client) { .build(); } - public static MetalakeUpdateRequest toMetalakeUpdateRequest(MetalakeChange change) { + static MetalakeUpdateRequest toMetalakeUpdateRequest(MetalakeChange change) { if (change instanceof MetalakeChange.RenameMetalake) { return new MetalakeUpdateRequest.RenameMetalakeRequest( ((MetalakeChange.RenameMetalake) change).getNewName()); diff --git a/integration/build.gradle.kts b/integration-test/build.gradle.kts similarity index 90% rename from integration/build.gradle.kts rename to integration-test/build.gradle.kts index adeb894d2f0..c2567553f8b 100644 --- a/integration/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -27,6 +27,7 @@ dependencies { tasks { val integrationTest by creating(Test::class) { + environment("GRAVITON_HOME", rootDir.path + "/distribution/package") useJUnitPlatform() } } \ No newline at end of file diff --git a/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java b/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java new file mode 100644 index 00000000000..070c4497ef0 --- /dev/null +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java @@ -0,0 +1,112 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.integration.e2e; + +import com.datastrato.graviton.MetalakeChange; +import com.datastrato.graviton.NameIdentifier; +import com.datastrato.graviton.client.GravitonMetaLake; +import com.datastrato.graviton.dto.MetalakeDTO; +import com.datastrato.graviton.exceptions.MetalakeAlreadyExistsException; +import com.datastrato.graviton.exceptions.NoSuchMetalakeException; +import com.datastrato.graviton.integration.util.AbstractIT; +import com.datastrato.graviton.integration.util.GravitonITUtils; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class MetalakeIT extends AbstractIT { + public static String metalakeName = GravitonITUtils.genRandomName(); + + @BeforeAll + private static void start() { + createMetalake(); + } + + @AfterAll + private static void stop() { + dropMetalake(); + } + + @Order(1) + @Test + public void testListMetalake() { + GravitonMetaLake[] metaLakes = client.listMetalakes(); + List result = + Arrays.stream(metaLakes) + .filter(metalakeDTO -> metalakeDTO.name().equals(metalakeName)) + .collect(Collectors.toList()); + + Assertions.assertEquals(result.size(), 1); + } + + @Order(2) + @Test + public void testLoadMetalake() { + GravitonMetaLake metaLake = client.loadMetalake(NameIdentifier.of(metalakeName)); + Assertions.assertEquals(metaLake.name(), metalakeName); + } + + @Order(3) + @Test + public void testAlterMetalake() { + String alterMetalakeName = GravitonITUtils.genRandomName(); + + MetalakeChange[] changes1 = + new MetalakeChange[] { + MetalakeChange.rename(alterMetalakeName), MetalakeChange.updateComment("newComment") + }; + GravitonMetaLake metaLake = client.alterMetalake(NameIdentifier.of(metalakeName), changes1); + Assertions.assertEquals(alterMetalakeName, metaLake.name()); + Assertions.assertEquals("newComment", metaLake.comment()); + Assertions.assertEquals("graviton", metaLake.auditInfo().creator()); + + // Test return not found + Throwable excep = + Assertions.assertThrows( + NoSuchMetalakeException.class, + () -> client.alterMetalake(NameIdentifier.of(metalakeName + "mock"), changes1)); + Assertions.assertTrue(excep.getMessage().contains("does not exist")); + + // Restore test record + MetalakeChange[] changes2 = new MetalakeChange[] {MetalakeChange.rename(metalakeName)}; + client.alterMetalake(NameIdentifier.of(alterMetalakeName), changes2); + } + + public static void createMetalake() { + GravitonMetaLake metaLake = + client.createMetalake( + NameIdentifier.parse(metalakeName), "comment", Collections.emptyMap()); + Assertions.assertEquals(metalakeName, metaLake.name()); + Assertions.assertEquals("comment", metaLake.comment()); + Assertions.assertEquals("graviton", metaLake.auditInfo().creator()); + + // Test metalake name already exists + Throwable excep = + Assertions.assertThrows( + MetalakeAlreadyExistsException.class, + () -> + client.createMetalake( + NameIdentifier.parse(metalakeName), "comment", Collections.emptyMap())); + Assertions.assertTrue(excep.getMessage().contains("already exists")); + } + + public static void dropMetalake() { + Assertions.assertTrue(client.dropMetalake(NameIdentifier.of(metalakeName))); + + // Test illegal metalake name identifier + Assertions.assertThrows( + IllegalArgumentException.class, + () -> client.dropMetalake(NameIdentifier.parse(metalakeName + "." + metalakeName))); + } +} diff --git a/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java b/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java new file mode 100644 index 00000000000..67fb3c2051f --- /dev/null +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.integration.util; + +import com.datastrato.graviton.client.GravitonClient; +import java.io.IOException; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AbstractIT { + public static final Logger LOG = LoggerFactory.getLogger(AbstractIT.class); + private static final int port = 8090; + protected static GravitonClient client; + + @BeforeAll + public static void startIntegrationTest() { + LOG.info("Starting up Graviton Server"); + GravitonITUtils.startGravitonServer(); + + client = GravitonClient.builder("http://127.0.0.1:" + port).build(); + } + + @AfterAll + public static void stopIntegrationTest() throws IOException { + client.close(); + GravitonITUtils.stopGravitonServer(); + LOG.info("Tearing down Graviton Server"); + } +} diff --git a/integration/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java b/integration-test/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java similarity index 100% rename from integration/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java rename to integration-test/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java diff --git a/integration/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java b/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java similarity index 83% rename from integration/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java rename to integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java index 8761e19d49e..225b2c81ca2 100644 --- a/integration/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java @@ -13,14 +13,18 @@ public class GravitonITUtils { public static void startGravitonServer() { CommandExecutor.executeCommandLocalHost( - "../distribution/package/bin/graviton.sh start", false, ProcessData.TypesOfData.OUTPUT); + System.getenv("GRAVITON_HOME") + "/bin/graviton.sh start", + false, + ProcessData.TypesOfData.OUTPUT); // wait for server to start. sleep(100, false); } public static void stopGravitonServer() { CommandExecutor.executeCommandLocalHost( - "../distribution/package/bin/graviton.sh stop", false, ProcessData.TypesOfData.OUTPUT); + System.getenv("GRAVITON_HOME") + "/bin/graviton.sh stop", + false, + ProcessData.TypesOfData.OUTPUT); // wait for server to stop. sleep(100, false); } diff --git a/integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java b/integration-test/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java similarity index 91% rename from integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java rename to integration-test/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java index 6edd5ac2212..b9caffd65ae 100644 --- a/integration/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java @@ -38,29 +38,29 @@ public enum TypesOfData { public static final Logger LOG = LoggerFactory.getLogger(ProcessData.class); - private Process checked_process; + private Process checkedProcess; private boolean printToConsole = false; public ProcessData( Process connected_process, boolean printToConsole, int silenceTimeout, TimeUnit timeUnit) { - this.checked_process = connected_process; + this.checkedProcess = connected_process; this.printToConsole = printToConsole; this.silenceTimeout = TimeUnit.MILLISECONDS.convert(silenceTimeout, timeUnit); } public ProcessData(Process connected_process, boolean printToConsole, int silenceTimeoutSec) { - this.checked_process = connected_process; + this.checkedProcess = connected_process; this.printToConsole = printToConsole; this.silenceTimeout = TimeUnit.MILLISECONDS.convert(silenceTimeoutSec, TimeUnit.SECONDS); } public ProcessData(Process connected_process, boolean printToConsole) { - this.checked_process = connected_process; + this.checkedProcess = connected_process; this.printToConsole = printToConsole; } - public ProcessData(Process connected_process) { - this.checked_process = connected_process; + public ProcessData(Process connectedProcess) { + this.checkedProcess = connectedProcess; this.printToConsole = true; } @@ -115,14 +115,14 @@ public Object getData(TypesOfData type) { public int getExitCodeValue() { try { if (!returnCodeRetrieved) { - this.checked_process.waitFor(); - this.returnCode = this.checked_process.exitValue(); + this.checkedProcess.waitFor(); + this.returnCode = this.checkedProcess.exitValue(); this.returnCodeRetrieved = true; - this.checked_process.destroy(); + this.checkedProcess.destroy(); } } catch (Exception inter) { throw new RuntimeException( - "Couldn't finish waiting for process " + this.checked_process + " termination", inter); + "Couldn't finish waiting for process " + this.checkedProcess + " termination", inter); } return this.returnCode; } @@ -133,7 +133,7 @@ public String getOutPutStream() { buildOutputAndErrorStreamData(); } catch (Exception e) { throw new RuntimeException( - "Couldn't retrieve Output Stream data from process: " + this.checked_process.toString(), + "Couldn't retrieve Output Stream data from process: " + this.checkedProcess.toString(), e); } } @@ -152,7 +152,7 @@ public String getErrorStream() { buildOutputAndErrorStreamData(); } catch (Exception e) { throw new RuntimeException( - "Couldn't retrieve Error Stream data from process: " + this.checked_process.toString(), + "Couldn't retrieve Error Stream data from process: " + this.checkedProcess.toString(), e); } } @@ -176,12 +176,11 @@ private void buildOutputAndErrorStreamData() throws IOException { StringBuilder sbErrorStream = new StringBuilder(); try { - InputStream in = this.checked_process.getInputStream(); - InputStream inErrors = this.checked_process.getErrorStream(); + InputStream in = this.checkedProcess.getInputStream(); + InputStream inErrors = this.checkedProcess.getErrorStream(); BufferedReader inReader = new BufferedReader(new InputStreamReader(in)); BufferedReader inReaderErrors = new BufferedReader(new InputStreamReader(inErrors)); - LOG.trace( - "Started retrieving data from streams of attached process: " + this.checked_process); + LOG.trace("Started retrieving data from streams of attached process: " + this.checkedProcess); // Store start time to be able to finish method if command hangs long lastStreamDataTime = System.currentTimeMillis(); @@ -197,7 +196,7 @@ private void buildOutputAndErrorStreamData() throws IOException { // Continue if process is alive or some output was produced on previous iteration // and there may be still some data to read. - while (isRunning(this.checked_process) || outputProduced) { + while (isRunning(this.checkedProcess) || outputProduced) { outputProduced = false; // Some local commands can exit fast, but immediate stream reading will give no output and @@ -268,7 +267,7 @@ private void buildOutputAndErrorStreamData() throws IOException { + (System.currentTimeMillis() - lastStreamDataTime > silenceTimeout) + " " + (System.currentTimeMillis() > unconditionalExitTime)); - this.checked_process.destroy(); + this.checkedProcess.destroy(); try { if ((System.currentTimeMillis() > unconditionalExitTime)) { LOG.error( diff --git a/integration/src/test/java/resources/log4j2.properties b/integration-test/src/test/java/resources/log4j2.properties similarity index 100% rename from integration/src/test/java/resources/log4j2.properties rename to integration-test/src/test/java/resources/log4j2.properties diff --git a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java b/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java deleted file mode 100644 index 7930b3e4dec..00000000000 --- a/integration/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright 2023 Datastrato. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.graviton.integration.e2e; - -import com.datastrato.graviton.MetalakeChange; -import com.datastrato.graviton.NameIdentifier; -import com.datastrato.graviton.client.DTOConverters; -import com.datastrato.graviton.client.ErrorHandlers; -import com.datastrato.graviton.client.GravitonMetaLake; -import com.datastrato.graviton.dto.MetalakeDTO; -import com.datastrato.graviton.dto.requests.MetalakeCreateRequest; -import com.datastrato.graviton.dto.requests.MetalakeUpdatesRequest; -import com.datastrato.graviton.dto.responses.DropResponse; -import com.datastrato.graviton.dto.responses.ErrorResponse; -import com.datastrato.graviton.dto.responses.MetalakeListResponse; -import com.datastrato.graviton.dto.responses.MetalakeResponse; -import com.datastrato.graviton.exceptions.MetalakeAlreadyExistsException; -import com.datastrato.graviton.exceptions.NoSuchMetalakeException; -import com.datastrato.graviton.integration.util.AbstractIT; -import com.datastrato.graviton.integration.util.GravitonITUtils; -import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import org.apache.hc.core5.http.Method; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; - -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class MetalakeIT extends AbstractIT { - public static String metalakeName_RESTful = GravitonITUtils.genRandomName(); - public static String metalakeName_API = GravitonITUtils.genRandomName(); - - static String reqPath = "api/metalakes"; - - static Consumer onError = ErrorHandlers.restErrorHandler(); - - public static MetalakeResponse createDefMetalake(String metalakeName) { - MetalakeCreateRequest reqBody = - new MetalakeCreateRequest(metalakeName, "comment", ImmutableMap.of("key", "value")); - - MetalakeResponse successResponse = - doExecuteRequest(Method.POST, reqPath, reqBody, MetalakeResponse.class, onError, h -> {}); - return successResponse; - } - - @BeforeAll - private static void startup() { - MetalakeResponse successResponse = createDefMetalake(metalakeName_RESTful); - Assertions.assertEquals(successResponse.getMetalake().name(), metalakeName_RESTful); - } - - @AfterAll - private static void teardown() throws IOException { - DropResponse response = - doExecuteRequest( - Method.DELETE, - reqPath + File.separator + metalakeName_RESTful, - null, - DropResponse.class, - onError, - h -> {}); - Assertions.assertEquals(response.dropped(), true); - } - - @Order(1) - @Test - public void testListMetalakeRestful() { - MetalakeListResponse listResponse = - doExecuteRequest(Method.GET, reqPath, null, MetalakeListResponse.class, onError, h -> {}); - - List result = - Arrays.stream(listResponse.getMetalakes()) - .filter(metalakeDTO -> metalakeDTO.name().equals(metalakeName_RESTful)) - .collect(Collectors.toList()); - - Assertions.assertEquals(result.size(), 1); - - Assertions.assertEquals(result.get(0).name(), metalakeName_RESTful); - Assertions.assertEquals("comment", result.get(0).comment()); - } - - @Order(2) - @Test - public void testPutMetalakeRestful() { - String putMetalakeName = GravitonITUtils.genRandomName(); - String putReqPath = reqPath + File.separator + metalakeName_RESTful; - MetalakeChange[] changes1 = - new MetalakeChange[] { - MetalakeChange.rename(putMetalakeName), MetalakeChange.updateComment("newComment") - }; - - MetalakeUpdatesRequest reqUpdates1 = - new MetalakeUpdatesRequest( - Arrays.stream(changes1) - .map(DTOConverters::toMetalakeUpdateRequest) - .collect(Collectors.toList())); - - MetalakeResponse responseUpdates = - doExecuteRequest( - Method.PUT, putReqPath, reqUpdates1, MetalakeResponse.class, onError, h -> {}); - Assertions.assertEquals(responseUpdates.getMetalake().name(), putMetalakeName); - Assertions.assertEquals("newComment", responseUpdates.getMetalake().comment()); - - // Restore test record - putReqPath = reqPath + File.separator + putMetalakeName; - MetalakeChange[] changes2 = new MetalakeChange[] {MetalakeChange.rename(metalakeName_RESTful)}; - - MetalakeUpdatesRequest reqUpdates2 = - new MetalakeUpdatesRequest( - Arrays.stream(changes2) - .map(DTOConverters::toMetalakeUpdateRequest) - .collect(Collectors.toList())); - - doExecuteRequest(Method.PUT, putReqPath, reqUpdates2, MetalakeResponse.class, onError, h -> {}); - } - - @Order(4) - @Test - public void testCreateMetalakeAPI() { - GravitonMetaLake metaLake = - client.createMetalake( - NameIdentifier.parse(metalakeName_API), "comment", Collections.emptyMap()); - Assertions.assertEquals(metalakeName_API, metaLake.name()); - Assertions.assertEquals("comment", metaLake.comment()); - Assertions.assertEquals("graviton", metaLake.auditInfo().creator()); - - // Test metalake name already exists - Throwable excep = - Assertions.assertThrows( - MetalakeAlreadyExistsException.class, - () -> - client.createMetalake( - NameIdentifier.parse(metalakeName_API), "comment", Collections.emptyMap())); - Assertions.assertTrue(excep.getMessage().contains("already exists")); - } - - @Order(5) - @Test - public void testListMetalakeAPI() { - GravitonMetaLake[] metaLakes = client.listMetalakes(); - List result = - Arrays.stream(metaLakes) - .filter(metalakeDTO -> metalakeDTO.name().equals(metalakeName_API)) - .collect(Collectors.toList()); - - Assertions.assertEquals(result.size(), 1); - } - - @Order(6) - @Test - public void testLoadMetalakeAPI() { - GravitonMetaLake metaLake = client.loadMetalake(NameIdentifier.of(metalakeName_API)); - Assertions.assertEquals(metaLake.name(), metalakeName_API); - } - - @Order(7) - @Test - public void testAlterMetalakeAPI() { - String alterMetalakeName = GravitonITUtils.genRandomName(); - - MetalakeChange[] changes1 = - new MetalakeChange[] { - MetalakeChange.rename(alterMetalakeName), MetalakeChange.updateComment("newComment") - }; - GravitonMetaLake metaLake = client.alterMetalake(NameIdentifier.of(metalakeName_API), changes1); - Assertions.assertEquals(alterMetalakeName, metaLake.name()); - Assertions.assertEquals("newComment", metaLake.comment()); - Assertions.assertEquals("graviton", metaLake.auditInfo().creator()); - - // Test return not found - Throwable excep = - Assertions.assertThrows( - NoSuchMetalakeException.class, - () -> client.alterMetalake(NameIdentifier.of(metalakeName_API + "mock"), changes1)); - Assertions.assertTrue(excep.getMessage().contains("does not exist")); - - // Restore test record - MetalakeChange[] changes2 = new MetalakeChange[] {MetalakeChange.rename(metalakeName_API)}; - client.alterMetalake(NameIdentifier.of(alterMetalakeName), changes2); - } - - @Order(8) - @Test - public void testDropMetalakeAPI() { - Assertions.assertTrue(client.dropMetalake(NameIdentifier.of(metalakeName_API))); - - // Test illegal metalake name identifier - Throwable excep1 = - Assertions.assertThrows( - IllegalArgumentException.class, - () -> - client.dropMetalake( - NameIdentifier.parse(metalakeName_API + "." + metalakeName_API))); - Assertions.assertTrue(excep1.getMessage().contains("namespace should be empty")); - } -} diff --git a/integration/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java b/integration/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java deleted file mode 100644 index 0fe5b79ca5c..00000000000 --- a/integration/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2023 Datastrato. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.graviton.integration.util; - -import com.datastrato.graviton.client.GravitonClient; -import com.datastrato.graviton.client.HTTPClient; -import com.datastrato.graviton.client.RESTClient; -import com.datastrato.graviton.dto.responses.ErrorResponse; -import com.datastrato.graviton.rest.RESTRequest; -import com.datastrato.graviton.rest.RESTResponse; -import com.google.common.collect.ImmutableMap; -import java.io.IOException; -import java.util.Map; -import java.util.function.Consumer; -import org.apache.hc.core5.http.Method; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class AbstractIT { - public static final Logger LOG = LoggerFactory.getLogger(AbstractIT.class); - private static RESTClient restClient; - - private static final int port = 8090; - private static final String URI = String.format("http://127.0.0.1:%d", port); - protected static GravitonClient client; - - @BeforeAll - public static void startUp() { - LOG.info("Starting up Graviton Server"); - GravitonITUtils.startGravitonServer(); - restClient = HTTPClient.builder(ImmutableMap.of()).uri(URI).build(); - - client = GravitonClient.builder("http://127.0.0.1:" + port).build(); - } - - @AfterAll - public static void tearDown() throws IOException { - restClient.close(); - GravitonITUtils.stopGravitonServer(); - LOG.info("Tearing down Graviton Server"); - } - - protected static T doExecuteRequest( - Method method, - String path, - RESTRequest body, - Class responseType, - Consumer onError, - Consumer> responseHeaders) { - Map headers = ImmutableMap.of(); - switch (method) { - case POST: - return restClient.post(path, body, responseType, headers, onError, responseHeaders); - case PUT: - return restClient.put(path, body, responseType, headers, onError, responseHeaders); - case GET: - return restClient.get(path, responseType, headers, onError); - case HEAD: - restClient.head(path, headers, onError); - return null; - case DELETE: - return restClient.delete(path, responseType, () -> headers, onError); - default: - throw new IllegalArgumentException(String.format("Invalid method: %s", method)); - } - } -} diff --git a/settings.gradle.kts b/settings.gradle.kts index 0b6c37de384..746e40a7236 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -4,4 +4,4 @@ */ rootProject.name = "graviton" -include("api", "client-java", "common", "core", "meta", "server", "catalog-hive", "integration") +include("api", "client-java", "common", "core", "meta", "server", "catalog-hive", "integration-test") From 71f37db667dd89f2671da4682266cba4919211e2 Mon Sep 17 00:00:00 2001 From: xunliu Date: Mon, 28 Aug 2023 15:08:32 +0800 Subject: [PATCH 09/16] removed hard code for Graviton restful uri --- integration-test/build.gradle.kts | 2 ++ .../graviton/integration/util/AbstractIT.java | 19 ++++++++++++++----- .../graviton/server/GravitonServer.java | 2 +- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index c2567553f8b..2541f9db065 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -13,7 +13,9 @@ plugins { dependencies { implementation(project(":api")) implementation(project(":common")) + implementation(project(":core")) implementation(project(":client-java")) + implementation(project(":server")) testCompileOnly(libs.lombok) testAnnotationProcessor(libs.lombok) diff --git a/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java b/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java index 67fb3c2051f..fd4aea36689 100644 --- a/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java @@ -4,8 +4,10 @@ */ package com.datastrato.graviton.integration.util; +import com.datastrato.graviton.Config; import com.datastrato.graviton.client.GravitonClient; -import java.io.IOException; +import com.datastrato.graviton.server.GravitonServer; +import com.datastrato.graviton.server.ServerConfig; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.slf4j.Logger; @@ -13,19 +15,26 @@ public class AbstractIT { public static final Logger LOG = LoggerFactory.getLogger(AbstractIT.class); - private static final int port = 8090; protected static GravitonClient client; @BeforeAll - public static void startIntegrationTest() { + public static void startIntegrationTest() throws Exception { LOG.info("Starting up Graviton Server"); GravitonITUtils.startGravitonServer(); - client = GravitonClient.builder("http://127.0.0.1:" + port).build(); + Config serverConfig = new ServerConfig(); + serverConfig.loadFromFile(GravitonServer.CONF_FILE); + + String uri = + "http://" + + serverConfig.get(ServerConfig.WEBSERVER_HOST) + + ":" + + serverConfig.get(ServerConfig.WEBSERVER_HTTP_PORT); + client = GravitonClient.builder(uri).build(); } @AfterAll - public static void stopIntegrationTest() throws IOException { + public static void stopIntegrationTest() { client.close(); GravitonITUtils.stopGravitonServer(); LOG.info("Tearing down Graviton Server"); diff --git a/server/src/main/java/com/datastrato/graviton/server/GravitonServer.java b/server/src/main/java/com/datastrato/graviton/server/GravitonServer.java index 6f838824ca7..0c8f3dc7604 100644 --- a/server/src/main/java/com/datastrato/graviton/server/GravitonServer.java +++ b/server/src/main/java/com/datastrato/graviton/server/GravitonServer.java @@ -23,7 +23,7 @@ public class GravitonServer extends ResourceConfig { private static final Logger LOG = LoggerFactory.getLogger(GravitonServer.class); - private static final String CONF_FILE = "graviton.conf"; + public static final String CONF_FILE = "graviton.conf"; private final ServerConfig serverConfig; From 12b9c960377bb4e48ef9435bcec74cb3721e512a Mon Sep 17 00:00:00 2001 From: xunliu Date: Mon, 28 Aug 2023 15:12:19 +0800 Subject: [PATCH 10/16] Remove test code --- .github/workflows/integration.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 70c0c8bd8d1..9d181ec59e8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -44,6 +44,4 @@ jobs: - name: Graviton Integration Tests run: | - ls -lt ./ - ls -lt distribution/package - gradle integrationTest \ No newline at end of file + gradle integrationTest \ No newline at end of file From 6702bc1f783ed82efe5bf6e44f642641f78acddb Mon Sep 17 00:00:00 2001 From: xunliu Date: Tue, 29 Aug 2023 09:51:52 +0800 Subject: [PATCH 11/16] fix --- .../datastrato/graviton/integration/util/GravitonITUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java b/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java index 225b2c81ca2..44974e81b68 100644 --- a/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java @@ -31,7 +31,7 @@ public static void stopGravitonServer() { public static void sleep(long millis, boolean logOutput) { if (logOutput) { - LOG.info("Starting sleeping for " + (millis / 1000) + " seconds..."); + LOG.info("Starting sleeping for " + millis + " milliseconds..."); LOG.info("Caller: " + Thread.currentThread().getStackTrace()[2]); } try { From df1b21c86cb6a4dcc8d57d4255fa4a2987aa2e52 Mon Sep 17 00:00:00 2001 From: xunliu Date: Tue, 29 Aug 2023 09:58:37 +0800 Subject: [PATCH 12/16] trigger CI --- .../datastrato/graviton/integration/util/GravitonITUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java b/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java index 44974e81b68..a381159be9f 100644 --- a/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java @@ -31,7 +31,7 @@ public static void stopGravitonServer() { public static void sleep(long millis, boolean logOutput) { if (logOutput) { - LOG.info("Starting sleeping for " + millis + " milliseconds..."); + LOG.info("Starting sleeping for " + millis + " milliseconds ..."); LOG.info("Caller: " + Thread.currentThread().getStackTrace()[2]); } try { From 309999d1e915853453fb700ec673300850e4cd08 Mon Sep 17 00:00:00 2001 From: xunliu Date: Wed, 30 Aug 2023 10:36:01 +0800 Subject: [PATCH 13/16] fixed --- .../main/java/com/datastrato/graviton/client/DTOConverters.java | 2 +- .../datastrato/graviton/integration/util/GravitonITUtils.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java b/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java index cc100a6ca5e..cbf4dad50f7 100644 --- a/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java +++ b/client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java @@ -17,7 +17,7 @@ import com.datastrato.graviton.rel.SchemaChange; import com.datastrato.graviton.rel.TableChange; -public class DTOConverters { +class DTOConverters { private DTOConverters() {} static GravitonMetaLake toMetaLake(MetalakeDTO metalake, RESTClient client) { diff --git a/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java b/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java index a381159be9f..eeac3121443 100644 --- a/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java @@ -37,7 +37,7 @@ public static void sleep(long millis, boolean logOutput) { try { Thread.sleep(millis); } catch (InterruptedException e) { - LOG.error("Exception in WebDriverManager while getWebDriver ", e); + LOG.error("Exception in sleep() ", e); } if (logOutput) { LOG.info("Finished."); From e6a53b8cab7c933c288d58f2390472b0cb700990 Mon Sep 17 00:00:00 2001 From: xunliu Date: Wed, 30 Aug 2023 11:42:28 +0800 Subject: [PATCH 14/16] Add some comments --- .../com/datastrato/graviton/integration/e2e/MetalakeIT.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java b/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java index 070c4497ef0..4e9ed70282a 100644 --- a/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java @@ -30,11 +30,17 @@ public class MetalakeIT extends AbstractIT { @BeforeAll private static void start() { + // Prepare create a metalake test record, + // This way it can support `Run all tests` or `Run test in separated` to test `list`, `load`, + // `alter`, `drop` methods. createMetalake(); } @AfterAll private static void stop() { + // Always drop the test record here. + // This allows metalake to be safe deleted metalake when `Run all tests` or `Run test in + // separated` dropMetalake(); } From 278f33cba92682d28a9583b69ba6ece705e185ed Mon Sep 17 00:00:00 2001 From: xunliu Date: Wed, 30 Aug 2023 12:51:37 +0800 Subject: [PATCH 15/16] Refinement test case --- integration-test/build.gradle.kts | 1 + .../graviton/integration/e2e/MetalakeIT.java | 12 +++++++++--- .../graviton/integration/util/AbstractIT.java | 11 ++++++++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 2541f9db065..f6131a24c40 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -17,6 +17,7 @@ dependencies { implementation(project(":client-java")) implementation(project(":server")) + testCompileOnly(libs.commons.io) testCompileOnly(libs.lombok) testAnnotationProcessor(libs.lombok) testImplementation(libs.guava) diff --git a/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java b/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java index 4e9ed70282a..fa3a7de68f3 100644 --- a/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java @@ -68,6 +68,7 @@ public void testLoadMetalake() { public void testAlterMetalake() { String alterMetalakeName = GravitonITUtils.genRandomName(); + // TODO: Add more test cases for alter metalake MetalakeChange[] changes1 = new MetalakeChange[] { MetalakeChange.rename(alterMetalakeName), MetalakeChange.updateComment("newComment") @@ -77,6 +78,12 @@ public void testAlterMetalake() { Assertions.assertEquals("newComment", metaLake.comment()); Assertions.assertEquals("graviton", metaLake.auditInfo().creator()); + // Reload metatada from backend to check if the changes are applied + GravitonMetaLake metaLake1 = client.loadMetalake(NameIdentifier.of(alterMetalakeName)); + Assertions.assertEquals(alterMetalakeName, metaLake1.name()); + Assertions.assertEquals("newComment", metaLake1.comment()); + Assertions.assertEquals("graviton", metaLake1.auditInfo().creator()); + // Test return not found Throwable excep = Assertions.assertThrows( @@ -110,9 +117,8 @@ public static void createMetalake() { public static void dropMetalake() { Assertions.assertTrue(client.dropMetalake(NameIdentifier.of(metalakeName))); - // Test illegal metalake name identifier + // Reload metatada from backend to check if the drop are applied Assertions.assertThrows( - IllegalArgumentException.class, - () -> client.dropMetalake(NameIdentifier.parse(metalakeName + "." + metalakeName))); + NoSuchMetalakeException.class, () -> client.loadMetalake(NameIdentifier.of(metalakeName))); } } diff --git a/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java b/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java index fd4aea36689..96bac83af90 100644 --- a/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java +++ b/integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java @@ -4,10 +4,13 @@ */ package com.datastrato.graviton.integration.util; +import static com.datastrato.graviton.Configs.ENTRY_KV_ROCKSDB_BACKEND_PATH; + import com.datastrato.graviton.Config; import com.datastrato.graviton.client.GravitonClient; import com.datastrato.graviton.server.GravitonServer; import com.datastrato.graviton.server.ServerConfig; +import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.slf4j.Logger; @@ -20,11 +23,17 @@ public class AbstractIT { @BeforeAll public static void startIntegrationTest() throws Exception { LOG.info("Starting up Graviton Server"); - GravitonITUtils.startGravitonServer(); Config serverConfig = new ServerConfig(); serverConfig.loadFromFile(GravitonServer.CONF_FILE); + try { + FileUtils.deleteDirectory(FileUtils.getFile(serverConfig.get(ENTRY_KV_ROCKSDB_BACKEND_PATH))); + } catch (Exception e) { + // Ignore + } + + GravitonITUtils.startGravitonServer(); String uri = "http://" + serverConfig.get(ServerConfig.WEBSERVER_HOST) From e9a832949585956009c9154648087fb6891663b7 Mon Sep 17 00:00:00 2001 From: xunliu Date: Wed, 30 Aug 2023 18:03:28 +0800 Subject: [PATCH 16/16] Add Apache Zeppelin license --- LICENSE | 3 +++ NOTICE | 3 +++ 2 files changed, 6 insertions(+) diff --git a/LICENSE b/LICENSE index ab02b859260..f837ef50d13 100644 --- a/LICENSE +++ b/LICENSE @@ -231,3 +231,6 @@ Apache Hive ./catalog-hive/src/test/resources/hive-schema-3.1.0.derby.sql + Apache Zeppelin + ./integration-test/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java + ./integration-test/src/test/java/com/datastrato/graviton/integration/util/ProcessData.java diff --git a/NOTICE b/NOTICE index 958cbd098fa..91411f95e56 100644 --- a/NOTICE +++ b/NOTICE @@ -16,5 +16,8 @@ Copyright 2014 and onwards The Apache Software Foundation. Apache Hive Copyright 2008-2023 The Apache Software Foundation +Apache Zeppelin +Copyright 2016-2023 The Apache Software Foundation + This product includes software developed at The Apache Software Foundation (http://www.apache.org/).