[#216][#217] test: add e2e integration test framework & metalake integration test

.github/workflows/integration.yml

@@ -40,4 +40,8 @@ jobs:
       - name: Package Graviton
         run: |
           gradle build
-          gradle compileDistribution
+          gradle compileDistribution
+
+      - name: Graviton Integration Tests
+        run: |
+          gradle integrationTest

build.gradle.kts

@@ -40,8 +40,11 @@ subprojects {
   }
 
   tasks.configureEach<Test> {
-    useJUnitPlatform()
-    finalizedBy(tasks.getByName("jacocoTestReport"))
+    // Integration test module are tested sepatately
+    if (project.name != "integration-test") {
+      useJUnitPlatform()
+      finalizedBy(tasks.getByName("jacocoTestReport"))
+    }
   }
 
   tasks.withType<JacocoReport> {
@@ -59,8 +62,10 @@ subprojects {
 
   tasks.withType<Jar> {
     archiveFileName.set("${rootProject.name.lowercase(Locale.getDefault())}-${project.name}-$version.jar")
-    exclude("log4j2.properties")
-    exclude("test/**")
+    if (project.name != "integration-test") {
+      exclude("log4j2.properties")
+      exclude("test/**")
+    }
   }
 
   plugins.withType<SpotlessPlugin>().configureEach {
@@ -229,25 +234,10 @@ tasks {
     }
   }
 
-  // Print all dependencies of all subprojects, `./gradlew allDeps`
-  task("allDeps") {
-    doLast {
-      subprojects.forEach { project ->
-        println("Dependencies for project: ${project.name}")
-        project.configurations.forEach { configuration ->
-          configuration.allDependencies.forEach { dependency ->
-            println("- ${dependency.group}:${dependency.name}:${dependency.version}")
-          }
-        }
-        println()
-      }
-    }
+  task("integrationTest") {
+    dependsOn(":integration-test:integrationTest")
   }
 
-//  assemble {
-//    finalizedBy(assembleDistribution)
-//  }
-
   clean {
     dependsOn(cleanDistribution)
   }

client-java/src/main/java/com/datastrato/graviton/client/DTOConverters.java

@@ -17,7 +17,7 @@
 import com.datastrato.graviton.rel.SchemaChange;
 import com.datastrato.graviton.rel.TableChange;
 
-class DTOConverters {
+public class DTOConverters {
   private DTOConverters() {}
 
   static GravitonMetaLake toMetaLake(MetalakeDTO metalake, RESTClient client) {

integration-test/build.gradle.kts

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2023 Datastrato.
+ * This software is licensed under the Apache License version 2.
+ */
+
+plugins {
+  `maven-publish`
+  id("java")
+  id("idea")
+  id("com.diffplug.spotless")
+}
+
+dependencies {
+  implementation(project(":api"))
+  implementation(project(":common"))
+  implementation(project(":core"))
+  implementation(project(":client-java"))
+  implementation(project(":server"))
+
+  testCompileOnly(libs.lombok)
+  testAnnotationProcessor(libs.lombok)
+  testImplementation(libs.guava)
+  testImplementation(libs.commons.lang3)
+  testImplementation(libs.junit.jupiter.api)
+  testImplementation(libs.junit.jupiter.params)
+  testImplementation(libs.httpclient5)
+  testRuntimeOnly(libs.junit.jupiter.engine)
+}
+
+tasks {
+  val integrationTest by creating(Test::class) {
+    environment("GRAVITON_HOME", rootDir.path + "/distribution/package")
+    useJUnitPlatform()
+  }
+}

integration-test/src/test/java/com/datastrato/graviton/integration/e2e/MetalakeIT.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2023 Datastrato.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.graviton.integration.e2e;
+
+import com.datastrato.graviton.MetalakeChange;
+import com.datastrato.graviton.NameIdentifier;
+import com.datastrato.graviton.client.GravitonMetaLake;
+import com.datastrato.graviton.dto.MetalakeDTO;
+import com.datastrato.graviton.exceptions.MetalakeAlreadyExistsException;
+import com.datastrato.graviton.exceptions.NoSuchMetalakeException;
+import com.datastrato.graviton.integration.util.AbstractIT;
+import com.datastrato.graviton.integration.util.GravitonITUtils;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class MetalakeIT extends AbstractIT {
+  public static String metalakeName = GravitonITUtils.genRandomName();
+
+  @BeforeAll
+  private static void start() {
+    createMetalake();
+  }
+
+  @AfterAll
+  private static void stop() {
+    dropMetalake();
+  }
+
+  @Order(1)
+  @Test
+  public void testListMetalake() {
+    GravitonMetaLake[] metaLakes = client.listMetalakes();
+    List<MetalakeDTO> result =
+        Arrays.stream(metaLakes)
+            .filter(metalakeDTO -> metalakeDTO.name().equals(metalakeName))
+            .collect(Collectors.toList());
+
+    Assertions.assertEquals(result.size(), 1);
+  }
+
+  @Order(2)
+  @Test
+  public void testLoadMetalake() {
+    GravitonMetaLake metaLake = client.loadMetalake(NameIdentifier.of(metalakeName));
+    Assertions.assertEquals(metaLake.name(), metalakeName);
+  }
+
+  @Order(3)
+  @Test
+  public void testAlterMetalake() {
+    String alterMetalakeName = GravitonITUtils.genRandomName();
+
+    MetalakeChange[] changes1 =
+        new MetalakeChange[] {
+          MetalakeChange.rename(alterMetalakeName), MetalakeChange.updateComment("newComment")
+        };
+    GravitonMetaLake metaLake = client.alterMetalake(NameIdentifier.of(metalakeName), changes1);
+    Assertions.assertEquals(alterMetalakeName, metaLake.name());
+    Assertions.assertEquals("newComment", metaLake.comment());
+    Assertions.assertEquals("graviton", metaLake.auditInfo().creator());
+
+    // Test return not found
+    Throwable excep =
+        Assertions.assertThrows(
+            NoSuchMetalakeException.class,
+            () -> client.alterMetalake(NameIdentifier.of(metalakeName + "mock"), changes1));
+    Assertions.assertTrue(excep.getMessage().contains("does not exist"));
+
+    // Restore test record
+    MetalakeChange[] changes2 = new MetalakeChange[] {MetalakeChange.rename(metalakeName)};
+    client.alterMetalake(NameIdentifier.of(alterMetalakeName), changes2);
+  }
+
+  public static void createMetalake() {
+    GravitonMetaLake metaLake =
+        client.createMetalake(
+            NameIdentifier.parse(metalakeName), "comment", Collections.emptyMap());
+    Assertions.assertEquals(metalakeName, metaLake.name());
+    Assertions.assertEquals("comment", metaLake.comment());
+    Assertions.assertEquals("graviton", metaLake.auditInfo().creator());
+
+    // Test metalake name already exists
+    Throwable excep =
+        Assertions.assertThrows(
+            MetalakeAlreadyExistsException.class,
+            () ->
+                client.createMetalake(
+                    NameIdentifier.parse(metalakeName), "comment", Collections.emptyMap()));
+    Assertions.assertTrue(excep.getMessage().contains("already exists"));
+  }
+
+  public static void dropMetalake() {
+    Assertions.assertTrue(client.dropMetalake(NameIdentifier.of(metalakeName)));
+
+    // Test illegal metalake name identifier
+    Assertions.assertThrows(
+        IllegalArgumentException.class,
+        () -> client.dropMetalake(NameIdentifier.parse(metalakeName + "." + metalakeName)));
+  }
+}

integration-test/src/test/java/com/datastrato/graviton/integration/util/AbstractIT.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2023 Datastrato.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.graviton.integration.util;
+
+import com.datastrato.graviton.Config;
+import com.datastrato.graviton.client.GravitonClient;
+import com.datastrato.graviton.server.GravitonServer;
+import com.datastrato.graviton.server.ServerConfig;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AbstractIT {
+  public static final Logger LOG = LoggerFactory.getLogger(AbstractIT.class);
+  protected static GravitonClient client;
+
+  @BeforeAll
+  public static void startIntegrationTest() throws Exception {
+    LOG.info("Starting up Graviton Server");
+    GravitonITUtils.startGravitonServer();
+
+    Config serverConfig = new ServerConfig();
+    serverConfig.loadFromFile(GravitonServer.CONF_FILE);
+
+    String uri =
+        "http://"
+            + serverConfig.get(ServerConfig.WEBSERVER_HOST)
+            + ":"
+            + serverConfig.get(ServerConfig.WEBSERVER_HTTP_PORT);
+    client = GravitonClient.builder(uri).build();
+  }
+
+  @AfterAll
+  public static void stopIntegrationTest() {
+    client.close();
+    GravitonITUtils.stopGravitonServer();
+    LOG.info("Tearing down Graviton Server");
+  }
+}

integration-test/src/test/java/com/datastrato/graviton/integration/util/CommandExecutor.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datastrato.graviton.integration.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// zeppelin-integration/src/test/java/org/apache/zeppelin/CommandExecutor.java
+public class CommandExecutor {
+  public static final Logger LOG = LoggerFactory.getLogger(CommandExecutor.class);
+
+  public enum IGNORE_ERRORS {
+    TRUE,
+    FALSE
+  }
+
+  public static int NORMAL_EXIT = 0;
+
+  private static IGNORE_ERRORS DEFAULT_BEHAVIOUR_ON_ERRORS = IGNORE_ERRORS.TRUE;
+
+  public static Object executeCommandLocalHost(
+      String[] command,
+      boolean printToConsole,
+      ProcessData.TypesOfData type,
+      IGNORE_ERRORS ignore_errors) {
+    List<String> subCommandsAsList = new ArrayList<>(Arrays.asList(command));
+    String mergedCommand = StringUtils.join(subCommandsAsList, " ");
+
+    LOG.info("Sending command \"" + mergedCommand + "\" to localhost");
+
+    ProcessBuilder processBuilder = new ProcessBuilder(command);
+    Process process = null;
+    try {
+      process = processBuilder.start();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+
+    ProcessData dataOfProcess = new ProcessData(process, printToConsole);
+    Object outputOfProcess = dataOfProcess.getData(type);
+    int exit_code = dataOfProcess.getExitCodeValue();
+
+    if (!printToConsole) LOG.trace(outputOfProcess.toString());
+    else LOG.debug(outputOfProcess.toString());
+    if (ignore_errors == IGNORE_ERRORS.FALSE && exit_code != NORMAL_EXIT) {
+      LOG.error(String.format("Command '%s' failed with exit code %s", mergedCommand, exit_code));
+    }
+    return outputOfProcess;
+  }
+
+  public static Object executeCommandLocalHost(
+      String command, boolean printToConsole, ProcessData.TypesOfData type) {
+    return executeCommandLocalHost(
+        new String[] {"bash", "-c", command}, printToConsole, type, DEFAULT_BEHAVIOUR_ON_ERRORS);
+  }
+}

integration-test/src/test/java/com/datastrato/graviton/integration/util/GravitonITUtils.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2023 Datastrato.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.graviton.integration.util;
+
+import java.util.UUID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GravitonITUtils {
+  public static final Logger LOG = LoggerFactory.getLogger(GravitonITUtils.class);
+
+  public static void startGravitonServer() {
+    CommandExecutor.executeCommandLocalHost(
+        System.getenv("GRAVITON_HOME") + "/bin/graviton.sh start",
+        false,
+        ProcessData.TypesOfData.OUTPUT);
+    // wait for server to start.
+    sleep(100, false);
+  }
+
+  public static void stopGravitonServer() {
+    CommandExecutor.executeCommandLocalHost(
+        System.getenv("GRAVITON_HOME") + "/bin/graviton.sh stop",
+        false,
+        ProcessData.TypesOfData.OUTPUT);
+    // wait for server to stop.
+    sleep(100, false);
+  }
+
+  public static void sleep(long millis, boolean logOutput) {
+    if (logOutput) {
+      LOG.info("Starting sleeping for " + millis + " milliseconds ...");
+      LOG.info("Caller: " + Thread.currentThread().getStackTrace()[2]);
+    }
+    try {
+      Thread.sleep(millis);
+    } catch (InterruptedException e) {
+      LOG.error("Exception in WebDriverManager while getWebDriver ", e);
+    }
+    if (logOutput) {
+      LOG.info("Finished.");
+    }
+  }
+
+  public static String genRandomName() {
+    return UUID.randomUUID().toString().replace("-", "");
+  }
+}