From f81227557babd5ae9450d64aa29a5f11fcc7ffa6 Mon Sep 17 00:00:00 2001 From: CHEYNE Date: Tue, 9 Apr 2024 18:31:05 +0800 Subject: [PATCH] [#2636][#2637][#2638][#2639][#2776] improve(IT): test create different providers for the catalogs (#2741) ### What changes were proposed in this pull request? Improve by creating different providers for the catalogs. - replace hivecontainer to trinocontainer - create iceberg catalog - create mysql catalog - create postgresql catalog - create fileset catalog - test left side tree list - test tree node refresh ### Why are the changes needed? Fix: #2636 Fix: #2637 Fix: #2638 Fix: #2639 Fix: #2776 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? IT ![o1](https://github.com/datastrato/gravitino/assets/17310559/6c55d1c8-a44a-4b8e-906e-e8856565ab83) --- .../test/web/ui/CatalogsPageTest.java | 417 +++++++++++++++--- .../test/web/ui/pages/CatalogsPage.java | 199 ++++++++- .../app/metalakes/metalake/MetalakeTree.js | 10 +- .../rightContent/CreateCatalogDialog.js | 2 + web/src/app/rootLayout/AppBar.js | 1 + 5 files changed, 539 insertions(+), 90 deletions(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java index cb1a1e88210..133af12d523 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java @@ -5,12 +5,22 @@ package com.datastrato.gravitino.integration.test.web.ui; +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.client.GravitinoAdminClient; +import com.datastrato.gravitino.client.GravitinoMetalake; import com.datastrato.gravitino.integration.test.container.ContainerSuite; -import com.datastrato.gravitino.integration.test.container.HiveContainer; +import com.datastrato.gravitino.integration.test.container.TrinoITContainers; +import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.web.ui.pages.CatalogsPage; import com.datastrato.gravitino.integration.test.web.ui.pages.MetalakePage; import com.datastrato.gravitino.integration.test.web.ui.utils.AbstractWebIT; -import org.apache.hadoop.hive.conf.HiveConf; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.types.Types; +import com.google.common.collect.Maps; +import java.util.Arrays; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -23,84 +33,222 @@ @Tag("gravitino-docker-it") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class CatalogsPageTest extends AbstractWebIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); MetalakePage metalakePage = new MetalakePage(); CatalogsPage catalogsPage = new CatalogsPage(); - private static final String metalakeName = "metalake_name"; - private static final String metalakeSelectName = "metalake_select_name"; - String catalogName = "catalog_name"; - String catalogType = "relational"; - String modifiedCatalogName = catalogName + "_edited"; - String schemaName = "default"; - String tableName = "employee"; + protected static TrinoITContainers trinoITContainers; + protected static GravitinoAdminClient gravitinoClient; + private static GravitinoMetalake metalake; + private static Catalog catalog; + + protected static String gravitinoUri = "http://127.0.0.1:8090"; + protected static String trinoUri = "http://127.0.0.1:8080"; + protected static String hiveMetastoreUri = "thrift://127.0.0.1:9083"; + protected static String hdfsUri = "hdfs://127.0.0.1:9000"; + protected static String mysqlUri = "jdbc:mysql://127.0.0.1"; + protected static String postgresqlUri = "jdbc:postgresql://127.0.0.1"; + + private static final String WEB_TITLE = "Gravitino"; + private static final String CATALOG_TABLE_TITLE = "Schemas"; + private static final String SCHEMA_TABLE_TITLE = "Tables"; + private static final String TABLE_TABLE_TITLE = "Columns"; + private static final String METALAKE_NAME = "test"; + private static final String METALAKE_SELECT_NAME = "metalake_select_name"; + private static final String CATALOG_TYPE = "relational"; + private static final String DEFAULT_CATALOG_NAME = "default_catalog"; + private static final String HIVE_CATALOG_NAME = "catalog_hive"; + private static final String MODIFIED_CATALOG_NAME = HIVE_CATALOG_NAME + "_edited"; + private static final String ICEBERG_CATALOG_NAME = "catalog_iceberg"; + private static final String FILESET_CATALOG_NAME = "catalog_fileset"; + private static final String SCHEMA_NAME = "default"; + private static final String TABLE_NAME = "table1"; + private static final String TABLE_NAME_2 = "table2"; + private static final String COLUMN_NAME = "column"; + private static final String COLUMN_NAME_2 = "column_2"; + + private static final String MYSQL_CATALOG_NAME = "catalog_mysql"; + private static final String MYSQL_JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; + + private static final String PG_CATALOG_NAME = "catalog_pg"; + private static final String PG_JDBC_DRIVER = "org.postgresql.Driver"; + private static final String PG_JDBC_DB = "gt_db"; + + private static final String COMMON_JDBC_USER = "trino"; + private static final String COMMON_JDBC_PWD = "ds123"; @BeforeAll - public static void before() { - containerSuite.startHiveContainer(); + public static void before() throws Exception { + gravitinoClient = AbstractIT.getGravitinoClient(); - // Initial hive client - HiveConf hiveConf = new HiveConf(); - String hiveMetastoreUris = - String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); - hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, hiveMetastoreUris); + gravitinoUri = String.format("http://127.0.0.1:%d", AbstractIT.getGravitinoServerPort()); + + trinoITContainers = ContainerSuite.getTrinoITContainers(); + trinoITContainers.launch(AbstractIT.getGravitinoServerPort()); + + trinoUri = trinoITContainers.getTrinoUri(); + hiveMetastoreUri = trinoITContainers.getHiveMetastoreUri(); + hdfsUri = trinoITContainers.getHdfsUri(); + mysqlUri = trinoITContainers.getMysqlUri(); + postgresqlUri = trinoITContainers.getPostgresqlUri(); + } + + void createTableAndColumn( + String metalakeName, + String catalogName, + String schemaName, + String tableName, + String colName) { + Map properties = Maps.newHashMap(); + Column column = Column.of(colName, Types.IntegerType.get(), "column comment"); + catalog + .asTableCatalog() + .createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + new Column[] {column}, + "comment", + properties); } @AfterAll public static void after() { try { - closer.close(); + if (trinoITContainers != null) trinoITContainers.shutdown(); } catch (Exception e) { - LOG.error(e.getMessage(), e); + LOG.error("shutdown trino containers error", e); } } @Test - @Order(1) - public void testCreateHiveCatalog() throws InterruptedException { - // Create metalake first + @Order(0) + public void testDeleteCatalog() throws InterruptedException { + // create metalake clickAndWait(metalakePage.createMetalakeBtn); - metalakePage.setMetalakeNameField(metalakeName); + metalakePage.setMetalakeNameField(METALAKE_NAME); clickAndWait(metalakePage.submitHandleMetalakeBtn); - // Create another metalake for select option clickAndWait(metalakePage.createMetalakeBtn); - metalakePage.setMetalakeNameField(metalakeSelectName); + metalakePage.setMetalakeNameField(METALAKE_SELECT_NAME); clickAndWait(metalakePage.submitHandleMetalakeBtn); + // load metalake + metalake = gravitinoClient.loadMetalake(NameIdentifier.of(METALAKE_NAME)); + metalakePage.clickMetalakeLink(METALAKE_NAME); + // create catalog + clickAndWait(catalogsPage.createCatalogBtn); + catalogsPage.setCatalogNameField(DEFAULT_CATALOG_NAME); + catalogsPage.setCatalogFixedProp("metastore.uris", hiveMetastoreUri); + clickAndWait(catalogsPage.handleSubmitCatalogBtn); + // delete catalog + catalogsPage.clickDeleteCatalogBtn(DEFAULT_CATALOG_NAME); + clickAndWait(catalogsPage.confirmDeleteBtn); + Assertions.assertTrue(catalogsPage.verifyEmptyCatalog()); + } - metalakePage.clickMetalakeLink(metalakeName); + @Test + @Order(1) + public void testCreateHiveCatalog() throws InterruptedException { // Create catalog clickAndWait(catalogsPage.createCatalogBtn); - catalogsPage.setCatalogNameField(catalogName); + catalogsPage.setCatalogNameField(HIVE_CATALOG_NAME); catalogsPage.setCatalogCommentField("catalog comment"); - String hiveMetastoreUris = - String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); - catalogsPage.setHiveCatalogURI(hiveMetastoreUris); + catalogsPage.setCatalogFixedProp("metastore.uris", hiveMetastoreUri); catalogsPage.addCatalogPropsBtn.click(); - catalogsPage.setCatalogProps(1, "key1", "value1"); + catalogsPage.setCatalogPropsAt(1, "key1", "value1"); catalogsPage.addCatalogPropsBtn.click(); - catalogsPage.setCatalogProps(2, "key2", "value2"); + catalogsPage.setCatalogPropsAt(2, "key2", "value2"); clickAndWait(catalogsPage.handleSubmitCatalogBtn); + // load catalog + catalog = metalake.loadCatalog(NameIdentifier.of(METALAKE_NAME, HIVE_CATALOG_NAME)); - Assertions.assertTrue(catalogsPage.verifyCreateHiveCatalog(catalogName)); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(HIVE_CATALOG_NAME)); } @Test @Order(2) + public void testCreateIcebergCatalog() throws InterruptedException { + clickAndWait(catalogsPage.createCatalogBtn); + catalogsPage.setCatalogNameField(ICEBERG_CATALOG_NAME); + // select provider as iceberg + clickAndWait(catalogsPage.catalogProviderSelector); + catalogsPage.clickSelectProvider("lakehouse-iceberg"); + catalogsPage.setCatalogCommentField("iceberg catalog comment"); + // set iceberg uri + catalogsPage.setCatalogFixedProp("uri", hiveMetastoreUri); + // set iceberg warehouse + catalogsPage.setCatalogFixedProp("warehouse", hdfsUri); + clickAndWait(catalogsPage.handleSubmitCatalogBtn); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(ICEBERG_CATALOG_NAME)); + } + + @Test + @Order(3) + public void testCreateMysqlCatalog() throws InterruptedException { + // create mysql catalog actions + clickAndWait(catalogsPage.createCatalogBtn); + catalogsPage.setCatalogNameField(MYSQL_CATALOG_NAME); + // select provider as mysql + clickAndWait(catalogsPage.catalogProviderSelector); + catalogsPage.clickSelectProvider("jdbc-mysql"); + catalogsPage.setCatalogCommentField("mysql catalog comment"); + // set mysql catalog props + catalogsPage.setCatalogFixedProp("jdbc-driver", MYSQL_JDBC_DRIVER); + catalogsPage.setCatalogFixedProp("jdbc-url", mysqlUri); + catalogsPage.setCatalogFixedProp("jdbc-user", COMMON_JDBC_USER); + catalogsPage.setCatalogFixedProp("jdbc-password", COMMON_JDBC_PWD); + clickAndWait(catalogsPage.handleSubmitCatalogBtn); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(MYSQL_CATALOG_NAME)); + } + + @Test + @Order(4) + public void testCreatePgCatalog() throws InterruptedException { + // create postgresql catalog actions + clickAndWait(catalogsPage.createCatalogBtn); + catalogsPage.setCatalogNameField(PG_CATALOG_NAME); + // select provider as mysql + clickAndWait(catalogsPage.catalogProviderSelector); + catalogsPage.clickSelectProvider("jdbc-postgresql"); + catalogsPage.setCatalogCommentField("postgresql catalog comment"); + // set mysql catalog props + catalogsPage.setCatalogFixedProp("jdbc-driver", PG_JDBC_DRIVER); + catalogsPage.setCatalogFixedProp("jdbc-url", postgresqlUri + ":5432/" + PG_JDBC_DB); + catalogsPage.setCatalogFixedProp("jdbc-user", COMMON_JDBC_USER); + catalogsPage.setCatalogFixedProp("jdbc-password", COMMON_JDBC_PWD); + catalogsPage.setCatalogFixedProp("jdbc-database", PG_JDBC_DB); + + clickAndWait(catalogsPage.handleSubmitCatalogBtn); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(PG_CATALOG_NAME)); + } + + @Test + @Order(5) + public void testCreateFilesetCatalog() throws InterruptedException { + clickAndWait(catalogsPage.createCatalogBtn); + catalogsPage.setCatalogNameField(FILESET_CATALOG_NAME); + clickAndWait(catalogsPage.catalogTypeSelector); + catalogsPage.clickSelectType("fileset"); + catalogsPage.setCatalogCommentField("fileset catalog comment"); + clickAndWait(catalogsPage.handleSubmitCatalogBtn); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(FILESET_CATALOG_NAME)); + } + + @Test + @Order(6) public void testRefreshPage() { driver.navigate().refresh(); - Assertions.assertEquals(driver.getTitle(), "Gravitino"); + Assertions.assertEquals(driver.getTitle(), WEB_TITLE); Assertions.assertTrue(catalogsPage.verifyRefreshPage()); + List catalogsNames = + Arrays.asList( + HIVE_CATALOG_NAME, + ICEBERG_CATALOG_NAME, + MYSQL_CATALOG_NAME, + PG_CATALOG_NAME, + FILESET_CATALOG_NAME); + Assertions.assertTrue(catalogsPage.verifyCreatedCatalogs(catalogsNames)); } @Test - @Order(3) + @Order(7) public void testViewTabMetalakeDetails() throws InterruptedException { clickAndWait(catalogsPage.tabDetailsBtn); Assertions.assertTrue(catalogsPage.verifyShowDetailsContent()); @@ -109,70 +257,193 @@ public void testViewTabMetalakeDetails() throws InterruptedException { } @Test - @Order(4) + @Order(8) public void testViewCatalogDetails() throws InterruptedException { - catalogsPage.clickViewCatalogBtn(catalogName); - String hiveMetastoreUris = - String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); - Assertions.assertTrue(catalogsPage.verifyShowCatalogDetails(catalogName, hiveMetastoreUris)); + catalogsPage.clickViewCatalogBtn(HIVE_CATALOG_NAME); + Assertions.assertTrue( + catalogsPage.verifyShowCatalogDetails(HIVE_CATALOG_NAME, hiveMetastoreUri)); } @Test - @Order(5) + @Order(9) public void testEditCatalog() throws InterruptedException { - catalogsPage.clickEditCatalogBtn(catalogName); - catalogsPage.setCatalogNameField(modifiedCatalogName); + catalogsPage.clickEditCatalogBtn(HIVE_CATALOG_NAME); + catalogsPage.setCatalogNameField(MODIFIED_CATALOG_NAME); clickAndWait(catalogsPage.handleSubmitCatalogBtn); - Assertions.assertTrue(catalogsPage.verifyEditedCatalog(modifiedCatalogName)); + Assertions.assertTrue(catalogsPage.verifyEditedCatalog(MODIFIED_CATALOG_NAME)); } + // test catalog show schema list @Test - @Order(6) + @Order(10) public void testClickCatalogLink() { - catalogsPage.clickCatalogLink(metalakeName, modifiedCatalogName, catalogType); - Assertions.assertTrue(catalogsPage.verifyShowTableTitle("Schemas")); + catalogsPage.clickCatalogLink(METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME)); + Assertions.assertTrue(catalogsPage.verifySelectedNode(MODIFIED_CATALOG_NAME)); } @Test - @Order(7) + @Order(11) + public void testRefreshCatalogPage() { + driver.navigate().refresh(); + Assertions.assertEquals(driver.getTitle(), WEB_TITLE); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(SCHEMA_NAME)); + List treeNodes = + Arrays.asList( + MODIFIED_CATALOG_NAME, + SCHEMA_NAME, + ICEBERG_CATALOG_NAME, + MYSQL_CATALOG_NAME, + PG_CATALOG_NAME, + FILESET_CATALOG_NAME); + Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); + Assertions.assertTrue(catalogsPage.verifySelectedNode(MODIFIED_CATALOG_NAME)); + } + + // test schema show table list + @Test + @Order(12) public void testClickSchemaLink() { - catalogsPage.clickSchemaLink(metalakeName, modifiedCatalogName, catalogType, schemaName); - Assertions.assertTrue(catalogsPage.verifyShowTableTitle("Tables")); + // create table + createTableAndColumn( + METALAKE_NAME, MODIFIED_CATALOG_NAME, SCHEMA_NAME, TABLE_NAME, COLUMN_NAME); + catalogsPage.clickSchemaLink(METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME)); + Assertions.assertTrue(catalogsPage.verifySelectedNode(SCHEMA_NAME)); } @Test - @Order(8) + @Order(13) + public void testRefreshSchemaPage() { + driver.navigate().refresh(); + Assertions.assertEquals(driver.getTitle(), WEB_TITLE); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME)); + List treeNodes = + Arrays.asList( + MODIFIED_CATALOG_NAME, + SCHEMA_NAME, + TABLE_NAME, + ICEBERG_CATALOG_NAME, + MYSQL_CATALOG_NAME, + PG_CATALOG_NAME, + FILESET_CATALOG_NAME); + Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); + Assertions.assertTrue(catalogsPage.verifySelectedNode(SCHEMA_NAME)); + } + + // test table show column list + @Test + @Order(14) public void testClickTableLink() { catalogsPage.clickTableLink( - metalakeName, modifiedCatalogName, catalogType, schemaName, tableName); - Assertions.assertTrue(catalogsPage.verifyShowTableTitle("Columns")); + METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME, TABLE_NAME); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); Assertions.assertTrue(catalogsPage.verifyTableColumns()); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME)); + Assertions.assertTrue(catalogsPage.verifySelectedNode(TABLE_NAME)); } @Test - @Order(9) + @Order(15) + public void testRefreshTablePage() { + driver.navigate().refresh(); + Assertions.assertEquals(driver.getTitle(), WEB_TITLE); + Assertions.assertTrue(catalogsPage.verifyRefreshPage()); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyTableColumns()); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME)); + List treeNodes = + Arrays.asList( + MODIFIED_CATALOG_NAME, + SCHEMA_NAME, + TABLE_NAME, + ICEBERG_CATALOG_NAME, + MYSQL_CATALOG_NAME, + PG_CATALOG_NAME, + FILESET_CATALOG_NAME); + Assertions.assertTrue(catalogsPage.verifyTreeNodes(treeNodes)); + } + + @Test + @Order(16) public void testSelectMetalake() throws InterruptedException { - catalogsPage.metalakeSelectChange(metalakeSelectName); + catalogsPage.metalakeSelectChange(METALAKE_SELECT_NAME); Assertions.assertTrue(catalogsPage.verifyEmptyCatalog()); - catalogsPage.metalakeSelectChange(metalakeName); - Assertions.assertTrue(catalogsPage.verifyCreateHiveCatalog(modifiedCatalogName)); + catalogsPage.metalakeSelectChange(METALAKE_NAME); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(MODIFIED_CATALOG_NAME)); } @Test - @Order(10) - public void testDeleteCatalog() throws InterruptedException { - catalogsPage.clickBreadCrumbsToCatalogs(); - catalogsPage.clickDeleteCatalogBtn(modifiedCatalogName); - clickAndWait(catalogsPage.confirmDeleteBtn); - Assertions.assertTrue(catalogsPage.verifyEmptyCatalog()); + @Order(17) + public void testClickTreeList() throws InterruptedException { + String icebergNode = + String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, ICEBERG_CATALOG_NAME, CATALOG_TYPE); + catalogsPage.clickTreeNode(icebergNode); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(ICEBERG_CATALOG_NAME)); + String mysqlNode = + String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, MYSQL_CATALOG_NAME, CATALOG_TYPE); + catalogsPage.clickTreeNode(mysqlNode); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(MYSQL_CATALOG_NAME)); + String pgNode = + String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, PG_CATALOG_NAME, CATALOG_TYPE); + catalogsPage.clickTreeNode(pgNode); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(PG_CATALOG_NAME)); + String filesetNode = + String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, FILESET_CATALOG_NAME, "fileset"); + catalogsPage.clickTreeNode(filesetNode); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(FILESET_CATALOG_NAME)); + String hiveNode = + String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE); + catalogsPage.clickTreeNode(hiveNode); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(CATALOG_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyGetCatalog(MODIFIED_CATALOG_NAME)); + String schemaNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME); + catalogsPage.clickTreeNode(schemaNode); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(SCHEMA_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(TABLE_NAME)); + String tableNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME, TABLE_NAME); + catalogsPage.clickTreeNode(tableNode); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME)); + Assertions.assertTrue(catalogsPage.verifyTableColumns()); } @Test - @Order(11) + @Order(18) + public void testTreeNodeRefresh() throws InterruptedException { + createTableAndColumn( + METALAKE_NAME, MODIFIED_CATALOG_NAME, SCHEMA_NAME, TABLE_NAME_2, COLUMN_NAME_2); + String hiveNode = + String.format("{{%s}}{{%s}}{{%s}}", METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE); + catalogsPage.clickTreeNode(hiveNode); + String schemaNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME); + catalogsPage.clickTreeNodeRefresh(schemaNode); + String tableNode = + String.format( + "{{%s}}{{%s}}{{%s}}{{%s}}{{%s}}", + METALAKE_NAME, MODIFIED_CATALOG_NAME, CATALOG_TYPE, SCHEMA_NAME, TABLE_NAME_2); + catalogsPage.clickTreeNode(tableNode); + Assertions.assertTrue(catalogsPage.verifyShowTableTitle(TABLE_TABLE_TITLE)); + Assertions.assertTrue(catalogsPage.verifyShowDataItemInList(COLUMN_NAME_2)); + Assertions.assertTrue(catalogsPage.verifyTableColumns()); + } + + @Test + @Order(19) public void testBackHomePage() throws InterruptedException { clickAndWait(catalogsPage.backHomeBtn); Assertions.assertTrue(catalogsPage.verifyBackHomePage()); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java index 1921f561a05..17b9c2d6aa3 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java @@ -6,6 +6,7 @@ package com.datastrato.gravitino.integration.test.web.ui.pages; import com.datastrato.gravitino.integration.test.web.ui.utils.AbstractWebIT; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -19,7 +20,6 @@ import org.openqa.selenium.support.ui.WebDriverWait; public class CatalogsPage extends AbstractWebIT { - @FindBy(xpath = "//*[@data-refer='back-home-btn']") public WebElement backHomeBtn; @@ -77,9 +77,6 @@ public class CatalogsPage extends AbstractWebIT { @FindBy(xpath = "//a[@data-refer='metalake-name-link']") public WebElement metalakeNameLink; - @FindBy(xpath = "//div[@data-prev-refer='props-metastore.uris']//input[@name='value']") - public WebElement hiveCatalogURIInput; - @FindBy(xpath = "//div[contains(@class, 'MuiDataGrid-columnHeaders')]//div[@role='row']") public WebElement columnHeaders; @@ -89,6 +86,18 @@ public class CatalogsPage extends AbstractWebIT { @FindBy(xpath = "//*[@data-refer='details-props-table']") public WebElement detailsPropsTable; + @FindBy(xpath = "//*[@data-refer='catalog-provider-selector']") + public WebElement catalogProviderSelector; + + @FindBy(xpath = "//*[@data-refer='catalog-type-selector']") + public WebElement catalogTypeSelector; + + @FindBy(xpath = "//ul[@aria-labelledby='select-catalog-provider']") + public WebElement catalogProviderList; + + @FindBy(xpath = "//ul[@aria-labelledby='select-catalog-type']") + public WebElement catalogTypeList; + public CatalogsPage() { PageFactory.initElements(driver, this); } @@ -129,15 +138,19 @@ public void setCatalogCommentField(String nameField) { } } - public void setHiveCatalogURI(String value) { + // set the required fixed catalog properties + public void setCatalogFixedProp(String key, String value) { try { - hiveCatalogURIInput.sendKeys(value); + String xpath = "//div[@data-prev-refer='props-" + key + "']//input[@name='value']"; + WebElement propItem = driver.findElement(By.xpath(xpath)); + propItem.sendKeys(value); } catch (Exception e) { LOG.error(e.getMessage(), e); } } - public void setCatalogProps(int index, String key, String value) { + // set the indexed catalog properties + public void setCatalogPropsAt(int index, String key, String value) { try { // Set the indexed props key String keyPath = "//div[@data-refer='props-key-" + index + "']//input[@name='key']"; @@ -267,14 +280,49 @@ public void clickBreadCrumbsToCatalogs() { } } - public boolean verifyCreateHiveCatalog(String name) { + public void clickSelectProvider(String provider) throws InterruptedException { + WebElement providerItem = + catalogProviderList.findElement(By.xpath(".//li[@data-value='" + provider + "']")); + clickAndWait(providerItem); + } + + public void clickSelectType(String type) throws InterruptedException { + WebElement typeItem = + catalogTypeList.findElement(By.xpath(".//li[@data-value='" + type + "']")); + clickAndWait(typeItem); + } + + public void clickTreeNode(String nodeKey) throws InterruptedException { + WebElement treeNode = driver.findElement(By.xpath("//p[@data-refer-node='" + nodeKey + "']")); + clickAndWait(treeNode); + } + + public void clickTreeNodeRefresh(String nodeKey) throws InterruptedException { + WebElement treeNodeRefreshBtn = + driver.findElement(By.xpath("//button[@data-refer='tree-node-refresh-" + nodeKey + "']")); + try { + int reTry = 3; + for (int i = 0; i < reTry; i++) { + clickAndWait(treeNodeRefreshBtn); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + public boolean verifyGetCatalog(String name) { try { String xpath = "//div[contains(@class, 'ant-tree-treenode')]//span[@title='" + name + "']//p[@data-refer='tree-node']"; WebElement treeNode = treeView.findElement(By.xpath(xpath)); - return Objects.equals(treeNode.getText(), name); + boolean match = Objects.equals(treeNode.getText(), name); + if (!match) { + LOG.error("tree node: {} does not match with name: {}", treeNode.getText(), name); + return false; + } + return true; } catch (Exception e) { LOG.error(e.getMessage(), e); return false; @@ -325,7 +373,17 @@ public boolean verifyShowCatalogDetails(String name, String hiveMetastoreUris) By.xpath( ".//*[@data-prev-refer='details-props-key-gravitino.bypass.hive.metastore.client.capability.check']")); - return isVisible && isText && isHiveURIS && isShowCheck; + boolean verifyAll = isVisible && isText && isHiveURIS && isShowCheck; + if (!verifyAll) { + LOG.error( + "not verified all - isVisible: {}, isText: {}, isHiveURIS: {}, isShowCheck: {}", + isVisible, + isText, + isHiveURIS, + isShowCheck); + return false; + } + return true; } catch (Exception e) { LOG.error(e.getMessage(), e); return false; @@ -345,7 +403,12 @@ public boolean verifyEditedCatalog(String name) { wait.until(ExpectedConditions.visibilityOf(treeNode)); // Check if the link text is match with name - return Objects.equals(treeNode.getText(), name); + boolean match = Objects.equals(treeNode.getText(), name); + if (!match) { + LOG.error("tree node {} does not match with name: {}", treeNode.getText(), name); + return false; + } + return true; } catch (Exception e) { LOG.error(e.getMessage(), e); return false; @@ -357,7 +420,10 @@ public boolean verifyEmptyCatalog() { // Check is empty table boolean isNoRows = waitShowText("No rows", tableWrapper); if (!isNoRows) { - LOG.error(tableWrapper.getText(), tableWrapper); + LOG.error( + "is not empty catalog list, tableWrapper text: {}, tableWrapper: {}", + tableWrapper.getText(), + tableWrapper); return false; } return true; @@ -372,7 +438,36 @@ public boolean verifyShowTableTitle(String title) { WebElement text = tabTableBtn.findElement(By.tagName("p")); WebDriverWait wait = new WebDriverWait(driver, MAX_TIMEOUT); wait.until(ExpectedConditions.visibilityOf(text)); - return Objects.equals(text.getText(), title); + boolean matchTitle = Objects.equals(text.getText(), title); + if (!matchTitle) { + LOG.error("table title: {} does not match with title: {}", text.getText(), title); + return false; + } + return true; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + return false; + } + } + + public boolean verifyShowDataItemInList(String itemName) { + try { + Thread.sleep(ACTION_SLEEP_MILLIS); + List list = + driver.findElements( + By.xpath( + "//div[@data-refer='table-grid']//div[contains(@class, 'MuiDataGrid-main')]/div[contains(@class, 'MuiDataGrid-virtualScroller')]/div/div[@role='rowgroup']//div[@data-field='name']")); + List texts = new ArrayList<>(); + for (WebElement element : list) { + texts.add(element.getText()); + } + + if (!texts.contains(itemName)) { + LOG.error("table list: {} does not include itemName: {}", texts, itemName); + return false; + } + + return true; } catch (Exception e) { LOG.error(e.getMessage(), e); return false; @@ -386,14 +481,14 @@ public boolean verifyTableColumns() { List columnHeadersRows = columnHeaders.findElements(By.xpath("./div[@role='columnheader']")); if (columnHeadersRows.size() != columns.size()) { - LOG.error("Column headers count does not match expected: {}", columns.size()); + LOG.error("Column headers count does not match, expected: {}", columns.size()); return false; } for (int i = 0; i < columnHeadersRows.size(); i++) { String headerText = columnHeadersRows.get(i).getText(); if (!headerText.equals(columns.get(i))) { - LOG.error("Column header '{}' does not match expected '{}'", headerText, columns.get(i)); + LOG.error("Column header '{}' does not match, expected '{}'", headerText, columns.get(i)); return false; } } @@ -409,7 +504,13 @@ public boolean verifyBackHomePage() { try { WebDriverWait wait = new WebDriverWait(driver, MAX_TIMEOUT); wait.until(ExpectedConditions.visibilityOf(metalakePageTitle)); - return Objects.equals(metalakePageTitle.getText(), "Metalakes"); + boolean matchTitle = Objects.equals(metalakePageTitle.getText(), "Metalakes"); + if (!matchTitle) { + LOG.error( + "metalakePageTitle: {} does not match with Metalakes", metalakePageTitle.getText()); + return false; + } + return true; } catch (Exception e) { LOG.error(e.getMessage(), e); return false; @@ -436,4 +537,70 @@ public boolean verifyRefreshPage() { return false; } } + + public boolean verifyCreatedCatalogs(List catalogNames) { + try { + List list = + tableGrid.findElements( + By.xpath( + "./div[contains(@class, 'MuiDataGrid-main')]/div[contains(@class, 'MuiDataGrid-virtualScroller')]/div/div[@role='rowgroup']//div[@data-field='name']")); + List texts = new ArrayList<>(); + for (WebElement webElement : list) { + String rowItemColName = webElement.getText(); + texts.add(rowItemColName); + } + if (!texts.containsAll(catalogNames)) { + LOG.error("table list: {} does not containsAll catalogNames: {}", texts, catalogNames); + return false; + } + return true; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + return false; + } + } + + public boolean verifyTreeNodes(List treeNodes) { + try { + Thread.sleep(ACTION_SLEEP_MILLIS); + List list = + driver.findElements( + By.xpath( + "//div[@data-refer='tree-view']//div[@class='ant-tree-list-holder']/div/div[@class='ant-tree-list-holder-inner']/div[contains(@class, 'ant-tree-treenode')]")); + List texts = new ArrayList<>(); + for (WebElement webElement : list) { + String nodeName = + webElement.findElement(By.xpath(".//span[@class='ant-tree-title']")).getText(); + texts.add(nodeName); + } + if (!treeNodes.containsAll(texts)) { + LOG.error("tree nodes list: {} does not containsAll treeNodes: {}", texts, treeNodes); + return false; + } + return true; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + return false; + } + } + + public boolean verifySelectedNode(String nodeName) { + try { + Thread.sleep(ACTION_SLEEP_MILLIS); + WebElement selectedNode = + driver.findElement( + By.xpath( + "//div[@data-refer='tree-view']//div[contains(@class, 'ant-tree-treenode-selected')]//span[@class='ant-tree-title']")); + waitShowText(nodeName, selectedNode); + if (!selectedNode.getText().equals(nodeName)) { + LOG.error( + "selectedNode: {} does not match with nodeName: {}", selectedNode.getText(), nodeName); + return false; + } + return true; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + return false; + } + } } diff --git a/web/src/app/metalakes/metalake/MetalakeTree.js b/web/src/app/metalakes/metalake/MetalakeTree.js index a024b6e8b9d..ec4e9c9af22 100644 --- a/web/src/app/metalakes/metalake/MetalakeTree.js +++ b/web/src/app/metalakes/metalake/MetalakeTree.js @@ -141,6 +141,7 @@ const MetalakeTree = props => { onClick={e => handleClickIcon(e, nodeProps)} onMouseEnter={e => onMouseEnter(e, nodeProps)} onMouseLeave={e => onMouseLeave(e, nodeProps)} + data-refer={`tree-node-refresh-${nodeProps.data.key}`} > { onClick={e => handleClickIcon(e, nodeProps)} onMouseEnter={e => onMouseEnter(e, nodeProps)} onMouseLeave={e => onMouseLeave(e, nodeProps)} + data-refer={`tree-node-refresh-${nodeProps.data.key}`} > @@ -174,6 +176,7 @@ const MetalakeTree = props => { onClick={e => handleClickIcon(e, nodeProps)} onMouseEnter={e => onMouseEnter(e, nodeProps)} onMouseLeave={e => onMouseLeave(e, nodeProps)} + data-refer={`tree-node-refresh-${nodeProps.data.key}`} > @@ -187,6 +190,7 @@ const MetalakeTree = props => { onClick={e => handleClickIcon(e, nodeProps)} onMouseEnter={e => onMouseEnter(e, nodeProps)} onMouseLeave={e => onMouseLeave(e, nodeProps)} + data-refer={`tree-node-refresh-${nodeProps.data.key}`} > @@ -200,7 +204,11 @@ const MetalakeTree = props => { const renderNode = nodeData => { if (nodeData.path) { return ( - theme.palette.text.secondary }} data-refer='tree-node'> + theme.palette.text.secondary }} + data-refer='tree-node' + data-refer-node={nodeData.key} + > {nodeData.title} ) diff --git a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js index 63963eb1e0d..8e6bbde8244 100644 --- a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js +++ b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js @@ -423,6 +423,7 @@ const CreateCatalogDialog = props => { error={Boolean(errors.type)} labelId='select-catalog-type' disabled={type === 'update'} + data-refer='catalog-type-selector' > relational fileset @@ -451,6 +452,7 @@ const CreateCatalogDialog = props => { error={Boolean(errors.provider)} labelId='select-catalog-provider' disabled={type === 'update'} + data-refer='catalog-provider-selector' > {providerTypes.map(item => { return ( diff --git a/web/src/app/rootLayout/AppBar.js b/web/src/app/rootLayout/AppBar.js index 765fb47050e..0998151dadd 100644 --- a/web/src/app/rootLayout/AppBar.js +++ b/web/src/app/rootLayout/AppBar.js @@ -43,6 +43,7 @@ const AppBar = () => { } const metalakeItems = store.metalakes.map(i => i.name) setMetalakes(metalakeItems) + // eslint-disable-next-line react-hooks/exhaustive-deps }, [store.metalakes]) return (